172 static void	sleepq_resume_thread(struct sleepqueue *sq, struct thread *td,
174 static void	sleepq_remove_thread(struct sleepqueue *sq, struct thread *td);
250 sleepq_free(struct sleepqueue *sq)  in sleepq_free()  argument
253 	uma_zfree(sleepq_zone, sq);  in sleepq_free()
277 	struct sleepqueue *sq;  in sleepq_lookup()  local
282 	LIST_FOREACH(sq, &sc->sc_queues, sq_hash)  in sleepq_lookup()
283 		if (sq->sq_wchan == wchan)  in sleepq_lookup()
284 			return (sq);  in sleepq_lookup()
311 	struct sleepqueue *sq;  in sleepq_add()  local
332 	sq = sleepq_lookup(wchan);  in sleepq_add()
339 	if (sq == NULL) {  in sleepq_add()
343 		sq = td->td_sleepqueue;  in sleepq_add()
345 			KASSERT(TAILQ_EMPTY(&sq->sq_blocked[i]),  in sleepq_add()
347 			KASSERT(sq->sq_blockedcnt[i] == 0,  in sleepq_add()
350 		KASSERT(LIST_EMPTY(&sq->sq_free),  in sleepq_add()
352 		KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));  in sleepq_add()
353 		sq->sq_lock = lock;  in sleepq_add()
363 		sq = td->td_sleepqueue;  in sleepq_add()
364 		LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);  in sleepq_add()
365 		sq->sq_wchan = wchan;  in sleepq_add()
366 		sq->sq_type = flags & SLEEPQ_TYPE;  in sleepq_add()
368 		MPASS(wchan == sq->sq_wchan);  in sleepq_add()
369 		MPASS(lock == sq->sq_lock);  in sleepq_add()
370 		MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);  in sleepq_add()
371 		LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);  in sleepq_add()
374 	TAILQ_INSERT_TAIL(&sq->sq_blocked[queue], td, td_slpq);  in sleepq_add()
375 	sq->sq_blockedcnt[queue]++;  in sleepq_add()
425 	struct sleepqueue *sq;  in sleepq_sleepcnt()  local
429 	sq = sleepq_lookup(wchan);  in sleepq_sleepcnt()
430 	if (sq == NULL)  in sleepq_sleepcnt()
432 	return (sq->sq_blockedcnt[queue]);  in sleepq_sleepcnt()
506 	struct sleepqueue *sq;  in sleepq_catch_signals()  local
530 			sq = sleepq_lookup(wchan);  in sleepq_catch_signals()
531 			sleepq_remove_thread(sq, td);  in sleepq_catch_signals()
549 	struct sleepqueue *sq;  in sleepq_switch()  local
593 		sq = sleepq_lookup(wchan);  in sleepq_switch()
594 		sleepq_remove_thread(sq, td);  in sleepq_switch()
721 	struct sleepqueue *sq;  in sleepq_type()  local
726 	sq = sleepq_lookup(wchan);  in sleepq_type()
727 	if (sq == NULL)  in sleepq_type()
729 	type = sq->sq_type;  in sleepq_type()
741 sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri,  in sleepq_resume_thread()  argument
748 	MPASS(sq->sq_wchan != NULL);  in sleepq_resume_thread()
749 	MPASS(td->td_wchan == sq->sq_wchan);  in sleepq_resume_thread()
751 	sc = SC_LOOKUP(sq->sq_wchan);  in sleepq_resume_thread()
770 	sleepq_remove_thread(sq, td);  in sleepq_resume_thread()
799 sleepq_remove_thread(struct sleepqueue *sq, struct thread *td)  in sleepq_remove_thread()  argument
804 	MPASS(sq->sq_wchan != NULL);  in sleepq_remove_thread()
805 	MPASS(td->td_wchan == sq->sq_wchan);  in sleepq_remove_thread()
808 	sc = SC_LOOKUP(sq->sq_wchan);  in sleepq_remove_thread()
814 	sq->sq_blockedcnt[td->td_sqqueue]--;  in sleepq_remove_thread()
815 	TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq);  in sleepq_remove_thread()
822 	if (LIST_EMPTY(&sq->sq_free)) {  in sleepq_remove_thread()
823 		td->td_sleepqueue = sq;  in sleepq_remove_thread()
825 		sq->sq_wchan = NULL;  in sleepq_remove_thread()
831 		td->td_sleepqueue = LIST_FIRST(&sq->sq_free);  in sleepq_remove_thread()
868 	struct sleepqueue *sq;  in sleepq_remove_nested()  local
876 	sq = sleepq_lookup(wchan);  in sleepq_remove_nested()
877 	MPASS(sq != NULL);  in sleepq_remove_nested()
879 	sleepq_remove_thread(sq, td);  in sleepq_remove_nested()
891 	struct sleepqueue *sq;  in sleepq_dtor()  local
894 	sq = mem;  in sleepq_dtor()
896 		MPASS(TAILQ_EMPTY(&sq->sq_blocked[i]));  in sleepq_dtor()
897 		MPASS(sq->sq_blockedcnt[i] == 0);  in sleepq_dtor()
908 	struct sleepqueue *sq;  in sleepq_init()  local
912 	sq = mem;  in sleepq_init()
914 		TAILQ_INIT(&sq->sq_blocked[i]);  in sleepq_init()
915 		sq->sq_blockedcnt[i] = 0;  in sleepq_init()
917 	LIST_INIT(&sq->sq_free);  in sleepq_init()
928 	struct sleepqueue *sq;  in sleepq_signal()  local
935 	sq = sleepq_lookup(wchan);  in sleepq_signal()
936 	if (sq == NULL) {  in sleepq_signal()
941 	KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),  in sleepq_signal()
944 	head = &sq->sq_blocked[queue];  in sleepq_signal()
973 	sleepq_resume_thread(sq, besttd, pri,  in sleepq_signal()
990 	struct sleepqueue *sq;  in sleepq_broadcast()  local
995 	sq = sleepq_lookup(wchan);  in sleepq_broadcast()
996 	if (sq != NULL) {  in sleepq_broadcast()
997 		KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),  in sleepq_broadcast()
1000 		sleepq_remove_matching(sq, queue, match_any, pri);  in sleepq_broadcast()
1008 sleepq_remove_matching(struct sleepqueue *sq, int queue,  in sleepq_remove_matching()  argument
1019 	TAILQ_FOREACH_SAFE(td, &sq->sq_blocked[queue], td_slpq, tdn) {  in sleepq_remove_matching()
1021 			sleepq_resume_thread(sq, td, pri, SRQ_HOLD);  in sleepq_remove_matching()
1033 	struct sleepqueue *sq;  in sleepq_timeout()  local
1055 		sq = sleepq_lookup(wchan);  in sleepq_timeout()
1056 		MPASS(sq != NULL);  in sleepq_timeout()
1058 		sleepq_resume_thread(sq, td, 0, 0);  in sleepq_timeout()
1080 	struct sleepqueue *sq;  in sleepq_remove()  local
1102 	sq = sleepq_lookup(wchan);  in sleepq_remove()
1103 	MPASS(sq != NULL);  in sleepq_remove()
1105 	sleepq_resume_thread(sq, td, 0, 0);  in sleepq_remove()
1117 	struct sleepqueue *sq;  in sleepq_abort()  local
1150 	sq = sleepq_lookup(wchan);  in sleepq_abort()
1151 	MPASS(sq != NULL);  in sleepq_abort()
1154 	sleepq_resume_thread(sq, td, 0, 0);  in sleepq_abort()
1161 	struct sleepqueue *sq, *sq1;  in sleepq_chains_remove_matching()  local
1169 		LIST_FOREACH_SAFE(sq, &sc->sc_queues, sq_hash, sq1) {  in sleepq_chains_remove_matching()
1171 				sleepq_remove_matching(sq, i, matches, 0);  in sleepq_chains_remove_matching()
1189 	struct sleepqueue *sq;  in sleepq_sbuf_print_stacks()  local
1231 		sq = sleepq_lookup(wchan);  in sleepq_sbuf_print_stacks()
1232 		if (sq == NULL) {  in sleepq_sbuf_print_stacks()
1242 		TAILQ_FOREACH_SAFE(td, &sq->sq_blocked[queue], td_slpq,  in sleepq_sbuf_print_stacks()
1447 	struct sleepqueue *sq;  in DB_SHOW_COMMAND()  local
1464 	LIST_FOREACH(sq, &sc->sc_queues, sq_hash)  in DB_SHOW_COMMAND()
1465 		if (sq->sq_wchan == wchan)  in DB_SHOW_COMMAND()
1473 		LIST_FOREACH(sq, &sleepq_chains[i].sc_queues, sq_hash) {  in DB_SHOW_COMMAND()
1474 			if (sq == (struct sleepqueue *)addr)  in DB_SHOW_COMMAND()
1481 	db_printf("Wait channel: %p\n", sq->sq_wchan);  in DB_SHOW_COMMAND()
1482 	db_printf("Queue type: %d\n", sq->sq_type);  in DB_SHOW_COMMAND()
1484 	if (sq->sq_lock) {  in DB_SHOW_COMMAND()
1485 		lock = sq->sq_lock;  in DB_SHOW_COMMAND()
1493 		if (TAILQ_EMPTY(&sq->sq_blocked[i]))  in DB_SHOW_COMMAND()
1496 			TAILQ_FOREACH(td, &sq->sq_blocked[i],  in DB_SHOW_COMMAND()
1502 		db_printf("(expected: %u)\n", sq->sq_blockedcnt[i]);  in DB_SHOW_COMMAND()