xref: /freebsd/sys/kern/kern_condvar.c (revision e3043798aa79660648a80166c1b48461668c4887)
1238510fcSJason Evans /*-
2238510fcSJason Evans  * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>.
3238510fcSJason Evans  * All rights reserved.
4238510fcSJason Evans  *
5238510fcSJason Evans  * Redistribution and use in source and binary forms, with or without
6238510fcSJason Evans  * modification, are permitted provided that the following conditions
7238510fcSJason Evans  * are met:
8238510fcSJason Evans  * 1. Redistributions of source code must retain the above copyright
9238510fcSJason Evans  *    notice, this list of conditions and the following disclaimer.
10238510fcSJason Evans  * 2. Redistributions in binary form must reproduce the above copyright
11238510fcSJason Evans  *    notice, this list of conditions and the following disclaimer in the
12238510fcSJason Evans  *    documentation and/or other materials provided with the distribution.
13238510fcSJason Evans  *
14238510fcSJason Evans  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15238510fcSJason Evans  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16238510fcSJason Evans  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17238510fcSJason Evans  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18238510fcSJason Evans  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19238510fcSJason Evans  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20238510fcSJason Evans  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21238510fcSJason Evans  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22238510fcSJason Evans  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23238510fcSJason Evans  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24238510fcSJason Evans  * SUCH DAMAGE.
25238510fcSJason Evans  */
26238510fcSJason Evans 
27677b542eSDavid E. O'Brien #include <sys/cdefs.h>
28677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
29677b542eSDavid E. O'Brien 
30238510fcSJason Evans #include "opt_ktrace.h"
31238510fcSJason Evans 
32238510fcSJason Evans #include <sys/param.h>
33238510fcSJason Evans #include <sys/systm.h>
3411f9ca69SMark Johnston #include <sys/limits.h>
35fb919e4dSMark Murray #include <sys/lock.h>
36fb919e4dSMark Murray #include <sys/mutex.h>
37238510fcSJason Evans #include <sys/proc.h>
38238510fcSJason Evans #include <sys/kernel.h>
39238510fcSJason Evans #include <sys/ktr.h>
40238510fcSJason Evans #include <sys/condvar.h>
414e997f4bSJeff Roberson #include <sys/sched.h>
42238510fcSJason Evans #include <sys/signalvar.h>
4344f3b092SJohn Baldwin #include <sys/sleepqueue.h>
44238510fcSJason Evans #include <sys/resourcevar.h>
45238510fcSJason Evans #ifdef KTRACE
46238510fcSJason Evans #include <sys/uio.h>
47238510fcSJason Evans #include <sys/ktrace.h>
48238510fcSJason Evans #endif
49238510fcSJason Evans 
50238510fcSJason Evans /*
5111f9ca69SMark Johnston  * A bound below which cv_waiters is valid.  Once cv_waiters reaches this bound,
5211f9ca69SMark Johnston  * cv_signal must manually check the wait queue for threads.
5311f9ca69SMark Johnston  */
5411f9ca69SMark Johnston #define	CV_WAITERS_BOUND	INT_MAX
5511f9ca69SMark Johnston 
5611f9ca69SMark Johnston #define	CV_WAITERS_INC(cvp) do {					\
5711f9ca69SMark Johnston 	if ((cvp)->cv_waiters < CV_WAITERS_BOUND)			\
5811f9ca69SMark Johnston 		(cvp)->cv_waiters++;					\
5911f9ca69SMark Johnston } while (0)
6011f9ca69SMark Johnston 
6111f9ca69SMark Johnston /*
62238510fcSJason Evans  * Common sanity checks for cv_wait* functions.
63238510fcSJason Evans  */
648f27b08eSJohn Baldwin #define	CV_ASSERT(cvp, lock, td) do {					\
650a15e5d3SAttilio Rao 	KASSERT((td) != NULL, ("%s: td NULL", __func__));		\
6671fad9fdSJulian Elischer 	KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__));	\
67a48740b6SDavid E. O'Brien 	KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__));		\
688f27b08eSJohn Baldwin 	KASSERT((lock) != NULL, ("%s: lock NULL", __func__));		\
69238510fcSJason Evans } while (0)
70238510fcSJason Evans 
71238510fcSJason Evans /*
72238510fcSJason Evans  * Initialize a condition variable.  Must be called before use.
73238510fcSJason Evans  */
74238510fcSJason Evans void
75238510fcSJason Evans cv_init(struct cv *cvp, const char *desc)
76238510fcSJason Evans {
77238510fcSJason Evans 
78238510fcSJason Evans 	cvp->cv_description = desc;
799000d57dSJohn Baldwin 	cvp->cv_waiters = 0;
80238510fcSJason Evans }
81238510fcSJason Evans 
82238510fcSJason Evans /*
83238510fcSJason Evans  * Destroy a condition variable.  The condition variable must be re-initialized
84238510fcSJason Evans  * in order to be re-used.
85238510fcSJason Evans  */
86238510fcSJason Evans void
87238510fcSJason Evans cv_destroy(struct cv *cvp)
88238510fcSJason Evans {
8944f3b092SJohn Baldwin #ifdef INVARIANTS
9044f3b092SJohn Baldwin 	struct sleepqueue *sq;
91238510fcSJason Evans 
922ff0e645SJohn Baldwin 	sleepq_lock(cvp);
9344f3b092SJohn Baldwin 	sq = sleepq_lookup(cvp);
9444f3b092SJohn Baldwin 	sleepq_release(cvp);
9544f3b092SJohn Baldwin 	KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__));
9644f3b092SJohn Baldwin #endif
97238510fcSJason Evans }
98238510fcSJason Evans 
99238510fcSJason Evans /*
100b40ce416SJulian Elischer  * Wait on a condition variable.  The current thread is placed on the condition
101238510fcSJason Evans  * variable's wait queue and suspended.  A cv_signal or cv_broadcast on the same
102b40ce416SJulian Elischer  * condition variable will resume the thread.  The mutex is released before
103238510fcSJason Evans  * sleeping and will be held on return.  It is recommended that the mutex be
104238510fcSJason Evans  * held when cv_signal or cv_broadcast are called.
105238510fcSJason Evans  */
106238510fcSJason Evans void
1078f27b08eSJohn Baldwin _cv_wait(struct cv *cvp, struct lock_object *lock)
108238510fcSJason Evans {
1098f27b08eSJohn Baldwin 	WITNESS_SAVE_DECL(lock_witness);
1108f27b08eSJohn Baldwin 	struct lock_class *class;
111503916a7SJohn Baldwin 	struct thread *td;
1127faf4d90SDavide Italiano 	uintptr_t lock_state;
113238510fcSJason Evans 
114503916a7SJohn Baldwin 	td = curthread;
115414e7679SJohn Baldwin 	lock_state = 0;
116503916a7SJohn Baldwin #ifdef KTRACE
117503916a7SJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
11888bf5036SJohn Baldwin 		ktrcsw(1, 0, cv_wmesg(cvp));
119503916a7SJohn Baldwin #endif
1208f27b08eSJohn Baldwin 	CV_ASSERT(cvp, lock, td);
1218f27b08eSJohn Baldwin 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
122503916a7SJohn Baldwin 	    "Waiting on \"%s\"", cvp->cv_description);
1238f27b08eSJohn Baldwin 	class = LOCK_CLASS(lock);
124238510fcSJason Evans 
125b4f1d267SJohn Baldwin 	if (SCHEDULER_STOPPED())
126238510fcSJason Evans 		return;
1274bc37205SJeffrey Hsu 
128503916a7SJohn Baldwin 	sleepq_lock(cvp);
129503916a7SJohn Baldwin 
13011f9ca69SMark Johnston 	CV_WAITERS_INC(cvp);
1317d43ca69SJohn Baldwin 	if (lock == &Giant.lock_object)
1327d43ca69SJohn Baldwin 		mtx_assert(&Giant, MA_OWNED);
133503916a7SJohn Baldwin 	DROP_GIANT();
134503916a7SJohn Baldwin 
1358f27b08eSJohn Baldwin 	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
136414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
1379fa7ce0fSJohn Baldwin 		if (class->lc_flags & LC_SLEEPABLE)
1389fa7ce0fSJohn Baldwin 			sleepq_release(cvp);
1397d43ca69SJohn Baldwin 		WITNESS_SAVE(lock, lock_witness);
1409fa7ce0fSJohn Baldwin 		lock_state = class->lc_unlock(lock);
1419fa7ce0fSJohn Baldwin 		if (class->lc_flags & LC_SLEEPABLE)
1429fa7ce0fSJohn Baldwin 			sleepq_lock(cvp);
143414e7679SJohn Baldwin 	}
144c5aa6b58SJeff Roberson 	sleepq_wait(cvp, 0);
145503916a7SJohn Baldwin 
146503916a7SJohn Baldwin #ifdef KTRACE
147503916a7SJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
14888bf5036SJohn Baldwin 		ktrcsw(0, 0, cv_wmesg(cvp));
149503916a7SJohn Baldwin #endif
150503916a7SJohn Baldwin 	PICKUP_GIANT();
151414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
1528f27b08eSJohn Baldwin 		class->lc_lock(lock, lock_state);
1538f27b08eSJohn Baldwin 		WITNESS_RESTORE(lock, lock_witness);
15492f44a3fSCraig Rodrigues 	}
155414e7679SJohn Baldwin }
15692f44a3fSCraig Rodrigues 
15792f44a3fSCraig Rodrigues /*
15892f44a3fSCraig Rodrigues  * Wait on a condition variable.  This function differs from cv_wait by
159*e3043798SPedro F. Giffuni  * not acquiring the mutex after condition variable was signaled.
16092f44a3fSCraig Rodrigues  */
16192f44a3fSCraig Rodrigues void
1628f27b08eSJohn Baldwin _cv_wait_unlock(struct cv *cvp, struct lock_object *lock)
16392f44a3fSCraig Rodrigues {
1648f27b08eSJohn Baldwin 	struct lock_class *class;
16592f44a3fSCraig Rodrigues 	struct thread *td;
16692f44a3fSCraig Rodrigues 
16792f44a3fSCraig Rodrigues 	td = curthread;
16892f44a3fSCraig Rodrigues #ifdef KTRACE
16992f44a3fSCraig Rodrigues 	if (KTRPOINT(td, KTR_CSW))
17088bf5036SJohn Baldwin 		ktrcsw(1, 0, cv_wmesg(cvp));
17192f44a3fSCraig Rodrigues #endif
1728f27b08eSJohn Baldwin 	CV_ASSERT(cvp, lock, td);
1738f27b08eSJohn Baldwin 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
17492f44a3fSCraig Rodrigues 	    "Waiting on \"%s\"", cvp->cv_description);
175414e7679SJohn Baldwin 	KASSERT(lock != &Giant.lock_object,
176414e7679SJohn Baldwin 	    ("cv_wait_unlock cannot be used with Giant"));
1778f27b08eSJohn Baldwin 	class = LOCK_CLASS(lock);
17892f44a3fSCraig Rodrigues 
179b4f1d267SJohn Baldwin 	if (SCHEDULER_STOPPED()) {
1808f27b08eSJohn Baldwin 		class->lc_unlock(lock);
18192f44a3fSCraig Rodrigues 		return;
18292f44a3fSCraig Rodrigues 	}
18392f44a3fSCraig Rodrigues 
1842ff0e645SJohn Baldwin 	sleepq_lock(cvp);
185238510fcSJason Evans 
18611f9ca69SMark Johnston 	CV_WAITERS_INC(cvp);
187c86b6ff5SJohn Baldwin 	DROP_GIANT();
188238510fcSJason Evans 
1898f27b08eSJohn Baldwin 	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
1909fa7ce0fSJohn Baldwin 	if (class->lc_flags & LC_SLEEPABLE)
1919fa7ce0fSJohn Baldwin 		sleepq_release(cvp);
1929fa7ce0fSJohn Baldwin 	class->lc_unlock(lock);
1939fa7ce0fSJohn Baldwin 	if (class->lc_flags & LC_SLEEPABLE)
1949fa7ce0fSJohn Baldwin 		sleepq_lock(cvp);
195c5aa6b58SJeff Roberson 	sleepq_wait(cvp, 0);
196238510fcSJason Evans 
197503916a7SJohn Baldwin #ifdef KTRACE
198503916a7SJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
19988bf5036SJohn Baldwin 		ktrcsw(0, 0, cv_wmesg(cvp));
200503916a7SJohn Baldwin #endif
201238510fcSJason Evans 	PICKUP_GIANT();
202238510fcSJason Evans }
203238510fcSJason Evans 
204238510fcSJason Evans /*
205238510fcSJason Evans  * Wait on a condition variable, allowing interruption by signals.  Return 0 if
206b40ce416SJulian Elischer  * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if
207238510fcSJason Evans  * a signal was caught.  If ERESTART is returned the system call should be
208238510fcSJason Evans  * restarted if possible.
209238510fcSJason Evans  */
210238510fcSJason Evans int
2118f27b08eSJohn Baldwin _cv_wait_sig(struct cv *cvp, struct lock_object *lock)
212238510fcSJason Evans {
2138f27b08eSJohn Baldwin 	WITNESS_SAVE_DECL(lock_witness);
2148f27b08eSJohn Baldwin 	struct lock_class *class;
215b40ce416SJulian Elischer 	struct thread *td;
2167faf4d90SDavide Italiano 	uintptr_t lock_state;
2177faf4d90SDavide Italiano 	int rval;
218238510fcSJason Evans 
219b40ce416SJulian Elischer 	td = curthread;
220414e7679SJohn Baldwin 	lock_state = 0;
221238510fcSJason Evans #ifdef KTRACE
2229ba7fe1bSJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
22388bf5036SJohn Baldwin 		ktrcsw(1, 0, cv_wmesg(cvp));
224238510fcSJason Evans #endif
2258f27b08eSJohn Baldwin 	CV_ASSERT(cvp, lock, td);
2268f27b08eSJohn Baldwin 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
22726306795SJohn Baldwin 	    "Waiting on \"%s\"", cvp->cv_description);
2288f27b08eSJohn Baldwin 	class = LOCK_CLASS(lock);
229238510fcSJason Evans 
230b4f1d267SJohn Baldwin 	if (SCHEDULER_STOPPED())
231274f8f48SJohn Baldwin 		return (0);
2324bc37205SJeffrey Hsu 
2332ff0e645SJohn Baldwin 	sleepq_lock(cvp);
2344bc37205SJeffrey Hsu 
23511f9ca69SMark Johnston 	CV_WAITERS_INC(cvp);
2367d43ca69SJohn Baldwin 	if (lock == &Giant.lock_object)
2377d43ca69SJohn Baldwin 		mtx_assert(&Giant, MA_OWNED);
238c86b6ff5SJohn Baldwin 	DROP_GIANT();
239238510fcSJason Evans 
2408f27b08eSJohn Baldwin 	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
2416cbb70e2SKip Macy 	    SLEEPQ_INTERRUPTIBLE, 0);
242414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
2439fa7ce0fSJohn Baldwin 		if (class->lc_flags & LC_SLEEPABLE)
2449fa7ce0fSJohn Baldwin 			sleepq_release(cvp);
2457d43ca69SJohn Baldwin 		WITNESS_SAVE(lock, lock_witness);
2469fa7ce0fSJohn Baldwin 		lock_state = class->lc_unlock(lock);
2479fa7ce0fSJohn Baldwin 		if (class->lc_flags & LC_SLEEPABLE)
2489fa7ce0fSJohn Baldwin 			sleepq_lock(cvp);
249414e7679SJohn Baldwin 	}
250c5aa6b58SJeff Roberson 	rval = sleepq_wait_sig(cvp, 0);
251238510fcSJason Evans 
252238510fcSJason Evans #ifdef KTRACE
2539ba7fe1bSJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
25488bf5036SJohn Baldwin 		ktrcsw(0, 0, cv_wmesg(cvp));
255238510fcSJason Evans #endif
2569ba7fe1bSJohn Baldwin 	PICKUP_GIANT();
257414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
2588f27b08eSJohn Baldwin 		class->lc_lock(lock, lock_state);
2598f27b08eSJohn Baldwin 		WITNESS_RESTORE(lock, lock_witness);
260414e7679SJohn Baldwin 	}
261238510fcSJason Evans 
262238510fcSJason Evans 	return (rval);
263238510fcSJason Evans }
264238510fcSJason Evans 
265238510fcSJason Evans /*
26646153735SDavide Italiano  * Wait on a condition variable for (at most) the value specified in sbt
26746153735SDavide Italiano  * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast,
26846153735SDavide Italiano  * EWOULDBLOCK if the timeout expires.
269238510fcSJason Evans  */
270238510fcSJason Evans int
27146153735SDavide Italiano _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
27246153735SDavide Italiano     sbintime_t pr, int flags)
273238510fcSJason Evans {
2748f27b08eSJohn Baldwin 	WITNESS_SAVE_DECL(lock_witness);
2758f27b08eSJohn Baldwin 	struct lock_class *class;
276b40ce416SJulian Elischer 	struct thread *td;
2778f27b08eSJohn Baldwin 	int lock_state, rval;
278238510fcSJason Evans 
279b40ce416SJulian Elischer 	td = curthread;
280414e7679SJohn Baldwin 	lock_state = 0;
281238510fcSJason Evans #ifdef KTRACE
2829ba7fe1bSJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
28388bf5036SJohn Baldwin 		ktrcsw(1, 0, cv_wmesg(cvp));
284238510fcSJason Evans #endif
2858f27b08eSJohn Baldwin 	CV_ASSERT(cvp, lock, td);
2868f27b08eSJohn Baldwin 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
28726306795SJohn Baldwin 	    "Waiting on \"%s\"", cvp->cv_description);
2888f27b08eSJohn Baldwin 	class = LOCK_CLASS(lock);
289238510fcSJason Evans 
290b4f1d267SJohn Baldwin 	if (SCHEDULER_STOPPED())
291b4f1d267SJohn Baldwin 		return (0);
2924bc37205SJeffrey Hsu 
2932ff0e645SJohn Baldwin 	sleepq_lock(cvp);
294238510fcSJason Evans 
29511f9ca69SMark Johnston 	CV_WAITERS_INC(cvp);
2967d43ca69SJohn Baldwin 	if (lock == &Giant.lock_object)
2977d43ca69SJohn Baldwin 		mtx_assert(&Giant, MA_OWNED);
298c86b6ff5SJohn Baldwin 	DROP_GIANT();
299238510fcSJason Evans 
3008f27b08eSJohn Baldwin 	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
30146153735SDavide Italiano 	sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
302414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
303a115fb62SHans Petter Selasky 		if (class->lc_flags & LC_SLEEPABLE)
304a115fb62SHans Petter Selasky 			sleepq_release(cvp);
3057d43ca69SJohn Baldwin 		WITNESS_SAVE(lock, lock_witness);
3069fa7ce0fSJohn Baldwin 		lock_state = class->lc_unlock(lock);
307a115fb62SHans Petter Selasky 		if (class->lc_flags & LC_SLEEPABLE)
3081a26c3c0SHans Petter Selasky 			sleepq_lock(cvp);
309a115fb62SHans Petter Selasky 	}
310c5aa6b58SJeff Roberson 	rval = sleepq_timedwait(cvp, 0);
311238510fcSJason Evans 
312238510fcSJason Evans #ifdef KTRACE
3139ba7fe1bSJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
31488bf5036SJohn Baldwin 		ktrcsw(0, 0, cv_wmesg(cvp));
315238510fcSJason Evans #endif
316238510fcSJason Evans 	PICKUP_GIANT();
317414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
3188f27b08eSJohn Baldwin 		class->lc_lock(lock, lock_state);
3198f27b08eSJohn Baldwin 		WITNESS_RESTORE(lock, lock_witness);
320414e7679SJohn Baldwin 	}
321238510fcSJason Evans 
322238510fcSJason Evans 	return (rval);
323238510fcSJason Evans }
324238510fcSJason Evans 
325238510fcSJason Evans /*
32646153735SDavide Italiano  * Wait on a condition variable for (at most) the value specified in sbt
32746153735SDavide Italiano  * argument, allowing interruption by signals.
32846153735SDavide Italiano  * Returns 0 if the thread was resumed by cv_signal or cv_broadcast,
32946153735SDavide Italiano  * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal
33046153735SDavide Italiano  * was caught.
331238510fcSJason Evans  */
332238510fcSJason Evans int
33346153735SDavide Italiano _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
33446153735SDavide Italiano     sbintime_t sbt, sbintime_t pr, int flags)
335238510fcSJason Evans {
3368f27b08eSJohn Baldwin 	WITNESS_SAVE_DECL(lock_witness);
3378f27b08eSJohn Baldwin 	struct lock_class *class;
338b40ce416SJulian Elischer 	struct thread *td;
3398f27b08eSJohn Baldwin 	int lock_state, rval;
340238510fcSJason Evans 
341b40ce416SJulian Elischer 	td = curthread;
342414e7679SJohn Baldwin 	lock_state = 0;
343238510fcSJason Evans #ifdef KTRACE
3449ba7fe1bSJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
34588bf5036SJohn Baldwin 		ktrcsw(1, 0, cv_wmesg(cvp));
346238510fcSJason Evans #endif
3478f27b08eSJohn Baldwin 	CV_ASSERT(cvp, lock, td);
3488f27b08eSJohn Baldwin 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
34926306795SJohn Baldwin 	    "Waiting on \"%s\"", cvp->cv_description);
3508f27b08eSJohn Baldwin 	class = LOCK_CLASS(lock);
351238510fcSJason Evans 
352b4f1d267SJohn Baldwin 	if (SCHEDULER_STOPPED())
353b4f1d267SJohn Baldwin 		return (0);
3544bc37205SJeffrey Hsu 
3552ff0e645SJohn Baldwin 	sleepq_lock(cvp);
356238510fcSJason Evans 
35711f9ca69SMark Johnston 	CV_WAITERS_INC(cvp);
3587d43ca69SJohn Baldwin 	if (lock == &Giant.lock_object)
3597d43ca69SJohn Baldwin 		mtx_assert(&Giant, MA_OWNED);
360c86b6ff5SJohn Baldwin 	DROP_GIANT();
361238510fcSJason Evans 
3628f27b08eSJohn Baldwin 	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
3636cbb70e2SKip Macy 	    SLEEPQ_INTERRUPTIBLE, 0);
36446153735SDavide Italiano 	sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
365414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
366a115fb62SHans Petter Selasky 		if (class->lc_flags & LC_SLEEPABLE)
367a115fb62SHans Petter Selasky 			sleepq_release(cvp);
3687d43ca69SJohn Baldwin 		WITNESS_SAVE(lock, lock_witness);
3699fa7ce0fSJohn Baldwin 		lock_state = class->lc_unlock(lock);
370a115fb62SHans Petter Selasky 		if (class->lc_flags & LC_SLEEPABLE)
3711a26c3c0SHans Petter Selasky 			sleepq_lock(cvp);
372a115fb62SHans Petter Selasky 	}
373c5aa6b58SJeff Roberson 	rval = sleepq_timedwait_sig(cvp, 0);
374238510fcSJason Evans 
375238510fcSJason Evans #ifdef KTRACE
3769ba7fe1bSJohn Baldwin 	if (KTRPOINT(td, KTR_CSW))
37788bf5036SJohn Baldwin 		ktrcsw(0, 0, cv_wmesg(cvp));
378238510fcSJason Evans #endif
3799ba7fe1bSJohn Baldwin 	PICKUP_GIANT();
380414e7679SJohn Baldwin 	if (lock != &Giant.lock_object) {
3818f27b08eSJohn Baldwin 		class->lc_lock(lock, lock_state);
3828f27b08eSJohn Baldwin 		WITNESS_RESTORE(lock, lock_witness);
383414e7679SJohn Baldwin 	}
384238510fcSJason Evans 
385238510fcSJason Evans 	return (rval);
386238510fcSJason Evans }
387238510fcSJason Evans 
388238510fcSJason Evans /*
389b40ce416SJulian Elischer  * Signal a condition variable, wakes up one waiting thread.  Will also wakeup
390238510fcSJason Evans  * the swapper if the process is not in memory, so that it can bring the
391b40ce416SJulian Elischer  * sleeping process in.  Note that this may also result in additional threads
392238510fcSJason Evans  * being made runnable.  Should be called with the same mutex as was passed to
393238510fcSJason Evans  * cv_wait held.
394238510fcSJason Evans  */
395238510fcSJason Evans void
396238510fcSJason Evans cv_signal(struct cv *cvp)
397238510fcSJason Evans {
398da7bbd2cSJohn Baldwin 	int wakeup_swapper;
399238510fcSJason Evans 
400da7bbd2cSJohn Baldwin 	wakeup_swapper = 0;
4012ff0e645SJohn Baldwin 	sleepq_lock(cvp);
402c636f94bSJohn Baldwin 	if (cvp->cv_waiters > 0) {
40311f9ca69SMark Johnston 		if (cvp->cv_waiters == CV_WAITERS_BOUND &&
40411f9ca69SMark Johnston 		    sleepq_lookup(cvp) == NULL) {
40511f9ca69SMark Johnston 			cvp->cv_waiters = 0;
40611f9ca69SMark Johnston 		} else {
40711f9ca69SMark Johnston 			if (cvp->cv_waiters < CV_WAITERS_BOUND)
408c636f94bSJohn Baldwin 				cvp->cv_waiters--;
40911f9ca69SMark Johnston 			wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0,
41011f9ca69SMark Johnston 			    0);
41111f9ca69SMark Johnston 		}
412c636f94bSJohn Baldwin 	}
4132ff0e645SJohn Baldwin 	sleepq_release(cvp);
414da7bbd2cSJohn Baldwin 	if (wakeup_swapper)
415da7bbd2cSJohn Baldwin 		kick_proc0();
4169000d57dSJohn Baldwin }
417238510fcSJason Evans 
418238510fcSJason Evans /*
419b40ce416SJulian Elischer  * Broadcast a signal to a condition variable.  Wakes up all waiting threads.
420238510fcSJason Evans  * Should be called with the same mutex as was passed to cv_wait held.
421238510fcSJason Evans  */
422238510fcSJason Evans void
423512824f8SSeigo Tanimura cv_broadcastpri(struct cv *cvp, int pri)
424238510fcSJason Evans {
425da7bbd2cSJohn Baldwin 	int wakeup_swapper;
426da7bbd2cSJohn Baldwin 
427c5aa6b58SJeff Roberson 	/*
428c5aa6b58SJeff Roberson 	 * XXX sleepq_broadcast pri argument changed from -1 meaning
429c5aa6b58SJeff Roberson 	 * no pri to 0 meaning no pri.
430c5aa6b58SJeff Roberson 	 */
431da7bbd2cSJohn Baldwin 	wakeup_swapper = 0;
432c5aa6b58SJeff Roberson 	if (pri == -1)
433c5aa6b58SJeff Roberson 		pri = 0;
4342ff0e645SJohn Baldwin 	sleepq_lock(cvp);
435c636f94bSJohn Baldwin 	if (cvp->cv_waiters > 0) {
436c636f94bSJohn Baldwin 		cvp->cv_waiters = 0;
437da7bbd2cSJohn Baldwin 		wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
438c636f94bSJohn Baldwin 	}
4392ff0e645SJohn Baldwin 	sleepq_release(cvp);
440da7bbd2cSJohn Baldwin 	if (wakeup_swapper)
441da7bbd2cSJohn Baldwin 		kick_proc0();
4429000d57dSJohn Baldwin }
443