xref: /freebsd/sys/kern/kern_lock.c (revision 84887fa3620e529b82b433f946a3af761d18e34d)
19454b2d8SWarner Losh /*-
253bf4bb2SPeter Wemm  * Copyright (c) 1995
353bf4bb2SPeter Wemm  *	The Regents of the University of California.  All rights reserved.
453bf4bb2SPeter Wemm  *
503e9c6c1SJohn Dyson  * Copyright (C) 1997
603e9c6c1SJohn Dyson  *	John S. Dyson.  All rights reserved.
703e9c6c1SJohn Dyson  *
853bf4bb2SPeter Wemm  * This code contains ideas from software contributed to Berkeley by
953bf4bb2SPeter Wemm  * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
1053bf4bb2SPeter Wemm  * System project at Carnegie-Mellon University.
1153bf4bb2SPeter Wemm  *
1253bf4bb2SPeter Wemm  * Redistribution and use in source and binary forms, with or without
1353bf4bb2SPeter Wemm  * modification, are permitted provided that the following conditions
1453bf4bb2SPeter Wemm  * are met:
1553bf4bb2SPeter Wemm  * 1. Redistributions of source code must retain the above copyright
1653bf4bb2SPeter Wemm  *    notice, this list of conditions and the following disclaimer.
1753bf4bb2SPeter Wemm  * 2. Redistributions in binary form must reproduce the above copyright
1853bf4bb2SPeter Wemm  *    notice, this list of conditions and the following disclaimer in the
1953bf4bb2SPeter Wemm  *    documentation and/or other materials provided with the distribution.
2053bf4bb2SPeter Wemm  * 3. All advertising materials mentioning features or use of this software
2153bf4bb2SPeter Wemm  *    must display the following acknowledgement:
2253bf4bb2SPeter Wemm  *	This product includes software developed by the University of
2353bf4bb2SPeter Wemm  *	California, Berkeley and its contributors.
2453bf4bb2SPeter Wemm  * 4. Neither the name of the University nor the names of its contributors
2553bf4bb2SPeter Wemm  *    may be used to endorse or promote products derived from this software
2653bf4bb2SPeter Wemm  *    without specific prior written permission.
2753bf4bb2SPeter Wemm  *
2853bf4bb2SPeter Wemm  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
2953bf4bb2SPeter Wemm  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
3053bf4bb2SPeter Wemm  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
3153bf4bb2SPeter Wemm  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
3253bf4bb2SPeter Wemm  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
3353bf4bb2SPeter Wemm  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
3453bf4bb2SPeter Wemm  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
3553bf4bb2SPeter Wemm  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
3653bf4bb2SPeter Wemm  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
3753bf4bb2SPeter Wemm  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
3853bf4bb2SPeter Wemm  * SUCH DAMAGE.
3953bf4bb2SPeter Wemm  *
4053bf4bb2SPeter Wemm  *	@(#)kern_lock.c	8.18 (Berkeley) 5/21/95
4153bf4bb2SPeter Wemm  */
4253bf4bb2SPeter Wemm 
43677b542eSDavid E. O'Brien #include <sys/cdefs.h>
44677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
45677b542eSDavid E. O'Brien 
46be6847d7SJohn Baldwin #include "opt_ddb.h"
477c0435b9SKip Macy #include "opt_global.h"
48be6847d7SJohn Baldwin 
4953bf4bb2SPeter Wemm #include <sys/param.h>
50c30bf5c3SRobert Watson #include <sys/kdb.h>
519722d88fSJason Evans #include <sys/kernel.h>
5261d80e90SJohn Baldwin #include <sys/ktr.h>
5353bf4bb2SPeter Wemm #include <sys/lock.h>
548302d183SBruce Evans #include <sys/lockmgr.h>
55d8881ca3SJohn Baldwin #include <sys/mutex.h>
568302d183SBruce Evans #include <sys/proc.h>
574bdb9b11SPeter Wemm #include <sys/systm.h>
587c0435b9SKip Macy #include <sys/lock_profile.h>
59e8ddb61dSJeff Roberson #ifdef DEBUG_LOCKS
60e8ddb61dSJeff Roberson #include <sys/stack.h>
61e8ddb61dSJeff Roberson #endif
6253bf4bb2SPeter Wemm 
636efc8a16SAttilio Rao #define	LOCKMGR_TRYOP(x)	((x) & LK_NOWAIT)
646efc8a16SAttilio Rao #define	LOCKMGR_TRYW(x)		(LOCKMGR_TRYOP((x)) ? LOP_TRYLOCK : 0)
6584887fa3SAttilio Rao #define	LOCKMGR_UNHELD(x)	(((x) & (LK_HAVE_EXCL | LK_SHARE_NONZERO)) == 0)
6684887fa3SAttilio Rao #define	LOCKMGR_NOTOWNER(td)	((td) != curthread && (td) != LK_KERNPROC)
676efc8a16SAttilio Rao 
68f9721b43SAttilio Rao static void	assert_lockmgr(struct lock_object *lock, int what);
69be6847d7SJohn Baldwin #ifdef DDB
70be6847d7SJohn Baldwin #include <ddb/ddb.h>
7161bd5e21SKip Macy static void	db_show_lockmgr(struct lock_object *lock);
72be6847d7SJohn Baldwin #endif
736e21afd4SJohn Baldwin static void	lock_lockmgr(struct lock_object *lock, int how);
746e21afd4SJohn Baldwin static int	unlock_lockmgr(struct lock_object *lock);
7561bd5e21SKip Macy 
7661bd5e21SKip Macy struct lock_class lock_class_lockmgr = {
773ff6d229SJohn Baldwin 	.lc_name = "lockmgr",
783ff6d229SJohn Baldwin 	.lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
79f9721b43SAttilio Rao 	.lc_assert = assert_lockmgr,
8061bd5e21SKip Macy #ifdef DDB
816e21afd4SJohn Baldwin 	.lc_ddb_show = db_show_lockmgr,
8261bd5e21SKip Macy #endif
836e21afd4SJohn Baldwin 	.lc_lock = lock_lockmgr,
846e21afd4SJohn Baldwin 	.lc_unlock = unlock_lockmgr,
8561bd5e21SKip Macy };
8661bd5e21SKip Macy 
8784887fa3SAttilio Rao #ifndef INVARIANTS
8884887fa3SAttilio Rao #define	_lockmgr_assert(lkp, what, file, line)
8984887fa3SAttilio Rao #endif
9084887fa3SAttilio Rao 
9153bf4bb2SPeter Wemm /*
9253bf4bb2SPeter Wemm  * Locking primitives implementation.
9353bf4bb2SPeter Wemm  * Locks provide shared/exclusive sychronization.
9453bf4bb2SPeter Wemm  */
9553bf4bb2SPeter Wemm 
966e21afd4SJohn Baldwin void
97f9721b43SAttilio Rao assert_lockmgr(struct lock_object *lock, int what)
98f9721b43SAttilio Rao {
99f9721b43SAttilio Rao 
100f9721b43SAttilio Rao 	panic("lockmgr locks do not support assertions");
101f9721b43SAttilio Rao }
102f9721b43SAttilio Rao 
103f9721b43SAttilio Rao void
1046e21afd4SJohn Baldwin lock_lockmgr(struct lock_object *lock, int how)
1056e21afd4SJohn Baldwin {
1066e21afd4SJohn Baldwin 
1076e21afd4SJohn Baldwin 	panic("lockmgr locks do not support sleep interlocking");
1086e21afd4SJohn Baldwin }
1096e21afd4SJohn Baldwin 
1106e21afd4SJohn Baldwin int
1116e21afd4SJohn Baldwin unlock_lockmgr(struct lock_object *lock)
1126e21afd4SJohn Baldwin {
1136e21afd4SJohn Baldwin 
1146e21afd4SJohn Baldwin 	panic("lockmgr locks do not support sleep interlocking");
1156e21afd4SJohn Baldwin }
1166e21afd4SJohn Baldwin 
117d7a7e179SAttilio Rao #define	COUNT(td, x)	((td)->td_locks += (x))
11899448ed1SJohn Dyson #define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \
11999448ed1SJohn Dyson 	LK_SHARE_NONZERO | LK_WAIT_NONZERO)
12099448ed1SJohn Dyson 
121fe68a916SKip Macy static int acquire(struct lock **lkpp, int extflags, int wanted, int *contested, uint64_t *waittime);
12299448ed1SJohn Dyson static int acquiredrain(struct lock *lkp, int extflags) ;
12303e9c6c1SJohn Dyson 
124a96ab770SJeff Roberson static __inline void
125f158df07SJeff Roberson sharelock(struct thread *td, struct lock *lkp, int incr) {
12603e9c6c1SJohn Dyson 	lkp->lk_flags |= LK_SHARE_NONZERO;
12703e9c6c1SJohn Dyson 	lkp->lk_sharecount += incr;
128f158df07SJeff Roberson 	COUNT(td, incr);
12903e9c6c1SJohn Dyson }
13003e9c6c1SJohn Dyson 
131a96ab770SJeff Roberson static __inline void
132f158df07SJeff Roberson shareunlock(struct thread *td, struct lock *lkp, int decr) {
133219cbf59SEivind Eklund 
1345526d2d9SEivind Eklund 	KASSERT(lkp->lk_sharecount >= decr, ("shareunlock: count < decr"));
13503e9c6c1SJohn Dyson 
136f158df07SJeff Roberson 	COUNT(td, -decr);
1379b2e5badSJohn Dyson 	if (lkp->lk_sharecount == decr) {
13803e9c6c1SJohn Dyson 		lkp->lk_flags &= ~LK_SHARE_NONZERO;
1399b2e5badSJohn Dyson 		if (lkp->lk_flags & (LK_WANT_UPGRADE | LK_WANT_EXCL)) {
1409b2e5badSJohn Dyson 			wakeup(lkp);
1419b2e5badSJohn Dyson 		}
1429b2e5badSJohn Dyson 		lkp->lk_sharecount = 0;
1439b2e5badSJohn Dyson 	} else {
1449b2e5badSJohn Dyson 		lkp->lk_sharecount -= decr;
1459b2e5badSJohn Dyson 	}
14603e9c6c1SJohn Dyson }
14703e9c6c1SJohn Dyson 
14803e9c6c1SJohn Dyson static int
149fe68a916SKip Macy acquire(struct lock **lkpp, int extflags, int wanted, int *contested, uint64_t *waittime)
15041bd6c15SJeff Roberson {
151c6964d3bSKirk McKusick 	struct lock *lkp = *lkpp;
15220728d8fSJeff Roberson 	int error;
153c06394f5SJohn Baldwin 	CTR3(KTR_LOCK,
154ff381670SRobert Watson 	    "acquire(): lkp == %p, extflags == 0x%x, wanted == 0x%x",
155a18b1f1dSJason Evans 	    lkp, extflags, wanted);
156a18b1f1dSJason Evans 
15720728d8fSJeff Roberson 	if ((extflags & LK_NOWAIT) && (lkp->lk_flags & wanted))
15803e9c6c1SJohn Dyson 		return EBUSY;
15920728d8fSJeff Roberson 	error = 0;
160fe68a916SKip Macy 	if ((lkp->lk_flags & wanted) != 0)
161fe68a916SKip Macy 		lock_profile_obtain_lock_failed(&lkp->lk_object, contested, waittime);
162fe68a916SKip Macy 
16303e9c6c1SJohn Dyson 	while ((lkp->lk_flags & wanted) != 0) {
16420728d8fSJeff Roberson 		CTR2(KTR_LOCK,
16520728d8fSJeff Roberson 		    "acquire(): lkp == %p, lk_flags == 0x%x sleeping",
16620728d8fSJeff Roberson 		    lkp, lkp->lk_flags);
16703e9c6c1SJohn Dyson 		lkp->lk_flags |= LK_WAIT_NONZERO;
16803e9c6c1SJohn Dyson 		lkp->lk_waitcount++;
16996fde7daSJake Burkholder 		error = msleep(lkp, lkp->lk_interlock, lkp->lk_prio,
17023b59018SMatthew Dillon 		    lkp->lk_wmesg,
17123b59018SMatthew Dillon 		    ((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0));
1729b2e5badSJohn Dyson 		lkp->lk_waitcount--;
17320728d8fSJeff Roberson 		if (lkp->lk_waitcount == 0)
17420728d8fSJeff Roberson 			lkp->lk_flags &= ~LK_WAIT_NONZERO;
17520728d8fSJeff Roberson 		if (error)
17620728d8fSJeff Roberson 			break;
17703e9c6c1SJohn Dyson 		if (extflags & LK_SLEEPFAIL) {
17820728d8fSJeff Roberson 			error = ENOLCK;
17920728d8fSJeff Roberson 			break;
18003e9c6c1SJohn Dyson 		}
181c6964d3bSKirk McKusick 		if (lkp->lk_newlock != NULL) {
182c6964d3bSKirk McKusick 			mtx_lock(lkp->lk_newlock->lk_interlock);
183c6964d3bSKirk McKusick 			mtx_unlock(lkp->lk_interlock);
184c6964d3bSKirk McKusick 			if (lkp->lk_waitcount == 0)
185c6964d3bSKirk McKusick 				wakeup((void *)(&lkp->lk_newlock));
186c6964d3bSKirk McKusick 			*lkpp = lkp = lkp->lk_newlock;
187c6964d3bSKirk McKusick 		}
18803e9c6c1SJohn Dyson 	}
18920728d8fSJeff Roberson 	mtx_assert(lkp->lk_interlock, MA_OWNED);
19020728d8fSJeff Roberson 	return (error);
19103e9c6c1SJohn Dyson }
19203e9c6c1SJohn Dyson 
19353bf4bb2SPeter Wemm /*
19453bf4bb2SPeter Wemm  * Set, change, or release a lock.
19553bf4bb2SPeter Wemm  *
19653bf4bb2SPeter Wemm  * Shared requests increment the shared count. Exclusive requests set the
19753bf4bb2SPeter Wemm  * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
19853bf4bb2SPeter Wemm  * accepted shared locks and shared-to-exclusive upgrades to go away.
19953bf4bb2SPeter Wemm  */
20053bf4bb2SPeter Wemm int
2010e9eb108SAttilio Rao _lockmgr(struct lock *lkp, u_int flags, struct mtx *interlkp, char *file,
2020e9eb108SAttilio Rao     int line)
2037c0435b9SKip Macy 
20453bf4bb2SPeter Wemm {
2050e9eb108SAttilio Rao 	struct thread *td;
20653bf4bb2SPeter Wemm 	int error;
207635962afSJohn Baldwin 	int extflags, lockflags;
208fe68a916SKip Macy 	int contested = 0;
209fe68a916SKip Macy 	uint64_t waitstart = 0;
21053bf4bb2SPeter Wemm 
21153bf4bb2SPeter Wemm 	error = 0;
2120e9eb108SAttilio Rao 	td = curthread;
21303e9c6c1SJohn Dyson 
21484887fa3SAttilio Rao #ifdef INVARIANTS
21584887fa3SAttilio Rao 	if (lkp->lk_flags & LK_DESTROYED) {
21684887fa3SAttilio Rao 		if (flags & LK_INTERLOCK)
21784887fa3SAttilio Rao 			mtx_unlock(interlkp);
21884887fa3SAttilio Rao 		if (panicstr != NULL)
21984887fa3SAttilio Rao 			return (0);
22084887fa3SAttilio Rao 		panic("%s: %p lockmgr is destroyed", __func__, lkp);
22184887fa3SAttilio Rao 	}
22284887fa3SAttilio Rao #endif
22317661e5aSJeff Roberson 	if ((flags & LK_INTERNAL) == 0)
2249ed346baSBosko Milekic 		mtx_lock(lkp->lk_interlock);
22541bd6c15SJeff Roberson 	CTR6(KTR_LOCK,
22641bd6c15SJeff Roberson 	    "lockmgr(): lkp == %p (lk_wmesg == \"%s\"), owner == %p, exclusivecount == %d, flags == 0x%x, "
22741bd6c15SJeff Roberson 	    "td == %p", lkp, lkp->lk_wmesg, lkp->lk_lockholder,
22841bd6c15SJeff Roberson 	    lkp->lk_exclusivecount, flags, td);
229e8ddb61dSJeff Roberson #ifdef DEBUG_LOCKS
230e8ddb61dSJeff Roberson 	{
231e8ddb61dSJeff Roberson 		struct stack stack; /* XXX */
232e8ddb61dSJeff Roberson 		stack_save(&stack);
233e37a4994SPawel Jakub Dawidek 		CTRSTACK(KTR_LOCK, &stack, 0, 1);
234e8ddb61dSJeff Roberson 	}
23541bd6c15SJeff Roberson #endif
23641bd6c15SJeff Roberson 
23798689e1eSAlfred Perlstein 	if (flags & LK_INTERLOCK) {
2386157b69fSAlfred Perlstein 		mtx_assert(interlkp, MA_OWNED | MA_NOTRECURSED);
2399ed346baSBosko Milekic 		mtx_unlock(interlkp);
24098689e1eSAlfred Perlstein 	}
24103e9c6c1SJohn Dyson 
24217661e5aSJeff Roberson 	if ((flags & (LK_NOWAIT|LK_RELEASE)) == 0)
24326306795SJohn Baldwin 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK,
244aa89d8cdSJohn Baldwin 		    &lkp->lk_interlock->lock_object,
24526306795SJohn Baldwin 		    "Acquiring lockmgr lock \"%s\"", lkp->lk_wmesg);
24617661e5aSJeff Roberson 
2473f085c22SJohn Baldwin 	if (panicstr != NULL) {
2483f085c22SJohn Baldwin 		mtx_unlock(lkp->lk_interlock);
2493f085c22SJohn Baldwin 		return (0);
2503f085c22SJohn Baldwin 	}
251c4c0ec5bSJeff Roberson 	if ((lkp->lk_flags & LK_NOSHARE) &&
252c4c0ec5bSJeff Roberson 	    (flags & LK_TYPE_MASK) == LK_SHARED) {
253c4c0ec5bSJeff Roberson 		flags &= ~LK_TYPE_MASK;
254c4c0ec5bSJeff Roberson 		flags |= LK_EXCLUSIVE;
255c4c0ec5bSJeff Roberson 	}
25653bf4bb2SPeter Wemm 	extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK;
25753bf4bb2SPeter Wemm 
25853bf4bb2SPeter Wemm 	switch (flags & LK_TYPE_MASK) {
25953bf4bb2SPeter Wemm 
26053bf4bb2SPeter Wemm 	case LK_SHARED:
2616efc8a16SAttilio Rao 		if (!LOCKMGR_TRYOP(extflags))
2626efc8a16SAttilio Rao 			WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER, file,
2636efc8a16SAttilio Rao 			    line);
264beef8a36SJulian Elischer 		/*
265beef8a36SJulian Elischer 		 * If we are not the exclusive lock holder, we have to block
266beef8a36SJulian Elischer 		 * while there is an exclusive lock holder or while an
267beef8a36SJulian Elischer 		 * exclusive lock request or upgrade request is in progress.
268beef8a36SJulian Elischer 		 *
269fa2a4d05STim J. Robbins 		 * However, if TDP_DEADLKTREAT is set, we override exclusive
270beef8a36SJulian Elischer 		 * lock requests or upgrade requests ( but not the exclusive
271beef8a36SJulian Elischer 		 * lock itself ).
272beef8a36SJulian Elischer 		 */
273d7a7e179SAttilio Rao 		if (lkp->lk_lockholder != td) {
274635962afSJohn Baldwin 			lockflags = LK_HAVE_EXCL;
2759032b51eSAttilio Rao 			if (!(td->td_pflags & TDP_DEADLKTREAT))
276bce98419SJohn Baldwin 				lockflags |= LK_WANT_EXCL | LK_WANT_UPGRADE;
277fe68a916SKip Macy 			error = acquire(&lkp, extflags, lockflags, &contested, &waitstart);
27853bf4bb2SPeter Wemm 			if (error)
27953bf4bb2SPeter Wemm 				break;
280f158df07SJeff Roberson 			sharelock(td, lkp, 1);
2817c0435b9SKip Macy 			if (lkp->lk_sharecount == 1)
282fe68a916SKip Macy 				lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
2836efc8a16SAttilio Rao 			WITNESS_LOCK(&lkp->lk_object, LOCKMGR_TRYW(extflags),
2846efc8a16SAttilio Rao 			    file, line);
2857c0435b9SKip Macy 
2867181624aSJeff Roberson #if defined(DEBUG_LOCKS)
2877499fd8dSJeff Roberson 			stack_save(&lkp->lk_stack);
2887181624aSJeff Roberson #endif
28953bf4bb2SPeter Wemm 			break;
29053bf4bb2SPeter Wemm 		}
29153bf4bb2SPeter Wemm 		/*
29253bf4bb2SPeter Wemm 		 * We hold an exclusive lock, so downgrade it to shared.
29353bf4bb2SPeter Wemm 		 * An alternative would be to fail with EDEADLK.
29453bf4bb2SPeter Wemm 		 */
29593b0017fSPhilippe Charnier 		/* FALLTHROUGH downgrade */
29653bf4bb2SPeter Wemm 
29753bf4bb2SPeter Wemm 	case LK_DOWNGRADE:
29884887fa3SAttilio Rao 		_lockmgr_assert(lkp, KA_XLOCKED, file, line);
299f158df07SJeff Roberson 		sharelock(td, lkp, lkp->lk_exclusivecount);
3006efc8a16SAttilio Rao 		WITNESS_DOWNGRADE(&lkp->lk_object, 0, file, line);
301f158df07SJeff Roberson 		COUNT(td, -lkp->lk_exclusivecount);
30253bf4bb2SPeter Wemm 		lkp->lk_exclusivecount = 0;
30353bf4bb2SPeter Wemm 		lkp->lk_flags &= ~LK_HAVE_EXCL;
3046f8132a8SJulian Elischer 		lkp->lk_lockholder = LK_NOPROC;
30553bf4bb2SPeter Wemm 		if (lkp->lk_waitcount)
30653bf4bb2SPeter Wemm 			wakeup((void *)lkp);
30753bf4bb2SPeter Wemm 		break;
30853bf4bb2SPeter Wemm 
30953bf4bb2SPeter Wemm 	case LK_UPGRADE:
31053bf4bb2SPeter Wemm 		/*
31153bf4bb2SPeter Wemm 		 * Upgrade a shared lock to an exclusive one. If another
31253bf4bb2SPeter Wemm 		 * shared lock has already requested an upgrade to an
31353bf4bb2SPeter Wemm 		 * exclusive lock, our shared lock is released and an
31453bf4bb2SPeter Wemm 		 * exclusive lock is requested (which will be granted
31553bf4bb2SPeter Wemm 		 * after the upgrade). If we return an error, the file
31653bf4bb2SPeter Wemm 		 * will always be unlocked.
31753bf4bb2SPeter Wemm 		 */
31884887fa3SAttilio Rao 		_lockmgr_assert(lkp, KA_SLOCKED, file, line);
319f158df07SJeff Roberson 		shareunlock(td, lkp, 1);
3207c0435b9SKip Macy 		if (lkp->lk_sharecount == 0)
3217c0435b9SKip Macy 			lock_profile_release_lock(&lkp->lk_object);
32253bf4bb2SPeter Wemm 		/*
32353bf4bb2SPeter Wemm 		 * If we are just polling, check to see if we will block.
32453bf4bb2SPeter Wemm 		 */
32553bf4bb2SPeter Wemm 		if ((extflags & LK_NOWAIT) &&
32653bf4bb2SPeter Wemm 		    ((lkp->lk_flags & LK_WANT_UPGRADE) ||
32753bf4bb2SPeter Wemm 		     lkp->lk_sharecount > 1)) {
32853bf4bb2SPeter Wemm 			error = EBUSY;
3296efc8a16SAttilio Rao 			WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
33053bf4bb2SPeter Wemm 			break;
33153bf4bb2SPeter Wemm 		}
33253bf4bb2SPeter Wemm 		if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
33353bf4bb2SPeter Wemm 			/*
33453bf4bb2SPeter Wemm 			 * We are first shared lock to request an upgrade, so
33553bf4bb2SPeter Wemm 			 * request upgrade and wait for the shared count to
33653bf4bb2SPeter Wemm 			 * drop to zero, then take exclusive lock.
33753bf4bb2SPeter Wemm 			 */
33853bf4bb2SPeter Wemm 			lkp->lk_flags |= LK_WANT_UPGRADE;
339fe68a916SKip Macy 			error = acquire(&lkp, extflags, LK_SHARE_NONZERO, &contested, &waitstart);
34053bf4bb2SPeter Wemm 			lkp->lk_flags &= ~LK_WANT_UPGRADE;
3419b2e5badSJohn Dyson 
3424cef6d5aSAlexander Kabaev 			if (error) {
3434cef6d5aSAlexander Kabaev 			         if ((lkp->lk_flags & ( LK_WANT_EXCL | LK_WAIT_NONZERO)) == (LK_WANT_EXCL | LK_WAIT_NONZERO))
3444cef6d5aSAlexander Kabaev 			                   wakeup((void *)lkp);
3456efc8a16SAttilio Rao 				WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
34653bf4bb2SPeter Wemm 			         break;
3474cef6d5aSAlexander Kabaev 			}
34853bf4bb2SPeter Wemm 			if (lkp->lk_exclusivecount != 0)
34953bf4bb2SPeter Wemm 				panic("lockmgr: non-zero exclusive count");
350d8b8e875SPaul Saab 			lkp->lk_flags |= LK_HAVE_EXCL;
351d7a7e179SAttilio Rao 			lkp->lk_lockholder = td;
35253bf4bb2SPeter Wemm 			lkp->lk_exclusivecount = 1;
3536efc8a16SAttilio Rao 			WITNESS_UPGRADE(&lkp->lk_object, LOP_EXCLUSIVE |
3546efc8a16SAttilio Rao 			    LOP_TRYLOCK, file, line);
355f158df07SJeff Roberson 			COUNT(td, 1);
356fe68a916SKip Macy 			lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
35715a1057cSEivind Eklund #if defined(DEBUG_LOCKS)
3587499fd8dSJeff Roberson 			stack_save(&lkp->lk_stack);
35915a1057cSEivind Eklund #endif
36053bf4bb2SPeter Wemm 			break;
36153bf4bb2SPeter Wemm 		}
36253bf4bb2SPeter Wemm 		/*
36353bf4bb2SPeter Wemm 		 * Someone else has requested upgrade. Release our shared
36453bf4bb2SPeter Wemm 		 * lock, awaken upgrade requestor if we are the last shared
36553bf4bb2SPeter Wemm 		 * lock, then request an exclusive lock.
36653bf4bb2SPeter Wemm 		 */
3676efc8a16SAttilio Rao 		WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
36803e9c6c1SJohn Dyson 		if ( (lkp->lk_flags & (LK_SHARE_NONZERO|LK_WAIT_NONZERO)) ==
36903e9c6c1SJohn Dyson 			LK_WAIT_NONZERO)
37053bf4bb2SPeter Wemm 			wakeup((void *)lkp);
37193b0017fSPhilippe Charnier 		/* FALLTHROUGH exclusive request */
37253bf4bb2SPeter Wemm 
37353bf4bb2SPeter Wemm 	case LK_EXCLUSIVE:
3746efc8a16SAttilio Rao 		if (!LOCKMGR_TRYOP(extflags))
3756efc8a16SAttilio Rao 			WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER |
3766efc8a16SAttilio Rao 			    LOP_EXCLUSIVE, file, line);
377d7a7e179SAttilio Rao 		if (lkp->lk_lockholder == td) {
37853bf4bb2SPeter Wemm 			/*
37953bf4bb2SPeter Wemm 			 *	Recursive lock.
38053bf4bb2SPeter Wemm 			 */
38133638e93SKirk McKusick 			if ((extflags & (LK_NOWAIT | LK_CANRECURSE)) == 0)
38253bf4bb2SPeter Wemm 				panic("lockmgr: locking against myself");
38333638e93SKirk McKusick 			if ((extflags & LK_CANRECURSE) != 0) {
38453bf4bb2SPeter Wemm 				lkp->lk_exclusivecount++;
3856efc8a16SAttilio Rao 				WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE |
3866efc8a16SAttilio Rao 				    LOCKMGR_TRYW(extflags), file, line);
387f158df07SJeff Roberson 				COUNT(td, 1);
38853bf4bb2SPeter Wemm 				break;
38953bf4bb2SPeter Wemm 			}
39033638e93SKirk McKusick 		}
39153bf4bb2SPeter Wemm 		/*
39253bf4bb2SPeter Wemm 		 * If we are just polling, check to see if we will sleep.
39353bf4bb2SPeter Wemm 		 */
39403e9c6c1SJohn Dyson 		if ((extflags & LK_NOWAIT) &&
39503e9c6c1SJohn Dyson 		    (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO))) {
39653bf4bb2SPeter Wemm 			error = EBUSY;
39753bf4bb2SPeter Wemm 			break;
39853bf4bb2SPeter Wemm 		}
39953bf4bb2SPeter Wemm 		/*
40053bf4bb2SPeter Wemm 		 * Try to acquire the want_exclusive flag.
40153bf4bb2SPeter Wemm 		 */
402fe68a916SKip Macy 		error = acquire(&lkp, extflags, (LK_HAVE_EXCL | LK_WANT_EXCL), &contested, &waitstart);
40353bf4bb2SPeter Wemm 		if (error)
40453bf4bb2SPeter Wemm 			break;
40553bf4bb2SPeter Wemm 		lkp->lk_flags |= LK_WANT_EXCL;
40653bf4bb2SPeter Wemm 		/*
40753bf4bb2SPeter Wemm 		 * Wait for shared locks and upgrades to finish.
40853bf4bb2SPeter Wemm 		 */
409fe68a916SKip Macy 		error = acquire(&lkp, extflags, LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, &contested, &waitstart);
41053bf4bb2SPeter Wemm 		lkp->lk_flags &= ~LK_WANT_EXCL;
4114cef6d5aSAlexander Kabaev 		if (error) {
4124cef6d5aSAlexander Kabaev 			if (lkp->lk_flags & LK_WAIT_NONZERO)
4134cef6d5aSAlexander Kabaev 			         wakeup((void *)lkp);
41453bf4bb2SPeter Wemm 			break;
4154cef6d5aSAlexander Kabaev 		}
41653bf4bb2SPeter Wemm 		lkp->lk_flags |= LK_HAVE_EXCL;
417d7a7e179SAttilio Rao 		lkp->lk_lockholder = td;
41853bf4bb2SPeter Wemm 		if (lkp->lk_exclusivecount != 0)
41953bf4bb2SPeter Wemm 			panic("lockmgr: non-zero exclusive count");
42053bf4bb2SPeter Wemm 		lkp->lk_exclusivecount = 1;
4216efc8a16SAttilio Rao 		WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE |
4226efc8a16SAttilio Rao 		    LOCKMGR_TRYW(extflags), file, line);
423f158df07SJeff Roberson 		COUNT(td, 1);
424fe68a916SKip Macy 		lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
42515a1057cSEivind Eklund #if defined(DEBUG_LOCKS)
4267499fd8dSJeff Roberson 		stack_save(&lkp->lk_stack);
42715a1057cSEivind Eklund #endif
42853bf4bb2SPeter Wemm 		break;
42953bf4bb2SPeter Wemm 
43053bf4bb2SPeter Wemm 	case LK_RELEASE:
43184887fa3SAttilio Rao 		_lockmgr_assert(lkp, KA_LOCKED, file, line);
43253bf4bb2SPeter Wemm 		if (lkp->lk_exclusivecount != 0) {
4336efc8a16SAttilio Rao 			if (lkp->lk_lockholder != LK_KERNPROC) {
4346efc8a16SAttilio Rao 				WITNESS_UNLOCK(&lkp->lk_object, LOP_EXCLUSIVE,
4356efc8a16SAttilio Rao 				    file, line);
436f158df07SJeff Roberson 				COUNT(td, -1);
4376efc8a16SAttilio Rao 			}
43884887fa3SAttilio Rao 			if (lkp->lk_exclusivecount-- == 1) {
43953bf4bb2SPeter Wemm 				lkp->lk_flags &= ~LK_HAVE_EXCL;
44053bf4bb2SPeter Wemm 				lkp->lk_lockholder = LK_NOPROC;
4417c0435b9SKip Macy 				lock_profile_release_lock(&lkp->lk_object);
44253bf4bb2SPeter Wemm 			}
4436efc8a16SAttilio Rao 		} else if (lkp->lk_flags & LK_SHARE_NONZERO) {
4446efc8a16SAttilio Rao 			WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
445f158df07SJeff Roberson 			shareunlock(td, lkp, 1);
4461f71de49SSuleiman Souhlal 		}
4471f71de49SSuleiman Souhlal 
44803e9c6c1SJohn Dyson 		if (lkp->lk_flags & LK_WAIT_NONZERO)
44953bf4bb2SPeter Wemm 			wakeup((void *)lkp);
45053bf4bb2SPeter Wemm 		break;
45153bf4bb2SPeter Wemm 
45253bf4bb2SPeter Wemm 	case LK_DRAIN:
45353bf4bb2SPeter Wemm 		/*
45453bf4bb2SPeter Wemm 		 * Check that we do not already hold the lock, as it can
45553bf4bb2SPeter Wemm 		 * never drain if we do. Unfortunately, we have no way to
45653bf4bb2SPeter Wemm 		 * check for holding a shared lock, but at least we can
45753bf4bb2SPeter Wemm 		 * check for an exclusive one.
45853bf4bb2SPeter Wemm 		 */
4596efc8a16SAttilio Rao 		if (!LOCKMGR_TRYOP(extflags))
4606efc8a16SAttilio Rao 			WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER |
4616efc8a16SAttilio Rao 			    LOP_EXCLUSIVE, file, line);
462d7a7e179SAttilio Rao 		if (lkp->lk_lockholder == td)
46353bf4bb2SPeter Wemm 			panic("lockmgr: draining against myself");
46403e9c6c1SJohn Dyson 
46503e9c6c1SJohn Dyson 		error = acquiredrain(lkp, extflags);
46603e9c6c1SJohn Dyson 		if (error)
46753bf4bb2SPeter Wemm 			break;
46853bf4bb2SPeter Wemm 		lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL;
469d7a7e179SAttilio Rao 		lkp->lk_lockholder = td;
47053bf4bb2SPeter Wemm 		lkp->lk_exclusivecount = 1;
4716efc8a16SAttilio Rao 		WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE |
4726efc8a16SAttilio Rao 		    LOCKMGR_TRYW(extflags), file, line);
473f158df07SJeff Roberson 		COUNT(td, 1);
47415a1057cSEivind Eklund #if defined(DEBUG_LOCKS)
4757499fd8dSJeff Roberson 		stack_save(&lkp->lk_stack);
47615a1057cSEivind Eklund #endif
47753bf4bb2SPeter Wemm 		break;
47853bf4bb2SPeter Wemm 
47953bf4bb2SPeter Wemm 	default:
4809ed346baSBosko Milekic 		mtx_unlock(lkp->lk_interlock);
48153bf4bb2SPeter Wemm 		panic("lockmgr: unknown locktype request %d",
48253bf4bb2SPeter Wemm 		    flags & LK_TYPE_MASK);
48353bf4bb2SPeter Wemm 		/* NOTREACHED */
48453bf4bb2SPeter Wemm 	}
48503e9c6c1SJohn Dyson 	if ((lkp->lk_flags & LK_WAITDRAIN) &&
48603e9c6c1SJohn Dyson 	    (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE |
48703e9c6c1SJohn Dyson 		LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0) {
48853bf4bb2SPeter Wemm 		lkp->lk_flags &= ~LK_WAITDRAIN;
48953bf4bb2SPeter Wemm 		wakeup((void *)&lkp->lk_flags);
49053bf4bb2SPeter Wemm 	}
4919ed346baSBosko Milekic 	mtx_unlock(lkp->lk_interlock);
49253bf4bb2SPeter Wemm 	return (error);
49353bf4bb2SPeter Wemm }
49453bf4bb2SPeter Wemm 
49599448ed1SJohn Dyson static int
49699448ed1SJohn Dyson acquiredrain(struct lock *lkp, int extflags) {
49799448ed1SJohn Dyson 	int error;
49899448ed1SJohn Dyson 
49999448ed1SJohn Dyson 	if ((extflags & LK_NOWAIT) && (lkp->lk_flags & LK_ALL)) {
50099448ed1SJohn Dyson 		return EBUSY;
50199448ed1SJohn Dyson 	}
50299448ed1SJohn Dyson 	while (lkp->lk_flags & LK_ALL) {
50399448ed1SJohn Dyson 		lkp->lk_flags |= LK_WAITDRAIN;
50496fde7daSJake Burkholder 		error = msleep(&lkp->lk_flags, lkp->lk_interlock, lkp->lk_prio,
50523b59018SMatthew Dillon 			lkp->lk_wmesg,
50623b59018SMatthew Dillon 			((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0));
50799448ed1SJohn Dyson 		if (error)
50899448ed1SJohn Dyson 			return error;
50999448ed1SJohn Dyson 		if (extflags & LK_SLEEPFAIL) {
51099448ed1SJohn Dyson 			return ENOLCK;
51199448ed1SJohn Dyson 		}
51299448ed1SJohn Dyson 	}
51399448ed1SJohn Dyson 	return 0;
51499448ed1SJohn Dyson }
51599448ed1SJohn Dyson 
51699448ed1SJohn Dyson /*
51799448ed1SJohn Dyson  * Initialize a lock; required before use.
51899448ed1SJohn Dyson  */
51999448ed1SJohn Dyson void
52099448ed1SJohn Dyson lockinit(lkp, prio, wmesg, timo, flags)
52199448ed1SJohn Dyson 	struct lock *lkp;
52299448ed1SJohn Dyson 	int prio;
52304858e7eSEivind Eklund 	const char *wmesg;
52499448ed1SJohn Dyson 	int timo;
52599448ed1SJohn Dyson 	int flags;
52699448ed1SJohn Dyson {
5276efc8a16SAttilio Rao 	int iflags;
5286efc8a16SAttilio Rao 
529c06394f5SJohn Baldwin 	CTR5(KTR_LOCK, "lockinit(): lkp == %p, prio == %d, wmesg == \"%s\", "
530a18b1f1dSJason Evans 	    "timo == %d, flags = 0x%x\n", lkp, prio, wmesg, timo, flags);
53199448ed1SJohn Dyson 
532857d9c60SDon Lewis 	lkp->lk_interlock = mtx_pool_alloc(mtxpool_lockbuilder);
5336efc8a16SAttilio Rao 	lkp->lk_flags = (flags & LK_EXTFLG_MASK) & ~(LK_NOWITNESS | LK_NODUP);
53499448ed1SJohn Dyson 	lkp->lk_sharecount = 0;
53599448ed1SJohn Dyson 	lkp->lk_waitcount = 0;
53699448ed1SJohn Dyson 	lkp->lk_exclusivecount = 0;
53799448ed1SJohn Dyson 	lkp->lk_prio = prio;
53899448ed1SJohn Dyson 	lkp->lk_timo = timo;
53999448ed1SJohn Dyson 	lkp->lk_lockholder = LK_NOPROC;
540c6964d3bSKirk McKusick 	lkp->lk_newlock = NULL;
5416efc8a16SAttilio Rao 	iflags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE;
5426efc8a16SAttilio Rao 	if (!(flags & LK_NODUP))
5436efc8a16SAttilio Rao 		iflags |= LO_DUPOK;
5446efc8a16SAttilio Rao 	if (!(flags & LK_NOWITNESS))
5456efc8a16SAttilio Rao 		iflags |= LO_WITNESS;
5463a096f6cSKirk McKusick #ifdef DEBUG_LOCKS
5477499fd8dSJeff Roberson 	stack_zero(&lkp->lk_stack);
5483a096f6cSKirk McKusick #endif
5496efc8a16SAttilio Rao 	lock_init(&lkp->lk_object, &lock_class_lockmgr, wmesg, NULL, iflags);
55099448ed1SJohn Dyson }
55199448ed1SJohn Dyson 
55299448ed1SJohn Dyson /*
553a18b1f1dSJason Evans  * Destroy a lock.
554a18b1f1dSJason Evans  */
555a18b1f1dSJason Evans void
556a18b1f1dSJason Evans lockdestroy(lkp)
557a18b1f1dSJason Evans 	struct lock *lkp;
558a18b1f1dSJason Evans {
559c91fcee7SJohn Baldwin 
560c06394f5SJohn Baldwin 	CTR2(KTR_LOCK, "lockdestroy(): lkp == %p (lk_wmesg == \"%s\")",
561a18b1f1dSJason Evans 	    lkp, lkp->lk_wmesg);
56284887fa3SAttilio Rao 	KASSERT((lkp->lk_flags & (LK_HAVE_EXCL | LK_SHARE_NONZERO)) == 0,
56384887fa3SAttilio Rao 	    ("lockmgr still held"));
56484887fa3SAttilio Rao 	KASSERT(lkp->lk_exclusivecount == 0, ("lockmgr still recursed"));
56584887fa3SAttilio Rao 	lkp->lk_flags = LK_DESTROYED;
566ab2dab16SJohn Baldwin 	lock_destroy(&lkp->lk_object);
567a18b1f1dSJason Evans }
568a18b1f1dSJason Evans 
569a18b1f1dSJason Evans /*
570d7a7e179SAttilio Rao  * Disown the lockmgr.
571d7a7e179SAttilio Rao  */
572d7a7e179SAttilio Rao void
5736efc8a16SAttilio Rao _lockmgr_disown(struct lock *lkp, const char *file, int line)
574d7a7e179SAttilio Rao {
575d7a7e179SAttilio Rao 	struct thread *td;
576d7a7e179SAttilio Rao 
577d7a7e179SAttilio Rao 	td = curthread;
57884887fa3SAttilio Rao 	KASSERT(panicstr != NULL || (lkp->lk_flags & LK_DESTROYED) == 0,
57984887fa3SAttilio Rao 	    ("%s: %p lockmgr is destroyed", __func__, lkp));
58084887fa3SAttilio Rao 	_lockmgr_assert(lkp, KA_XLOCKED | KA_NOTRECURSED, file, line);
581d7a7e179SAttilio Rao 
582d7a7e179SAttilio Rao 	/*
583d7a7e179SAttilio Rao 	 * Drop the lock reference and switch the owner.  This will result
584d7a7e179SAttilio Rao 	 * in an atomic operation like td_lock is only accessed by curthread
585d1127e66SAttilio Rao 	 * and lk_lockholder only needs one write.  Note also that the lock
586d1127e66SAttilio Rao 	 * owner can be alredy KERNPROC, so in that case just skip the
587d1127e66SAttilio Rao 	 * decrement.
588d7a7e179SAttilio Rao 	 */
5896efc8a16SAttilio Rao 	if (lkp->lk_lockholder == td) {
5906efc8a16SAttilio Rao 		WITNESS_UNLOCK(&lkp->lk_object, LOP_EXCLUSIVE, file, line);
591d7a7e179SAttilio Rao 		td->td_locks--;
5926efc8a16SAttilio Rao 	}
593d7a7e179SAttilio Rao 	lkp->lk_lockholder = LK_KERNPROC;
594d7a7e179SAttilio Rao }
595d7a7e179SAttilio Rao 
596d7a7e179SAttilio Rao /*
59799448ed1SJohn Dyson  * Determine the status of a lock.
59899448ed1SJohn Dyson  */
59999448ed1SJohn Dyson int
600b40ce416SJulian Elischer lockstatus(lkp, td)
60199448ed1SJohn Dyson 	struct lock *lkp;
602b40ce416SJulian Elischer 	struct thread *td;
60399448ed1SJohn Dyson {
60499448ed1SJohn Dyson 	int lock_type = 0;
6052b59d50cSRobert Watson 	int interlocked;
60699448ed1SJohn Dyson 
6072433c488SAttilio Rao 	KASSERT(td == curthread,
6080e9eb108SAttilio Rao 	    ("%s: thread passed argument (%p) is not valid", __func__, td));
60984887fa3SAttilio Rao 	KASSERT((lkp->lk_flags & LK_DESTROYED) == 0,
61084887fa3SAttilio Rao 	    ("%s: %p lockmgr is destroyed", __func__, lkp));
6110e9eb108SAttilio Rao 
6122b59d50cSRobert Watson 	if (!kdb_active) {
6132b59d50cSRobert Watson 		interlocked = 1;
6149ed346baSBosko Milekic 		mtx_lock(lkp->lk_interlock);
6152b59d50cSRobert Watson 	} else
6162b59d50cSRobert Watson 		interlocked = 0;
6176bdfe06aSEivind Eklund 	if (lkp->lk_exclusivecount != 0) {
6182433c488SAttilio Rao 		if (lkp->lk_lockholder == td)
61999448ed1SJohn Dyson 			lock_type = LK_EXCLUSIVE;
6206bdfe06aSEivind Eklund 		else
6216bdfe06aSEivind Eklund 			lock_type = LK_EXCLOTHER;
6226bdfe06aSEivind Eklund 	} else if (lkp->lk_sharecount != 0)
62399448ed1SJohn Dyson 		lock_type = LK_SHARED;
6242b59d50cSRobert Watson 	if (interlocked)
6259ed346baSBosko Milekic 		mtx_unlock(lkp->lk_interlock);
62699448ed1SJohn Dyson 	return (lock_type);
62799448ed1SJohn Dyson }
62899448ed1SJohn Dyson 
62953bf4bb2SPeter Wemm /*
63004aa807cSTor Egge  * Determine the number of waiters on a lock.
63104aa807cSTor Egge  */
63204aa807cSTor Egge int
63304aa807cSTor Egge lockwaiters(lkp)
63404aa807cSTor Egge 	struct lock *lkp;
63504aa807cSTor Egge {
63604aa807cSTor Egge 	int count;
63704aa807cSTor Egge 
63884887fa3SAttilio Rao 	KASSERT((lkp->lk_flags & LK_DESTROYED) == 0,
63984887fa3SAttilio Rao 	    ("%s: %p lockmgr is destroyed", __func__, lkp));
64004aa807cSTor Egge 	mtx_lock(lkp->lk_interlock);
64104aa807cSTor Egge 	count = lkp->lk_waitcount;
64204aa807cSTor Egge 	mtx_unlock(lkp->lk_interlock);
64304aa807cSTor Egge 	return (count);
64404aa807cSTor Egge }
64504aa807cSTor Egge 
64604aa807cSTor Egge /*
64753bf4bb2SPeter Wemm  * Print out information about state of a lock. Used by VOP_PRINT
6480e61ac7bSPoul-Henning Kamp  * routines to display status about contained locks.
64953bf4bb2SPeter Wemm  */
650a1ce9d5cSPeter Wemm void
65153bf4bb2SPeter Wemm lockmgr_printinfo(lkp)
65253bf4bb2SPeter Wemm 	struct lock *lkp;
65353bf4bb2SPeter Wemm {
65453bf4bb2SPeter Wemm 
65553bf4bb2SPeter Wemm 	if (lkp->lk_sharecount)
65653bf4bb2SPeter Wemm 		printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg,
65753bf4bb2SPeter Wemm 		    lkp->lk_sharecount);
65853bf4bb2SPeter Wemm 	else if (lkp->lk_flags & LK_HAVE_EXCL)
659c969c60cSAlexander Kabaev 		printf(" lock type %s: EXCL (count %d) by thread %p (pid %d)",
660c969c60cSAlexander Kabaev 		    lkp->lk_wmesg, lkp->lk_exclusivecount,
661c969c60cSAlexander Kabaev 		    lkp->lk_lockholder, lkp->lk_lockholder->td_proc->p_pid);
66253bf4bb2SPeter Wemm 	if (lkp->lk_waitcount > 0)
66353bf4bb2SPeter Wemm 		printf(" with %d pending", lkp->lk_waitcount);
6647499fd8dSJeff Roberson #ifdef DEBUG_LOCKS
6659ccca7d1SRobert Watson 	stack_print_ddb(&lkp->lk_stack);
6667499fd8dSJeff Roberson #endif
66753bf4bb2SPeter Wemm }
668be6847d7SJohn Baldwin 
66984887fa3SAttilio Rao #ifdef INVARIANT_SUPPORT
67084887fa3SAttilio Rao #ifndef INVARIANTS
67184887fa3SAttilio Rao #undef _lockmgr_assert
67284887fa3SAttilio Rao #endif
67384887fa3SAttilio Rao 
67484887fa3SAttilio Rao void
67584887fa3SAttilio Rao _lockmgr_assert(struct lock *lkp, int what, const char *file, int line)
67684887fa3SAttilio Rao {
67784887fa3SAttilio Rao 	struct thread *td;
67884887fa3SAttilio Rao 	u_int x;
67984887fa3SAttilio Rao 	int slocked = 0;
68084887fa3SAttilio Rao 
68184887fa3SAttilio Rao 	x = lkp->lk_flags;
68284887fa3SAttilio Rao 	td = lkp->lk_lockholder;
68384887fa3SAttilio Rao 	if (panicstr != NULL)
68484887fa3SAttilio Rao 		return;
68584887fa3SAttilio Rao 	switch (what) {
68684887fa3SAttilio Rao 	case KA_SLOCKED:
68784887fa3SAttilio Rao 	case KA_SLOCKED | KA_NOTRECURSED:
68884887fa3SAttilio Rao 	case KA_SLOCKED | KA_RECURSED:
68984887fa3SAttilio Rao 		slocked = 1;
69084887fa3SAttilio Rao 	case KA_LOCKED:
69184887fa3SAttilio Rao 	case KA_LOCKED | KA_NOTRECURSED:
69284887fa3SAttilio Rao 	case KA_LOCKED | KA_RECURSED:
69384887fa3SAttilio Rao #ifdef WITNESS
69484887fa3SAttilio Rao 		/*
69584887fa3SAttilio Rao 		 * We cannot trust WITNESS if the lock is held in
69684887fa3SAttilio Rao 		 * exclusive mode and a call to lockmgr_disown() happened.
69784887fa3SAttilio Rao 		 * Workaround this skipping the check if the lock is
69884887fa3SAttilio Rao 		 * held in exclusive mode even for the KA_LOCKED case.
69984887fa3SAttilio Rao 		 */
70084887fa3SAttilio Rao 		if (slocked || (x & LK_HAVE_EXCL) == 0) {
70184887fa3SAttilio Rao 			witness_assert(&lkp->lk_object, what, file, line);
70284887fa3SAttilio Rao 			break;
70384887fa3SAttilio Rao 		}
70484887fa3SAttilio Rao #endif
70584887fa3SAttilio Rao 		if (LOCKMGR_UNHELD(x) || ((x & LK_SHARE_NONZERO) == 0 &&
70684887fa3SAttilio Rao 		    (slocked || LOCKMGR_NOTOWNER(td))))
70784887fa3SAttilio Rao 			panic("Lock %s not %slocked @ %s:%d\n",
70884887fa3SAttilio Rao 			    lkp->lk_object.lo_name, slocked ? "share " : "",
70984887fa3SAttilio Rao 			    file, line);
71084887fa3SAttilio Rao 		if ((x & LK_SHARE_NONZERO) == 0) {
71184887fa3SAttilio Rao 			if (lockmgr_recursed(lkp)) {
71284887fa3SAttilio Rao 				if (what & KA_NOTRECURSED)
71384887fa3SAttilio Rao 					panic("Lock %s recursed @ %s:%d\n",
71484887fa3SAttilio Rao 					    lkp->lk_object.lo_name, file, line);
71584887fa3SAttilio Rao 			} else if (what & KA_RECURSED)
71684887fa3SAttilio Rao 				panic("Lock %s not recursed @ %s:%d\n",
71784887fa3SAttilio Rao 				    lkp->lk_object.lo_name, file, line);
71884887fa3SAttilio Rao 		}
71984887fa3SAttilio Rao 		break;
72084887fa3SAttilio Rao 	case KA_XLOCKED:
72184887fa3SAttilio Rao 	case KA_XLOCKED | KA_NOTRECURSED:
72284887fa3SAttilio Rao 	case KA_XLOCKED | KA_RECURSED:
72384887fa3SAttilio Rao 		if ((x & LK_HAVE_EXCL) == 0 || LOCKMGR_NOTOWNER(td))
72484887fa3SAttilio Rao 			panic("Lock %s not exclusively locked @ %s:%d\n",
72584887fa3SAttilio Rao 			    lkp->lk_object.lo_name, file, line);
72684887fa3SAttilio Rao 		if (lockmgr_recursed(lkp)) {
72784887fa3SAttilio Rao 			if (what & KA_NOTRECURSED)
72884887fa3SAttilio Rao 				panic("Lock %s recursed @ %s:%d\n",
72984887fa3SAttilio Rao 				    lkp->lk_object.lo_name, file, line);
73084887fa3SAttilio Rao 		} else if (what & KA_RECURSED)
73184887fa3SAttilio Rao 			panic("Lock %s not recursed @ %s:%d\n",
73284887fa3SAttilio Rao 			    lkp->lk_object.lo_name, file, line);
73384887fa3SAttilio Rao 		break;
73484887fa3SAttilio Rao 	case KA_UNLOCKED:
73584887fa3SAttilio Rao 		if (td == curthread || td == LK_KERNPROC)
73684887fa3SAttilio Rao 			panic("Lock %s exclusively locked @ %s:%d\n",
73784887fa3SAttilio Rao 			    lkp->lk_object.lo_name, file, line);
73884887fa3SAttilio Rao 		break;
73984887fa3SAttilio Rao 	case KA_HELD:
74084887fa3SAttilio Rao 	case KA_UNHELD:
74184887fa3SAttilio Rao 		if (LOCKMGR_UNHELD(x)) {
74284887fa3SAttilio Rao 			if (what & KA_HELD)
74384887fa3SAttilio Rao 				panic("Lock %s not locked by anyone @ %s:%d\n",
74484887fa3SAttilio Rao 				    lkp->lk_object.lo_name, file, line);
74584887fa3SAttilio Rao 		} else if (what & KA_UNHELD)
74684887fa3SAttilio Rao 			panic("Lock %s locked by someone @ %s:%d\n",
74784887fa3SAttilio Rao 			    lkp->lk_object.lo_name, file, line);
74884887fa3SAttilio Rao 		break;
74984887fa3SAttilio Rao 	default:
75084887fa3SAttilio Rao 		panic("Unknown lockmgr lock assertion: 0x%x @ %s:%d", what,
75184887fa3SAttilio Rao 		    file, line);
75284887fa3SAttilio Rao 	}
75384887fa3SAttilio Rao }
75484887fa3SAttilio Rao #endif	/* INVARIANT_SUPPORT */
75584887fa3SAttilio Rao 
756be6847d7SJohn Baldwin #ifdef DDB
757462a7addSJohn Baldwin /*
758462a7addSJohn Baldwin  * Check to see if a thread that is blocked on a sleep queue is actually
759462a7addSJohn Baldwin  * blocked on a 'struct lock'.  If so, output some details and return true.
760462a7addSJohn Baldwin  * If the lock has an exclusive owner, return that in *ownerp.
761462a7addSJohn Baldwin  */
762462a7addSJohn Baldwin int
763462a7addSJohn Baldwin lockmgr_chain(struct thread *td, struct thread **ownerp)
764462a7addSJohn Baldwin {
765462a7addSJohn Baldwin 	struct lock *lkp;
766462a7addSJohn Baldwin 
767462a7addSJohn Baldwin 	lkp = td->td_wchan;
768462a7addSJohn Baldwin 
769462a7addSJohn Baldwin 	/* Simple test to see if wchan points to a lockmgr lock. */
770ab2dab16SJohn Baldwin 	if (LOCK_CLASS(&lkp->lk_object) == &lock_class_lockmgr &&
771ab2dab16SJohn Baldwin 	    lkp->lk_wmesg == td->td_wmesg)
7726d257b6eSJohn Baldwin 		goto ok;
7736d257b6eSJohn Baldwin 
7746d257b6eSJohn Baldwin 	/*
7756d257b6eSJohn Baldwin 	 * If this thread is doing a DRAIN, then it would be asleep on
7766d257b6eSJohn Baldwin 	 * &lkp->lk_flags rather than lkp.
7776d257b6eSJohn Baldwin 	 */
7786d257b6eSJohn Baldwin 	lkp = (struct lock *)((char *)td->td_wchan -
7796d257b6eSJohn Baldwin 	    offsetof(struct lock, lk_flags));
780ab2dab16SJohn Baldwin 	if (LOCK_CLASS(&lkp->lk_object) == &lock_class_lockmgr &&
781ab2dab16SJohn Baldwin 	    lkp->lk_wmesg == td->td_wmesg && (lkp->lk_flags & LK_WAITDRAIN))
7826d257b6eSJohn Baldwin 		goto ok;
7836d257b6eSJohn Baldwin 
7846d257b6eSJohn Baldwin 	/* Doen't seem to be a lockmgr lock. */
785462a7addSJohn Baldwin 	return (0);
786462a7addSJohn Baldwin 
7876d257b6eSJohn Baldwin ok:
788462a7addSJohn Baldwin 	/* Ok, we think we have a lockmgr lock, so output some details. */
789462a7addSJohn Baldwin 	db_printf("blocked on lk \"%s\" ", lkp->lk_wmesg);
790462a7addSJohn Baldwin 	if (lkp->lk_sharecount) {
791462a7addSJohn Baldwin 		db_printf("SHARED (count %d)\n", lkp->lk_sharecount);
792462a7addSJohn Baldwin 		*ownerp = NULL;
793462a7addSJohn Baldwin 	} else {
794462a7addSJohn Baldwin 		db_printf("EXCL (count %d)\n", lkp->lk_exclusivecount);
795462a7addSJohn Baldwin 		*ownerp = lkp->lk_lockholder;
796462a7addSJohn Baldwin 	}
797462a7addSJohn Baldwin 	return (1);
798462a7addSJohn Baldwin }
799462a7addSJohn Baldwin 
80061bd5e21SKip Macy void
80161bd5e21SKip Macy db_show_lockmgr(struct lock_object *lock)
802be6847d7SJohn Baldwin {
803be6847d7SJohn Baldwin 	struct thread *td;
804be6847d7SJohn Baldwin 	struct lock *lkp;
805be6847d7SJohn Baldwin 
80661bd5e21SKip Macy 	lkp = (struct lock *)lock;
807be6847d7SJohn Baldwin 
808be6847d7SJohn Baldwin 	db_printf(" lock type: %s\n", lkp->lk_wmesg);
809be6847d7SJohn Baldwin 	db_printf(" state: ");
810be6847d7SJohn Baldwin 	if (lkp->lk_sharecount)
811be6847d7SJohn Baldwin 		db_printf("SHARED (count %d)\n", lkp->lk_sharecount);
812be6847d7SJohn Baldwin 	else if (lkp->lk_flags & LK_HAVE_EXCL) {
813be6847d7SJohn Baldwin 		td = lkp->lk_lockholder;
814be6847d7SJohn Baldwin 		db_printf("EXCL (count %d) %p ", lkp->lk_exclusivecount, td);
815be6847d7SJohn Baldwin 		db_printf("(tid %d, pid %d, \"%s\")\n", td->td_tid,
816431f8906SJulian Elischer 		    td->td_proc->p_pid, td->td_name);
817be6847d7SJohn Baldwin 	} else
818be6847d7SJohn Baldwin 		db_printf("UNLOCKED\n");
819be6847d7SJohn Baldwin 	if (lkp->lk_waitcount > 0)
820be6847d7SJohn Baldwin 		db_printf(" waiters: %d\n", lkp->lk_waitcount);
821be6847d7SJohn Baldwin }
822be6847d7SJohn Baldwin #endif
823