xref: /freebsd/sys/kern/kern_lock.c (revision 10391db5300b72ff2918fbeecb0b6ab20f4dc373)
19454b2d8SWarner Losh /*-
28a36da99SPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
38a36da99SPedro F. Giffuni  *
4047dd67eSAttilio Rao  * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
5047dd67eSAttilio Rao  * All rights reserved.
653bf4bb2SPeter Wemm  *
753bf4bb2SPeter Wemm  * Redistribution and use in source and binary forms, with or without
853bf4bb2SPeter Wemm  * modification, are permitted provided that the following conditions
953bf4bb2SPeter Wemm  * are met:
1053bf4bb2SPeter Wemm  * 1. Redistributions of source code must retain the above copyright
11047dd67eSAttilio Rao  *    notice(s), this list of conditions and the following disclaimer as
12047dd67eSAttilio Rao  *    the first lines of this file unmodified other than the possible
13047dd67eSAttilio Rao  *    addition of one or more copyright notices.
1453bf4bb2SPeter Wemm  * 2. Redistributions in binary form must reproduce the above copyright
15047dd67eSAttilio Rao  *    notice(s), this list of conditions and the following disclaimer in the
1653bf4bb2SPeter Wemm  *    documentation and/or other materials provided with the distribution.
1753bf4bb2SPeter Wemm  *
18047dd67eSAttilio Rao  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
19047dd67eSAttilio Rao  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20047dd67eSAttilio Rao  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21047dd67eSAttilio Rao  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
22047dd67eSAttilio Rao  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23047dd67eSAttilio Rao  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24047dd67eSAttilio Rao  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25047dd67eSAttilio Rao  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2653bf4bb2SPeter Wemm  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27047dd67eSAttilio Rao  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
28047dd67eSAttilio Rao  * DAMAGE.
2953bf4bb2SPeter Wemm  */
3053bf4bb2SPeter Wemm 
31047dd67eSAttilio Rao #include "opt_ddb.h"
32f5f9340bSFabien Thomas #include "opt_hwpmc_hooks.h"
33047dd67eSAttilio Rao 
34677b542eSDavid E. O'Brien #include <sys/cdefs.h>
35677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
36677b542eSDavid E. O'Brien 
3753bf4bb2SPeter Wemm #include <sys/param.h>
38cd2fe4e6SAttilio Rao #include <sys/kdb.h>
3961d80e90SJohn Baldwin #include <sys/ktr.h>
4053bf4bb2SPeter Wemm #include <sys/lock.h>
41047dd67eSAttilio Rao #include <sys/lock_profile.h>
428302d183SBruce Evans #include <sys/lockmgr.h>
43d8881ca3SJohn Baldwin #include <sys/mutex.h>
448302d183SBruce Evans #include <sys/proc.h>
45047dd67eSAttilio Rao #include <sys/sleepqueue.h>
46e8ddb61dSJeff Roberson #ifdef DEBUG_LOCKS
47e8ddb61dSJeff Roberson #include <sys/stack.h>
48e8ddb61dSJeff Roberson #endif
49651175c9SAttilio Rao #include <sys/sysctl.h>
50047dd67eSAttilio Rao #include <sys/systm.h>
5153bf4bb2SPeter Wemm 
52047dd67eSAttilio Rao #include <machine/cpu.h>
536efc8a16SAttilio Rao 
54be6847d7SJohn Baldwin #ifdef DDB
55be6847d7SJohn Baldwin #include <ddb/ddb.h>
56047dd67eSAttilio Rao #endif
57047dd67eSAttilio Rao 
58f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
59f5f9340bSFabien Thomas #include <sys/pmckern.h>
60f5f9340bSFabien Thomas PMC_SOFT_DECLARE( , , lock, failed);
61f5f9340bSFabien Thomas #endif
62f5f9340bSFabien Thomas 
63651175c9SAttilio Rao CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) ==
64651175c9SAttilio Rao     (LK_ADAPTIVE | LK_NOSHARE));
65651175c9SAttilio Rao CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
66651175c9SAttilio Rao     ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
67047dd67eSAttilio Rao 
68047dd67eSAttilio Rao #define	SQ_EXCLUSIVE_QUEUE	0
69047dd67eSAttilio Rao #define	SQ_SHARED_QUEUE		1
70047dd67eSAttilio Rao 
71047dd67eSAttilio Rao #ifndef INVARIANTS
72047dd67eSAttilio Rao #define	_lockmgr_assert(lk, what, file, line)
73047dd67eSAttilio Rao #endif
74ce1c953eSMark Johnston 
75047dd67eSAttilio Rao #define	TD_SLOCKS_INC(td)	((td)->td_lk_slocks++)
76047dd67eSAttilio Rao #define	TD_SLOCKS_DEC(td)	((td)->td_lk_slocks--)
77047dd67eSAttilio Rao 
78047dd67eSAttilio Rao #ifndef DEBUG_LOCKS
79047dd67eSAttilio Rao #define	STACK_PRINT(lk)
80047dd67eSAttilio Rao #define	STACK_SAVE(lk)
81047dd67eSAttilio Rao #define	STACK_ZERO(lk)
82047dd67eSAttilio Rao #else
83047dd67eSAttilio Rao #define	STACK_PRINT(lk)	stack_print_ddb(&(lk)->lk_stack)
84047dd67eSAttilio Rao #define	STACK_SAVE(lk)	stack_save(&(lk)->lk_stack)
85047dd67eSAttilio Rao #define	STACK_ZERO(lk)	stack_zero(&(lk)->lk_stack)
86047dd67eSAttilio Rao #endif
87047dd67eSAttilio Rao 
88047dd67eSAttilio Rao #define	LOCK_LOG2(lk, string, arg1, arg2)				\
89047dd67eSAttilio Rao 	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
90047dd67eSAttilio Rao 		CTR2(KTR_LOCK, (string), (arg1), (arg2))
91047dd67eSAttilio Rao #define	LOCK_LOG3(lk, string, arg1, arg2, arg3)				\
92047dd67eSAttilio Rao 	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
93047dd67eSAttilio Rao 		CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
94047dd67eSAttilio Rao 
95e5f94314SAttilio Rao #define	GIANT_DECLARE							\
96e5f94314SAttilio Rao 	int _i = 0;							\
97e5f94314SAttilio Rao 	WITNESS_SAVE_DECL(Giant)
98e5f94314SAttilio Rao #define	GIANT_RESTORE() do {						\
99e5f94314SAttilio Rao 	if (_i > 0) {							\
100e5f94314SAttilio Rao 		while (_i--)						\
101e5f94314SAttilio Rao 			mtx_lock(&Giant);				\
102e5f94314SAttilio Rao 		WITNESS_RESTORE(&Giant.lock_object, Giant);		\
103e5f94314SAttilio Rao 	}								\
104e5f94314SAttilio Rao } while (0)
105e5f94314SAttilio Rao #define	GIANT_SAVE() do {						\
106e5f94314SAttilio Rao 	if (mtx_owned(&Giant)) {					\
107e5f94314SAttilio Rao 		WITNESS_SAVE(&Giant.lock_object, Giant);		\
108e5f94314SAttilio Rao 		while (mtx_owned(&Giant)) {				\
109e5f94314SAttilio Rao 			_i++;						\
110e5f94314SAttilio Rao 			mtx_unlock(&Giant);				\
111e5f94314SAttilio Rao 		}							\
112e5f94314SAttilio Rao 	}								\
113e5f94314SAttilio Rao } while (0)
114e5f94314SAttilio Rao 
11572ba3c08SKonstantin Belousov #define	LK_CAN_SHARE(x, flags)						\
11672ba3c08SKonstantin Belousov 	(((x) & LK_SHARE) &&						\
11772ba3c08SKonstantin Belousov 	(((x) & (LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) == 0 ||	\
11872ba3c08SKonstantin Belousov 	(curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||	\
11972ba3c08SKonstantin Belousov 	(curthread->td_pflags & TDP_DEADLKTREAT)))
120e5f94314SAttilio Rao #define	LK_TRYOP(x)							\
121e5f94314SAttilio Rao 	((x) & LK_NOWAIT)
122e5f94314SAttilio Rao 
123e5f94314SAttilio Rao #define	LK_CAN_WITNESS(x)						\
124e5f94314SAttilio Rao 	(((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
125e5f94314SAttilio Rao #define	LK_TRYWIT(x)							\
126e5f94314SAttilio Rao 	(LK_TRYOP(x) ? LOP_TRYLOCK : 0)
127047dd67eSAttilio Rao 
128651175c9SAttilio Rao #define	LK_CAN_ADAPT(lk, f)						\
129651175c9SAttilio Rao 	(((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 &&		\
130651175c9SAttilio Rao 	((f) & LK_SLEEPFAIL) == 0)
131651175c9SAttilio Rao 
132047dd67eSAttilio Rao #define	lockmgr_disowned(lk)						\
133047dd67eSAttilio Rao 	(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
134047dd67eSAttilio Rao 
135*10391db5SMateusz Guzik #define	lockmgr_xlocked_v(v)						\
136*10391db5SMateusz Guzik 	(((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
137*10391db5SMateusz Guzik 
138*10391db5SMateusz Guzik #define	lockmgr_xlocked(lk) lockmgr_xlocked_v((lk)->lk_lock)
139047dd67eSAttilio Rao 
140d576deedSPawel Jakub Dawidek static void	assert_lockmgr(const struct lock_object *lock, int how);
141047dd67eSAttilio Rao #ifdef DDB
142d576deedSPawel Jakub Dawidek static void	db_show_lockmgr(const struct lock_object *lock);
143be6847d7SJohn Baldwin #endif
1447faf4d90SDavide Italiano static void	lock_lockmgr(struct lock_object *lock, uintptr_t how);
145a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
146d576deedSPawel Jakub Dawidek static int	owner_lockmgr(const struct lock_object *lock,
147d576deedSPawel Jakub Dawidek 		    struct thread **owner);
148a5aedd68SStacey Son #endif
1497faf4d90SDavide Italiano static uintptr_t unlock_lockmgr(struct lock_object *lock);
15061bd5e21SKip Macy 
15161bd5e21SKip Macy struct lock_class lock_class_lockmgr = {
1523ff6d229SJohn Baldwin 	.lc_name = "lockmgr",
153047dd67eSAttilio Rao 	.lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
154f9721b43SAttilio Rao 	.lc_assert = assert_lockmgr,
15561bd5e21SKip Macy #ifdef DDB
1566e21afd4SJohn Baldwin 	.lc_ddb_show = db_show_lockmgr,
15761bd5e21SKip Macy #endif
1586e21afd4SJohn Baldwin 	.lc_lock = lock_lockmgr,
159a5aedd68SStacey Son 	.lc_unlock = unlock_lockmgr,
160a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
161a5aedd68SStacey Son 	.lc_owner = owner_lockmgr,
162a5aedd68SStacey Son #endif
16361bd5e21SKip Macy };
16461bd5e21SKip Macy 
1651c6987ebSMateusz Guzik struct lockmgr_wait {
1661c6987ebSMateusz Guzik 	const char *iwmesg;
1671c6987ebSMateusz Guzik 	int ipri;
1681c6987ebSMateusz Guzik 	int itimo;
1691c6987ebSMateusz Guzik };
1701c6987ebSMateusz Guzik 
171c4a48867SMateusz Guzik static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
172c4a48867SMateusz Guzik     int flags);
1731c6987ebSMateusz Guzik static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp);
1741c6987ebSMateusz Guzik 
1751c6987ebSMateusz Guzik static void
1761c6987ebSMateusz Guzik lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper)
1771c6987ebSMateusz Guzik {
1781c6987ebSMateusz Guzik 	struct lock_class *class;
1791c6987ebSMateusz Guzik 
1801c6987ebSMateusz Guzik 	if (flags & LK_INTERLOCK) {
1811c6987ebSMateusz Guzik 		class = LOCK_CLASS(ilk);
1821c6987ebSMateusz Guzik 		class->lc_unlock(ilk);
1831c6987ebSMateusz Guzik 	}
1841c6987ebSMateusz Guzik 
1851c6987ebSMateusz Guzik 	if (__predict_false(wakeup_swapper))
1861c6987ebSMateusz Guzik 		kick_proc0();
1871c6987ebSMateusz Guzik }
188c4a48867SMateusz Guzik 
189c4a48867SMateusz Guzik static void
190c4a48867SMateusz Guzik lockmgr_note_shared_acquire(struct lock *lk, int contested,
191c4a48867SMateusz Guzik     uint64_t waittime, const char *file, int line, int flags)
192c4a48867SMateusz Guzik {
193c4a48867SMateusz Guzik 
194c4a48867SMateusz Guzik 	lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
195c4a48867SMateusz Guzik 	    file, line);
196c4a48867SMateusz Guzik 	LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
197c4a48867SMateusz Guzik 	WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
198c4a48867SMateusz Guzik 	TD_LOCKS_INC(curthread);
199c4a48867SMateusz Guzik 	TD_SLOCKS_INC(curthread);
200c4a48867SMateusz Guzik 	STACK_SAVE(lk);
201c4a48867SMateusz Guzik }
202c4a48867SMateusz Guzik 
203c4a48867SMateusz Guzik static void
204c4a48867SMateusz Guzik lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
205c4a48867SMateusz Guzik {
206c4a48867SMateusz Guzik 
207c4a48867SMateusz Guzik 	lock_profile_release_lock(&lk->lock_object);
208c4a48867SMateusz Guzik 	WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
209c4a48867SMateusz Guzik 	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
210c4a48867SMateusz Guzik 	TD_LOCKS_DEC(curthread);
211c4a48867SMateusz Guzik 	TD_SLOCKS_DEC(curthread);
212c4a48867SMateusz Guzik }
213c4a48867SMateusz Guzik 
214c4a48867SMateusz Guzik static void
215c4a48867SMateusz Guzik lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
216c4a48867SMateusz Guzik     uint64_t waittime, const char *file, int line, int flags)
217c4a48867SMateusz Guzik {
218c4a48867SMateusz Guzik 
219c4a48867SMateusz Guzik 	lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
220c4a48867SMateusz Guzik 	    file, line);
221c4a48867SMateusz Guzik 	LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
222c4a48867SMateusz Guzik 	WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
223c4a48867SMateusz Guzik 	    line);
224c4a48867SMateusz Guzik 	TD_LOCKS_INC(curthread);
225c4a48867SMateusz Guzik 	STACK_SAVE(lk);
226c4a48867SMateusz Guzik }
227c4a48867SMateusz Guzik 
228c4a48867SMateusz Guzik static void
229c4a48867SMateusz Guzik lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
230c4a48867SMateusz Guzik {
231c4a48867SMateusz Guzik 
232c4a48867SMateusz Guzik 	lock_profile_release_lock(&lk->lock_object);
233c4a48867SMateusz Guzik 	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
234c4a48867SMateusz Guzik 	    line);
235c4a48867SMateusz Guzik 	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
236c4a48867SMateusz Guzik 	TD_LOCKS_DEC(curthread);
237c4a48867SMateusz Guzik }
238c4a48867SMateusz Guzik 
239047dd67eSAttilio Rao static __inline struct thread *
240d576deedSPawel Jakub Dawidek lockmgr_xholder(const struct lock *lk)
241047dd67eSAttilio Rao {
242047dd67eSAttilio Rao 	uintptr_t x;
243047dd67eSAttilio Rao 
244047dd67eSAttilio Rao 	x = lk->lk_lock;
245047dd67eSAttilio Rao 	return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
246047dd67eSAttilio Rao }
24784887fa3SAttilio Rao 
24853bf4bb2SPeter Wemm /*
249047dd67eSAttilio Rao  * It assumes sleepq_lock held and returns with this one unheld.
250047dd67eSAttilio Rao  * It also assumes the generic interlock is sane and previously checked.
251047dd67eSAttilio Rao  * If LK_INTERLOCK is specified the interlock is not reacquired after the
252047dd67eSAttilio Rao  * sleep.
25353bf4bb2SPeter Wemm  */
254047dd67eSAttilio Rao static __inline int
255047dd67eSAttilio Rao sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
256047dd67eSAttilio Rao     const char *wmesg, int pri, int timo, int queue)
257047dd67eSAttilio Rao {
258e5f94314SAttilio Rao 	GIANT_DECLARE;
259047dd67eSAttilio Rao 	struct lock_class *class;
260047dd67eSAttilio Rao 	int catch, error;
26153bf4bb2SPeter Wemm 
262047dd67eSAttilio Rao 	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
2635047a8fdSAttilio Rao 	catch = pri & PCATCH;
264047dd67eSAttilio Rao 	pri &= PRIMASK;
265047dd67eSAttilio Rao 	error = 0;
266047dd67eSAttilio Rao 
267047dd67eSAttilio Rao 	LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
268047dd67eSAttilio Rao 	    (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
269047dd67eSAttilio Rao 
270047dd67eSAttilio Rao 	if (flags & LK_INTERLOCK)
271047dd67eSAttilio Rao 		class->lc_unlock(ilk);
2722028867dSAttilio Rao 	if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0)
2732028867dSAttilio Rao 		lk->lk_exslpfail++;
274e5f94314SAttilio Rao 	GIANT_SAVE();
275047dd67eSAttilio Rao 	sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
276047dd67eSAttilio Rao 	    SLEEPQ_INTERRUPTIBLE : 0), queue);
277a115fb62SHans Petter Selasky 	if ((flags & LK_TIMELOCK) && timo)
278047dd67eSAttilio Rao 		sleepq_set_timeout(&lk->lock_object, timo);
279a115fb62SHans Petter Selasky 
280047dd67eSAttilio Rao 	/*
281047dd67eSAttilio Rao 	 * Decisional switch for real sleeping.
282047dd67eSAttilio Rao 	 */
283047dd67eSAttilio Rao 	if ((flags & LK_TIMELOCK) && timo && catch)
284047dd67eSAttilio Rao 		error = sleepq_timedwait_sig(&lk->lock_object, pri);
285047dd67eSAttilio Rao 	else if ((flags & LK_TIMELOCK) && timo)
286047dd67eSAttilio Rao 		error = sleepq_timedwait(&lk->lock_object, pri);
287047dd67eSAttilio Rao 	else if (catch)
288047dd67eSAttilio Rao 		error = sleepq_wait_sig(&lk->lock_object, pri);
289047dd67eSAttilio Rao 	else
290047dd67eSAttilio Rao 		sleepq_wait(&lk->lock_object, pri);
291e5f94314SAttilio Rao 	GIANT_RESTORE();
292047dd67eSAttilio Rao 	if ((flags & LK_SLEEPFAIL) && error == 0)
293047dd67eSAttilio Rao 		error = ENOLCK;
294047dd67eSAttilio Rao 
295047dd67eSAttilio Rao 	return (error);
296047dd67eSAttilio Rao }
297047dd67eSAttilio Rao 
298da7bbd2cSJohn Baldwin static __inline int
299047dd67eSAttilio Rao wakeupshlk(struct lock *lk, const char *file, int line)
300047dd67eSAttilio Rao {
3010ad122a9SMateusz Guzik 	uintptr_t v, x, orig_x;
3022028867dSAttilio Rao 	u_int realexslp;
303da7bbd2cSJohn Baldwin 	int queue, wakeup_swapper;
304047dd67eSAttilio Rao 
305da7bbd2cSJohn Baldwin 	wakeup_swapper = 0;
306047dd67eSAttilio Rao 	for (;;) {
307047dd67eSAttilio Rao 		x = lk->lk_lock;
3081c6987ebSMateusz Guzik 		if (lockmgr_sunlock_try(lk, &x))
309047dd67eSAttilio Rao 			break;
310047dd67eSAttilio Rao 
311047dd67eSAttilio Rao 		/*
312047dd67eSAttilio Rao 		 * We should have a sharer with waiters, so enter the hard
313047dd67eSAttilio Rao 		 * path in order to handle wakeups correctly.
314047dd67eSAttilio Rao 		 */
315047dd67eSAttilio Rao 		sleepq_lock(&lk->lock_object);
3160ad122a9SMateusz Guzik 		orig_x = lk->lk_lock;
3170ad122a9SMateusz Guzik retry_sleepq:
3180ad122a9SMateusz Guzik 		x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
319047dd67eSAttilio Rao 		v = LK_UNLOCKED;
320047dd67eSAttilio Rao 
321047dd67eSAttilio Rao 		/*
322047dd67eSAttilio Rao 		 * If the lock has exclusive waiters, give them preference in
323047dd67eSAttilio Rao 		 * order to avoid deadlock with shared runners up.
3242028867dSAttilio Rao 		 * If interruptible sleeps left the exclusive queue empty
3252028867dSAttilio Rao 		 * avoid a starvation for the threads sleeping on the shared
3262028867dSAttilio Rao 		 * queue by giving them precedence and cleaning up the
3272028867dSAttilio Rao 		 * exclusive waiters bit anyway.
328c636ba83SAttilio Rao 		 * Please note that lk_exslpfail count may be lying about
329c636ba83SAttilio Rao 		 * the real number of waiters with the LK_SLEEPFAIL flag on
330e3043798SPedro F. Giffuni 		 * because they may be used in conjunction with interruptible
331aab9c8c2SAttilio Rao 		 * sleeps so lk_exslpfail might be considered an 'upper limit'
332aab9c8c2SAttilio Rao 		 * bound, including the edge cases.
333047dd67eSAttilio Rao 		 */
3342028867dSAttilio Rao 		realexslp = sleepq_sleepcnt(&lk->lock_object,
3352028867dSAttilio Rao 		    SQ_EXCLUSIVE_QUEUE);
3362028867dSAttilio Rao 		if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
3372028867dSAttilio Rao 			if (lk->lk_exslpfail < realexslp) {
3382028867dSAttilio Rao 				lk->lk_exslpfail = 0;
339047dd67eSAttilio Rao 				queue = SQ_EXCLUSIVE_QUEUE;
340047dd67eSAttilio Rao 				v |= (x & LK_SHARED_WAITERS);
341047dd67eSAttilio Rao 			} else {
3422028867dSAttilio Rao 				lk->lk_exslpfail = 0;
3432028867dSAttilio Rao 				LOCK_LOG2(lk,
3442028867dSAttilio Rao 				    "%s: %p has only LK_SLEEPFAIL sleepers",
3452028867dSAttilio Rao 				    __func__, lk);
3462028867dSAttilio Rao 				LOCK_LOG2(lk,
3472028867dSAttilio Rao 			    "%s: %p waking up threads on the exclusive queue",
3482028867dSAttilio Rao 				    __func__, lk);
3492028867dSAttilio Rao 				wakeup_swapper =
3502028867dSAttilio Rao 				    sleepq_broadcast(&lk->lock_object,
3512028867dSAttilio Rao 				    SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
3522028867dSAttilio Rao 				queue = SQ_SHARED_QUEUE;
3532028867dSAttilio Rao 			}
3542028867dSAttilio Rao 
3552028867dSAttilio Rao 		} else {
3569dbf7a62SAttilio Rao 
3579dbf7a62SAttilio Rao 			/*
3589dbf7a62SAttilio Rao 			 * Exclusive waiters sleeping with LK_SLEEPFAIL on
3599dbf7a62SAttilio Rao 			 * and using interruptible sleeps/timeout may have
3609dbf7a62SAttilio Rao 			 * left spourious lk_exslpfail counts on, so clean
3619dbf7a62SAttilio Rao 			 * it up anyway.
3629dbf7a62SAttilio Rao 			 */
3639dbf7a62SAttilio Rao 			lk->lk_exslpfail = 0;
364047dd67eSAttilio Rao 			queue = SQ_SHARED_QUEUE;
365047dd67eSAttilio Rao 		}
366047dd67eSAttilio Rao 
3670ad122a9SMateusz Guzik 		if (lockmgr_sunlock_try(lk, &orig_x)) {
368047dd67eSAttilio Rao 			sleepq_release(&lk->lock_object);
3690ad122a9SMateusz Guzik 			break;
3700ad122a9SMateusz Guzik 		}
3710ad122a9SMateusz Guzik 
3720ad122a9SMateusz Guzik 		x |= LK_SHARERS_LOCK(1);
3730ad122a9SMateusz Guzik 		if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
3740ad122a9SMateusz Guzik 			orig_x = x;
3750ad122a9SMateusz Guzik 			goto retry_sleepq;
376047dd67eSAttilio Rao 		}
377047dd67eSAttilio Rao 		LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
378047dd67eSAttilio Rao 		    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
379047dd67eSAttilio Rao 		    "exclusive");
3802028867dSAttilio Rao 		wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
381da7bbd2cSJohn Baldwin 		    0, queue);
382047dd67eSAttilio Rao 		sleepq_release(&lk->lock_object);
383047dd67eSAttilio Rao 		break;
384047dd67eSAttilio Rao 	}
385047dd67eSAttilio Rao 
386c4a48867SMateusz Guzik 	lockmgr_note_shared_release(lk, file, line);
387da7bbd2cSJohn Baldwin 	return (wakeup_swapper);
388047dd67eSAttilio Rao }
389047dd67eSAttilio Rao 
390047dd67eSAttilio Rao static void
391d576deedSPawel Jakub Dawidek assert_lockmgr(const struct lock_object *lock, int what)
392f9721b43SAttilio Rao {
393f9721b43SAttilio Rao 
394f9721b43SAttilio Rao 	panic("lockmgr locks do not support assertions");
395f9721b43SAttilio Rao }
396f9721b43SAttilio Rao 
397047dd67eSAttilio Rao static void
3987faf4d90SDavide Italiano lock_lockmgr(struct lock_object *lock, uintptr_t how)
3996e21afd4SJohn Baldwin {
4006e21afd4SJohn Baldwin 
4016e21afd4SJohn Baldwin 	panic("lockmgr locks do not support sleep interlocking");
4026e21afd4SJohn Baldwin }
4036e21afd4SJohn Baldwin 
4047faf4d90SDavide Italiano static uintptr_t
4056e21afd4SJohn Baldwin unlock_lockmgr(struct lock_object *lock)
4066e21afd4SJohn Baldwin {
4076e21afd4SJohn Baldwin 
4086e21afd4SJohn Baldwin 	panic("lockmgr locks do not support sleep interlocking");
4096e21afd4SJohn Baldwin }
4106e21afd4SJohn Baldwin 
411a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
412a5aedd68SStacey Son static int
413d576deedSPawel Jakub Dawidek owner_lockmgr(const struct lock_object *lock, struct thread **owner)
414a5aedd68SStacey Son {
415a5aedd68SStacey Son 
416a5aedd68SStacey Son 	panic("lockmgr locks do not support owner inquiring");
417a5aedd68SStacey Son }
418a5aedd68SStacey Son #endif
419a5aedd68SStacey Son 
42099448ed1SJohn Dyson void
421047dd67eSAttilio Rao lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
42299448ed1SJohn Dyson {
4236efc8a16SAttilio Rao 	int iflags;
4246efc8a16SAttilio Rao 
425047dd67eSAttilio Rao 	MPASS((flags & ~LK_INIT_MASK) == 0);
426353998acSAttilio Rao 	ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
427353998acSAttilio Rao             ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
428353998acSAttilio Rao             &lk->lk_lock));
42999448ed1SJohn Dyson 
430f0830182SAttilio Rao 	iflags = LO_SLEEPABLE | LO_UPGRADABLE;
431f0830182SAttilio Rao 	if (flags & LK_CANRECURSE)
432f0830182SAttilio Rao 		iflags |= LO_RECURSABLE;
433047dd67eSAttilio Rao 	if ((flags & LK_NODUP) == 0)
4346efc8a16SAttilio Rao 		iflags |= LO_DUPOK;
4357fbfba7bSAttilio Rao 	if (flags & LK_NOPROFILE)
4367fbfba7bSAttilio Rao 		iflags |= LO_NOPROFILE;
437047dd67eSAttilio Rao 	if ((flags & LK_NOWITNESS) == 0)
4386efc8a16SAttilio Rao 		iflags |= LO_WITNESS;
4397fbfba7bSAttilio Rao 	if (flags & LK_QUIET)
4407fbfba7bSAttilio Rao 		iflags |= LO_QUIET;
441e63091eaSMarcel Moolenaar 	if (flags & LK_IS_VNODE)
442e63091eaSMarcel Moolenaar 		iflags |= LO_IS_VNODE;
443651175c9SAttilio Rao 	iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE);
444047dd67eSAttilio Rao 
445b5fb43e5SJohn Baldwin 	lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
446047dd67eSAttilio Rao 	lk->lk_lock = LK_UNLOCKED;
447047dd67eSAttilio Rao 	lk->lk_recurse = 0;
4482028867dSAttilio Rao 	lk->lk_exslpfail = 0;
449047dd67eSAttilio Rao 	lk->lk_timo = timo;
450047dd67eSAttilio Rao 	lk->lk_pri = pri;
451047dd67eSAttilio Rao 	STACK_ZERO(lk);
45299448ed1SJohn Dyson }
45399448ed1SJohn Dyson 
4543634d5b2SJohn Baldwin /*
4553634d5b2SJohn Baldwin  * XXX: Gross hacks to manipulate external lock flags after
4563634d5b2SJohn Baldwin  * initialization.  Used for certain vnode and buf locks.
4573634d5b2SJohn Baldwin  */
4583634d5b2SJohn Baldwin void
4593634d5b2SJohn Baldwin lockallowshare(struct lock *lk)
4603634d5b2SJohn Baldwin {
4613634d5b2SJohn Baldwin 
4623634d5b2SJohn Baldwin 	lockmgr_assert(lk, KA_XLOCKED);
4633634d5b2SJohn Baldwin 	lk->lock_object.lo_flags &= ~LK_NOSHARE;
4643634d5b2SJohn Baldwin }
4653634d5b2SJohn Baldwin 
4663634d5b2SJohn Baldwin void
467575e02d9SKonstantin Belousov lockdisableshare(struct lock *lk)
468575e02d9SKonstantin Belousov {
469575e02d9SKonstantin Belousov 
470575e02d9SKonstantin Belousov 	lockmgr_assert(lk, KA_XLOCKED);
471575e02d9SKonstantin Belousov 	lk->lock_object.lo_flags |= LK_NOSHARE;
472575e02d9SKonstantin Belousov }
473575e02d9SKonstantin Belousov 
474575e02d9SKonstantin Belousov void
4753634d5b2SJohn Baldwin lockallowrecurse(struct lock *lk)
4763634d5b2SJohn Baldwin {
4773634d5b2SJohn Baldwin 
4783634d5b2SJohn Baldwin 	lockmgr_assert(lk, KA_XLOCKED);
4793634d5b2SJohn Baldwin 	lk->lock_object.lo_flags |= LO_RECURSABLE;
4803634d5b2SJohn Baldwin }
4813634d5b2SJohn Baldwin 
4823634d5b2SJohn Baldwin void
4833634d5b2SJohn Baldwin lockdisablerecurse(struct lock *lk)
4843634d5b2SJohn Baldwin {
4853634d5b2SJohn Baldwin 
4863634d5b2SJohn Baldwin 	lockmgr_assert(lk, KA_XLOCKED);
4873634d5b2SJohn Baldwin 	lk->lock_object.lo_flags &= ~LO_RECURSABLE;
4883634d5b2SJohn Baldwin }
4893634d5b2SJohn Baldwin 
490a18b1f1dSJason Evans void
491047dd67eSAttilio Rao lockdestroy(struct lock *lk)
492a18b1f1dSJason Evans {
493c91fcee7SJohn Baldwin 
494047dd67eSAttilio Rao 	KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
495047dd67eSAttilio Rao 	KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
4962028867dSAttilio Rao 	KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
497047dd67eSAttilio Rao 	lock_destroy(&lk->lock_object);
498047dd67eSAttilio Rao }
499047dd67eSAttilio Rao 
500c4a48867SMateusz Guzik static bool __always_inline
501c4a48867SMateusz Guzik lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags)
502c4a48867SMateusz Guzik {
503c4a48867SMateusz Guzik 
504c4a48867SMateusz Guzik 	/*
505c4a48867SMateusz Guzik 	 * If no other thread has an exclusive lock, or
506c4a48867SMateusz Guzik 	 * no exclusive waiter is present, bump the count of
507c4a48867SMateusz Guzik 	 * sharers.  Since we have to preserve the state of
508c4a48867SMateusz Guzik 	 * waiters, if we fail to acquire the shared lock
509c4a48867SMateusz Guzik 	 * loop back and retry.
510c4a48867SMateusz Guzik 	 */
511c4a48867SMateusz Guzik 	*xp = lk->lk_lock;
512c4a48867SMateusz Guzik 	while (LK_CAN_SHARE(*xp, flags)) {
513c4a48867SMateusz Guzik 		if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
514c4a48867SMateusz Guzik 		    *xp + LK_ONE_SHARER)) {
515c4a48867SMateusz Guzik 			return (true);
516c4a48867SMateusz Guzik 		}
517c4a48867SMateusz Guzik 	}
518c4a48867SMateusz Guzik 	return (false);
519c4a48867SMateusz Guzik }
520c4a48867SMateusz Guzik 
521c4a48867SMateusz Guzik static bool __always_inline
5221c6987ebSMateusz Guzik lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp)
523c4a48867SMateusz Guzik {
524c4a48867SMateusz Guzik 
525c4a48867SMateusz Guzik 	for (;;) {
526c4a48867SMateusz Guzik 		/*
527c4a48867SMateusz Guzik 		 * If there is more than one shared lock held, just drop one
528c4a48867SMateusz Guzik 		 * and return.
529c4a48867SMateusz Guzik 		 */
5301c6987ebSMateusz Guzik 		if (LK_SHARERS(*xp) > 1) {
5311c6987ebSMateusz Guzik 			if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
5321c6987ebSMateusz Guzik 			    *xp - LK_ONE_SHARER))
533c4a48867SMateusz Guzik 				return (true);
534c4a48867SMateusz Guzik 			continue;
535c4a48867SMateusz Guzik 		}
536c4a48867SMateusz Guzik 
537c4a48867SMateusz Guzik 		/*
538c4a48867SMateusz Guzik 		 * If there are not waiters on the exclusive queue, drop the
539c4a48867SMateusz Guzik 		 * lock quickly.
540c4a48867SMateusz Guzik 		 */
5411c6987ebSMateusz Guzik 		if ((*xp & LK_ALL_WAITERS) == 0) {
5421c6987ebSMateusz Guzik 			MPASS((*xp & ~LK_EXCLUSIVE_SPINNERS) ==
543c4a48867SMateusz Guzik 			    LK_SHARERS_LOCK(1));
5441c6987ebSMateusz Guzik 			if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
545c4a48867SMateusz Guzik 			    LK_UNLOCKED))
546c4a48867SMateusz Guzik 				return (true);
547c4a48867SMateusz Guzik 			continue;
548c4a48867SMateusz Guzik 		}
549c4a48867SMateusz Guzik 		break;
550c4a48867SMateusz Guzik 	}
551c4a48867SMateusz Guzik 	return (false);
552c4a48867SMateusz Guzik }
553c4a48867SMateusz Guzik 
5541c6987ebSMateusz Guzik static __noinline int
5551c6987ebSMateusz Guzik lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
5561c6987ebSMateusz Guzik     const char *file, int line, struct lockmgr_wait *lwa)
557c4a48867SMateusz Guzik {
5581c6987ebSMateusz Guzik 	uintptr_t tid, x;
5591c6987ebSMateusz Guzik 	int error = 0;
560047dd67eSAttilio Rao 	const char *iwmesg;
5611c6987ebSMateusz Guzik 	int ipri, itimo;
5621c6987ebSMateusz Guzik 
5631723a064SJeff Roberson #ifdef LOCK_PROFILING
5641723a064SJeff Roberson 	uint64_t waittime = 0;
5651723a064SJeff Roberson 	int contested = 0;
5661723a064SJeff Roberson #endif
567047dd67eSAttilio Rao 
5681c6987ebSMateusz Guzik 	if (__predict_false(panicstr != NULL))
5691c6987ebSMateusz Guzik 		goto out;
5701c6987ebSMateusz Guzik 
571047dd67eSAttilio Rao 	tid = (uintptr_t)curthread;
572047dd67eSAttilio Rao 
573e5f94314SAttilio Rao 	if (LK_CAN_WITNESS(flags))
574e5f94314SAttilio Rao 		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
57524150d37SJohn Baldwin 		    file, line, flags & LK_INTERLOCK ? ilk : NULL);
576047dd67eSAttilio Rao 	for (;;) {
577c4a48867SMateusz Guzik 		if (lockmgr_slock_try(lk, &x, flags))
578047dd67eSAttilio Rao 			break;
579f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
580f5f9340bSFabien Thomas 		PMC_SOFT_CALL( , , lock, failed);
581f5f9340bSFabien Thomas #endif
582047dd67eSAttilio Rao 		lock_profile_obtain_lock_failed(&lk->lock_object,
583047dd67eSAttilio Rao 		    &contested, &waittime);
584047dd67eSAttilio Rao 
585047dd67eSAttilio Rao 		/*
58696f1567fSKonstantin Belousov 		 * If the lock is already held by curthread in
587047dd67eSAttilio Rao 		 * exclusive way avoid a deadlock.
588047dd67eSAttilio Rao 		 */
589047dd67eSAttilio Rao 		if (LK_HOLDER(x) == tid) {
590047dd67eSAttilio Rao 			LOCK_LOG2(lk,
59196f1567fSKonstantin Belousov 			    "%s: %p already held in exclusive mode",
592047dd67eSAttilio Rao 			    __func__, lk);
593047dd67eSAttilio Rao 			error = EDEADLK;
594047dd67eSAttilio Rao 			break;
595a18b1f1dSJason Evans 		}
596a18b1f1dSJason Evans 
597a18b1f1dSJason Evans 		/*
598047dd67eSAttilio Rao 		 * If the lock is expected to not sleep just give up
599047dd67eSAttilio Rao 		 * and return.
600d7a7e179SAttilio Rao 		 */
601047dd67eSAttilio Rao 		if (LK_TRYOP(flags)) {
602047dd67eSAttilio Rao 			LOCK_LOG2(lk, "%s: %p fails the try operation",
603047dd67eSAttilio Rao 			    __func__, lk);
604047dd67eSAttilio Rao 			error = EBUSY;
605047dd67eSAttilio Rao 			break;
606047dd67eSAttilio Rao 		}
607047dd67eSAttilio Rao 
608047dd67eSAttilio Rao 		/*
609047dd67eSAttilio Rao 		 * Acquire the sleepqueue chain lock because we
610047dd67eSAttilio Rao 		 * probabilly will need to manipulate waiters flags.
611047dd67eSAttilio Rao 		 */
612047dd67eSAttilio Rao 		sleepq_lock(&lk->lock_object);
613047dd67eSAttilio Rao 		x = lk->lk_lock;
6140ad122a9SMateusz Guzik retry_sleepq:
615047dd67eSAttilio Rao 
616047dd67eSAttilio Rao 		/*
617047dd67eSAttilio Rao 		 * if the lock can be acquired in shared mode, try
618047dd67eSAttilio Rao 		 * again.
619047dd67eSAttilio Rao 		 */
62072ba3c08SKonstantin Belousov 		if (LK_CAN_SHARE(x, flags)) {
621047dd67eSAttilio Rao 			sleepq_release(&lk->lock_object);
622047dd67eSAttilio Rao 			continue;
623047dd67eSAttilio Rao 		}
624047dd67eSAttilio Rao 
625047dd67eSAttilio Rao 		/*
626047dd67eSAttilio Rao 		 * Try to set the LK_SHARED_WAITERS flag.  If we fail,
627047dd67eSAttilio Rao 		 * loop back and retry.
628047dd67eSAttilio Rao 		 */
629047dd67eSAttilio Rao 		if ((x & LK_SHARED_WAITERS) == 0) {
6300ad122a9SMateusz Guzik 			if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
631047dd67eSAttilio Rao 			    x | LK_SHARED_WAITERS)) {
6320ad122a9SMateusz Guzik 				goto retry_sleepq;
633047dd67eSAttilio Rao 			}
634047dd67eSAttilio Rao 			LOCK_LOG2(lk, "%s: %p set shared waiters flag",
635047dd67eSAttilio Rao 			    __func__, lk);
636047dd67eSAttilio Rao 		}
637047dd67eSAttilio Rao 
6381c6987ebSMateusz Guzik 		if (lwa == NULL) {
6391c6987ebSMateusz Guzik 			iwmesg = lk->lock_object.lo_name;
6401c6987ebSMateusz Guzik 			ipri = lk->lk_pri;
6411c6987ebSMateusz Guzik 			itimo = lk->lk_timo;
6421c6987ebSMateusz Guzik 		} else {
6431c6987ebSMateusz Guzik 			iwmesg = lwa->iwmesg;
6441c6987ebSMateusz Guzik 			ipri = lwa->ipri;
6451c6987ebSMateusz Guzik 			itimo = lwa->itimo;
6461c6987ebSMateusz Guzik 		}
6471c6987ebSMateusz Guzik 
648047dd67eSAttilio Rao 		/*
649047dd67eSAttilio Rao 		 * As far as we have been unable to acquire the
650047dd67eSAttilio Rao 		 * shared lock and the shared waiters flag is set,
651047dd67eSAttilio Rao 		 * we will sleep.
652047dd67eSAttilio Rao 		 */
653047dd67eSAttilio Rao 		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
654047dd67eSAttilio Rao 		    SQ_SHARED_QUEUE);
655047dd67eSAttilio Rao 		flags &= ~LK_INTERLOCK;
656047dd67eSAttilio Rao 		if (error) {
657047dd67eSAttilio Rao 			LOCK_LOG3(lk,
658047dd67eSAttilio Rao 			    "%s: interrupted sleep for %p with %d",
659047dd67eSAttilio Rao 			    __func__, lk, error);
660047dd67eSAttilio Rao 			break;
661047dd67eSAttilio Rao 		}
662047dd67eSAttilio Rao 		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
663047dd67eSAttilio Rao 		    __func__, lk);
664047dd67eSAttilio Rao 	}
665047dd67eSAttilio Rao 	if (error == 0) {
666c4a48867SMateusz Guzik #ifdef LOCK_PROFILING
667c4a48867SMateusz Guzik 		lockmgr_note_shared_acquire(lk, contested, waittime,
668c4a48867SMateusz Guzik 		    file, line, flags);
669c4a48867SMateusz Guzik #else
670c4a48867SMateusz Guzik 		lockmgr_note_shared_acquire(lk, 0, 0, file, line,
671c4a48867SMateusz Guzik 		    flags);
672c4a48867SMateusz Guzik #endif
673047dd67eSAttilio Rao 	}
674047dd67eSAttilio Rao 
6751c6987ebSMateusz Guzik out:
6761c6987ebSMateusz Guzik 	lockmgr_exit(flags, ilk, 0);
6771c6987ebSMateusz Guzik 	return (error);
678047dd67eSAttilio Rao }
679047dd67eSAttilio Rao 
6801c6987ebSMateusz Guzik static __noinline int
6811c6987ebSMateusz Guzik lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
6821c6987ebSMateusz Guzik     const char *file, int line, struct lockmgr_wait *lwa)
6831c6987ebSMateusz Guzik {
6841c6987ebSMateusz Guzik 	struct lock_class *class;
6851c6987ebSMateusz Guzik 	uintptr_t tid, x, v;
6861c6987ebSMateusz Guzik 	int error = 0;
6871c6987ebSMateusz Guzik 	const char *iwmesg;
6881c6987ebSMateusz Guzik 	int ipri, itimo;
6897c6fe803SKonstantin Belousov 
6901c6987ebSMateusz Guzik #ifdef LOCK_PROFILING
6911c6987ebSMateusz Guzik 	uint64_t waittime = 0;
6921c6987ebSMateusz Guzik 	int contested = 0;
6931c6987ebSMateusz Guzik #endif
694047dd67eSAttilio Rao 
6951c6987ebSMateusz Guzik 	if (__predict_false(panicstr != NULL))
6961c6987ebSMateusz Guzik 		goto out;
6971c6987ebSMateusz Guzik 
6981c6987ebSMateusz Guzik 	tid = (uintptr_t)curthread;
6991c6987ebSMateusz Guzik 
700e5f94314SAttilio Rao 	if (LK_CAN_WITNESS(flags))
701e5f94314SAttilio Rao 		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
70224150d37SJohn Baldwin 		    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
70324150d37SJohn Baldwin 		    ilk : NULL);
704047dd67eSAttilio Rao 
705047dd67eSAttilio Rao 	/*
70696f1567fSKonstantin Belousov 	 * If curthread already holds the lock and this one is
707047dd67eSAttilio Rao 	 * allowed to recurse, simply recurse on it.
708047dd67eSAttilio Rao 	 */
709047dd67eSAttilio Rao 	if (lockmgr_xlocked(lk)) {
710047dd67eSAttilio Rao 		if ((flags & LK_CANRECURSE) == 0 &&
711f0830182SAttilio Rao 		    (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
712047dd67eSAttilio Rao 			/*
713047dd67eSAttilio Rao 			 * If the lock is expected to not panic just
714047dd67eSAttilio Rao 			 * give up and return.
715047dd67eSAttilio Rao 			 */
716047dd67eSAttilio Rao 			if (LK_TRYOP(flags)) {
717047dd67eSAttilio Rao 				LOCK_LOG2(lk,
718047dd67eSAttilio Rao 				    "%s: %p fails the try operation",
719047dd67eSAttilio Rao 				    __func__, lk);
720047dd67eSAttilio Rao 				error = EBUSY;
7211c6987ebSMateusz Guzik 				goto out;
722047dd67eSAttilio Rao 			}
7231c6987ebSMateusz Guzik 			if (flags & LK_INTERLOCK) {
7241c6987ebSMateusz Guzik 				class = LOCK_CLASS(ilk);
725047dd67eSAttilio Rao 				class->lc_unlock(ilk);
7261c6987ebSMateusz Guzik 			}
72783fc34eaSGleb Smirnoff 			panic("%s: recursing on non recursive lockmgr %p "
72883fc34eaSGleb Smirnoff 			    "@ %s:%d\n", __func__, lk, file, line);
729047dd67eSAttilio Rao 		}
730047dd67eSAttilio Rao 		lk->lk_recurse++;
731047dd67eSAttilio Rao 		LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
732047dd67eSAttilio Rao 		LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
733047dd67eSAttilio Rao 		    lk->lk_recurse, file, line);
734e5f94314SAttilio Rao 		WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
735e5f94314SAttilio Rao 		    LK_TRYWIT(flags), file, line);
736047dd67eSAttilio Rao 		TD_LOCKS_INC(curthread);
7371c6987ebSMateusz Guzik 		goto out;
738047dd67eSAttilio Rao 	}
739047dd67eSAttilio Rao 
740fc4f686dSMateusz Guzik 	for (;;) {
741fc4f686dSMateusz Guzik 		if (lk->lk_lock == LK_UNLOCKED &&
742fc4f686dSMateusz Guzik 		    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
743fc4f686dSMateusz Guzik 			break;
744f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
745f5f9340bSFabien Thomas 		PMC_SOFT_CALL( , , lock, failed);
746f5f9340bSFabien Thomas #endif
747047dd67eSAttilio Rao 		lock_profile_obtain_lock_failed(&lk->lock_object,
748047dd67eSAttilio Rao 		    &contested, &waittime);
749047dd67eSAttilio Rao 
750047dd67eSAttilio Rao 		/*
751047dd67eSAttilio Rao 		 * If the lock is expected to not sleep just give up
752047dd67eSAttilio Rao 		 * and return.
753047dd67eSAttilio Rao 		 */
754047dd67eSAttilio Rao 		if (LK_TRYOP(flags)) {
755047dd67eSAttilio Rao 			LOCK_LOG2(lk, "%s: %p fails the try operation",
756047dd67eSAttilio Rao 			    __func__, lk);
757047dd67eSAttilio Rao 			error = EBUSY;
758047dd67eSAttilio Rao 			break;
759047dd67eSAttilio Rao 		}
760047dd67eSAttilio Rao 
761047dd67eSAttilio Rao 		/*
762047dd67eSAttilio Rao 		 * Acquire the sleepqueue chain lock because we
763047dd67eSAttilio Rao 		 * probabilly will need to manipulate waiters flags.
764047dd67eSAttilio Rao 		 */
765047dd67eSAttilio Rao 		sleepq_lock(&lk->lock_object);
766047dd67eSAttilio Rao 		x = lk->lk_lock;
7670ad122a9SMateusz Guzik retry_sleepq:
768047dd67eSAttilio Rao 
769047dd67eSAttilio Rao 		/*
770047dd67eSAttilio Rao 		 * if the lock has been released while we spun on
771047dd67eSAttilio Rao 		 * the sleepqueue chain lock just try again.
772047dd67eSAttilio Rao 		 */
773047dd67eSAttilio Rao 		if (x == LK_UNLOCKED) {
774047dd67eSAttilio Rao 			sleepq_release(&lk->lock_object);
775047dd67eSAttilio Rao 			continue;
776047dd67eSAttilio Rao 		}
777047dd67eSAttilio Rao 
778047dd67eSAttilio Rao 		/*
779047dd67eSAttilio Rao 		 * The lock can be in the state where there is a
780047dd67eSAttilio Rao 		 * pending queue of waiters, but still no owner.
781047dd67eSAttilio Rao 		 * This happens when the lock is contested and an
782047dd67eSAttilio Rao 		 * owner is going to claim the lock.
783047dd67eSAttilio Rao 		 * If curthread is the one successfully acquiring it
784047dd67eSAttilio Rao 		 * claim lock ownership and return, preserving waiters
785047dd67eSAttilio Rao 		 * flags.
786047dd67eSAttilio Rao 		 */
787651175c9SAttilio Rao 		v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
788651175c9SAttilio Rao 		if ((x & ~v) == LK_UNLOCKED) {
789651175c9SAttilio Rao 			v &= ~LK_EXCLUSIVE_SPINNERS;
7900ad122a9SMateusz Guzik 			if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
791047dd67eSAttilio Rao 			    tid | v)) {
792047dd67eSAttilio Rao 				sleepq_release(&lk->lock_object);
793047dd67eSAttilio Rao 				LOCK_LOG2(lk,
794047dd67eSAttilio Rao 				    "%s: %p claimed by a new writer",
795047dd67eSAttilio Rao 				    __func__, lk);
796047dd67eSAttilio Rao 				break;
797047dd67eSAttilio Rao 			}
7980ad122a9SMateusz Guzik 			goto retry_sleepq;
799047dd67eSAttilio Rao 		}
800047dd67eSAttilio Rao 
801047dd67eSAttilio Rao 		/*
802047dd67eSAttilio Rao 		 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
803047dd67eSAttilio Rao 		 * fail, loop back and retry.
804047dd67eSAttilio Rao 		 */
805047dd67eSAttilio Rao 		if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
8060ad122a9SMateusz Guzik 			if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
807047dd67eSAttilio Rao 			    x | LK_EXCLUSIVE_WAITERS)) {
8080ad122a9SMateusz Guzik 				goto retry_sleepq;
809047dd67eSAttilio Rao 			}
810047dd67eSAttilio Rao 			LOCK_LOG2(lk, "%s: %p set excl waiters flag",
811047dd67eSAttilio Rao 			    __func__, lk);
812047dd67eSAttilio Rao 		}
813047dd67eSAttilio Rao 
8141c6987ebSMateusz Guzik 		if (lwa == NULL) {
8151c6987ebSMateusz Guzik 			iwmesg = lk->lock_object.lo_name;
8161c6987ebSMateusz Guzik 			ipri = lk->lk_pri;
8171c6987ebSMateusz Guzik 			itimo = lk->lk_timo;
8181c6987ebSMateusz Guzik 		} else {
8191c6987ebSMateusz Guzik 			iwmesg = lwa->iwmesg;
8201c6987ebSMateusz Guzik 			ipri = lwa->ipri;
8211c6987ebSMateusz Guzik 			itimo = lwa->itimo;
8221c6987ebSMateusz Guzik 		}
8231c6987ebSMateusz Guzik 
824047dd67eSAttilio Rao 		/*
825047dd67eSAttilio Rao 		 * As far as we have been unable to acquire the
826047dd67eSAttilio Rao 		 * exclusive lock and the exclusive waiters flag
827047dd67eSAttilio Rao 		 * is set, we will sleep.
828047dd67eSAttilio Rao 		 */
829047dd67eSAttilio Rao 		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
830047dd67eSAttilio Rao 		    SQ_EXCLUSIVE_QUEUE);
831047dd67eSAttilio Rao 		flags &= ~LK_INTERLOCK;
832047dd67eSAttilio Rao 		if (error) {
833047dd67eSAttilio Rao 			LOCK_LOG3(lk,
834047dd67eSAttilio Rao 			    "%s: interrupted sleep for %p with %d",
835047dd67eSAttilio Rao 			    __func__, lk, error);
836047dd67eSAttilio Rao 			break;
837047dd67eSAttilio Rao 		}
838047dd67eSAttilio Rao 		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
839047dd67eSAttilio Rao 		    __func__, lk);
840047dd67eSAttilio Rao 	}
841047dd67eSAttilio Rao 	if (error == 0) {
842c4a48867SMateusz Guzik #ifdef LOCK_PROFILING
843c4a48867SMateusz Guzik 		lockmgr_note_exclusive_acquire(lk, contested, waittime,
844c4a48867SMateusz Guzik 		    file, line, flags);
845c4a48867SMateusz Guzik #else
846c4a48867SMateusz Guzik 		lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
847c4a48867SMateusz Guzik 		    flags);
848c4a48867SMateusz Guzik #endif
849047dd67eSAttilio Rao 	}
8501c6987ebSMateusz Guzik 
8511c6987ebSMateusz Guzik out:
8521c6987ebSMateusz Guzik 	lockmgr_exit(flags, ilk, 0);
8531c6987ebSMateusz Guzik 	return (error);
8541c6987ebSMateusz Guzik }
8551c6987ebSMateusz Guzik 
8561c6987ebSMateusz Guzik static __noinline int
8571c6987ebSMateusz Guzik lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
8581c6987ebSMateusz Guzik     const char *file, int line, struct lockmgr_wait *lwa)
8591c6987ebSMateusz Guzik {
8601c6987ebSMateusz Guzik 	uintptr_t tid, x, v;
8611c6987ebSMateusz Guzik 	int error = 0;
8621c6987ebSMateusz Guzik 	int wakeup_swapper = 0;
8631c6987ebSMateusz Guzik 	int op;
8641c6987ebSMateusz Guzik 
8651c6987ebSMateusz Guzik 	if (__predict_false(panicstr != NULL))
8661c6987ebSMateusz Guzik 		goto out;
8671c6987ebSMateusz Guzik 
8681c6987ebSMateusz Guzik 	tid = (uintptr_t)curthread;
8691c6987ebSMateusz Guzik 
8701c6987ebSMateusz Guzik 	_lockmgr_assert(lk, KA_SLOCKED, file, line);
8711c6987ebSMateusz Guzik 	v = lk->lk_lock;
8721c6987ebSMateusz Guzik 	x = v & LK_ALL_WAITERS;
8731c6987ebSMateusz Guzik 	v &= LK_EXCLUSIVE_SPINNERS;
8741c6987ebSMateusz Guzik 
8751c6987ebSMateusz Guzik 	/*
8761c6987ebSMateusz Guzik 	 * Try to switch from one shared lock to an exclusive one.
8771c6987ebSMateusz Guzik 	 * We need to preserve waiters flags during the operation.
8781c6987ebSMateusz Guzik 	 */
8791c6987ebSMateusz Guzik 	if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
8801c6987ebSMateusz Guzik 	    tid | x)) {
8811c6987ebSMateusz Guzik 		LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
8821c6987ebSMateusz Guzik 		    line);
8831c6987ebSMateusz Guzik 		WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
8841c6987ebSMateusz Guzik 		    LK_TRYWIT(flags), file, line);
8851c6987ebSMateusz Guzik 		TD_SLOCKS_DEC(curthread);
8861c6987ebSMateusz Guzik 		goto out;
8871c6987ebSMateusz Guzik 	}
8881c6987ebSMateusz Guzik 
8891c6987ebSMateusz Guzik 	op = flags & LK_TYPE_MASK;
8901c6987ebSMateusz Guzik 
8911c6987ebSMateusz Guzik 	/*
8921c6987ebSMateusz Guzik 	 * In LK_TRYUPGRADE mode, do not drop the lock,
8931c6987ebSMateusz Guzik 	 * returning EBUSY instead.
8941c6987ebSMateusz Guzik 	 */
8951c6987ebSMateusz Guzik 	if (op == LK_TRYUPGRADE) {
8961c6987ebSMateusz Guzik 		LOCK_LOG2(lk, "%s: %p failed the nowait upgrade",
8971c6987ebSMateusz Guzik 		    __func__, lk);
8981c6987ebSMateusz Guzik 		error = EBUSY;
8991c6987ebSMateusz Guzik 		goto out;
9001c6987ebSMateusz Guzik 	}
9011c6987ebSMateusz Guzik 
9021c6987ebSMateusz Guzik 	/*
9031c6987ebSMateusz Guzik 	 * We have been unable to succeed in upgrading, so just
9041c6987ebSMateusz Guzik 	 * give up the shared lock.
9051c6987ebSMateusz Guzik 	 */
9061c6987ebSMateusz Guzik 	wakeup_swapper |= wakeupshlk(lk, file, line);
9071c6987ebSMateusz Guzik 	error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
9081c6987ebSMateusz Guzik 	flags &= ~LK_INTERLOCK;
9091c6987ebSMateusz Guzik out:
9101c6987ebSMateusz Guzik 	lockmgr_exit(flags, ilk, wakeup_swapper);
9111c6987ebSMateusz Guzik 	return (error);
9121c6987ebSMateusz Guzik }
9131c6987ebSMateusz Guzik 
9141c6987ebSMateusz Guzik int
9151c6987ebSMateusz Guzik lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk,
9161c6987ebSMateusz Guzik     const char *file, int line)
9171c6987ebSMateusz Guzik {
9181c6987ebSMateusz Guzik 	struct lock_class *class;
9191c6987ebSMateusz Guzik 	uintptr_t x, tid;
9201c6987ebSMateusz Guzik 	u_int op;
9211c6987ebSMateusz Guzik 	bool locked;
9221c6987ebSMateusz Guzik 
923b543c98cSConrad Meyer 	if (__predict_false(panicstr != NULL))
924b543c98cSConrad Meyer 		return (0);
925b543c98cSConrad Meyer 
9261c6987ebSMateusz Guzik 	op = flags & LK_TYPE_MASK;
9271c6987ebSMateusz Guzik 	locked = false;
9281c6987ebSMateusz Guzik 	switch (op) {
9291c6987ebSMateusz Guzik 	case LK_SHARED:
9301c6987ebSMateusz Guzik 		if (LK_CAN_WITNESS(flags))
9311c6987ebSMateusz Guzik 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
9321c6987ebSMateusz Guzik 			    file, line, flags & LK_INTERLOCK ? ilk : NULL);
9331c6987ebSMateusz Guzik 		if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
9341c6987ebSMateusz Guzik 			break;
9351c6987ebSMateusz Guzik 		if (lockmgr_slock_try(lk, &x, flags)) {
9361c6987ebSMateusz Guzik 			lockmgr_note_shared_acquire(lk, 0, 0,
9371c6987ebSMateusz Guzik 			    file, line, flags);
9381c6987ebSMateusz Guzik 			locked = true;
9391c6987ebSMateusz Guzik 		} else {
9401c6987ebSMateusz Guzik 			return (lockmgr_slock_hard(lk, flags, ilk, file, line,
9411c6987ebSMateusz Guzik 			    NULL));
9421c6987ebSMateusz Guzik 		}
9431c6987ebSMateusz Guzik 		break;
9441c6987ebSMateusz Guzik 	case LK_EXCLUSIVE:
9451c6987ebSMateusz Guzik 		if (LK_CAN_WITNESS(flags))
9461c6987ebSMateusz Guzik 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
9471c6987ebSMateusz Guzik 			    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
9481c6987ebSMateusz Guzik 			    ilk : NULL);
9491c6987ebSMateusz Guzik 		tid = (uintptr_t)curthread;
9501c6987ebSMateusz Guzik 		if (lk->lk_lock == LK_UNLOCKED &&
9511c6987ebSMateusz Guzik 		    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
9521c6987ebSMateusz Guzik 			lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
9531c6987ebSMateusz Guzik 			    flags);
9541c6987ebSMateusz Guzik 			locked = true;
9551c6987ebSMateusz Guzik 		} else {
9561c6987ebSMateusz Guzik 			return (lockmgr_xlock_hard(lk, flags, ilk, file, line,
9571c6987ebSMateusz Guzik 			    NULL));
9581c6987ebSMateusz Guzik 		}
9591c6987ebSMateusz Guzik 		break;
9601c6987ebSMateusz Guzik 	case LK_UPGRADE:
9611c6987ebSMateusz Guzik 	case LK_TRYUPGRADE:
9621c6987ebSMateusz Guzik 		return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL));
9631c6987ebSMateusz Guzik 	default:
9641c6987ebSMateusz Guzik 		break;
9651c6987ebSMateusz Guzik 	}
9661c6987ebSMateusz Guzik 	if (__predict_true(locked)) {
9671c6987ebSMateusz Guzik 		if (__predict_false(flags & LK_INTERLOCK)) {
9681c6987ebSMateusz Guzik 			class = LOCK_CLASS(ilk);
9691c6987ebSMateusz Guzik 			class->lc_unlock(ilk);
9701c6987ebSMateusz Guzik 		}
9711c6987ebSMateusz Guzik 		return (0);
9721c6987ebSMateusz Guzik 	} else {
9731c6987ebSMateusz Guzik 		return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
9741c6987ebSMateusz Guzik 		    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
9751c6987ebSMateusz Guzik 	}
9761c6987ebSMateusz Guzik }
9771c6987ebSMateusz Guzik 
9781c6987ebSMateusz Guzik static __noinline int
9791c6987ebSMateusz Guzik lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
9801c6987ebSMateusz Guzik     const char *file, int line)
9811c6987ebSMateusz Guzik 
9821c6987ebSMateusz Guzik {
9831c6987ebSMateusz Guzik 	int wakeup_swapper = 0;
9841c6987ebSMateusz Guzik 
9851c6987ebSMateusz Guzik 	if (__predict_false(panicstr != NULL))
9861c6987ebSMateusz Guzik 		goto out;
9871c6987ebSMateusz Guzik 
9881c6987ebSMateusz Guzik 	wakeup_swapper = wakeupshlk(lk, file, line);
9891c6987ebSMateusz Guzik 
9901c6987ebSMateusz Guzik out:
9911c6987ebSMateusz Guzik 	lockmgr_exit(flags, ilk, wakeup_swapper);
9921c6987ebSMateusz Guzik 	return (0);
9931c6987ebSMateusz Guzik }
9941c6987ebSMateusz Guzik 
9951c6987ebSMateusz Guzik static __noinline int
9961c6987ebSMateusz Guzik lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
9971c6987ebSMateusz Guzik     const char *file, int line)
9981c6987ebSMateusz Guzik {
9991c6987ebSMateusz Guzik 	uintptr_t tid, v;
10001c6987ebSMateusz Guzik 	int wakeup_swapper = 0;
10011c6987ebSMateusz Guzik 	u_int realexslp;
10021c6987ebSMateusz Guzik 	int queue;
10031c6987ebSMateusz Guzik 
10041c6987ebSMateusz Guzik 	if (__predict_false(panicstr != NULL))
10051c6987ebSMateusz Guzik 		goto out;
10061c6987ebSMateusz Guzik 
10071c6987ebSMateusz Guzik 	tid = (uintptr_t)curthread;
10081c6987ebSMateusz Guzik 
10091c6987ebSMateusz Guzik 	/*
10101c6987ebSMateusz Guzik 	 * As first option, treact the lock as if it has not
10111c6987ebSMateusz Guzik 	 * any waiter.
10121c6987ebSMateusz Guzik 	 * Fix-up the tid var if the lock has been disowned.
10131c6987ebSMateusz Guzik 	 */
10141c6987ebSMateusz Guzik 	if (LK_HOLDER(x) == LK_KERNPROC)
10151c6987ebSMateusz Guzik 		tid = LK_KERNPROC;
10161c6987ebSMateusz Guzik 	else {
10171c6987ebSMateusz Guzik 		WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
10181c6987ebSMateusz Guzik 		TD_LOCKS_DEC(curthread);
10191c6987ebSMateusz Guzik 	}
10201c6987ebSMateusz Guzik 	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
10211c6987ebSMateusz Guzik 
10221c6987ebSMateusz Guzik 	/*
10231c6987ebSMateusz Guzik 	 * The lock is held in exclusive mode.
10241c6987ebSMateusz Guzik 	 * If the lock is recursed also, then unrecurse it.
10251c6987ebSMateusz Guzik 	 */
1026*10391db5SMateusz Guzik 	if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) {
10271c6987ebSMateusz Guzik 		LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);
10281c6987ebSMateusz Guzik 		lk->lk_recurse--;
10291c6987ebSMateusz Guzik 		goto out;
10301c6987ebSMateusz Guzik 	}
10311c6987ebSMateusz Guzik 	if (tid != LK_KERNPROC)
10321c6987ebSMateusz Guzik 		lock_profile_release_lock(&lk->lock_object);
10331c6987ebSMateusz Guzik 
1034*10391db5SMateusz Guzik 	if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
10351c6987ebSMateusz Guzik 		goto out;
10361c6987ebSMateusz Guzik 
10371c6987ebSMateusz Guzik 	sleepq_lock(&lk->lock_object);
10381c6987ebSMateusz Guzik 	x = lk->lk_lock;
10391c6987ebSMateusz Guzik 	v = LK_UNLOCKED;
10401c6987ebSMateusz Guzik 
10411c6987ebSMateusz Guzik 	/*
10421c6987ebSMateusz Guzik 	 * If the lock has exclusive waiters, give them
10431c6987ebSMateusz Guzik 	 * preference in order to avoid deadlock with
10441c6987ebSMateusz Guzik 	 * shared runners up.
10451c6987ebSMateusz Guzik 	 * If interruptible sleeps left the exclusive queue
10461c6987ebSMateusz Guzik 	 * empty avoid a starvation for the threads sleeping
10471c6987ebSMateusz Guzik 	 * on the shared queue by giving them precedence
10481c6987ebSMateusz Guzik 	 * and cleaning up the exclusive waiters bit anyway.
10491c6987ebSMateusz Guzik 	 * Please note that lk_exslpfail count may be lying
10501c6987ebSMateusz Guzik 	 * about the real number of waiters with the
10511c6987ebSMateusz Guzik 	 * LK_SLEEPFAIL flag on because they may be used in
10521c6987ebSMateusz Guzik 	 * conjunction with interruptible sleeps so
10531c6987ebSMateusz Guzik 	 * lk_exslpfail might be considered an 'upper limit'
10541c6987ebSMateusz Guzik 	 * bound, including the edge cases.
10551c6987ebSMateusz Guzik 	 */
10561c6987ebSMateusz Guzik 	MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
10571c6987ebSMateusz Guzik 	realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
10581c6987ebSMateusz Guzik 	if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
10591c6987ebSMateusz Guzik 		if (lk->lk_exslpfail < realexslp) {
10601c6987ebSMateusz Guzik 			lk->lk_exslpfail = 0;
10611c6987ebSMateusz Guzik 			queue = SQ_EXCLUSIVE_QUEUE;
10621c6987ebSMateusz Guzik 			v |= (x & LK_SHARED_WAITERS);
10631c6987ebSMateusz Guzik 		} else {
10641c6987ebSMateusz Guzik 			lk->lk_exslpfail = 0;
10651c6987ebSMateusz Guzik 			LOCK_LOG2(lk,
10661c6987ebSMateusz Guzik 			    "%s: %p has only LK_SLEEPFAIL sleepers",
10671c6987ebSMateusz Guzik 			    __func__, lk);
10681c6987ebSMateusz Guzik 			LOCK_LOG2(lk,
10691c6987ebSMateusz Guzik 			    "%s: %p waking up threads on the exclusive queue",
10701c6987ebSMateusz Guzik 			    __func__, lk);
10711c6987ebSMateusz Guzik 			wakeup_swapper = sleepq_broadcast(&lk->lock_object,
10721c6987ebSMateusz Guzik 			    SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
10731c6987ebSMateusz Guzik 			queue = SQ_SHARED_QUEUE;
10741c6987ebSMateusz Guzik 		}
10751c6987ebSMateusz Guzik 	} else {
10761c6987ebSMateusz Guzik 
10771c6987ebSMateusz Guzik 		/*
10781c6987ebSMateusz Guzik 		 * Exclusive waiters sleeping with LK_SLEEPFAIL
10791c6987ebSMateusz Guzik 		 * on and using interruptible sleeps/timeout
10801c6987ebSMateusz Guzik 		 * may have left spourious lk_exslpfail counts
10811c6987ebSMateusz Guzik 		 * on, so clean it up anyway.
10821c6987ebSMateusz Guzik 		 */
10831c6987ebSMateusz Guzik 		lk->lk_exslpfail = 0;
10841c6987ebSMateusz Guzik 		queue = SQ_SHARED_QUEUE;
10851c6987ebSMateusz Guzik 	}
10861c6987ebSMateusz Guzik 
10871c6987ebSMateusz Guzik 	LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
10881c6987ebSMateusz Guzik 	    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
10891c6987ebSMateusz Guzik 	    "exclusive");
10901c6987ebSMateusz Guzik 	atomic_store_rel_ptr(&lk->lk_lock, v);
10911c6987ebSMateusz Guzik 	wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
10921c6987ebSMateusz Guzik 	sleepq_release(&lk->lock_object);
10931c6987ebSMateusz Guzik 
10941c6987ebSMateusz Guzik out:
10951c6987ebSMateusz Guzik 	lockmgr_exit(flags, ilk, wakeup_swapper);
10961c6987ebSMateusz Guzik 	return (0);
10971c6987ebSMateusz Guzik }
10981c6987ebSMateusz Guzik 
10991c6987ebSMateusz Guzik int
11001c6987ebSMateusz Guzik lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk)
11011c6987ebSMateusz Guzik {
11021c6987ebSMateusz Guzik 	struct lock_class *class;
11031c6987ebSMateusz Guzik 	uintptr_t x, tid;
11041c6987ebSMateusz Guzik 	bool unlocked;
11051c6987ebSMateusz Guzik 	const char *file;
11061c6987ebSMateusz Guzik 	int line;
11071c6987ebSMateusz Guzik 
1108b543c98cSConrad Meyer 	if (__predict_false(panicstr != NULL))
1109b543c98cSConrad Meyer 		return (0);
1110b543c98cSConrad Meyer 
11111c6987ebSMateusz Guzik 	file = __FILE__;
11121c6987ebSMateusz Guzik 	line = __LINE__;
11131c6987ebSMateusz Guzik 
11141c6987ebSMateusz Guzik 	_lockmgr_assert(lk, KA_LOCKED, file, line);
11151c6987ebSMateusz Guzik 	unlocked = false;
11161c6987ebSMateusz Guzik 	x = lk->lk_lock;
11171c6987ebSMateusz Guzik 	if (__predict_true(x & LK_SHARE) != 0) {
11181c6987ebSMateusz Guzik 		if (lockmgr_sunlock_try(lk, &x)) {
11191c6987ebSMateusz Guzik 			lockmgr_note_shared_release(lk, file, line);
11201c6987ebSMateusz Guzik 			unlocked = true;
11211c6987ebSMateusz Guzik 		} else {
11221c6987ebSMateusz Guzik 			return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
11231c6987ebSMateusz Guzik 		}
11241c6987ebSMateusz Guzik 	} else {
11251c6987ebSMateusz Guzik 		tid = (uintptr_t)curthread;
11261c6987ebSMateusz Guzik 		if (!lockmgr_recursed(lk) &&
11271c6987ebSMateusz Guzik 		    atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
11281c6987ebSMateusz Guzik 			lockmgr_note_exclusive_release(lk, file, line);
11291c6987ebSMateusz Guzik 			unlocked = true;
11301c6987ebSMateusz Guzik 		} else {
11311c6987ebSMateusz Guzik 			return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
11321c6987ebSMateusz Guzik 		}
11331c6987ebSMateusz Guzik 	}
11341c6987ebSMateusz Guzik 	if (__predict_false(flags & LK_INTERLOCK)) {
11351c6987ebSMateusz Guzik 		class = LOCK_CLASS(ilk);
11361c6987ebSMateusz Guzik 		class->lc_unlock(ilk);
11371c6987ebSMateusz Guzik 	}
11381c6987ebSMateusz Guzik 	return (0);
11391c6987ebSMateusz Guzik }
11401c6987ebSMateusz Guzik 
11411c6987ebSMateusz Guzik int
11421c6987ebSMateusz Guzik __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
11431c6987ebSMateusz Guzik     const char *wmesg, int pri, int timo, const char *file, int line)
11441c6987ebSMateusz Guzik {
11451c6987ebSMateusz Guzik 	GIANT_DECLARE;
11461c6987ebSMateusz Guzik 	struct lockmgr_wait lwa;
11471c6987ebSMateusz Guzik 	struct lock_class *class;
11481c6987ebSMateusz Guzik 	const char *iwmesg;
11491c6987ebSMateusz Guzik 	uintptr_t tid, v, x;
11501c6987ebSMateusz Guzik 	u_int op, realexslp;
11511c6987ebSMateusz Guzik 	int error, ipri, itimo, queue, wakeup_swapper;
11521c6987ebSMateusz Guzik #ifdef LOCK_PROFILING
11531c6987ebSMateusz Guzik 	uint64_t waittime = 0;
11541c6987ebSMateusz Guzik 	int contested = 0;
11551c6987ebSMateusz Guzik #endif
11561c6987ebSMateusz Guzik 
1157b543c98cSConrad Meyer 	if (panicstr != NULL)
1158b543c98cSConrad Meyer 		return (0);
1159b543c98cSConrad Meyer 
11601c6987ebSMateusz Guzik 	error = 0;
11611c6987ebSMateusz Guzik 	tid = (uintptr_t)curthread;
11621c6987ebSMateusz Guzik 	op = (flags & LK_TYPE_MASK);
11631c6987ebSMateusz Guzik 	iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
11641c6987ebSMateusz Guzik 	ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
11651c6987ebSMateusz Guzik 	itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
11661c6987ebSMateusz Guzik 
11671c6987ebSMateusz Guzik 	lwa.iwmesg = iwmesg;
11681c6987ebSMateusz Guzik 	lwa.ipri = ipri;
11691c6987ebSMateusz Guzik 	lwa.itimo = itimo;
11701c6987ebSMateusz Guzik 
11711c6987ebSMateusz Guzik 	MPASS((flags & ~LK_TOTAL_MASK) == 0);
11721c6987ebSMateusz Guzik 	KASSERT((op & (op - 1)) == 0,
11731c6987ebSMateusz Guzik 	    ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
11741c6987ebSMateusz Guzik 	KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
11751c6987ebSMateusz Guzik 	    (op != LK_DOWNGRADE && op != LK_RELEASE),
11761c6987ebSMateusz Guzik 	    ("%s: Invalid flags in regard of the operation desired @ %s:%d",
11771c6987ebSMateusz Guzik 	    __func__, file, line));
11781c6987ebSMateusz Guzik 	KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
11791c6987ebSMateusz Guzik 	    ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
11801c6987ebSMateusz Guzik 	    __func__, file, line));
11811c6987ebSMateusz Guzik 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
11821c6987ebSMateusz Guzik 	    ("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread,
11831c6987ebSMateusz Guzik 	    lk->lock_object.lo_name, file, line));
11841c6987ebSMateusz Guzik 
11851c6987ebSMateusz Guzik 	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
11861c6987ebSMateusz Guzik 
11871c6987ebSMateusz Guzik 	if (lk->lock_object.lo_flags & LK_NOSHARE) {
11881c6987ebSMateusz Guzik 		switch (op) {
11891c6987ebSMateusz Guzik 		case LK_SHARED:
11901c6987ebSMateusz Guzik 			op = LK_EXCLUSIVE;
11911c6987ebSMateusz Guzik 			break;
11921c6987ebSMateusz Guzik 		case LK_UPGRADE:
11931c6987ebSMateusz Guzik 		case LK_TRYUPGRADE:
11941c6987ebSMateusz Guzik 		case LK_DOWNGRADE:
11951c6987ebSMateusz Guzik 			_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
11961c6987ebSMateusz Guzik 			    file, line);
11971c6987ebSMateusz Guzik 			if (flags & LK_INTERLOCK)
11981c6987ebSMateusz Guzik 				class->lc_unlock(ilk);
11991c6987ebSMateusz Guzik 			return (0);
12001c6987ebSMateusz Guzik 		}
12011c6987ebSMateusz Guzik 	}
12021c6987ebSMateusz Guzik 
12031c6987ebSMateusz Guzik 	wakeup_swapper = 0;
12041c6987ebSMateusz Guzik 	switch (op) {
12051c6987ebSMateusz Guzik 	case LK_SHARED:
12061c6987ebSMateusz Guzik 		return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
12071c6987ebSMateusz Guzik 		break;
12081c6987ebSMateusz Guzik 	case LK_UPGRADE:
12091c6987ebSMateusz Guzik 	case LK_TRYUPGRADE:
12101c6987ebSMateusz Guzik 		return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
12111c6987ebSMateusz Guzik 		break;
12121c6987ebSMateusz Guzik 	case LK_EXCLUSIVE:
12131c6987ebSMateusz Guzik 		return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
1214047dd67eSAttilio Rao 		break;
1215047dd67eSAttilio Rao 	case LK_DOWNGRADE:
12161c7d98d0SAttilio Rao 		_lockmgr_assert(lk, KA_XLOCKED, file, line);
1217e5f94314SAttilio Rao 		LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
1218e5f94314SAttilio Rao 		WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
12191c7d98d0SAttilio Rao 
12201c7d98d0SAttilio Rao 		/*
12211c7d98d0SAttilio Rao 		 * Panic if the lock is recursed.
12221c7d98d0SAttilio Rao 		 */
12231c7d98d0SAttilio Rao 		if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
12241c7d98d0SAttilio Rao 			if (flags & LK_INTERLOCK)
12251c7d98d0SAttilio Rao 				class->lc_unlock(ilk);
12261c7d98d0SAttilio Rao 			panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n",
12271c7d98d0SAttilio Rao 			    __func__, iwmesg, file, line);
12281c7d98d0SAttilio Rao 		}
1229e5f94314SAttilio Rao 		TD_SLOCKS_INC(curthread);
1230047dd67eSAttilio Rao 
1231047dd67eSAttilio Rao 		/*
1232047dd67eSAttilio Rao 		 * In order to preserve waiters flags, just spin.
1233047dd67eSAttilio Rao 		 */
1234047dd67eSAttilio Rao 		for (;;) {
1235651175c9SAttilio Rao 			x = lk->lk_lock;
1236651175c9SAttilio Rao 			MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1237651175c9SAttilio Rao 			x &= LK_ALL_WAITERS;
1238047dd67eSAttilio Rao 			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1239e5f94314SAttilio Rao 			    LK_SHARERS_LOCK(1) | x))
1240047dd67eSAttilio Rao 				break;
1241047dd67eSAttilio Rao 			cpu_spinwait();
1242047dd67eSAttilio Rao 		}
1243047dd67eSAttilio Rao 		break;
1244047dd67eSAttilio Rao 	case LK_RELEASE:
1245047dd67eSAttilio Rao 		_lockmgr_assert(lk, KA_LOCKED, file, line);
1246047dd67eSAttilio Rao 		x = lk->lk_lock;
1247047dd67eSAttilio Rao 
12481c6987ebSMateusz Guzik 		if (__predict_true(x & LK_SHARE) != 0) {
12491c6987ebSMateusz Guzik 			return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
1250047dd67eSAttilio Rao 		} else {
12511c6987ebSMateusz Guzik 			return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
12522028867dSAttilio Rao 		}
1253047dd67eSAttilio Rao 		break;
1254047dd67eSAttilio Rao 	case LK_DRAIN:
1255e5f94314SAttilio Rao 		if (LK_CAN_WITNESS(flags))
1256e5f94314SAttilio Rao 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
125724150d37SJohn Baldwin 			    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
125824150d37SJohn Baldwin 			    ilk : NULL);
1259047dd67eSAttilio Rao 
1260047dd67eSAttilio Rao 		/*
126196f1567fSKonstantin Belousov 		 * Trying to drain a lock we already own will result in a
1262047dd67eSAttilio Rao 		 * deadlock.
1263047dd67eSAttilio Rao 		 */
1264047dd67eSAttilio Rao 		if (lockmgr_xlocked(lk)) {
1265047dd67eSAttilio Rao 			if (flags & LK_INTERLOCK)
1266047dd67eSAttilio Rao 				class->lc_unlock(ilk);
1267047dd67eSAttilio Rao 			panic("%s: draining %s with the lock held @ %s:%d\n",
1268047dd67eSAttilio Rao 			    __func__, iwmesg, file, line);
1269047dd67eSAttilio Rao 		}
1270047dd67eSAttilio Rao 
1271fc4f686dSMateusz Guzik 		for (;;) {
1272fc4f686dSMateusz Guzik 			if (lk->lk_lock == LK_UNLOCKED &&
1273fc4f686dSMateusz Guzik 			    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
1274fc4f686dSMateusz Guzik 				break;
1275fc4f686dSMateusz Guzik 
1276f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
1277f5f9340bSFabien Thomas 			PMC_SOFT_CALL( , , lock, failed);
1278f5f9340bSFabien Thomas #endif
1279047dd67eSAttilio Rao 			lock_profile_obtain_lock_failed(&lk->lock_object,
1280047dd67eSAttilio Rao 			    &contested, &waittime);
1281047dd67eSAttilio Rao 
1282047dd67eSAttilio Rao 			/*
1283047dd67eSAttilio Rao 			 * If the lock is expected to not sleep just give up
1284047dd67eSAttilio Rao 			 * and return.
1285047dd67eSAttilio Rao 			 */
1286047dd67eSAttilio Rao 			if (LK_TRYOP(flags)) {
1287047dd67eSAttilio Rao 				LOCK_LOG2(lk, "%s: %p fails the try operation",
1288047dd67eSAttilio Rao 				    __func__, lk);
1289047dd67eSAttilio Rao 				error = EBUSY;
1290047dd67eSAttilio Rao 				break;
1291047dd67eSAttilio Rao 			}
1292047dd67eSAttilio Rao 
1293047dd67eSAttilio Rao 			/*
1294047dd67eSAttilio Rao 			 * Acquire the sleepqueue chain lock because we
1295047dd67eSAttilio Rao 			 * probabilly will need to manipulate waiters flags.
1296047dd67eSAttilio Rao 			 */
1297047dd67eSAttilio Rao 			sleepq_lock(&lk->lock_object);
1298047dd67eSAttilio Rao 			x = lk->lk_lock;
1299047dd67eSAttilio Rao 
1300047dd67eSAttilio Rao 			/*
1301047dd67eSAttilio Rao 			 * if the lock has been released while we spun on
1302047dd67eSAttilio Rao 			 * the sleepqueue chain lock just try again.
1303047dd67eSAttilio Rao 			 */
1304047dd67eSAttilio Rao 			if (x == LK_UNLOCKED) {
1305047dd67eSAttilio Rao 				sleepq_release(&lk->lock_object);
1306047dd67eSAttilio Rao 				continue;
1307047dd67eSAttilio Rao 			}
1308047dd67eSAttilio Rao 
1309651175c9SAttilio Rao 			v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
1310651175c9SAttilio Rao 			if ((x & ~v) == LK_UNLOCKED) {
1311651175c9SAttilio Rao 				v = (x & ~LK_EXCLUSIVE_SPINNERS);
13122028867dSAttilio Rao 
13132028867dSAttilio Rao 				/*
13142028867dSAttilio Rao 				 * If interruptible sleeps left the exclusive
13152028867dSAttilio Rao 				 * queue empty avoid a starvation for the
13162028867dSAttilio Rao 				 * threads sleeping on the shared queue by
13172028867dSAttilio Rao 				 * giving them precedence and cleaning up the
13182028867dSAttilio Rao 				 * exclusive waiters bit anyway.
1319c636ba83SAttilio Rao 				 * Please note that lk_exslpfail count may be
1320c636ba83SAttilio Rao 				 * lying about the real number of waiters with
1321c636ba83SAttilio Rao 				 * the LK_SLEEPFAIL flag on because they may
1322e3043798SPedro F. Giffuni 				 * be used in conjunction with interruptible
1323aab9c8c2SAttilio Rao 				 * sleeps so lk_exslpfail might be considered
1324aab9c8c2SAttilio Rao 				 * an 'upper limit' bound, including the edge
1325c636ba83SAttilio Rao 				 * cases.
13262028867dSAttilio Rao 				 */
1327047dd67eSAttilio Rao 				if (v & LK_EXCLUSIVE_WAITERS) {
1328047dd67eSAttilio Rao 					queue = SQ_EXCLUSIVE_QUEUE;
1329047dd67eSAttilio Rao 					v &= ~LK_EXCLUSIVE_WAITERS;
1330047dd67eSAttilio Rao 				} else {
13319dbf7a62SAttilio Rao 
13329dbf7a62SAttilio Rao 					/*
13339dbf7a62SAttilio Rao 					 * Exclusive waiters sleeping with
13349dbf7a62SAttilio Rao 					 * LK_SLEEPFAIL on and using
13359dbf7a62SAttilio Rao 					 * interruptible sleeps/timeout may
13369dbf7a62SAttilio Rao 					 * have left spourious lk_exslpfail
13379dbf7a62SAttilio Rao 					 * counts on, so clean it up anyway.
13389dbf7a62SAttilio Rao 					 */
1339047dd67eSAttilio Rao 					MPASS(v & LK_SHARED_WAITERS);
13409dbf7a62SAttilio Rao 					lk->lk_exslpfail = 0;
1341047dd67eSAttilio Rao 					queue = SQ_SHARED_QUEUE;
1342047dd67eSAttilio Rao 					v &= ~LK_SHARED_WAITERS;
1343047dd67eSAttilio Rao 				}
13442028867dSAttilio Rao 				if (queue == SQ_EXCLUSIVE_QUEUE) {
13452028867dSAttilio Rao 					realexslp =
13462028867dSAttilio Rao 					    sleepq_sleepcnt(&lk->lock_object,
13472028867dSAttilio Rao 					    SQ_EXCLUSIVE_QUEUE);
13482028867dSAttilio Rao 					if (lk->lk_exslpfail >= realexslp) {
13492028867dSAttilio Rao 						lk->lk_exslpfail = 0;
13502028867dSAttilio Rao 						queue = SQ_SHARED_QUEUE;
13512028867dSAttilio Rao 						v &= ~LK_SHARED_WAITERS;
13522028867dSAttilio Rao 						if (realexslp != 0) {
13532028867dSAttilio Rao 							LOCK_LOG2(lk,
13542028867dSAttilio Rao 					"%s: %p has only LK_SLEEPFAIL sleepers",
13552028867dSAttilio Rao 							    __func__, lk);
13562028867dSAttilio Rao 							LOCK_LOG2(lk,
13572028867dSAttilio Rao 			"%s: %p waking up threads on the exclusive queue",
13582028867dSAttilio Rao 							    __func__, lk);
13592028867dSAttilio Rao 							wakeup_swapper =
13602028867dSAttilio Rao 							    sleepq_broadcast(
13612028867dSAttilio Rao 							    &lk->lock_object,
13622028867dSAttilio Rao 							    SLEEPQ_LK, 0,
13632028867dSAttilio Rao 							    SQ_EXCLUSIVE_QUEUE);
13642028867dSAttilio Rao 						}
13652028867dSAttilio Rao 					} else
13662028867dSAttilio Rao 						lk->lk_exslpfail = 0;
13672028867dSAttilio Rao 				}
1368047dd67eSAttilio Rao 				if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
1369047dd67eSAttilio Rao 					sleepq_release(&lk->lock_object);
1370047dd67eSAttilio Rao 					continue;
1371047dd67eSAttilio Rao 				}
1372047dd67eSAttilio Rao 				LOCK_LOG3(lk,
1373047dd67eSAttilio Rao 				"%s: %p waking up all threads on the %s queue",
1374047dd67eSAttilio Rao 				    __func__, lk, queue == SQ_SHARED_QUEUE ?
1375047dd67eSAttilio Rao 				    "shared" : "exclusive");
1376814f26daSJohn Baldwin 				wakeup_swapper |= sleepq_broadcast(
1377da7bbd2cSJohn Baldwin 				    &lk->lock_object, SLEEPQ_LK, 0, queue);
1378047dd67eSAttilio Rao 
1379047dd67eSAttilio Rao 				/*
1380047dd67eSAttilio Rao 				 * If shared waiters have been woken up we need
1381047dd67eSAttilio Rao 				 * to wait for one of them to acquire the lock
1382047dd67eSAttilio Rao 				 * before to set the exclusive waiters in
1383047dd67eSAttilio Rao 				 * order to avoid a deadlock.
1384047dd67eSAttilio Rao 				 */
1385047dd67eSAttilio Rao 				if (queue == SQ_SHARED_QUEUE) {
1386047dd67eSAttilio Rao 					for (v = lk->lk_lock;
1387047dd67eSAttilio Rao 					    (v & LK_SHARE) && !LK_SHARERS(v);
1388047dd67eSAttilio Rao 					    v = lk->lk_lock)
1389047dd67eSAttilio Rao 						cpu_spinwait();
1390047dd67eSAttilio Rao 				}
1391047dd67eSAttilio Rao 			}
1392047dd67eSAttilio Rao 
1393047dd67eSAttilio Rao 			/*
1394047dd67eSAttilio Rao 			 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
1395047dd67eSAttilio Rao 			 * fail, loop back and retry.
1396047dd67eSAttilio Rao 			 */
1397047dd67eSAttilio Rao 			if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
1398047dd67eSAttilio Rao 				if (!atomic_cmpset_ptr(&lk->lk_lock, x,
1399047dd67eSAttilio Rao 				    x | LK_EXCLUSIVE_WAITERS)) {
1400047dd67eSAttilio Rao 					sleepq_release(&lk->lock_object);
1401047dd67eSAttilio Rao 					continue;
1402047dd67eSAttilio Rao 				}
1403047dd67eSAttilio Rao 				LOCK_LOG2(lk, "%s: %p set drain waiters flag",
1404047dd67eSAttilio Rao 				    __func__, lk);
1405047dd67eSAttilio Rao 			}
1406047dd67eSAttilio Rao 
1407047dd67eSAttilio Rao 			/*
1408047dd67eSAttilio Rao 			 * As far as we have been unable to acquire the
1409047dd67eSAttilio Rao 			 * exclusive lock and the exclusive waiters flag
1410047dd67eSAttilio Rao 			 * is set, we will sleep.
1411047dd67eSAttilio Rao 			 */
1412047dd67eSAttilio Rao 			if (flags & LK_INTERLOCK) {
1413047dd67eSAttilio Rao 				class->lc_unlock(ilk);
1414047dd67eSAttilio Rao 				flags &= ~LK_INTERLOCK;
1415047dd67eSAttilio Rao 			}
1416e5f94314SAttilio Rao 			GIANT_SAVE();
1417047dd67eSAttilio Rao 			sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
1418047dd67eSAttilio Rao 			    SQ_EXCLUSIVE_QUEUE);
1419047dd67eSAttilio Rao 			sleepq_wait(&lk->lock_object, ipri & PRIMASK);
1420e5f94314SAttilio Rao 			GIANT_RESTORE();
1421047dd67eSAttilio Rao 			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
1422047dd67eSAttilio Rao 			    __func__, lk);
1423047dd67eSAttilio Rao 		}
1424047dd67eSAttilio Rao 
1425047dd67eSAttilio Rao 		if (error == 0) {
1426047dd67eSAttilio Rao 			lock_profile_obtain_lock_success(&lk->lock_object,
1427047dd67eSAttilio Rao 			    contested, waittime, file, line);
1428047dd67eSAttilio Rao 			LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
1429047dd67eSAttilio Rao 			    lk->lk_recurse, file, line);
1430e5f94314SAttilio Rao 			WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
1431e5f94314SAttilio Rao 			    LK_TRYWIT(flags), file, line);
1432047dd67eSAttilio Rao 			TD_LOCKS_INC(curthread);
1433047dd67eSAttilio Rao 			STACK_SAVE(lk);
1434047dd67eSAttilio Rao 		}
1435047dd67eSAttilio Rao 		break;
1436047dd67eSAttilio Rao 	default:
1437047dd67eSAttilio Rao 		if (flags & LK_INTERLOCK)
1438047dd67eSAttilio Rao 			class->lc_unlock(ilk);
1439047dd67eSAttilio Rao 		panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
1440047dd67eSAttilio Rao 	}
1441047dd67eSAttilio Rao 
1442047dd67eSAttilio Rao 	if (flags & LK_INTERLOCK)
1443047dd67eSAttilio Rao 		class->lc_unlock(ilk);
1444da7bbd2cSJohn Baldwin 	if (wakeup_swapper)
1445da7bbd2cSJohn Baldwin 		kick_proc0();
1446047dd67eSAttilio Rao 
1447047dd67eSAttilio Rao 	return (error);
1448047dd67eSAttilio Rao }
1449047dd67eSAttilio Rao 
1450d7a7e179SAttilio Rao void
1451047dd67eSAttilio Rao _lockmgr_disown(struct lock *lk, const char *file, int line)
1452047dd67eSAttilio Rao {
1453047dd67eSAttilio Rao 	uintptr_t tid, x;
1454047dd67eSAttilio Rao 
145535370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
145635370593SAndriy Gapon 		return;
145735370593SAndriy Gapon 
1458047dd67eSAttilio Rao 	tid = (uintptr_t)curthread;
14591c7d98d0SAttilio Rao 	_lockmgr_assert(lk, KA_XLOCKED, file, line);
14601c7d98d0SAttilio Rao 
14611c7d98d0SAttilio Rao 	/*
14621c7d98d0SAttilio Rao 	 * Panic if the lock is recursed.
14631c7d98d0SAttilio Rao 	 */
14641c7d98d0SAttilio Rao 	if (lockmgr_xlocked(lk) && lockmgr_recursed(lk))
14651c7d98d0SAttilio Rao 		panic("%s: disown a recursed lockmgr @ %s:%d\n",
14661c7d98d0SAttilio Rao 		    __func__,  file, line);
1467047dd67eSAttilio Rao 
1468047dd67eSAttilio Rao 	/*
146996f1567fSKonstantin Belousov 	 * If the owner is already LK_KERNPROC just skip the whole operation.
1470047dd67eSAttilio Rao 	 */
1471047dd67eSAttilio Rao 	if (LK_HOLDER(lk->lk_lock) != tid)
1472047dd67eSAttilio Rao 		return;
147304a28689SJeff Roberson 	lock_profile_release_lock(&lk->lock_object);
1474e5f94314SAttilio Rao 	LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
1475e5f94314SAttilio Rao 	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
1476e5f94314SAttilio Rao 	TD_LOCKS_DEC(curthread);
1477337c5ff4SAttilio Rao 	STACK_SAVE(lk);
1478047dd67eSAttilio Rao 
1479047dd67eSAttilio Rao 	/*
1480047dd67eSAttilio Rao 	 * In order to preserve waiters flags, just spin.
1481047dd67eSAttilio Rao 	 */
1482047dd67eSAttilio Rao 	for (;;) {
1483651175c9SAttilio Rao 		x = lk->lk_lock;
1484651175c9SAttilio Rao 		MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1485651175c9SAttilio Rao 		x &= LK_ALL_WAITERS;
148622dd228dSAttilio Rao 		if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1487e5f94314SAttilio Rao 		    LK_KERNPROC | x))
1488047dd67eSAttilio Rao 			return;
1489047dd67eSAttilio Rao 		cpu_spinwait();
1490047dd67eSAttilio Rao 	}
1491047dd67eSAttilio Rao }
1492047dd67eSAttilio Rao 
1493047dd67eSAttilio Rao void
1494d576deedSPawel Jakub Dawidek lockmgr_printinfo(const struct lock *lk)
1495d7a7e179SAttilio Rao {
1496d7a7e179SAttilio Rao 	struct thread *td;
1497047dd67eSAttilio Rao 	uintptr_t x;
1498d7a7e179SAttilio Rao 
1499047dd67eSAttilio Rao 	if (lk->lk_lock == LK_UNLOCKED)
1500047dd67eSAttilio Rao 		printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
1501047dd67eSAttilio Rao 	else if (lk->lk_lock & LK_SHARE)
1502047dd67eSAttilio Rao 		printf("lock type %s: SHARED (count %ju)\n",
1503047dd67eSAttilio Rao 		    lk->lock_object.lo_name,
1504047dd67eSAttilio Rao 		    (uintmax_t)LK_SHARERS(lk->lk_lock));
1505047dd67eSAttilio Rao 	else {
1506047dd67eSAttilio Rao 		td = lockmgr_xholder(lk);
1507e64b4fa8SKonstantin Belousov 		if (td == (struct thread *)LK_KERNPROC)
1508e64b4fa8SKonstantin Belousov 			printf("lock type %s: EXCL by KERNPROC\n",
1509e64b4fa8SKonstantin Belousov 			    lk->lock_object.lo_name);
1510e64b4fa8SKonstantin Belousov 		else
15112573ea5fSIvan Voras 			printf("lock type %s: EXCL by thread %p "
1512e64b4fa8SKonstantin Belousov 			    "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
1513e64b4fa8SKonstantin Belousov 			    td, td->td_proc->p_pid, td->td_proc->p_comm,
1514e64b4fa8SKonstantin Belousov 			    td->td_tid);
1515d7a7e179SAttilio Rao 	}
1516d7a7e179SAttilio Rao 
1517047dd67eSAttilio Rao 	x = lk->lk_lock;
1518047dd67eSAttilio Rao 	if (x & LK_EXCLUSIVE_WAITERS)
1519047dd67eSAttilio Rao 		printf(" with exclusive waiters pending\n");
1520047dd67eSAttilio Rao 	if (x & LK_SHARED_WAITERS)
1521047dd67eSAttilio Rao 		printf(" with shared waiters pending\n");
1522651175c9SAttilio Rao 	if (x & LK_EXCLUSIVE_SPINNERS)
1523651175c9SAttilio Rao 		printf(" with exclusive spinners pending\n");
1524047dd67eSAttilio Rao 
1525047dd67eSAttilio Rao 	STACK_PRINT(lk);
1526047dd67eSAttilio Rao }
1527047dd67eSAttilio Rao 
152899448ed1SJohn Dyson int
1529d576deedSPawel Jakub Dawidek lockstatus(const struct lock *lk)
153099448ed1SJohn Dyson {
1531047dd67eSAttilio Rao 	uintptr_t v, x;
1532047dd67eSAttilio Rao 	int ret;
153399448ed1SJohn Dyson 
1534047dd67eSAttilio Rao 	ret = LK_SHARED;
1535047dd67eSAttilio Rao 	x = lk->lk_lock;
1536047dd67eSAttilio Rao 	v = LK_HOLDER(x);
15370e9eb108SAttilio Rao 
1538047dd67eSAttilio Rao 	if ((x & LK_SHARE) == 0) {
1539047dd67eSAttilio Rao 		if (v == (uintptr_t)curthread || v == LK_KERNPROC)
1540047dd67eSAttilio Rao 			ret = LK_EXCLUSIVE;
15416bdfe06aSEivind Eklund 		else
1542047dd67eSAttilio Rao 			ret = LK_EXCLOTHER;
1543047dd67eSAttilio Rao 	} else if (x == LK_UNLOCKED)
1544047dd67eSAttilio Rao 		ret = 0;
154599448ed1SJohn Dyson 
1546047dd67eSAttilio Rao 	return (ret);
154753bf4bb2SPeter Wemm }
1548be6847d7SJohn Baldwin 
154984887fa3SAttilio Rao #ifdef INVARIANT_SUPPORT
1550de5b1952SAlexander Leidinger 
1551de5b1952SAlexander Leidinger FEATURE(invariant_support,
1552de5b1952SAlexander Leidinger     "Support for modules compiled with INVARIANTS option");
1553de5b1952SAlexander Leidinger 
155484887fa3SAttilio Rao #ifndef INVARIANTS
155584887fa3SAttilio Rao #undef	_lockmgr_assert
155684887fa3SAttilio Rao #endif
155784887fa3SAttilio Rao 
155884887fa3SAttilio Rao void
1559d576deedSPawel Jakub Dawidek _lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
156084887fa3SAttilio Rao {
156184887fa3SAttilio Rao 	int slocked = 0;
156284887fa3SAttilio Rao 
156384887fa3SAttilio Rao 	if (panicstr != NULL)
156484887fa3SAttilio Rao 		return;
156584887fa3SAttilio Rao 	switch (what) {
156684887fa3SAttilio Rao 	case KA_SLOCKED:
156784887fa3SAttilio Rao 	case KA_SLOCKED | KA_NOTRECURSED:
156884887fa3SAttilio Rao 	case KA_SLOCKED | KA_RECURSED:
156984887fa3SAttilio Rao 		slocked = 1;
157084887fa3SAttilio Rao 	case KA_LOCKED:
157184887fa3SAttilio Rao 	case KA_LOCKED | KA_NOTRECURSED:
157284887fa3SAttilio Rao 	case KA_LOCKED | KA_RECURSED:
1573e5f94314SAttilio Rao #ifdef WITNESS
1574e5f94314SAttilio Rao 
1575e5f94314SAttilio Rao 		/*
1576e5f94314SAttilio Rao 		 * We cannot trust WITNESS if the lock is held in exclusive
1577e5f94314SAttilio Rao 		 * mode and a call to lockmgr_disown() happened.
1578e5f94314SAttilio Rao 		 * Workaround this skipping the check if the lock is held in
1579e5f94314SAttilio Rao 		 * exclusive mode even for the KA_LOCKED case.
1580e5f94314SAttilio Rao 		 */
1581e5f94314SAttilio Rao 		if (slocked || (lk->lk_lock & LK_SHARE)) {
1582e5f94314SAttilio Rao 			witness_assert(&lk->lock_object, what, file, line);
1583e5f94314SAttilio Rao 			break;
1584e5f94314SAttilio Rao 		}
1585e5f94314SAttilio Rao #endif
1586047dd67eSAttilio Rao 		if (lk->lk_lock == LK_UNLOCKED ||
1587047dd67eSAttilio Rao 		    ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
1588047dd67eSAttilio Rao 		    (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
158984887fa3SAttilio Rao 			panic("Lock %s not %slocked @ %s:%d\n",
1590047dd67eSAttilio Rao 			    lk->lock_object.lo_name, slocked ? "share" : "",
159184887fa3SAttilio Rao 			    file, line);
1592047dd67eSAttilio Rao 
1593047dd67eSAttilio Rao 		if ((lk->lk_lock & LK_SHARE) == 0) {
1594047dd67eSAttilio Rao 			if (lockmgr_recursed(lk)) {
159584887fa3SAttilio Rao 				if (what & KA_NOTRECURSED)
159684887fa3SAttilio Rao 					panic("Lock %s recursed @ %s:%d\n",
1597047dd67eSAttilio Rao 					    lk->lock_object.lo_name, file,
1598047dd67eSAttilio Rao 					    line);
159984887fa3SAttilio Rao 			} else if (what & KA_RECURSED)
160084887fa3SAttilio Rao 				panic("Lock %s not recursed @ %s:%d\n",
1601047dd67eSAttilio Rao 				    lk->lock_object.lo_name, file, line);
160284887fa3SAttilio Rao 		}
160384887fa3SAttilio Rao 		break;
160484887fa3SAttilio Rao 	case KA_XLOCKED:
160584887fa3SAttilio Rao 	case KA_XLOCKED | KA_NOTRECURSED:
160684887fa3SAttilio Rao 	case KA_XLOCKED | KA_RECURSED:
1607047dd67eSAttilio Rao 		if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
160884887fa3SAttilio Rao 			panic("Lock %s not exclusively locked @ %s:%d\n",
1609047dd67eSAttilio Rao 			    lk->lock_object.lo_name, file, line);
1610047dd67eSAttilio Rao 		if (lockmgr_recursed(lk)) {
161184887fa3SAttilio Rao 			if (what & KA_NOTRECURSED)
161284887fa3SAttilio Rao 				panic("Lock %s recursed @ %s:%d\n",
1613047dd67eSAttilio Rao 				    lk->lock_object.lo_name, file, line);
161484887fa3SAttilio Rao 		} else if (what & KA_RECURSED)
161584887fa3SAttilio Rao 			panic("Lock %s not recursed @ %s:%d\n",
1616047dd67eSAttilio Rao 			    lk->lock_object.lo_name, file, line);
161784887fa3SAttilio Rao 		break;
161884887fa3SAttilio Rao 	case KA_UNLOCKED:
1619047dd67eSAttilio Rao 		if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
162084887fa3SAttilio Rao 			panic("Lock %s exclusively locked @ %s:%d\n",
1621047dd67eSAttilio Rao 			    lk->lock_object.lo_name, file, line);
162284887fa3SAttilio Rao 		break;
162384887fa3SAttilio Rao 	default:
1624047dd67eSAttilio Rao 		panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
1625047dd67eSAttilio Rao 		    line);
162684887fa3SAttilio Rao 	}
162784887fa3SAttilio Rao }
1628047dd67eSAttilio Rao #endif
162984887fa3SAttilio Rao 
1630be6847d7SJohn Baldwin #ifdef DDB
1631462a7addSJohn Baldwin int
1632462a7addSJohn Baldwin lockmgr_chain(struct thread *td, struct thread **ownerp)
1633462a7addSJohn Baldwin {
1634047dd67eSAttilio Rao 	struct lock *lk;
1635462a7addSJohn Baldwin 
1636047dd67eSAttilio Rao 	lk = td->td_wchan;
1637462a7addSJohn Baldwin 
1638047dd67eSAttilio Rao 	if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
1639462a7addSJohn Baldwin 		return (0);
1640047dd67eSAttilio Rao 	db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
1641047dd67eSAttilio Rao 	if (lk->lk_lock & LK_SHARE)
1642047dd67eSAttilio Rao 		db_printf("SHARED (count %ju)\n",
1643047dd67eSAttilio Rao 		    (uintmax_t)LK_SHARERS(lk->lk_lock));
1644047dd67eSAttilio Rao 	else
1645047dd67eSAttilio Rao 		db_printf("EXCL\n");
1646047dd67eSAttilio Rao 	*ownerp = lockmgr_xholder(lk);
1647462a7addSJohn Baldwin 
1648462a7addSJohn Baldwin 	return (1);
1649462a7addSJohn Baldwin }
1650462a7addSJohn Baldwin 
1651047dd67eSAttilio Rao static void
1652d576deedSPawel Jakub Dawidek db_show_lockmgr(const struct lock_object *lock)
1653be6847d7SJohn Baldwin {
1654be6847d7SJohn Baldwin 	struct thread *td;
1655d576deedSPawel Jakub Dawidek 	const struct lock *lk;
1656be6847d7SJohn Baldwin 
1657d576deedSPawel Jakub Dawidek 	lk = (const struct lock *)lock;
1658be6847d7SJohn Baldwin 
1659be6847d7SJohn Baldwin 	db_printf(" state: ");
1660047dd67eSAttilio Rao 	if (lk->lk_lock == LK_UNLOCKED)
1661be6847d7SJohn Baldwin 		db_printf("UNLOCKED\n");
1662047dd67eSAttilio Rao 	else if (lk->lk_lock & LK_SHARE)
1663047dd67eSAttilio Rao 		db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
1664047dd67eSAttilio Rao 	else {
1665047dd67eSAttilio Rao 		td = lockmgr_xholder(lk);
1666047dd67eSAttilio Rao 		if (td == (struct thread *)LK_KERNPROC)
1667047dd67eSAttilio Rao 			db_printf("XLOCK: LK_KERNPROC\n");
1668047dd67eSAttilio Rao 		else
1669047dd67eSAttilio Rao 			db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1670047dd67eSAttilio Rao 			    td->td_tid, td->td_proc->p_pid,
1671047dd67eSAttilio Rao 			    td->td_proc->p_comm);
1672047dd67eSAttilio Rao 		if (lockmgr_recursed(lk))
1673047dd67eSAttilio Rao 			db_printf(" recursed: %d\n", lk->lk_recurse);
1674047dd67eSAttilio Rao 	}
1675047dd67eSAttilio Rao 	db_printf(" waiters: ");
1676047dd67eSAttilio Rao 	switch (lk->lk_lock & LK_ALL_WAITERS) {
1677047dd67eSAttilio Rao 	case LK_SHARED_WAITERS:
1678047dd67eSAttilio Rao 		db_printf("shared\n");
1679e5023dd9SEdward Tomasz Napierala 		break;
1680047dd67eSAttilio Rao 	case LK_EXCLUSIVE_WAITERS:
1681047dd67eSAttilio Rao 		db_printf("exclusive\n");
1682047dd67eSAttilio Rao 		break;
1683047dd67eSAttilio Rao 	case LK_ALL_WAITERS:
1684047dd67eSAttilio Rao 		db_printf("shared and exclusive\n");
1685047dd67eSAttilio Rao 		break;
1686047dd67eSAttilio Rao 	default:
1687047dd67eSAttilio Rao 		db_printf("none\n");
1688047dd67eSAttilio Rao 	}
1689651175c9SAttilio Rao 	db_printf(" spinners: ");
1690651175c9SAttilio Rao 	if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
1691651175c9SAttilio Rao 		db_printf("exclusive\n");
1692651175c9SAttilio Rao 	else
1693651175c9SAttilio Rao 		db_printf("none\n");
1694be6847d7SJohn Baldwin }
1695be6847d7SJohn Baldwin #endif
1696