xref: /freebsd/sys/kern/kern_lock.c (revision d3d381b2b194b4d24853e92eecef55f262688d1a)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice(s), this list of conditions and the following disclaimer as
12  *    the first lines of this file unmodified other than the possible
13  *    addition of one or more copyright notices.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice(s), this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
19  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
22  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
28  * DAMAGE.
29  */
30 
31 #include "opt_ddb.h"
32 #include "opt_hwpmc_hooks.h"
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #include <sys/param.h>
38 #include <sys/kdb.h>
39 #include <sys/ktr.h>
40 #include <sys/lock.h>
41 #include <sys/lock_profile.h>
42 #include <sys/lockmgr.h>
43 #include <sys/mutex.h>
44 #include <sys/proc.h>
45 #include <sys/sleepqueue.h>
46 #ifdef DEBUG_LOCKS
47 #include <sys/stack.h>
48 #endif
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 
52 #include <machine/cpu.h>
53 
54 #ifdef DDB
55 #include <ddb/ddb.h>
56 #endif
57 
58 #ifdef HWPMC_HOOKS
59 #include <sys/pmckern.h>
60 PMC_SOFT_DECLARE( , , lock, failed);
61 #endif
62 
63 CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) ==
64     (LK_ADAPTIVE | LK_NOSHARE));
65 CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
66     ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
67 
68 #define	SQ_EXCLUSIVE_QUEUE	0
69 #define	SQ_SHARED_QUEUE		1
70 
71 #ifndef INVARIANTS
72 #define	_lockmgr_assert(lk, what, file, line)
73 #endif
74 
75 #define	TD_SLOCKS_INC(td)	((td)->td_lk_slocks++)
76 #define	TD_SLOCKS_DEC(td)	((td)->td_lk_slocks--)
77 
78 #ifndef DEBUG_LOCKS
79 #define	STACK_PRINT(lk)
80 #define	STACK_SAVE(lk)
81 #define	STACK_ZERO(lk)
82 #else
83 #define	STACK_PRINT(lk)	stack_print_ddb(&(lk)->lk_stack)
84 #define	STACK_SAVE(lk)	stack_save(&(lk)->lk_stack)
85 #define	STACK_ZERO(lk)	stack_zero(&(lk)->lk_stack)
86 #endif
87 
88 #define	LOCK_LOG2(lk, string, arg1, arg2)				\
89 	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
90 		CTR2(KTR_LOCK, (string), (arg1), (arg2))
91 #define	LOCK_LOG3(lk, string, arg1, arg2, arg3)				\
92 	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
93 		CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
94 
95 #define	GIANT_DECLARE							\
96 	int _i = 0;							\
97 	WITNESS_SAVE_DECL(Giant)
98 #define	GIANT_RESTORE() do {						\
99 	if (_i > 0) {							\
100 		while (_i--)						\
101 			mtx_lock(&Giant);				\
102 		WITNESS_RESTORE(&Giant.lock_object, Giant);		\
103 	}								\
104 } while (0)
105 #define	GIANT_SAVE() do {						\
106 	if (mtx_owned(&Giant)) {					\
107 		WITNESS_SAVE(&Giant.lock_object, Giant);		\
108 		while (mtx_owned(&Giant)) {				\
109 			_i++;						\
110 			mtx_unlock(&Giant);				\
111 		}							\
112 	}								\
113 } while (0)
114 
115 static bool __always_inline
116 LK_CAN_SHARE(uintptr_t x, int flags, bool fp)
117 {
118 
119 	if ((x & (LK_SHARE | LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) ==
120 	    LK_SHARE)
121 		return (true);
122 	if (fp || (!(x & LK_SHARE)))
123 		return (false);
124 	if ((curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||
125 	    (curthread->td_pflags & TDP_DEADLKTREAT))
126 		return (true);
127 	return (false);
128 }
129 
130 #define	LK_TRYOP(x)							\
131 	((x) & LK_NOWAIT)
132 
133 #define	LK_CAN_WITNESS(x)						\
134 	(((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
135 #define	LK_TRYWIT(x)							\
136 	(LK_TRYOP(x) ? LOP_TRYLOCK : 0)
137 
138 #define	LK_CAN_ADAPT(lk, f)						\
139 	(((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 &&		\
140 	((f) & LK_SLEEPFAIL) == 0)
141 
142 #define	lockmgr_disowned(lk)						\
143 	(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
144 
145 #define	lockmgr_xlocked_v(v)						\
146 	(((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
147 
148 #define	lockmgr_xlocked(lk) lockmgr_xlocked_v((lk)->lk_lock)
149 
150 static void	assert_lockmgr(const struct lock_object *lock, int how);
151 #ifdef DDB
152 static void	db_show_lockmgr(const struct lock_object *lock);
153 #endif
154 static void	lock_lockmgr(struct lock_object *lock, uintptr_t how);
155 #ifdef KDTRACE_HOOKS
156 static int	owner_lockmgr(const struct lock_object *lock,
157 		    struct thread **owner);
158 #endif
159 static uintptr_t unlock_lockmgr(struct lock_object *lock);
160 
161 struct lock_class lock_class_lockmgr = {
162 	.lc_name = "lockmgr",
163 	.lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
164 	.lc_assert = assert_lockmgr,
165 #ifdef DDB
166 	.lc_ddb_show = db_show_lockmgr,
167 #endif
168 	.lc_lock = lock_lockmgr,
169 	.lc_unlock = unlock_lockmgr,
170 #ifdef KDTRACE_HOOKS
171 	.lc_owner = owner_lockmgr,
172 #endif
173 };
174 
175 struct lockmgr_wait {
176 	const char *iwmesg;
177 	int ipri;
178 	int itimo;
179 };
180 
181 static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
182     int flags, bool fp);
183 static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp);
184 
185 static void
186 lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper)
187 {
188 	struct lock_class *class;
189 
190 	if (flags & LK_INTERLOCK) {
191 		class = LOCK_CLASS(ilk);
192 		class->lc_unlock(ilk);
193 	}
194 
195 	if (__predict_false(wakeup_swapper))
196 		kick_proc0();
197 }
198 
199 static void
200 lockmgr_note_shared_acquire(struct lock *lk, int contested,
201     uint64_t waittime, const char *file, int line, int flags)
202 {
203 
204 	lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
205 	    file, line);
206 	LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
207 	WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
208 	TD_LOCKS_INC(curthread);
209 	TD_SLOCKS_INC(curthread);
210 	STACK_SAVE(lk);
211 }
212 
213 static void
214 lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
215 {
216 
217 	lock_profile_release_lock(&lk->lock_object);
218 	WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
219 	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
220 	TD_LOCKS_DEC(curthread);
221 	TD_SLOCKS_DEC(curthread);
222 }
223 
224 static void
225 lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
226     uint64_t waittime, const char *file, int line, int flags)
227 {
228 
229 	lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
230 	    file, line);
231 	LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
232 	WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
233 	    line);
234 	TD_LOCKS_INC(curthread);
235 	STACK_SAVE(lk);
236 }
237 
238 static void
239 lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
240 {
241 
242 	lock_profile_release_lock(&lk->lock_object);
243 	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
244 	    line);
245 	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
246 	TD_LOCKS_DEC(curthread);
247 }
248 
249 static __inline struct thread *
250 lockmgr_xholder(const struct lock *lk)
251 {
252 	uintptr_t x;
253 
254 	x = lk->lk_lock;
255 	return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
256 }
257 
258 /*
259  * It assumes sleepq_lock held and returns with this one unheld.
260  * It also assumes the generic interlock is sane and previously checked.
261  * If LK_INTERLOCK is specified the interlock is not reacquired after the
262  * sleep.
263  */
264 static __inline int
265 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
266     const char *wmesg, int pri, int timo, int queue)
267 {
268 	GIANT_DECLARE;
269 	struct lock_class *class;
270 	int catch, error;
271 
272 	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
273 	catch = pri & PCATCH;
274 	pri &= PRIMASK;
275 	error = 0;
276 
277 	LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
278 	    (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
279 
280 	if (flags & LK_INTERLOCK)
281 		class->lc_unlock(ilk);
282 	if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0)
283 		lk->lk_exslpfail++;
284 	GIANT_SAVE();
285 	sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
286 	    SLEEPQ_INTERRUPTIBLE : 0), queue);
287 	if ((flags & LK_TIMELOCK) && timo)
288 		sleepq_set_timeout(&lk->lock_object, timo);
289 
290 	/*
291 	 * Decisional switch for real sleeping.
292 	 */
293 	if ((flags & LK_TIMELOCK) && timo && catch)
294 		error = sleepq_timedwait_sig(&lk->lock_object, pri);
295 	else if ((flags & LK_TIMELOCK) && timo)
296 		error = sleepq_timedwait(&lk->lock_object, pri);
297 	else if (catch)
298 		error = sleepq_wait_sig(&lk->lock_object, pri);
299 	else
300 		sleepq_wait(&lk->lock_object, pri);
301 	GIANT_RESTORE();
302 	if ((flags & LK_SLEEPFAIL) && error == 0)
303 		error = ENOLCK;
304 
305 	return (error);
306 }
307 
308 static __inline int
309 wakeupshlk(struct lock *lk, const char *file, int line)
310 {
311 	uintptr_t v, x, orig_x;
312 	u_int realexslp;
313 	int queue, wakeup_swapper;
314 
315 	wakeup_swapper = 0;
316 	for (;;) {
317 		x = lk->lk_lock;
318 		if (lockmgr_sunlock_try(lk, &x))
319 			break;
320 
321 		/*
322 		 * We should have a sharer with waiters, so enter the hard
323 		 * path in order to handle wakeups correctly.
324 		 */
325 		sleepq_lock(&lk->lock_object);
326 		orig_x = lk->lk_lock;
327 retry_sleepq:
328 		x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
329 		v = LK_UNLOCKED;
330 
331 		/*
332 		 * If the lock has exclusive waiters, give them preference in
333 		 * order to avoid deadlock with shared runners up.
334 		 * If interruptible sleeps left the exclusive queue empty
335 		 * avoid a starvation for the threads sleeping on the shared
336 		 * queue by giving them precedence and cleaning up the
337 		 * exclusive waiters bit anyway.
338 		 * Please note that lk_exslpfail count may be lying about
339 		 * the real number of waiters with the LK_SLEEPFAIL flag on
340 		 * because they may be used in conjunction with interruptible
341 		 * sleeps so lk_exslpfail might be considered an 'upper limit'
342 		 * bound, including the edge cases.
343 		 */
344 		realexslp = sleepq_sleepcnt(&lk->lock_object,
345 		    SQ_EXCLUSIVE_QUEUE);
346 		if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
347 			if (lk->lk_exslpfail < realexslp) {
348 				lk->lk_exslpfail = 0;
349 				queue = SQ_EXCLUSIVE_QUEUE;
350 				v |= (x & LK_SHARED_WAITERS);
351 			} else {
352 				lk->lk_exslpfail = 0;
353 				LOCK_LOG2(lk,
354 				    "%s: %p has only LK_SLEEPFAIL sleepers",
355 				    __func__, lk);
356 				LOCK_LOG2(lk,
357 			    "%s: %p waking up threads on the exclusive queue",
358 				    __func__, lk);
359 				wakeup_swapper =
360 				    sleepq_broadcast(&lk->lock_object,
361 				    SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
362 				queue = SQ_SHARED_QUEUE;
363 			}
364 
365 		} else {
366 
367 			/*
368 			 * Exclusive waiters sleeping with LK_SLEEPFAIL on
369 			 * and using interruptible sleeps/timeout may have
370 			 * left spourious lk_exslpfail counts on, so clean
371 			 * it up anyway.
372 			 */
373 			lk->lk_exslpfail = 0;
374 			queue = SQ_SHARED_QUEUE;
375 		}
376 
377 		if (lockmgr_sunlock_try(lk, &orig_x)) {
378 			sleepq_release(&lk->lock_object);
379 			break;
380 		}
381 
382 		x |= LK_SHARERS_LOCK(1);
383 		if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
384 			orig_x = x;
385 			goto retry_sleepq;
386 		}
387 		LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
388 		    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
389 		    "exclusive");
390 		wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
391 		    0, queue);
392 		sleepq_release(&lk->lock_object);
393 		break;
394 	}
395 
396 	lockmgr_note_shared_release(lk, file, line);
397 	return (wakeup_swapper);
398 }
399 
400 static void
401 assert_lockmgr(const struct lock_object *lock, int what)
402 {
403 
404 	panic("lockmgr locks do not support assertions");
405 }
406 
407 static void
408 lock_lockmgr(struct lock_object *lock, uintptr_t how)
409 {
410 
411 	panic("lockmgr locks do not support sleep interlocking");
412 }
413 
414 static uintptr_t
415 unlock_lockmgr(struct lock_object *lock)
416 {
417 
418 	panic("lockmgr locks do not support sleep interlocking");
419 }
420 
421 #ifdef KDTRACE_HOOKS
422 static int
423 owner_lockmgr(const struct lock_object *lock, struct thread **owner)
424 {
425 
426 	panic("lockmgr locks do not support owner inquiring");
427 }
428 #endif
429 
430 void
431 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
432 {
433 	int iflags;
434 
435 	MPASS((flags & ~LK_INIT_MASK) == 0);
436 	ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
437             ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
438             &lk->lk_lock));
439 
440 	iflags = LO_SLEEPABLE | LO_UPGRADABLE;
441 	if (flags & LK_CANRECURSE)
442 		iflags |= LO_RECURSABLE;
443 	if ((flags & LK_NODUP) == 0)
444 		iflags |= LO_DUPOK;
445 	if (flags & LK_NOPROFILE)
446 		iflags |= LO_NOPROFILE;
447 	if ((flags & LK_NOWITNESS) == 0)
448 		iflags |= LO_WITNESS;
449 	if (flags & LK_QUIET)
450 		iflags |= LO_QUIET;
451 	if (flags & LK_IS_VNODE)
452 		iflags |= LO_IS_VNODE;
453 	iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE);
454 
455 	lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
456 	lk->lk_lock = LK_UNLOCKED;
457 	lk->lk_recurse = 0;
458 	lk->lk_exslpfail = 0;
459 	lk->lk_timo = timo;
460 	lk->lk_pri = pri;
461 	STACK_ZERO(lk);
462 }
463 
464 /*
465  * XXX: Gross hacks to manipulate external lock flags after
466  * initialization.  Used for certain vnode and buf locks.
467  */
468 void
469 lockallowshare(struct lock *lk)
470 {
471 
472 	lockmgr_assert(lk, KA_XLOCKED);
473 	lk->lock_object.lo_flags &= ~LK_NOSHARE;
474 }
475 
476 void
477 lockdisableshare(struct lock *lk)
478 {
479 
480 	lockmgr_assert(lk, KA_XLOCKED);
481 	lk->lock_object.lo_flags |= LK_NOSHARE;
482 }
483 
484 void
485 lockallowrecurse(struct lock *lk)
486 {
487 
488 	lockmgr_assert(lk, KA_XLOCKED);
489 	lk->lock_object.lo_flags |= LO_RECURSABLE;
490 }
491 
492 void
493 lockdisablerecurse(struct lock *lk)
494 {
495 
496 	lockmgr_assert(lk, KA_XLOCKED);
497 	lk->lock_object.lo_flags &= ~LO_RECURSABLE;
498 }
499 
500 void
501 lockdestroy(struct lock *lk)
502 {
503 
504 	KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
505 	KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
506 	KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
507 	lock_destroy(&lk->lock_object);
508 }
509 
510 static bool __always_inline
511 lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags, bool fp)
512 {
513 
514 	/*
515 	 * If no other thread has an exclusive lock, or
516 	 * no exclusive waiter is present, bump the count of
517 	 * sharers.  Since we have to preserve the state of
518 	 * waiters, if we fail to acquire the shared lock
519 	 * loop back and retry.
520 	 */
521 	*xp = lk->lk_lock;
522 	while (LK_CAN_SHARE(*xp, flags, fp)) {
523 		if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
524 		    *xp + LK_ONE_SHARER)) {
525 			return (true);
526 		}
527 	}
528 	return (false);
529 }
530 
531 static bool __always_inline
532 lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp)
533 {
534 
535 	for (;;) {
536 		if (LK_SHARERS(*xp) > 1 || !(*xp & LK_ALL_WAITERS)) {
537 			if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
538 			    *xp - LK_ONE_SHARER))
539 				return (true);
540 			continue;
541 		}
542 		break;
543 	}
544 	return (false);
545 }
546 
547 static __noinline int
548 lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
549     const char *file, int line, struct lockmgr_wait *lwa)
550 {
551 	uintptr_t tid, x;
552 	int error = 0;
553 	const char *iwmesg;
554 	int ipri, itimo;
555 
556 #ifdef LOCK_PROFILING
557 	uint64_t waittime = 0;
558 	int contested = 0;
559 #endif
560 
561 	if (__predict_false(panicstr != NULL))
562 		goto out;
563 
564 	tid = (uintptr_t)curthread;
565 
566 	if (LK_CAN_WITNESS(flags))
567 		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
568 		    file, line, flags & LK_INTERLOCK ? ilk : NULL);
569 	for (;;) {
570 		if (lockmgr_slock_try(lk, &x, flags, false))
571 			break;
572 #ifdef HWPMC_HOOKS
573 		PMC_SOFT_CALL( , , lock, failed);
574 #endif
575 		lock_profile_obtain_lock_failed(&lk->lock_object,
576 		    &contested, &waittime);
577 
578 		/*
579 		 * If the lock is already held by curthread in
580 		 * exclusive way avoid a deadlock.
581 		 */
582 		if (LK_HOLDER(x) == tid) {
583 			LOCK_LOG2(lk,
584 			    "%s: %p already held in exclusive mode",
585 			    __func__, lk);
586 			error = EDEADLK;
587 			break;
588 		}
589 
590 		/*
591 		 * If the lock is expected to not sleep just give up
592 		 * and return.
593 		 */
594 		if (LK_TRYOP(flags)) {
595 			LOCK_LOG2(lk, "%s: %p fails the try operation",
596 			    __func__, lk);
597 			error = EBUSY;
598 			break;
599 		}
600 
601 		/*
602 		 * Acquire the sleepqueue chain lock because we
603 		 * probabilly will need to manipulate waiters flags.
604 		 */
605 		sleepq_lock(&lk->lock_object);
606 		x = lk->lk_lock;
607 retry_sleepq:
608 
609 		/*
610 		 * if the lock can be acquired in shared mode, try
611 		 * again.
612 		 */
613 		if (LK_CAN_SHARE(x, flags, false)) {
614 			sleepq_release(&lk->lock_object);
615 			continue;
616 		}
617 
618 		/*
619 		 * Try to set the LK_SHARED_WAITERS flag.  If we fail,
620 		 * loop back and retry.
621 		 */
622 		if ((x & LK_SHARED_WAITERS) == 0) {
623 			if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
624 			    x | LK_SHARED_WAITERS)) {
625 				goto retry_sleepq;
626 			}
627 			LOCK_LOG2(lk, "%s: %p set shared waiters flag",
628 			    __func__, lk);
629 		}
630 
631 		if (lwa == NULL) {
632 			iwmesg = lk->lock_object.lo_name;
633 			ipri = lk->lk_pri;
634 			itimo = lk->lk_timo;
635 		} else {
636 			iwmesg = lwa->iwmesg;
637 			ipri = lwa->ipri;
638 			itimo = lwa->itimo;
639 		}
640 
641 		/*
642 		 * As far as we have been unable to acquire the
643 		 * shared lock and the shared waiters flag is set,
644 		 * we will sleep.
645 		 */
646 		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
647 		    SQ_SHARED_QUEUE);
648 		flags &= ~LK_INTERLOCK;
649 		if (error) {
650 			LOCK_LOG3(lk,
651 			    "%s: interrupted sleep for %p with %d",
652 			    __func__, lk, error);
653 			break;
654 		}
655 		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
656 		    __func__, lk);
657 	}
658 	if (error == 0) {
659 #ifdef LOCK_PROFILING
660 		lockmgr_note_shared_acquire(lk, contested, waittime,
661 		    file, line, flags);
662 #else
663 		lockmgr_note_shared_acquire(lk, 0, 0, file, line,
664 		    flags);
665 #endif
666 	}
667 
668 out:
669 	lockmgr_exit(flags, ilk, 0);
670 	return (error);
671 }
672 
673 static __noinline int
674 lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
675     const char *file, int line, struct lockmgr_wait *lwa)
676 {
677 	struct lock_class *class;
678 	uintptr_t tid, x, v;
679 	int error = 0;
680 	const char *iwmesg;
681 	int ipri, itimo;
682 
683 #ifdef LOCK_PROFILING
684 	uint64_t waittime = 0;
685 	int contested = 0;
686 #endif
687 
688 	if (__predict_false(panicstr != NULL))
689 		goto out;
690 
691 	tid = (uintptr_t)curthread;
692 
693 	if (LK_CAN_WITNESS(flags))
694 		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
695 		    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
696 		    ilk : NULL);
697 
698 	/*
699 	 * If curthread already holds the lock and this one is
700 	 * allowed to recurse, simply recurse on it.
701 	 */
702 	if (lockmgr_xlocked(lk)) {
703 		if ((flags & LK_CANRECURSE) == 0 &&
704 		    (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
705 			/*
706 			 * If the lock is expected to not panic just
707 			 * give up and return.
708 			 */
709 			if (LK_TRYOP(flags)) {
710 				LOCK_LOG2(lk,
711 				    "%s: %p fails the try operation",
712 				    __func__, lk);
713 				error = EBUSY;
714 				goto out;
715 			}
716 			if (flags & LK_INTERLOCK) {
717 				class = LOCK_CLASS(ilk);
718 				class->lc_unlock(ilk);
719 			}
720 			panic("%s: recursing on non recursive lockmgr %p "
721 			    "@ %s:%d\n", __func__, lk, file, line);
722 		}
723 		lk->lk_recurse++;
724 		LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
725 		LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
726 		    lk->lk_recurse, file, line);
727 		WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
728 		    LK_TRYWIT(flags), file, line);
729 		TD_LOCKS_INC(curthread);
730 		goto out;
731 	}
732 
733 	for (;;) {
734 		if (lk->lk_lock == LK_UNLOCKED &&
735 		    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
736 			break;
737 #ifdef HWPMC_HOOKS
738 		PMC_SOFT_CALL( , , lock, failed);
739 #endif
740 		lock_profile_obtain_lock_failed(&lk->lock_object,
741 		    &contested, &waittime);
742 
743 		/*
744 		 * If the lock is expected to not sleep just give up
745 		 * and return.
746 		 */
747 		if (LK_TRYOP(flags)) {
748 			LOCK_LOG2(lk, "%s: %p fails the try operation",
749 			    __func__, lk);
750 			error = EBUSY;
751 			break;
752 		}
753 
754 		/*
755 		 * Acquire the sleepqueue chain lock because we
756 		 * probabilly will need to manipulate waiters flags.
757 		 */
758 		sleepq_lock(&lk->lock_object);
759 		x = lk->lk_lock;
760 retry_sleepq:
761 
762 		/*
763 		 * if the lock has been released while we spun on
764 		 * the sleepqueue chain lock just try again.
765 		 */
766 		if (x == LK_UNLOCKED) {
767 			sleepq_release(&lk->lock_object);
768 			continue;
769 		}
770 
771 		/*
772 		 * The lock can be in the state where there is a
773 		 * pending queue of waiters, but still no owner.
774 		 * This happens when the lock is contested and an
775 		 * owner is going to claim the lock.
776 		 * If curthread is the one successfully acquiring it
777 		 * claim lock ownership and return, preserving waiters
778 		 * flags.
779 		 */
780 		v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
781 		if ((x & ~v) == LK_UNLOCKED) {
782 			v &= ~LK_EXCLUSIVE_SPINNERS;
783 			if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
784 			    tid | v)) {
785 				sleepq_release(&lk->lock_object);
786 				LOCK_LOG2(lk,
787 				    "%s: %p claimed by a new writer",
788 				    __func__, lk);
789 				break;
790 			}
791 			goto retry_sleepq;
792 		}
793 
794 		/*
795 		 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
796 		 * fail, loop back and retry.
797 		 */
798 		if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
799 			if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
800 			    x | LK_EXCLUSIVE_WAITERS)) {
801 				goto retry_sleepq;
802 			}
803 			LOCK_LOG2(lk, "%s: %p set excl waiters flag",
804 			    __func__, lk);
805 		}
806 
807 		if (lwa == NULL) {
808 			iwmesg = lk->lock_object.lo_name;
809 			ipri = lk->lk_pri;
810 			itimo = lk->lk_timo;
811 		} else {
812 			iwmesg = lwa->iwmesg;
813 			ipri = lwa->ipri;
814 			itimo = lwa->itimo;
815 		}
816 
817 		/*
818 		 * As far as we have been unable to acquire the
819 		 * exclusive lock and the exclusive waiters flag
820 		 * is set, we will sleep.
821 		 */
822 		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
823 		    SQ_EXCLUSIVE_QUEUE);
824 		flags &= ~LK_INTERLOCK;
825 		if (error) {
826 			LOCK_LOG3(lk,
827 			    "%s: interrupted sleep for %p with %d",
828 			    __func__, lk, error);
829 			break;
830 		}
831 		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
832 		    __func__, lk);
833 	}
834 	if (error == 0) {
835 #ifdef LOCK_PROFILING
836 		lockmgr_note_exclusive_acquire(lk, contested, waittime,
837 		    file, line, flags);
838 #else
839 		lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
840 		    flags);
841 #endif
842 	}
843 
844 out:
845 	lockmgr_exit(flags, ilk, 0);
846 	return (error);
847 }
848 
849 static __noinline int
850 lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
851     const char *file, int line, struct lockmgr_wait *lwa)
852 {
853 	uintptr_t tid, x, v;
854 	int error = 0;
855 	int wakeup_swapper = 0;
856 	int op;
857 
858 	if (__predict_false(panicstr != NULL))
859 		goto out;
860 
861 	tid = (uintptr_t)curthread;
862 
863 	_lockmgr_assert(lk, KA_SLOCKED, file, line);
864 	v = lk->lk_lock;
865 	x = v & LK_ALL_WAITERS;
866 	v &= LK_EXCLUSIVE_SPINNERS;
867 
868 	/*
869 	 * Try to switch from one shared lock to an exclusive one.
870 	 * We need to preserve waiters flags during the operation.
871 	 */
872 	if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
873 	    tid | x)) {
874 		LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
875 		    line);
876 		WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
877 		    LK_TRYWIT(flags), file, line);
878 		TD_SLOCKS_DEC(curthread);
879 		goto out;
880 	}
881 
882 	op = flags & LK_TYPE_MASK;
883 
884 	/*
885 	 * In LK_TRYUPGRADE mode, do not drop the lock,
886 	 * returning EBUSY instead.
887 	 */
888 	if (op == LK_TRYUPGRADE) {
889 		LOCK_LOG2(lk, "%s: %p failed the nowait upgrade",
890 		    __func__, lk);
891 		error = EBUSY;
892 		goto out;
893 	}
894 
895 	/*
896 	 * We have been unable to succeed in upgrading, so just
897 	 * give up the shared lock.
898 	 */
899 	wakeup_swapper |= wakeupshlk(lk, file, line);
900 	error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
901 	flags &= ~LK_INTERLOCK;
902 out:
903 	lockmgr_exit(flags, ilk, wakeup_swapper);
904 	return (error);
905 }
906 
907 int
908 lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk,
909     const char *file, int line)
910 {
911 	struct lock_class *class;
912 	uintptr_t x, tid;
913 	u_int op;
914 	bool locked;
915 
916 	if (__predict_false(panicstr != NULL))
917 		return (0);
918 
919 	op = flags & LK_TYPE_MASK;
920 	locked = false;
921 	switch (op) {
922 	case LK_SHARED:
923 		if (LK_CAN_WITNESS(flags))
924 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
925 			    file, line, flags & LK_INTERLOCK ? ilk : NULL);
926 		if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
927 			break;
928 		if (lockmgr_slock_try(lk, &x, flags, true)) {
929 			lockmgr_note_shared_acquire(lk, 0, 0,
930 			    file, line, flags);
931 			locked = true;
932 		} else {
933 			return (lockmgr_slock_hard(lk, flags, ilk, file, line,
934 			    NULL));
935 		}
936 		break;
937 	case LK_EXCLUSIVE:
938 		if (LK_CAN_WITNESS(flags))
939 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
940 			    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
941 			    ilk : NULL);
942 		tid = (uintptr_t)curthread;
943 		if (lk->lk_lock == LK_UNLOCKED &&
944 		    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
945 			lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
946 			    flags);
947 			locked = true;
948 		} else {
949 			return (lockmgr_xlock_hard(lk, flags, ilk, file, line,
950 			    NULL));
951 		}
952 		break;
953 	case LK_UPGRADE:
954 	case LK_TRYUPGRADE:
955 		return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL));
956 	default:
957 		break;
958 	}
959 	if (__predict_true(locked)) {
960 		if (__predict_false(flags & LK_INTERLOCK)) {
961 			class = LOCK_CLASS(ilk);
962 			class->lc_unlock(ilk);
963 		}
964 		return (0);
965 	} else {
966 		return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
967 		    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
968 	}
969 }
970 
971 static __noinline int
972 lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
973     const char *file, int line)
974 
975 {
976 	int wakeup_swapper = 0;
977 
978 	if (__predict_false(panicstr != NULL))
979 		goto out;
980 
981 	wakeup_swapper = wakeupshlk(lk, file, line);
982 
983 out:
984 	lockmgr_exit(flags, ilk, wakeup_swapper);
985 	return (0);
986 }
987 
988 static __noinline int
989 lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
990     const char *file, int line)
991 {
992 	uintptr_t tid, v;
993 	int wakeup_swapper = 0;
994 	u_int realexslp;
995 	int queue;
996 
997 	if (__predict_false(panicstr != NULL))
998 		goto out;
999 
1000 	tid = (uintptr_t)curthread;
1001 
1002 	/*
1003 	 * As first option, treact the lock as if it has not
1004 	 * any waiter.
1005 	 * Fix-up the tid var if the lock has been disowned.
1006 	 */
1007 	if (LK_HOLDER(x) == LK_KERNPROC)
1008 		tid = LK_KERNPROC;
1009 	else {
1010 		WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
1011 		TD_LOCKS_DEC(curthread);
1012 	}
1013 	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
1014 
1015 	/*
1016 	 * The lock is held in exclusive mode.
1017 	 * If the lock is recursed also, then unrecurse it.
1018 	 */
1019 	if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) {
1020 		LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);
1021 		lk->lk_recurse--;
1022 		goto out;
1023 	}
1024 	if (tid != LK_KERNPROC)
1025 		lock_profile_release_lock(&lk->lock_object);
1026 
1027 	if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
1028 		goto out;
1029 
1030 	sleepq_lock(&lk->lock_object);
1031 	x = lk->lk_lock;
1032 	v = LK_UNLOCKED;
1033 
1034 	/*
1035 	 * If the lock has exclusive waiters, give them
1036 	 * preference in order to avoid deadlock with
1037 	 * shared runners up.
1038 	 * If interruptible sleeps left the exclusive queue
1039 	 * empty avoid a starvation for the threads sleeping
1040 	 * on the shared queue by giving them precedence
1041 	 * and cleaning up the exclusive waiters bit anyway.
1042 	 * Please note that lk_exslpfail count may be lying
1043 	 * about the real number of waiters with the
1044 	 * LK_SLEEPFAIL flag on because they may be used in
1045 	 * conjunction with interruptible sleeps so
1046 	 * lk_exslpfail might be considered an 'upper limit'
1047 	 * bound, including the edge cases.
1048 	 */
1049 	MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1050 	realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
1051 	if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
1052 		if (lk->lk_exslpfail < realexslp) {
1053 			lk->lk_exslpfail = 0;
1054 			queue = SQ_EXCLUSIVE_QUEUE;
1055 			v |= (x & LK_SHARED_WAITERS);
1056 		} else {
1057 			lk->lk_exslpfail = 0;
1058 			LOCK_LOG2(lk,
1059 			    "%s: %p has only LK_SLEEPFAIL sleepers",
1060 			    __func__, lk);
1061 			LOCK_LOG2(lk,
1062 			    "%s: %p waking up threads on the exclusive queue",
1063 			    __func__, lk);
1064 			wakeup_swapper = sleepq_broadcast(&lk->lock_object,
1065 			    SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
1066 			queue = SQ_SHARED_QUEUE;
1067 		}
1068 	} else {
1069 
1070 		/*
1071 		 * Exclusive waiters sleeping with LK_SLEEPFAIL
1072 		 * on and using interruptible sleeps/timeout
1073 		 * may have left spourious lk_exslpfail counts
1074 		 * on, so clean it up anyway.
1075 		 */
1076 		lk->lk_exslpfail = 0;
1077 		queue = SQ_SHARED_QUEUE;
1078 	}
1079 
1080 	LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
1081 	    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
1082 	    "exclusive");
1083 	atomic_store_rel_ptr(&lk->lk_lock, v);
1084 	wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
1085 	sleepq_release(&lk->lock_object);
1086 
1087 out:
1088 	lockmgr_exit(flags, ilk, wakeup_swapper);
1089 	return (0);
1090 }
1091 
1092 int
1093 lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk)
1094 {
1095 	struct lock_class *class;
1096 	uintptr_t x, tid;
1097 	const char *file;
1098 	int line;
1099 
1100 	if (__predict_false(panicstr != NULL))
1101 		return (0);
1102 
1103 	file = __FILE__;
1104 	line = __LINE__;
1105 
1106 	_lockmgr_assert(lk, KA_LOCKED, file, line);
1107 	x = lk->lk_lock;
1108 	if (__predict_true(x & LK_SHARE) != 0) {
1109 		if (lockmgr_sunlock_try(lk, &x)) {
1110 			lockmgr_note_shared_release(lk, file, line);
1111 		} else {
1112 			return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
1113 		}
1114 	} else {
1115 		tid = (uintptr_t)curthread;
1116 		if (!lockmgr_recursed(lk) &&
1117 		    atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
1118 			lockmgr_note_exclusive_release(lk, file, line);
1119 		} else {
1120 			return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
1121 		}
1122 	}
1123 	if (__predict_false(flags & LK_INTERLOCK)) {
1124 		class = LOCK_CLASS(ilk);
1125 		class->lc_unlock(ilk);
1126 	}
1127 	return (0);
1128 }
1129 
1130 int
1131 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
1132     const char *wmesg, int pri, int timo, const char *file, int line)
1133 {
1134 	GIANT_DECLARE;
1135 	struct lockmgr_wait lwa;
1136 	struct lock_class *class;
1137 	const char *iwmesg;
1138 	uintptr_t tid, v, x;
1139 	u_int op, realexslp;
1140 	int error, ipri, itimo, queue, wakeup_swapper;
1141 #ifdef LOCK_PROFILING
1142 	uint64_t waittime = 0;
1143 	int contested = 0;
1144 #endif
1145 
1146 	if (panicstr != NULL)
1147 		return (0);
1148 
1149 	error = 0;
1150 	tid = (uintptr_t)curthread;
1151 	op = (flags & LK_TYPE_MASK);
1152 	iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
1153 	ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
1154 	itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
1155 
1156 	lwa.iwmesg = iwmesg;
1157 	lwa.ipri = ipri;
1158 	lwa.itimo = itimo;
1159 
1160 	MPASS((flags & ~LK_TOTAL_MASK) == 0);
1161 	KASSERT((op & (op - 1)) == 0,
1162 	    ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
1163 	KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
1164 	    (op != LK_DOWNGRADE && op != LK_RELEASE),
1165 	    ("%s: Invalid flags in regard of the operation desired @ %s:%d",
1166 	    __func__, file, line));
1167 	KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
1168 	    ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
1169 	    __func__, file, line));
1170 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
1171 	    ("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread,
1172 	    lk->lock_object.lo_name, file, line));
1173 
1174 	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
1175 
1176 	if (lk->lock_object.lo_flags & LK_NOSHARE) {
1177 		switch (op) {
1178 		case LK_SHARED:
1179 			op = LK_EXCLUSIVE;
1180 			break;
1181 		case LK_UPGRADE:
1182 		case LK_TRYUPGRADE:
1183 		case LK_DOWNGRADE:
1184 			_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
1185 			    file, line);
1186 			if (flags & LK_INTERLOCK)
1187 				class->lc_unlock(ilk);
1188 			return (0);
1189 		}
1190 	}
1191 
1192 	wakeup_swapper = 0;
1193 	switch (op) {
1194 	case LK_SHARED:
1195 		return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
1196 		break;
1197 	case LK_UPGRADE:
1198 	case LK_TRYUPGRADE:
1199 		return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
1200 		break;
1201 	case LK_EXCLUSIVE:
1202 		return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
1203 		break;
1204 	case LK_DOWNGRADE:
1205 		_lockmgr_assert(lk, KA_XLOCKED, file, line);
1206 		LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
1207 		WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
1208 
1209 		/*
1210 		 * Panic if the lock is recursed.
1211 		 */
1212 		if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
1213 			if (flags & LK_INTERLOCK)
1214 				class->lc_unlock(ilk);
1215 			panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n",
1216 			    __func__, iwmesg, file, line);
1217 		}
1218 		TD_SLOCKS_INC(curthread);
1219 
1220 		/*
1221 		 * In order to preserve waiters flags, just spin.
1222 		 */
1223 		for (;;) {
1224 			x = lk->lk_lock;
1225 			MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1226 			x &= LK_ALL_WAITERS;
1227 			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1228 			    LK_SHARERS_LOCK(1) | x))
1229 				break;
1230 			cpu_spinwait();
1231 		}
1232 		break;
1233 	case LK_RELEASE:
1234 		_lockmgr_assert(lk, KA_LOCKED, file, line);
1235 		x = lk->lk_lock;
1236 
1237 		if (__predict_true(x & LK_SHARE) != 0) {
1238 			return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
1239 		} else {
1240 			return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
1241 		}
1242 		break;
1243 	case LK_DRAIN:
1244 		if (LK_CAN_WITNESS(flags))
1245 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1246 			    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
1247 			    ilk : NULL);
1248 
1249 		/*
1250 		 * Trying to drain a lock we already own will result in a
1251 		 * deadlock.
1252 		 */
1253 		if (lockmgr_xlocked(lk)) {
1254 			if (flags & LK_INTERLOCK)
1255 				class->lc_unlock(ilk);
1256 			panic("%s: draining %s with the lock held @ %s:%d\n",
1257 			    __func__, iwmesg, file, line);
1258 		}
1259 
1260 		for (;;) {
1261 			if (lk->lk_lock == LK_UNLOCKED &&
1262 			    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
1263 				break;
1264 
1265 #ifdef HWPMC_HOOKS
1266 			PMC_SOFT_CALL( , , lock, failed);
1267 #endif
1268 			lock_profile_obtain_lock_failed(&lk->lock_object,
1269 			    &contested, &waittime);
1270 
1271 			/*
1272 			 * If the lock is expected to not sleep just give up
1273 			 * and return.
1274 			 */
1275 			if (LK_TRYOP(flags)) {
1276 				LOCK_LOG2(lk, "%s: %p fails the try operation",
1277 				    __func__, lk);
1278 				error = EBUSY;
1279 				break;
1280 			}
1281 
1282 			/*
1283 			 * Acquire the sleepqueue chain lock because we
1284 			 * probabilly will need to manipulate waiters flags.
1285 			 */
1286 			sleepq_lock(&lk->lock_object);
1287 			x = lk->lk_lock;
1288 
1289 			/*
1290 			 * if the lock has been released while we spun on
1291 			 * the sleepqueue chain lock just try again.
1292 			 */
1293 			if (x == LK_UNLOCKED) {
1294 				sleepq_release(&lk->lock_object);
1295 				continue;
1296 			}
1297 
1298 			v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
1299 			if ((x & ~v) == LK_UNLOCKED) {
1300 				v = (x & ~LK_EXCLUSIVE_SPINNERS);
1301 
1302 				/*
1303 				 * If interruptible sleeps left the exclusive
1304 				 * queue empty avoid a starvation for the
1305 				 * threads sleeping on the shared queue by
1306 				 * giving them precedence and cleaning up the
1307 				 * exclusive waiters bit anyway.
1308 				 * Please note that lk_exslpfail count may be
1309 				 * lying about the real number of waiters with
1310 				 * the LK_SLEEPFAIL flag on because they may
1311 				 * be used in conjunction with interruptible
1312 				 * sleeps so lk_exslpfail might be considered
1313 				 * an 'upper limit' bound, including the edge
1314 				 * cases.
1315 				 */
1316 				if (v & LK_EXCLUSIVE_WAITERS) {
1317 					queue = SQ_EXCLUSIVE_QUEUE;
1318 					v &= ~LK_EXCLUSIVE_WAITERS;
1319 				} else {
1320 
1321 					/*
1322 					 * Exclusive waiters sleeping with
1323 					 * LK_SLEEPFAIL on and using
1324 					 * interruptible sleeps/timeout may
1325 					 * have left spourious lk_exslpfail
1326 					 * counts on, so clean it up anyway.
1327 					 */
1328 					MPASS(v & LK_SHARED_WAITERS);
1329 					lk->lk_exslpfail = 0;
1330 					queue = SQ_SHARED_QUEUE;
1331 					v &= ~LK_SHARED_WAITERS;
1332 				}
1333 				if (queue == SQ_EXCLUSIVE_QUEUE) {
1334 					realexslp =
1335 					    sleepq_sleepcnt(&lk->lock_object,
1336 					    SQ_EXCLUSIVE_QUEUE);
1337 					if (lk->lk_exslpfail >= realexslp) {
1338 						lk->lk_exslpfail = 0;
1339 						queue = SQ_SHARED_QUEUE;
1340 						v &= ~LK_SHARED_WAITERS;
1341 						if (realexslp != 0) {
1342 							LOCK_LOG2(lk,
1343 					"%s: %p has only LK_SLEEPFAIL sleepers",
1344 							    __func__, lk);
1345 							LOCK_LOG2(lk,
1346 			"%s: %p waking up threads on the exclusive queue",
1347 							    __func__, lk);
1348 							wakeup_swapper =
1349 							    sleepq_broadcast(
1350 							    &lk->lock_object,
1351 							    SLEEPQ_LK, 0,
1352 							    SQ_EXCLUSIVE_QUEUE);
1353 						}
1354 					} else
1355 						lk->lk_exslpfail = 0;
1356 				}
1357 				if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
1358 					sleepq_release(&lk->lock_object);
1359 					continue;
1360 				}
1361 				LOCK_LOG3(lk,
1362 				"%s: %p waking up all threads on the %s queue",
1363 				    __func__, lk, queue == SQ_SHARED_QUEUE ?
1364 				    "shared" : "exclusive");
1365 				wakeup_swapper |= sleepq_broadcast(
1366 				    &lk->lock_object, SLEEPQ_LK, 0, queue);
1367 
1368 				/*
1369 				 * If shared waiters have been woken up we need
1370 				 * to wait for one of them to acquire the lock
1371 				 * before to set the exclusive waiters in
1372 				 * order to avoid a deadlock.
1373 				 */
1374 				if (queue == SQ_SHARED_QUEUE) {
1375 					for (v = lk->lk_lock;
1376 					    (v & LK_SHARE) && !LK_SHARERS(v);
1377 					    v = lk->lk_lock)
1378 						cpu_spinwait();
1379 				}
1380 			}
1381 
1382 			/*
1383 			 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
1384 			 * fail, loop back and retry.
1385 			 */
1386 			if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
1387 				if (!atomic_cmpset_ptr(&lk->lk_lock, x,
1388 				    x | LK_EXCLUSIVE_WAITERS)) {
1389 					sleepq_release(&lk->lock_object);
1390 					continue;
1391 				}
1392 				LOCK_LOG2(lk, "%s: %p set drain waiters flag",
1393 				    __func__, lk);
1394 			}
1395 
1396 			/*
1397 			 * As far as we have been unable to acquire the
1398 			 * exclusive lock and the exclusive waiters flag
1399 			 * is set, we will sleep.
1400 			 */
1401 			if (flags & LK_INTERLOCK) {
1402 				class->lc_unlock(ilk);
1403 				flags &= ~LK_INTERLOCK;
1404 			}
1405 			GIANT_SAVE();
1406 			sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
1407 			    SQ_EXCLUSIVE_QUEUE);
1408 			sleepq_wait(&lk->lock_object, ipri & PRIMASK);
1409 			GIANT_RESTORE();
1410 			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
1411 			    __func__, lk);
1412 		}
1413 
1414 		if (error == 0) {
1415 			lock_profile_obtain_lock_success(&lk->lock_object,
1416 			    contested, waittime, file, line);
1417 			LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
1418 			    lk->lk_recurse, file, line);
1419 			WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
1420 			    LK_TRYWIT(flags), file, line);
1421 			TD_LOCKS_INC(curthread);
1422 			STACK_SAVE(lk);
1423 		}
1424 		break;
1425 	default:
1426 		if (flags & LK_INTERLOCK)
1427 			class->lc_unlock(ilk);
1428 		panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
1429 	}
1430 
1431 	if (flags & LK_INTERLOCK)
1432 		class->lc_unlock(ilk);
1433 	if (wakeup_swapper)
1434 		kick_proc0();
1435 
1436 	return (error);
1437 }
1438 
1439 void
1440 _lockmgr_disown(struct lock *lk, const char *file, int line)
1441 {
1442 	uintptr_t tid, x;
1443 
1444 	if (SCHEDULER_STOPPED())
1445 		return;
1446 
1447 	tid = (uintptr_t)curthread;
1448 	_lockmgr_assert(lk, KA_XLOCKED, file, line);
1449 
1450 	/*
1451 	 * Panic if the lock is recursed.
1452 	 */
1453 	if (lockmgr_xlocked(lk) && lockmgr_recursed(lk))
1454 		panic("%s: disown a recursed lockmgr @ %s:%d\n",
1455 		    __func__,  file, line);
1456 
1457 	/*
1458 	 * If the owner is already LK_KERNPROC just skip the whole operation.
1459 	 */
1460 	if (LK_HOLDER(lk->lk_lock) != tid)
1461 		return;
1462 	lock_profile_release_lock(&lk->lock_object);
1463 	LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
1464 	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
1465 	TD_LOCKS_DEC(curthread);
1466 	STACK_SAVE(lk);
1467 
1468 	/*
1469 	 * In order to preserve waiters flags, just spin.
1470 	 */
1471 	for (;;) {
1472 		x = lk->lk_lock;
1473 		MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1474 		x &= LK_ALL_WAITERS;
1475 		if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1476 		    LK_KERNPROC | x))
1477 			return;
1478 		cpu_spinwait();
1479 	}
1480 }
1481 
1482 void
1483 lockmgr_printinfo(const struct lock *lk)
1484 {
1485 	struct thread *td;
1486 	uintptr_t x;
1487 
1488 	if (lk->lk_lock == LK_UNLOCKED)
1489 		printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
1490 	else if (lk->lk_lock & LK_SHARE)
1491 		printf("lock type %s: SHARED (count %ju)\n",
1492 		    lk->lock_object.lo_name,
1493 		    (uintmax_t)LK_SHARERS(lk->lk_lock));
1494 	else {
1495 		td = lockmgr_xholder(lk);
1496 		if (td == (struct thread *)LK_KERNPROC)
1497 			printf("lock type %s: EXCL by KERNPROC\n",
1498 			    lk->lock_object.lo_name);
1499 		else
1500 			printf("lock type %s: EXCL by thread %p "
1501 			    "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
1502 			    td, td->td_proc->p_pid, td->td_proc->p_comm,
1503 			    td->td_tid);
1504 	}
1505 
1506 	x = lk->lk_lock;
1507 	if (x & LK_EXCLUSIVE_WAITERS)
1508 		printf(" with exclusive waiters pending\n");
1509 	if (x & LK_SHARED_WAITERS)
1510 		printf(" with shared waiters pending\n");
1511 	if (x & LK_EXCLUSIVE_SPINNERS)
1512 		printf(" with exclusive spinners pending\n");
1513 
1514 	STACK_PRINT(lk);
1515 }
1516 
1517 int
1518 lockstatus(const struct lock *lk)
1519 {
1520 	uintptr_t v, x;
1521 	int ret;
1522 
1523 	ret = LK_SHARED;
1524 	x = lk->lk_lock;
1525 	v = LK_HOLDER(x);
1526 
1527 	if ((x & LK_SHARE) == 0) {
1528 		if (v == (uintptr_t)curthread || v == LK_KERNPROC)
1529 			ret = LK_EXCLUSIVE;
1530 		else
1531 			ret = LK_EXCLOTHER;
1532 	} else if (x == LK_UNLOCKED)
1533 		ret = 0;
1534 
1535 	return (ret);
1536 }
1537 
1538 #ifdef INVARIANT_SUPPORT
1539 
1540 FEATURE(invariant_support,
1541     "Support for modules compiled with INVARIANTS option");
1542 
1543 #ifndef INVARIANTS
1544 #undef	_lockmgr_assert
1545 #endif
1546 
1547 void
1548 _lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
1549 {
1550 	int slocked = 0;
1551 
1552 	if (panicstr != NULL)
1553 		return;
1554 	switch (what) {
1555 	case KA_SLOCKED:
1556 	case KA_SLOCKED | KA_NOTRECURSED:
1557 	case KA_SLOCKED | KA_RECURSED:
1558 		slocked = 1;
1559 	case KA_LOCKED:
1560 	case KA_LOCKED | KA_NOTRECURSED:
1561 	case KA_LOCKED | KA_RECURSED:
1562 #ifdef WITNESS
1563 
1564 		/*
1565 		 * We cannot trust WITNESS if the lock is held in exclusive
1566 		 * mode and a call to lockmgr_disown() happened.
1567 		 * Workaround this skipping the check if the lock is held in
1568 		 * exclusive mode even for the KA_LOCKED case.
1569 		 */
1570 		if (slocked || (lk->lk_lock & LK_SHARE)) {
1571 			witness_assert(&lk->lock_object, what, file, line);
1572 			break;
1573 		}
1574 #endif
1575 		if (lk->lk_lock == LK_UNLOCKED ||
1576 		    ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
1577 		    (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
1578 			panic("Lock %s not %slocked @ %s:%d\n",
1579 			    lk->lock_object.lo_name, slocked ? "share" : "",
1580 			    file, line);
1581 
1582 		if ((lk->lk_lock & LK_SHARE) == 0) {
1583 			if (lockmgr_recursed(lk)) {
1584 				if (what & KA_NOTRECURSED)
1585 					panic("Lock %s recursed @ %s:%d\n",
1586 					    lk->lock_object.lo_name, file,
1587 					    line);
1588 			} else if (what & KA_RECURSED)
1589 				panic("Lock %s not recursed @ %s:%d\n",
1590 				    lk->lock_object.lo_name, file, line);
1591 		}
1592 		break;
1593 	case KA_XLOCKED:
1594 	case KA_XLOCKED | KA_NOTRECURSED:
1595 	case KA_XLOCKED | KA_RECURSED:
1596 		if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
1597 			panic("Lock %s not exclusively locked @ %s:%d\n",
1598 			    lk->lock_object.lo_name, file, line);
1599 		if (lockmgr_recursed(lk)) {
1600 			if (what & KA_NOTRECURSED)
1601 				panic("Lock %s recursed @ %s:%d\n",
1602 				    lk->lock_object.lo_name, file, line);
1603 		} else if (what & KA_RECURSED)
1604 			panic("Lock %s not recursed @ %s:%d\n",
1605 			    lk->lock_object.lo_name, file, line);
1606 		break;
1607 	case KA_UNLOCKED:
1608 		if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
1609 			panic("Lock %s exclusively locked @ %s:%d\n",
1610 			    lk->lock_object.lo_name, file, line);
1611 		break;
1612 	default:
1613 		panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
1614 		    line);
1615 	}
1616 }
1617 #endif
1618 
1619 #ifdef DDB
1620 int
1621 lockmgr_chain(struct thread *td, struct thread **ownerp)
1622 {
1623 	struct lock *lk;
1624 
1625 	lk = td->td_wchan;
1626 
1627 	if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
1628 		return (0);
1629 	db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
1630 	if (lk->lk_lock & LK_SHARE)
1631 		db_printf("SHARED (count %ju)\n",
1632 		    (uintmax_t)LK_SHARERS(lk->lk_lock));
1633 	else
1634 		db_printf("EXCL\n");
1635 	*ownerp = lockmgr_xholder(lk);
1636 
1637 	return (1);
1638 }
1639 
1640 static void
1641 db_show_lockmgr(const struct lock_object *lock)
1642 {
1643 	struct thread *td;
1644 	const struct lock *lk;
1645 
1646 	lk = (const struct lock *)lock;
1647 
1648 	db_printf(" state: ");
1649 	if (lk->lk_lock == LK_UNLOCKED)
1650 		db_printf("UNLOCKED\n");
1651 	else if (lk->lk_lock & LK_SHARE)
1652 		db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
1653 	else {
1654 		td = lockmgr_xholder(lk);
1655 		if (td == (struct thread *)LK_KERNPROC)
1656 			db_printf("XLOCK: LK_KERNPROC\n");
1657 		else
1658 			db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1659 			    td->td_tid, td->td_proc->p_pid,
1660 			    td->td_proc->p_comm);
1661 		if (lockmgr_recursed(lk))
1662 			db_printf(" recursed: %d\n", lk->lk_recurse);
1663 	}
1664 	db_printf(" waiters: ");
1665 	switch (lk->lk_lock & LK_ALL_WAITERS) {
1666 	case LK_SHARED_WAITERS:
1667 		db_printf("shared\n");
1668 		break;
1669 	case LK_EXCLUSIVE_WAITERS:
1670 		db_printf("exclusive\n");
1671 		break;
1672 	case LK_ALL_WAITERS:
1673 		db_printf("shared and exclusive\n");
1674 		break;
1675 	default:
1676 		db_printf("none\n");
1677 	}
1678 	db_printf(" spinners: ");
1679 	if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
1680 		db_printf("exclusive\n");
1681 	else
1682 		db_printf("none\n");
1683 }
1684 #endif
1685