xref: /freebsd/sys/kern/kern_lock.c (revision 195ebc7e9e4b129de810833791a19dfb4349d6a9)
1 /*-
2  * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice(s), this list of conditions and the following disclaimer as
10  *    the first lines of this file unmodified other than the possible
11  *    addition of one or more copyright notices.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice(s), this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
26  * DAMAGE.
27  */
28 
29 #include "opt_ddb.h"
30 #include "opt_kdtrace.h"
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include <sys/param.h>
36 #include <sys/ktr.h>
37 #include <sys/lock.h>
38 #include <sys/lock_profile.h>
39 #include <sys/lockmgr.h>
40 #include <sys/mutex.h>
41 #include <sys/proc.h>
42 #include <sys/sleepqueue.h>
43 #ifdef DEBUG_LOCKS
44 #include <sys/stack.h>
45 #endif
46 #include <sys/systm.h>
47 
48 #include <machine/cpu.h>
49 
50 #ifdef DDB
51 #include <ddb/ddb.h>
52 #endif
53 
54 CTASSERT((LK_NOSHARE & LO_CLASSFLAGS) == LK_NOSHARE);
55 
56 #define	SQ_EXCLUSIVE_QUEUE	0
57 #define	SQ_SHARED_QUEUE		1
58 
59 #ifndef INVARIANTS
60 #define	_lockmgr_assert(lk, what, file, line)
61 #define	TD_LOCKS_INC(td)
62 #define	TD_LOCKS_DEC(td)
63 #else
64 #define	TD_LOCKS_INC(td)	((td)->td_locks++)
65 #define	TD_LOCKS_DEC(td)	((td)->td_locks--)
66 #endif
67 #define	TD_SLOCKS_INC(td)	((td)->td_lk_slocks++)
68 #define	TD_SLOCKS_DEC(td)	((td)->td_lk_slocks--)
69 
70 #ifndef DEBUG_LOCKS
71 #define	STACK_PRINT(lk)
72 #define	STACK_SAVE(lk)
73 #define	STACK_ZERO(lk)
74 #else
75 #define	STACK_PRINT(lk)	stack_print_ddb(&(lk)->lk_stack)
76 #define	STACK_SAVE(lk)	stack_save(&(lk)->lk_stack)
77 #define	STACK_ZERO(lk)	stack_zero(&(lk)->lk_stack)
78 #endif
79 
80 #define	LOCK_LOG2(lk, string, arg1, arg2)				\
81 	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
82 		CTR2(KTR_LOCK, (string), (arg1), (arg2))
83 #define	LOCK_LOG3(lk, string, arg1, arg2, arg3)				\
84 	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
85 		CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
86 
87 #define	GIANT_DECLARE							\
88 	int _i = 0;							\
89 	WITNESS_SAVE_DECL(Giant)
90 #define	GIANT_RESTORE() do {						\
91 	if (_i > 0) {							\
92 		while (_i--)						\
93 			mtx_lock(&Giant);				\
94 		WITNESS_RESTORE(&Giant.lock_object, Giant);		\
95 	}								\
96 } while (0)
97 #define	GIANT_SAVE() do {						\
98 	if (mtx_owned(&Giant)) {					\
99 		WITNESS_SAVE(&Giant.lock_object, Giant);		\
100 		while (mtx_owned(&Giant)) {				\
101 			_i++;						\
102 			mtx_unlock(&Giant);				\
103 		}							\
104 	}								\
105 } while (0)
106 
107 #define	LK_CAN_SHARE(x)							\
108 	(((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 ||	\
109 	curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT)))
110 #define	LK_TRYOP(x)							\
111 	((x) & LK_NOWAIT)
112 
113 #define	LK_CAN_WITNESS(x)						\
114 	(((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
115 #define	LK_TRYWIT(x)							\
116 	(LK_TRYOP(x) ? LOP_TRYLOCK : 0)
117 
118 #define	lockmgr_disowned(lk)						\
119 	(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
120 
121 #define	lockmgr_xlocked(lk)						\
122 	(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
123 
124 static void	 assert_lockmgr(struct lock_object *lock, int how);
125 #ifdef DDB
126 static void	 db_show_lockmgr(struct lock_object *lock);
127 #endif
128 static void	 lock_lockmgr(struct lock_object *lock, int how);
129 #ifdef KDTRACE_HOOKS
130 static int	 owner_lockmgr(struct lock_object *lock, struct thread **owner);
131 #endif
132 static int	 unlock_lockmgr(struct lock_object *lock);
133 
134 struct lock_class lock_class_lockmgr = {
135 	.lc_name = "lockmgr",
136 	.lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
137 	.lc_assert = assert_lockmgr,
138 #ifdef DDB
139 	.lc_ddb_show = db_show_lockmgr,
140 #endif
141 	.lc_lock = lock_lockmgr,
142 	.lc_unlock = unlock_lockmgr,
143 #ifdef KDTRACE_HOOKS
144 	.lc_owner = owner_lockmgr,
145 #endif
146 };
147 
148 static __inline struct thread *
149 lockmgr_xholder(struct lock *lk)
150 {
151 	uintptr_t x;
152 
153 	x = lk->lk_lock;
154 	return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
155 }
156 
157 /*
158  * It assumes sleepq_lock held and returns with this one unheld.
159  * It also assumes the generic interlock is sane and previously checked.
160  * If LK_INTERLOCK is specified the interlock is not reacquired after the
161  * sleep.
162  */
163 static __inline int
164 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
165     const char *wmesg, int pri, int timo, int queue)
166 {
167 	GIANT_DECLARE;
168 	struct lock_class *class;
169 	int catch, error;
170 
171 	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
172 	catch = pri & PCATCH;
173 	pri &= PRIMASK;
174 	error = 0;
175 
176 	LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
177 	    (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
178 
179 	if (flags & LK_INTERLOCK)
180 		class->lc_unlock(ilk);
181 	GIANT_SAVE();
182 	sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
183 	    SLEEPQ_INTERRUPTIBLE : 0), queue);
184 	if ((flags & LK_TIMELOCK) && timo)
185 		sleepq_set_timeout(&lk->lock_object, timo);
186 
187 	/*
188 	 * Decisional switch for real sleeping.
189 	 */
190 	if ((flags & LK_TIMELOCK) && timo && catch)
191 		error = sleepq_timedwait_sig(&lk->lock_object, pri);
192 	else if ((flags & LK_TIMELOCK) && timo)
193 		error = sleepq_timedwait(&lk->lock_object, pri);
194 	else if (catch)
195 		error = sleepq_wait_sig(&lk->lock_object, pri);
196 	else
197 		sleepq_wait(&lk->lock_object, pri);
198 	GIANT_RESTORE();
199 	if ((flags & LK_SLEEPFAIL) && error == 0)
200 		error = ENOLCK;
201 
202 	return (error);
203 }
204 
205 static __inline int
206 wakeupshlk(struct lock *lk, const char *file, int line)
207 {
208 	uintptr_t v, x;
209 	int queue, wakeup_swapper;
210 
211 	TD_LOCKS_DEC(curthread);
212 	TD_SLOCKS_DEC(curthread);
213 	WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
214 	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
215 
216 	wakeup_swapper = 0;
217 	for (;;) {
218 		x = lk->lk_lock;
219 
220 		/*
221 		 * If there is more than one shared lock held, just drop one
222 		 * and return.
223 		 */
224 		if (LK_SHARERS(x) > 1) {
225 			if (atomic_cmpset_ptr(&lk->lk_lock, x,
226 			    x - LK_ONE_SHARER))
227 				break;
228 			continue;
229 		}
230 
231 		/*
232 		 * If there are not waiters on the exclusive queue, drop the
233 		 * lock quickly.
234 		 */
235 		if ((x & LK_ALL_WAITERS) == 0) {
236 			MPASS(x == LK_SHARERS_LOCK(1));
237 			if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1),
238 			    LK_UNLOCKED))
239 				break;
240 			continue;
241 		}
242 
243 		/*
244 		 * We should have a sharer with waiters, so enter the hard
245 		 * path in order to handle wakeups correctly.
246 		 */
247 		sleepq_lock(&lk->lock_object);
248 		x = lk->lk_lock & LK_ALL_WAITERS;
249 		v = LK_UNLOCKED;
250 
251 		/*
252 		 * If the lock has exclusive waiters, give them preference in
253 		 * order to avoid deadlock with shared runners up.
254 		 */
255 		if (x & LK_EXCLUSIVE_WAITERS) {
256 			queue = SQ_EXCLUSIVE_QUEUE;
257 			v |= (x & LK_SHARED_WAITERS);
258 		} else {
259 			MPASS(x == LK_SHARED_WAITERS);
260 			queue = SQ_SHARED_QUEUE;
261 		}
262 
263 		if (!atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
264 		    v)) {
265 			sleepq_release(&lk->lock_object);
266 			continue;
267 		}
268 		LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
269 		    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
270 		    "exclusive");
271 		wakeup_swapper = sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
272 		    0, queue);
273 		sleepq_release(&lk->lock_object);
274 		break;
275 	}
276 
277 	lock_profile_release_lock(&lk->lock_object);
278 	return (wakeup_swapper);
279 }
280 
281 static void
282 assert_lockmgr(struct lock_object *lock, int what)
283 {
284 
285 	panic("lockmgr locks do not support assertions");
286 }
287 
288 static void
289 lock_lockmgr(struct lock_object *lock, int how)
290 {
291 
292 	panic("lockmgr locks do not support sleep interlocking");
293 }
294 
295 static int
296 unlock_lockmgr(struct lock_object *lock)
297 {
298 
299 	panic("lockmgr locks do not support sleep interlocking");
300 }
301 
302 #ifdef KDTRACE_HOOKS
303 static int
304 owner_lockmgr(struct lock_object *lock, struct thread **owner)
305 {
306 
307 	panic("lockmgr locks do not support owner inquiring");
308 }
309 #endif
310 
311 void
312 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
313 {
314 	int iflags;
315 
316 	MPASS((flags & ~LK_INIT_MASK) == 0);
317 
318 	iflags = LO_SLEEPABLE | LO_UPGRADABLE;
319 	if (flags & LK_CANRECURSE)
320 		iflags |= LO_RECURSABLE;
321 	if ((flags & LK_NODUP) == 0)
322 		iflags |= LO_DUPOK;
323 	if (flags & LK_NOPROFILE)
324 		iflags |= LO_NOPROFILE;
325 	if ((flags & LK_NOWITNESS) == 0)
326 		iflags |= LO_WITNESS;
327 	if (flags & LK_QUIET)
328 		iflags |= LO_QUIET;
329 	iflags |= flags & LK_NOSHARE;
330 
331 	lk->lk_lock = LK_UNLOCKED;
332 	lk->lk_recurse = 0;
333 	lk->lk_timo = timo;
334 	lk->lk_pri = pri;
335 	lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
336 	STACK_ZERO(lk);
337 }
338 
339 void
340 lockdestroy(struct lock *lk)
341 {
342 
343 	KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
344 	KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
345 	lock_destroy(&lk->lock_object);
346 }
347 
348 int
349 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
350     const char *wmesg, int pri, int timo, const char *file, int line)
351 {
352 	GIANT_DECLARE;
353 	struct lock_class *class;
354 	const char *iwmesg;
355 	uintptr_t tid, v, x;
356 	u_int op;
357 	int error, ipri, itimo, queue, wakeup_swapper;
358 #ifdef LOCK_PROFILING
359 	uint64_t waittime = 0;
360 	int contested = 0;
361 #endif
362 
363 	error = 0;
364 	tid = (uintptr_t)curthread;
365 	op = (flags & LK_TYPE_MASK);
366 	iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
367 	ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
368 	itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
369 
370 	MPASS((flags & ~LK_TOTAL_MASK) == 0);
371 	KASSERT((op & (op - 1)) == 0,
372 	    ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
373 	KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
374 	    (op != LK_DOWNGRADE && op != LK_RELEASE),
375 	    ("%s: Invalid flags in regard of the operation desired @ %s:%d",
376 	    __func__, file, line));
377 	KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
378 	    ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
379 	    __func__, file, line));
380 
381 	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
382 	if (panicstr != NULL) {
383 		if (flags & LK_INTERLOCK)
384 			class->lc_unlock(ilk);
385 		return (0);
386 	}
387 
388 	if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE))
389 		op = LK_EXCLUSIVE;
390 
391 	wakeup_swapper = 0;
392 	switch (op) {
393 	case LK_SHARED:
394 		if (LK_CAN_WITNESS(flags))
395 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
396 			    file, line, ilk);
397 		for (;;) {
398 			x = lk->lk_lock;
399 
400 			/*
401 			 * If no other thread has an exclusive lock, or
402 			 * no exclusive waiter is present, bump the count of
403 			 * sharers.  Since we have to preserve the state of
404 			 * waiters, if we fail to acquire the shared lock
405 			 * loop back and retry.
406 			 */
407 			if (LK_CAN_SHARE(x)) {
408 				if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
409 				    x + LK_ONE_SHARER))
410 					break;
411 				continue;
412 			}
413 			lock_profile_obtain_lock_failed(&lk->lock_object,
414 			    &contested, &waittime);
415 
416 			/*
417 			 * If the lock is already held by curthread in
418 			 * exclusive way avoid a deadlock.
419 			 */
420 			if (LK_HOLDER(x) == tid) {
421 				LOCK_LOG2(lk,
422 				    "%s: %p already held in exclusive mode",
423 				    __func__, lk);
424 				error = EDEADLK;
425 				break;
426 			}
427 
428 			/*
429 			 * If the lock is expected to not sleep just give up
430 			 * and return.
431 			 */
432 			if (LK_TRYOP(flags)) {
433 				LOCK_LOG2(lk, "%s: %p fails the try operation",
434 				    __func__, lk);
435 				error = EBUSY;
436 				break;
437 			}
438 
439 			/*
440 			 * Acquire the sleepqueue chain lock because we
441 			 * probabilly will need to manipulate waiters flags.
442 			 */
443 			sleepq_lock(&lk->lock_object);
444 			x = lk->lk_lock;
445 
446 			/*
447 			 * if the lock can be acquired in shared mode, try
448 			 * again.
449 			 */
450 			if (LK_CAN_SHARE(x)) {
451 				sleepq_release(&lk->lock_object);
452 				continue;
453 			}
454 
455 			/*
456 			 * Try to set the LK_SHARED_WAITERS flag.  If we fail,
457 			 * loop back and retry.
458 			 */
459 			if ((x & LK_SHARED_WAITERS) == 0) {
460 				if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x,
461 				    x | LK_SHARED_WAITERS)) {
462 					sleepq_release(&lk->lock_object);
463 					continue;
464 				}
465 				LOCK_LOG2(lk, "%s: %p set shared waiters flag",
466 				    __func__, lk);
467 			}
468 
469 			/*
470 			 * As far as we have been unable to acquire the
471 			 * shared lock and the shared waiters flag is set,
472 			 * we will sleep.
473 			 */
474 			error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
475 			    SQ_SHARED_QUEUE);
476 			flags &= ~LK_INTERLOCK;
477 			if (error) {
478 				LOCK_LOG3(lk,
479 				    "%s: interrupted sleep for %p with %d",
480 				    __func__, lk, error);
481 				break;
482 			}
483 			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
484 			    __func__, lk);
485 		}
486 		if (error == 0) {
487 			lock_profile_obtain_lock_success(&lk->lock_object,
488 			    contested, waittime, file, line);
489 			LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file,
490 			    line);
491 			WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file,
492 			    line);
493 			TD_LOCKS_INC(curthread);
494 			TD_SLOCKS_INC(curthread);
495 			STACK_SAVE(lk);
496 		}
497 		break;
498 	case LK_UPGRADE:
499 		_lockmgr_assert(lk, KA_SLOCKED, file, line);
500 		x = lk->lk_lock & LK_ALL_WAITERS;
501 
502 		/*
503 		 * Try to switch from one shared lock to an exclusive one.
504 		 * We need to preserve waiters flags during the operation.
505 		 */
506 		if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
507 		    tid | x)) {
508 			LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
509 			    line);
510 			WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
511 			    LK_TRYWIT(flags), file, line);
512 			TD_SLOCKS_DEC(curthread);
513 			break;
514 		}
515 
516 		/*
517 		 * We have been unable to succeed in upgrading, so just
518 		 * give up the shared lock.
519 		 */
520 		wakeup_swapper |= wakeupshlk(lk, file, line);
521 
522 		/* FALLTHROUGH */
523 	case LK_EXCLUSIVE:
524 		if (LK_CAN_WITNESS(flags))
525 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
526 			    LOP_EXCLUSIVE, file, line, ilk);
527 
528 		/*
529 		 * If curthread already holds the lock and this one is
530 		 * allowed to recurse, simply recurse on it.
531 		 */
532 		if (lockmgr_xlocked(lk)) {
533 			if ((flags & LK_CANRECURSE) == 0 &&
534 			    (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
535 
536 				/*
537 				 * If the lock is expected to not panic just
538 				 * give up and return.
539 				 */
540 				if (LK_TRYOP(flags)) {
541 					LOCK_LOG2(lk,
542 					    "%s: %p fails the try operation",
543 					    __func__, lk);
544 					error = EBUSY;
545 					break;
546 				}
547 				if (flags & LK_INTERLOCK)
548 					class->lc_unlock(ilk);
549 		panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n",
550 				    __func__, iwmesg, file, line);
551 			}
552 			lk->lk_recurse++;
553 			LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
554 			LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
555 			    lk->lk_recurse, file, line);
556 			WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
557 			    LK_TRYWIT(flags), file, line);
558 			TD_LOCKS_INC(curthread);
559 			break;
560 		}
561 
562 		while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED,
563 		    tid)) {
564 			lock_profile_obtain_lock_failed(&lk->lock_object,
565 			    &contested, &waittime);
566 
567 			/*
568 			 * If the lock is expected to not sleep just give up
569 			 * and return.
570 			 */
571 			if (LK_TRYOP(flags)) {
572 				LOCK_LOG2(lk, "%s: %p fails the try operation",
573 				    __func__, lk);
574 				error = EBUSY;
575 				break;
576 			}
577 
578 			/*
579 			 * Acquire the sleepqueue chain lock because we
580 			 * probabilly will need to manipulate waiters flags.
581 			 */
582 			sleepq_lock(&lk->lock_object);
583 			x = lk->lk_lock;
584 			v = x & LK_ALL_WAITERS;
585 
586 			/*
587 			 * if the lock has been released while we spun on
588 			 * the sleepqueue chain lock just try again.
589 			 */
590 			if (x == LK_UNLOCKED) {
591 				sleepq_release(&lk->lock_object);
592 				continue;
593 			}
594 
595 			/*
596 			 * The lock can be in the state where there is a
597 			 * pending queue of waiters, but still no owner.
598 			 * This happens when the lock is contested and an
599 			 * owner is going to claim the lock.
600 			 * If curthread is the one successfully acquiring it
601 			 * claim lock ownership and return, preserving waiters
602 			 * flags.
603 			 */
604 			if (x == (LK_UNLOCKED | v)) {
605 				if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
606 				    tid | v)) {
607 					sleepq_release(&lk->lock_object);
608 					LOCK_LOG2(lk,
609 					    "%s: %p claimed by a new writer",
610 					    __func__, lk);
611 					break;
612 				}
613 				sleepq_release(&lk->lock_object);
614 				continue;
615 			}
616 
617 			/*
618 			 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
619 			 * fail, loop back and retry.
620 			 */
621 			if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
622 				if (!atomic_cmpset_ptr(&lk->lk_lock, x,
623 				    x | LK_EXCLUSIVE_WAITERS)) {
624 					sleepq_release(&lk->lock_object);
625 					continue;
626 				}
627 				LOCK_LOG2(lk, "%s: %p set excl waiters flag",
628 				    __func__, lk);
629 			}
630 
631 			/*
632 			 * As far as we have been unable to acquire the
633 			 * exclusive lock and the exclusive waiters flag
634 			 * is set, we will sleep.
635 			 */
636 			error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
637 			    SQ_EXCLUSIVE_QUEUE);
638 			flags &= ~LK_INTERLOCK;
639 			if (error) {
640 				LOCK_LOG3(lk,
641 				    "%s: interrupted sleep for %p with %d",
642 				    __func__, lk, error);
643 				break;
644 			}
645 			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
646 			    __func__, lk);
647 		}
648 		if (error == 0) {
649 			lock_profile_obtain_lock_success(&lk->lock_object,
650 			    contested, waittime, file, line);
651 			LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
652 			    lk->lk_recurse, file, line);
653 			WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
654 			    LK_TRYWIT(flags), file, line);
655 			TD_LOCKS_INC(curthread);
656 			STACK_SAVE(lk);
657 		}
658 		break;
659 	case LK_DOWNGRADE:
660 		_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
661 		LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
662 		WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
663 		TD_SLOCKS_INC(curthread);
664 
665 		/*
666 		 * In order to preserve waiters flags, just spin.
667 		 */
668 		for (;;) {
669 			x = lk->lk_lock & LK_ALL_WAITERS;
670 			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
671 			    LK_SHARERS_LOCK(1) | x))
672 				break;
673 			cpu_spinwait();
674 		}
675 		break;
676 	case LK_RELEASE:
677 		_lockmgr_assert(lk, KA_LOCKED, file, line);
678 		x = lk->lk_lock;
679 
680 		if ((x & LK_SHARE) == 0) {
681 
682 			/*
683 			 * As first option, treact the lock as if it has not
684 			 * any waiter.
685 			 * Fix-up the tid var if the lock has been disowned.
686 			 */
687 			if (LK_HOLDER(x) == LK_KERNPROC)
688 				tid = LK_KERNPROC;
689 			else {
690 				WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE,
691 				    file, line);
692 				TD_LOCKS_DEC(curthread);
693 			}
694 			LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0,
695 			    lk->lk_recurse, file, line);
696 
697 			/*
698 			 * The lock is held in exclusive mode.
699 			 * If the lock is recursed also, then unrecurse it.
700 			 */
701 			if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
702 				LOCK_LOG2(lk, "%s: %p unrecursing", __func__,
703 				    lk);
704 				lk->lk_recurse--;
705 				break;
706 			}
707 			if (tid != LK_KERNPROC)
708 				lock_profile_release_lock(&lk->lock_object);
709 
710 			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid,
711 			    LK_UNLOCKED))
712 				break;
713 
714 			sleepq_lock(&lk->lock_object);
715 			x = lk->lk_lock & LK_ALL_WAITERS;
716 			v = LK_UNLOCKED;
717 
718 			/*
719 		 	 * If the lock has exclusive waiters, give them
720 			 * preference in order to avoid deadlock with
721 			 * shared runners up.
722 			 */
723 			if (x & LK_EXCLUSIVE_WAITERS) {
724 				queue = SQ_EXCLUSIVE_QUEUE;
725 				v |= (x & LK_SHARED_WAITERS);
726 			} else {
727 				MPASS(x == LK_SHARED_WAITERS);
728 				queue = SQ_SHARED_QUEUE;
729 			}
730 
731 			LOCK_LOG3(lk,
732 			    "%s: %p waking up threads on the %s queue",
733 			    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
734 			    "exclusive");
735 			atomic_store_rel_ptr(&lk->lk_lock, v);
736 			wakeup_swapper = sleepq_broadcast(&lk->lock_object,
737 			    SLEEPQ_LK, 0, queue);
738 			sleepq_release(&lk->lock_object);
739 			break;
740 		} else
741 			wakeup_swapper = wakeupshlk(lk, file, line);
742 		break;
743 	case LK_DRAIN:
744 		if (LK_CAN_WITNESS(flags))
745 			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
746 			    LOP_EXCLUSIVE, file, line, ilk);
747 
748 		/*
749 		 * Trying to drain a lock we already own will result in a
750 		 * deadlock.
751 		 */
752 		if (lockmgr_xlocked(lk)) {
753 			if (flags & LK_INTERLOCK)
754 				class->lc_unlock(ilk);
755 			panic("%s: draining %s with the lock held @ %s:%d\n",
756 			    __func__, iwmesg, file, line);
757 		}
758 
759 		while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
760 			lock_profile_obtain_lock_failed(&lk->lock_object,
761 			    &contested, &waittime);
762 
763 			/*
764 			 * If the lock is expected to not sleep just give up
765 			 * and return.
766 			 */
767 			if (LK_TRYOP(flags)) {
768 				LOCK_LOG2(lk, "%s: %p fails the try operation",
769 				    __func__, lk);
770 				error = EBUSY;
771 				break;
772 			}
773 
774 			/*
775 			 * Acquire the sleepqueue chain lock because we
776 			 * probabilly will need to manipulate waiters flags.
777 			 */
778 			sleepq_lock(&lk->lock_object);
779 			x = lk->lk_lock;
780 			v = x & LK_ALL_WAITERS;
781 
782 			/*
783 			 * if the lock has been released while we spun on
784 			 * the sleepqueue chain lock just try again.
785 			 */
786 			if (x == LK_UNLOCKED) {
787 				sleepq_release(&lk->lock_object);
788 				continue;
789 			}
790 
791 			if (x == (LK_UNLOCKED | v)) {
792 				v = x;
793 				if (v & LK_EXCLUSIVE_WAITERS) {
794 					queue = SQ_EXCLUSIVE_QUEUE;
795 					v &= ~LK_EXCLUSIVE_WAITERS;
796 				} else {
797 					MPASS(v & LK_SHARED_WAITERS);
798 					queue = SQ_SHARED_QUEUE;
799 					v &= ~LK_SHARED_WAITERS;
800 				}
801 				if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
802 					sleepq_release(&lk->lock_object);
803 					continue;
804 				}
805 				LOCK_LOG3(lk,
806 				"%s: %p waking up all threads on the %s queue",
807 				    __func__, lk, queue == SQ_SHARED_QUEUE ?
808 				    "shared" : "exclusive");
809 				wakeup_swapper |= sleepq_broadcast(
810 				    &lk->lock_object, SLEEPQ_LK, 0, queue);
811 
812 				/*
813 				 * If shared waiters have been woken up we need
814 				 * to wait for one of them to acquire the lock
815 				 * before to set the exclusive waiters in
816 				 * order to avoid a deadlock.
817 				 */
818 				if (queue == SQ_SHARED_QUEUE) {
819 					for (v = lk->lk_lock;
820 					    (v & LK_SHARE) && !LK_SHARERS(v);
821 					    v = lk->lk_lock)
822 						cpu_spinwait();
823 				}
824 			}
825 
826 			/*
827 			 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
828 			 * fail, loop back and retry.
829 			 */
830 			if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
831 				if (!atomic_cmpset_ptr(&lk->lk_lock, x,
832 				    x | LK_EXCLUSIVE_WAITERS)) {
833 					sleepq_release(&lk->lock_object);
834 					continue;
835 				}
836 				LOCK_LOG2(lk, "%s: %p set drain waiters flag",
837 				    __func__, lk);
838 			}
839 
840 			/*
841 			 * As far as we have been unable to acquire the
842 			 * exclusive lock and the exclusive waiters flag
843 			 * is set, we will sleep.
844 			 */
845 			if (flags & LK_INTERLOCK) {
846 				class->lc_unlock(ilk);
847 				flags &= ~LK_INTERLOCK;
848 			}
849 			GIANT_SAVE();
850 			sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
851 			    SQ_EXCLUSIVE_QUEUE);
852 			sleepq_wait(&lk->lock_object, ipri & PRIMASK);
853 			GIANT_RESTORE();
854 			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
855 			    __func__, lk);
856 		}
857 
858 		if (error == 0) {
859 			lock_profile_obtain_lock_success(&lk->lock_object,
860 			    contested, waittime, file, line);
861 			LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
862 			    lk->lk_recurse, file, line);
863 			WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
864 			    LK_TRYWIT(flags), file, line);
865 			TD_LOCKS_INC(curthread);
866 			STACK_SAVE(lk);
867 		}
868 		break;
869 	default:
870 		if (flags & LK_INTERLOCK)
871 			class->lc_unlock(ilk);
872 		panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
873 	}
874 
875 	if (flags & LK_INTERLOCK)
876 		class->lc_unlock(ilk);
877 	if (wakeup_swapper)
878 		kick_proc0();
879 
880 	return (error);
881 }
882 
883 void
884 _lockmgr_disown(struct lock *lk, const char *file, int line)
885 {
886 	uintptr_t tid, x;
887 
888 	tid = (uintptr_t)curthread;
889 	_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
890 
891 	/*
892 	 * If the owner is already LK_KERNPROC just skip the whole operation.
893 	 */
894 	if (LK_HOLDER(lk->lk_lock) != tid)
895 		return;
896 	lock_profile_release_lock(&lk->lock_object);
897 	LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
898 	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
899 	TD_LOCKS_DEC(curthread);
900 
901 	/*
902 	 * In order to preserve waiters flags, just spin.
903 	 */
904 	for (;;) {
905 		x = lk->lk_lock & LK_ALL_WAITERS;
906 		if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
907 		    LK_KERNPROC | x))
908 			return;
909 		cpu_spinwait();
910 	}
911 }
912 
913 void
914 lockmgr_printinfo(struct lock *lk)
915 {
916 	struct thread *td;
917 	uintptr_t x;
918 
919 	if (lk->lk_lock == LK_UNLOCKED)
920 		printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
921 	else if (lk->lk_lock & LK_SHARE)
922 		printf("lock type %s: SHARED (count %ju)\n",
923 		    lk->lock_object.lo_name,
924 		    (uintmax_t)LK_SHARERS(lk->lk_lock));
925 	else {
926 		td = lockmgr_xholder(lk);
927 		printf("lock type %s: EXCL by thread %p (pid %d)\n",
928 		    lk->lock_object.lo_name, td, td->td_proc->p_pid);
929 	}
930 
931 	x = lk->lk_lock;
932 	if (x & LK_EXCLUSIVE_WAITERS)
933 		printf(" with exclusive waiters pending\n");
934 	if (x & LK_SHARED_WAITERS)
935 		printf(" with shared waiters pending\n");
936 
937 	STACK_PRINT(lk);
938 }
939 
940 int
941 lockstatus(struct lock *lk)
942 {
943 	uintptr_t v, x;
944 	int ret;
945 
946 	ret = LK_SHARED;
947 	x = lk->lk_lock;
948 	v = LK_HOLDER(x);
949 
950 	if ((x & LK_SHARE) == 0) {
951 		if (v == (uintptr_t)curthread || v == LK_KERNPROC)
952 			ret = LK_EXCLUSIVE;
953 		else
954 			ret = LK_EXCLOTHER;
955 	} else if (x == LK_UNLOCKED)
956 		ret = 0;
957 
958 	return (ret);
959 }
960 
961 #ifdef INVARIANT_SUPPORT
962 #ifndef INVARIANTS
963 #undef	_lockmgr_assert
964 #endif
965 
966 void
967 _lockmgr_assert(struct lock *lk, int what, const char *file, int line)
968 {
969 	int slocked = 0;
970 
971 	if (panicstr != NULL)
972 		return;
973 	switch (what) {
974 	case KA_SLOCKED:
975 	case KA_SLOCKED | KA_NOTRECURSED:
976 	case KA_SLOCKED | KA_RECURSED:
977 		slocked = 1;
978 	case KA_LOCKED:
979 	case KA_LOCKED | KA_NOTRECURSED:
980 	case KA_LOCKED | KA_RECURSED:
981 #ifdef WITNESS
982 
983 		/*
984 		 * We cannot trust WITNESS if the lock is held in exclusive
985 		 * mode and a call to lockmgr_disown() happened.
986 		 * Workaround this skipping the check if the lock is held in
987 		 * exclusive mode even for the KA_LOCKED case.
988 		 */
989 		if (slocked || (lk->lk_lock & LK_SHARE)) {
990 			witness_assert(&lk->lock_object, what, file, line);
991 			break;
992 		}
993 #endif
994 		if (lk->lk_lock == LK_UNLOCKED ||
995 		    ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
996 		    (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
997 			panic("Lock %s not %slocked @ %s:%d\n",
998 			    lk->lock_object.lo_name, slocked ? "share" : "",
999 			    file, line);
1000 
1001 		if ((lk->lk_lock & LK_SHARE) == 0) {
1002 			if (lockmgr_recursed(lk)) {
1003 				if (what & KA_NOTRECURSED)
1004 					panic("Lock %s recursed @ %s:%d\n",
1005 					    lk->lock_object.lo_name, file,
1006 					    line);
1007 			} else if (what & KA_RECURSED)
1008 				panic("Lock %s not recursed @ %s:%d\n",
1009 				    lk->lock_object.lo_name, file, line);
1010 		}
1011 		break;
1012 	case KA_XLOCKED:
1013 	case KA_XLOCKED | KA_NOTRECURSED:
1014 	case KA_XLOCKED | KA_RECURSED:
1015 		if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
1016 			panic("Lock %s not exclusively locked @ %s:%d\n",
1017 			    lk->lock_object.lo_name, file, line);
1018 		if (lockmgr_recursed(lk)) {
1019 			if (what & KA_NOTRECURSED)
1020 				panic("Lock %s recursed @ %s:%d\n",
1021 				    lk->lock_object.lo_name, file, line);
1022 		} else if (what & KA_RECURSED)
1023 			panic("Lock %s not recursed @ %s:%d\n",
1024 			    lk->lock_object.lo_name, file, line);
1025 		break;
1026 	case KA_UNLOCKED:
1027 		if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
1028 			panic("Lock %s exclusively locked @ %s:%d\n",
1029 			    lk->lock_object.lo_name, file, line);
1030 		break;
1031 	default:
1032 		panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
1033 		    line);
1034 	}
1035 }
1036 #endif
1037 
1038 #ifdef DDB
1039 int
1040 lockmgr_chain(struct thread *td, struct thread **ownerp)
1041 {
1042 	struct lock *lk;
1043 
1044 	lk = td->td_wchan;
1045 
1046 	if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
1047 		return (0);
1048 	db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
1049 	if (lk->lk_lock & LK_SHARE)
1050 		db_printf("SHARED (count %ju)\n",
1051 		    (uintmax_t)LK_SHARERS(lk->lk_lock));
1052 	else
1053 		db_printf("EXCL\n");
1054 	*ownerp = lockmgr_xholder(lk);
1055 
1056 	return (1);
1057 }
1058 
1059 static void
1060 db_show_lockmgr(struct lock_object *lock)
1061 {
1062 	struct thread *td;
1063 	struct lock *lk;
1064 
1065 	lk = (struct lock *)lock;
1066 
1067 	db_printf(" state: ");
1068 	if (lk->lk_lock == LK_UNLOCKED)
1069 		db_printf("UNLOCKED\n");
1070 	else if (lk->lk_lock & LK_SHARE)
1071 		db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
1072 	else {
1073 		td = lockmgr_xholder(lk);
1074 		if (td == (struct thread *)LK_KERNPROC)
1075 			db_printf("XLOCK: LK_KERNPROC\n");
1076 		else
1077 			db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1078 			    td->td_tid, td->td_proc->p_pid,
1079 			    td->td_proc->p_comm);
1080 		if (lockmgr_recursed(lk))
1081 			db_printf(" recursed: %d\n", lk->lk_recurse);
1082 	}
1083 	db_printf(" waiters: ");
1084 	switch (lk->lk_lock & LK_ALL_WAITERS) {
1085 	case LK_SHARED_WAITERS:
1086 		db_printf("shared\n");
1087 		break;
1088 	case LK_EXCLUSIVE_WAITERS:
1089 		db_printf("exclusive\n");
1090 		break;
1091 	case LK_ALL_WAITERS:
1092 		db_printf("shared and exclusive\n");
1093 		break;
1094 	default:
1095 		db_printf("none\n");
1096 	}
1097 }
1098 #endif
1099