1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice(s), this list of conditions and the following disclaimer as
12 * the first lines of this file unmodified other than the possible
13 * addition of one or more copyright notices.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice(s), this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
28 * DAMAGE.
29 */
30
31 #include "opt_ddb.h"
32 #include "opt_hwpmc_hooks.h"
33
34 #include <sys/param.h>
35 #include <sys/kdb.h>
36 #include <sys/ktr.h>
37 #include <sys/limits.h>
38 #include <sys/lock.h>
39 #include <sys/lock_profile.h>
40 #include <sys/lockmgr.h>
41 #include <sys/lockstat.h>
42 #include <sys/mutex.h>
43 #include <sys/proc.h>
44 #include <sys/sleepqueue.h>
45 #ifdef DEBUG_LOCKS
46 #include <sys/stack.h>
47 #endif
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
50
51 #include <machine/cpu.h>
52
53 #ifdef DDB
54 #include <ddb/ddb.h>
55 #endif
56
57 #ifdef HWPMC_HOOKS
58 #include <sys/pmckern.h>
59 PMC_SOFT_DECLARE( , , lock, failed);
60 #endif
61
62 /*
63 * Hack. There should be prio_t or similar so that this is not necessary.
64 */
65 _Static_assert((PRILASTFLAG * 2) - 1 <= USHRT_MAX,
66 "prio flags wont fit in u_short pri in struct lock");
67
68 CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
69 ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
70
71 #define SQ_EXCLUSIVE_QUEUE 0
72 #define SQ_SHARED_QUEUE 1
73
74 #ifndef INVARIANTS
75 #define _lockmgr_assert(lk, what, file, line)
76 #endif
77
78 #define TD_SLOCKS_INC(td) ((td)->td_lk_slocks++)
79 #define TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--)
80
81 #ifndef DEBUG_LOCKS
82 #define STACK_PRINT(lk)
83 #define STACK_SAVE(lk)
84 #define STACK_ZERO(lk)
85 #else
86 #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
87 #define STACK_SAVE(lk) stack_save(&(lk)->lk_stack)
88 #define STACK_ZERO(lk) stack_zero(&(lk)->lk_stack)
89 #endif
90
91 #define LOCK_LOG2(lk, string, arg1, arg2) \
92 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
93 CTR2(KTR_LOCK, (string), (arg1), (arg2))
94 #define LOCK_LOG3(lk, string, arg1, arg2, arg3) \
95 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
96 CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
97
98 #define GIANT_DECLARE \
99 int _i = 0; \
100 WITNESS_SAVE_DECL(Giant)
101 #define GIANT_RESTORE() do { \
102 if (__predict_false(_i > 0)) { \
103 while (_i--) \
104 mtx_lock(&Giant); \
105 WITNESS_RESTORE(&Giant.lock_object, Giant); \
106 } \
107 } while (0)
108 #define GIANT_SAVE() do { \
109 if (__predict_false(mtx_owned(&Giant))) { \
110 WITNESS_SAVE(&Giant.lock_object, Giant); \
111 while (mtx_owned(&Giant)) { \
112 _i++; \
113 mtx_unlock(&Giant); \
114 } \
115 } \
116 } while (0)
117
118 static __always_inline bool
LK_CAN_SHARE(uintptr_t x,int flags,bool fp)119 LK_CAN_SHARE(uintptr_t x, int flags, bool fp)
120 {
121
122 if ((x & (LK_SHARE | LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) ==
123 LK_SHARE)
124 return (true);
125 if (fp || (!(x & LK_SHARE)))
126 return (false);
127 if ((curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||
128 (curthread->td_pflags & TDP_DEADLKTREAT))
129 return (true);
130 return (false);
131 }
132
133 #define LK_TRYOP(x) \
134 ((x) & LK_NOWAIT)
135
136 #define LK_CAN_WITNESS(x) \
137 (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
138 #define LK_TRYWIT(x) \
139 (LK_TRYOP(x) ? LOP_TRYLOCK : 0)
140
141 #define lockmgr_xlocked_v(v) \
142 (((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
143
144 #define lockmgr_xlocked(lk) lockmgr_xlocked_v(lockmgr_read_value(lk))
145
146 static void assert_lockmgr(const struct lock_object *lock, int how);
147 #ifdef DDB
148 static void db_show_lockmgr(const struct lock_object *lock);
149 #endif
150 static void lock_lockmgr(struct lock_object *lock, uintptr_t how);
151 #ifdef KDTRACE_HOOKS
152 static int owner_lockmgr(const struct lock_object *lock,
153 struct thread **owner);
154 #endif
155 static uintptr_t unlock_lockmgr(struct lock_object *lock);
156
157 struct lock_class lock_class_lockmgr = {
158 .lc_name = "lockmgr",
159 .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
160 .lc_assert = assert_lockmgr,
161 #ifdef DDB
162 .lc_ddb_show = db_show_lockmgr,
163 #endif
164 .lc_lock = lock_lockmgr,
165 .lc_unlock = unlock_lockmgr,
166 #ifdef KDTRACE_HOOKS
167 .lc_owner = owner_lockmgr,
168 #endif
169 };
170
171 static __read_mostly bool lk_adaptive = true;
172 static SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging");
173 SYSCTL_BOOL(_debug_lockmgr, OID_AUTO, adaptive_spinning, CTLFLAG_RW, &lk_adaptive,
174 0, "");
175 #define lockmgr_delay locks_delay
176
177 struct lockmgr_wait {
178 const char *iwmesg;
179 int ipri;
180 int itimo;
181 };
182
183 static __always_inline bool lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
184 int flags, bool fp);
185 static __always_inline bool lockmgr_sunlock_try(struct lock *lk,
186 uintptr_t *xp);
187
188 static void
lockmgr_exit(u_int flags,struct lock_object * ilk)189 lockmgr_exit(u_int flags, struct lock_object *ilk)
190 {
191 struct lock_class *class;
192
193 if (flags & LK_INTERLOCK) {
194 class = LOCK_CLASS(ilk);
195 class->lc_unlock(ilk);
196 }
197 }
198
199 static void
lockmgr_note_shared_acquire(struct lock * lk,int contested,uint64_t waittime,const char * file,int line,int flags)200 lockmgr_note_shared_acquire(struct lock *lk, int contested,
201 uint64_t waittime, const char *file, int line, int flags)
202 {
203
204 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
205 waittime, file, line, LOCKSTAT_READER);
206 LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
207 WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
208 TD_LOCKS_INC(curthread);
209 TD_SLOCKS_INC(curthread);
210 STACK_SAVE(lk);
211 }
212
213 static void
lockmgr_note_shared_release(struct lock * lk,const char * file,int line)214 lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
215 {
216
217 WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
218 LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
219 TD_LOCKS_DEC(curthread);
220 TD_SLOCKS_DEC(curthread);
221 }
222
223 static void
lockmgr_note_exclusive_acquire(struct lock * lk,int contested,uint64_t waittime,const char * file,int line,int flags)224 lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
225 uint64_t waittime, const char *file, int line, int flags)
226 {
227
228 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
229 waittime, file, line, LOCKSTAT_WRITER);
230 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
231 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
232 line);
233 TD_LOCKS_INC(curthread);
234 STACK_SAVE(lk);
235 }
236
237 static void
lockmgr_note_exclusive_release(struct lock * lk,const char * file,int line)238 lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
239 {
240
241 if (!lockmgr_disowned(lk)) {
242 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
243 TD_LOCKS_DEC(curthread);
244 }
245 LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
246 line);
247 }
248
249 static __inline struct thread *
lockmgr_xholder(const struct lock * lk)250 lockmgr_xholder(const struct lock *lk)
251 {
252 uintptr_t x;
253
254 x = lockmgr_read_value(lk);
255 return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
256 }
257
258 /*
259 * It assumes sleepq_lock held and returns with this one unheld.
260 * It also assumes the generic interlock is sane and previously checked.
261 * If LK_INTERLOCK is specified the interlock is not reacquired after the
262 * sleep.
263 */
264 static __inline int
sleeplk(struct lock * lk,u_int flags,struct lock_object * ilk,const char * wmesg,int pri,int timo,int queue)265 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
266 const char *wmesg, int pri, int timo, int queue)
267 {
268 GIANT_DECLARE;
269 struct lock_class *class;
270 int catch, error;
271
272 class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
273 catch = pri & PCATCH;
274 pri &= PRIMASK;
275 error = 0;
276
277 LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
278 (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
279
280 if (flags & LK_INTERLOCK)
281 class->lc_unlock(ilk);
282 if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0) {
283 if (lk->lk_exslpfail < USHRT_MAX)
284 lk->lk_exslpfail++;
285 }
286 GIANT_SAVE();
287 sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
288 SLEEPQ_INTERRUPTIBLE : 0), queue);
289 if ((flags & LK_TIMELOCK) && timo)
290 sleepq_set_timeout(&lk->lock_object, timo);
291
292 /*
293 * Decisional switch for real sleeping.
294 */
295 if ((flags & LK_TIMELOCK) && timo && catch)
296 error = sleepq_timedwait_sig(&lk->lock_object, pri);
297 else if ((flags & LK_TIMELOCK) && timo)
298 error = sleepq_timedwait(&lk->lock_object, pri);
299 else if (catch)
300 error = sleepq_wait_sig(&lk->lock_object, pri);
301 else
302 sleepq_wait(&lk->lock_object, pri);
303 GIANT_RESTORE();
304 if ((flags & LK_SLEEPFAIL) && error == 0)
305 error = ENOLCK;
306
307 return (error);
308 }
309
310 static __inline void
wakeupshlk(struct lock * lk,const char * file,int line)311 wakeupshlk(struct lock *lk, const char *file, int line)
312 {
313 uintptr_t v, x, orig_x;
314 u_int realexslp;
315 int queue;
316
317 for (;;) {
318 x = lockmgr_read_value(lk);
319 if (lockmgr_sunlock_try(lk, &x))
320 break;
321
322 /*
323 * We should have a sharer with waiters, so enter the hard
324 * path in order to handle wakeups correctly.
325 */
326 sleepq_lock(&lk->lock_object);
327 orig_x = lockmgr_read_value(lk);
328 retry_sleepq:
329 x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
330 v = LK_UNLOCKED;
331
332 /*
333 * If the lock has exclusive waiters, give them preference in
334 * order to avoid deadlock with shared runners up.
335 * If interruptible sleeps left the exclusive queue empty
336 * avoid a starvation for the threads sleeping on the shared
337 * queue by giving them precedence and cleaning up the
338 * exclusive waiters bit anyway.
339 * Please note that lk_exslpfail count may be lying about
340 * the real number of waiters with the LK_SLEEPFAIL flag on
341 * because they may be used in conjunction with interruptible
342 * sleeps so lk_exslpfail might be considered an 'upper limit'
343 * bound, including the edge cases.
344 */
345 realexslp = sleepq_sleepcnt(&lk->lock_object,
346 SQ_EXCLUSIVE_QUEUE);
347 if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
348 if (lk->lk_exslpfail != USHRT_MAX && lk->lk_exslpfail < realexslp) {
349 lk->lk_exslpfail = 0;
350 queue = SQ_EXCLUSIVE_QUEUE;
351 v |= (x & LK_SHARED_WAITERS);
352 } else {
353 lk->lk_exslpfail = 0;
354 LOCK_LOG2(lk,
355 "%s: %p has only LK_SLEEPFAIL sleepers",
356 __func__, lk);
357 LOCK_LOG2(lk,
358 "%s: %p waking up threads on the exclusive queue",
359 __func__, lk);
360 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
361 SQ_EXCLUSIVE_QUEUE);
362 queue = SQ_SHARED_QUEUE;
363 }
364 } else {
365 /*
366 * Exclusive waiters sleeping with LK_SLEEPFAIL on
367 * and using interruptible sleeps/timeout may have
368 * left spourious lk_exslpfail counts on, so clean
369 * it up anyway.
370 */
371 lk->lk_exslpfail = 0;
372 queue = SQ_SHARED_QUEUE;
373 }
374
375 if (lockmgr_sunlock_try(lk, &orig_x)) {
376 sleepq_release(&lk->lock_object);
377 break;
378 }
379
380 x |= LK_SHARERS_LOCK(1);
381 if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
382 orig_x = x;
383 goto retry_sleepq;
384 }
385 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
386 __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
387 "exclusive");
388 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
389 sleepq_release(&lk->lock_object);
390 break;
391 }
392
393 LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
394 }
395
396 static void
assert_lockmgr(const struct lock_object * lock,int what)397 assert_lockmgr(const struct lock_object *lock, int what)
398 {
399
400 panic("lockmgr locks do not support assertions");
401 }
402
403 static void
lock_lockmgr(struct lock_object * lock,uintptr_t how)404 lock_lockmgr(struct lock_object *lock, uintptr_t how)
405 {
406
407 panic("lockmgr locks do not support sleep interlocking");
408 }
409
410 static uintptr_t
unlock_lockmgr(struct lock_object * lock)411 unlock_lockmgr(struct lock_object *lock)
412 {
413
414 panic("lockmgr locks do not support sleep interlocking");
415 }
416
417 #ifdef KDTRACE_HOOKS
418 static int
owner_lockmgr(const struct lock_object * lock,struct thread ** owner)419 owner_lockmgr(const struct lock_object *lock, struct thread **owner)
420 {
421
422 panic("lockmgr locks do not support owner inquiring");
423 }
424 #endif
425
426 void
lockinit(struct lock * lk,int pri,const char * wmesg,int timo,int flags)427 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
428 {
429 int iflags;
430
431 MPASS((flags & ~LK_INIT_MASK) == 0);
432 ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
433 ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
434 &lk->lk_lock));
435
436 iflags = LO_SLEEPABLE | LO_UPGRADABLE;
437 if (flags & LK_CANRECURSE)
438 iflags |= LO_RECURSABLE;
439 if ((flags & LK_NODUP) == 0)
440 iflags |= LO_DUPOK;
441 if (flags & LK_NOPROFILE)
442 iflags |= LO_NOPROFILE;
443 if ((flags & LK_NOWITNESS) == 0)
444 iflags |= LO_WITNESS;
445 if (flags & LK_QUIET)
446 iflags |= LO_QUIET;
447 if (flags & LK_IS_VNODE)
448 iflags |= LO_IS_VNODE;
449 if (flags & LK_NEW)
450 iflags |= LO_NEW;
451 iflags |= flags & LK_NOSHARE;
452
453 lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
454 lk->lk_lock = LK_UNLOCKED;
455 lk->lk_recurse = 0;
456 lk->lk_exslpfail = 0;
457 lk->lk_timo = timo;
458 lk->lk_pri = pri;
459 STACK_ZERO(lk);
460 }
461
462 /*
463 * XXX: Gross hacks to manipulate external lock flags after
464 * initialization. Used for certain vnode and buf locks.
465 */
466 void
lockallowshare(struct lock * lk)467 lockallowshare(struct lock *lk)
468 {
469
470 lockmgr_assert(lk, KA_XLOCKED);
471 lk->lock_object.lo_flags &= ~LK_NOSHARE;
472 }
473
474 void
lockdisableshare(struct lock * lk)475 lockdisableshare(struct lock *lk)
476 {
477
478 lockmgr_assert(lk, KA_XLOCKED);
479 lk->lock_object.lo_flags |= LK_NOSHARE;
480 }
481
482 void
lockallowrecurse(struct lock * lk)483 lockallowrecurse(struct lock *lk)
484 {
485
486 lockmgr_assert(lk, KA_XLOCKED);
487 lk->lock_object.lo_flags |= LO_RECURSABLE;
488 }
489
490 void
lockdisablerecurse(struct lock * lk)491 lockdisablerecurse(struct lock *lk)
492 {
493
494 lockmgr_assert(lk, KA_XLOCKED);
495 lk->lock_object.lo_flags &= ~LO_RECURSABLE;
496 }
497
498 bool
lockcanrecurse(struct lock * lk)499 lockcanrecurse(struct lock *lk)
500 {
501 lockmgr_assert(lk, KA_LOCKED);
502 return ((lk->lock_object.lo_flags & LO_RECURSABLE) != 0);
503 }
504
505 void
lockdestroy(struct lock * lk)506 lockdestroy(struct lock *lk)
507 {
508
509 KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
510 KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
511 KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
512 lock_destroy(&lk->lock_object);
513 }
514
515 static __always_inline bool
lockmgr_slock_try(struct lock * lk,uintptr_t * xp,int flags,bool fp)516 lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags, bool fp)
517 {
518
519 /*
520 * If no other thread has an exclusive lock, or
521 * no exclusive waiter is present, bump the count of
522 * sharers. Since we have to preserve the state of
523 * waiters, if we fail to acquire the shared lock
524 * loop back and retry.
525 */
526 while (LK_CAN_SHARE(*xp, flags, fp)) {
527 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
528 *xp + LK_ONE_SHARER)) {
529 return (true);
530 }
531 }
532 return (false);
533 }
534
535 static __always_inline bool
lockmgr_sunlock_try(struct lock * lk,uintptr_t * xp)536 lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp)
537 {
538
539 for (;;) {
540 if (LK_SHARERS(*xp) > 1 || !(*xp & LK_ALL_WAITERS)) {
541 if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
542 *xp - LK_ONE_SHARER))
543 return (true);
544 continue;
545 }
546 break;
547 }
548 return (false);
549 }
550
551 static bool
lockmgr_slock_adaptive(struct lock_delay_arg * lda,struct lock * lk,uintptr_t * xp,int flags)552 lockmgr_slock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp,
553 int flags)
554 {
555 struct thread *owner;
556 uintptr_t x;
557
558 x = *xp;
559 MPASS(x != LK_UNLOCKED);
560 owner = (struct thread *)LK_HOLDER(x);
561 for (;;) {
562 MPASS(owner != curthread);
563 if (owner == (struct thread *)LK_KERNPROC)
564 return (false);
565 if ((x & LK_SHARE) && LK_SHARERS(x) > 0)
566 return (false);
567 if (owner == NULL)
568 return (false);
569 if (!TD_IS_RUNNING(owner))
570 return (false);
571 if ((x & LK_ALL_WAITERS) != 0)
572 return (false);
573 lock_delay(lda);
574 x = lockmgr_read_value(lk);
575 if (LK_CAN_SHARE(x, flags, false)) {
576 *xp = x;
577 return (true);
578 }
579 owner = (struct thread *)LK_HOLDER(x);
580 }
581 }
582
583 static __noinline int
lockmgr_slock_hard(struct lock * lk,u_int flags,struct lock_object * ilk,const char * file,int line,struct lockmgr_wait * lwa)584 lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
585 const char *file, int line, struct lockmgr_wait *lwa)
586 {
587 uintptr_t tid, x;
588 int error = 0;
589 const char *iwmesg;
590 int ipri, itimo;
591
592 #ifdef KDTRACE_HOOKS
593 uint64_t sleep_time = 0;
594 #endif
595 #ifdef LOCK_PROFILING
596 uint64_t waittime = 0;
597 int contested = 0;
598 #endif
599 struct lock_delay_arg lda;
600
601 if (SCHEDULER_STOPPED())
602 goto out;
603
604 tid = (uintptr_t)curthread;
605
606 if (LK_CAN_WITNESS(flags))
607 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
608 file, line, flags & LK_INTERLOCK ? ilk : NULL);
609 x = lockmgr_read_value(lk);
610 lock_delay_arg_init(&lda, &lockmgr_delay);
611 if (!lk_adaptive)
612 flags &= ~LK_ADAPTIVE;
613 /*
614 * The lock may already be locked exclusive by curthread,
615 * avoid deadlock.
616 */
617 if (LK_HOLDER(x) == tid) {
618 LOCK_LOG2(lk,
619 "%s: %p already held in exclusive mode",
620 __func__, lk);
621 error = EDEADLK;
622 goto out;
623 }
624
625 for (;;) {
626 if (lockmgr_slock_try(lk, &x, flags, false))
627 break;
628
629 lock_profile_obtain_lock_failed(&lk->lock_object, false,
630 &contested, &waittime);
631
632 if ((flags & (LK_ADAPTIVE | LK_INTERLOCK)) == LK_ADAPTIVE) {
633 if (lockmgr_slock_adaptive(&lda, lk, &x, flags))
634 continue;
635 }
636
637 #ifdef HWPMC_HOOKS
638 PMC_SOFT_CALL( , , lock, failed);
639 #endif
640
641 /*
642 * If the lock is expected to not sleep just give up
643 * and return.
644 */
645 if (LK_TRYOP(flags)) {
646 LOCK_LOG2(lk, "%s: %p fails the try operation",
647 __func__, lk);
648 error = EBUSY;
649 break;
650 }
651
652 /*
653 * Acquire the sleepqueue chain lock because we
654 * probabilly will need to manipulate waiters flags.
655 */
656 sleepq_lock(&lk->lock_object);
657 x = lockmgr_read_value(lk);
658 retry_sleepq:
659
660 /*
661 * if the lock can be acquired in shared mode, try
662 * again.
663 */
664 if (LK_CAN_SHARE(x, flags, false)) {
665 sleepq_release(&lk->lock_object);
666 continue;
667 }
668
669 /*
670 * Try to set the LK_SHARED_WAITERS flag. If we fail,
671 * loop back and retry.
672 */
673 if ((x & LK_SHARED_WAITERS) == 0) {
674 if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
675 x | LK_SHARED_WAITERS)) {
676 goto retry_sleepq;
677 }
678 LOCK_LOG2(lk, "%s: %p set shared waiters flag",
679 __func__, lk);
680 }
681
682 if (lwa == NULL) {
683 iwmesg = lk->lock_object.lo_name;
684 ipri = lk->lk_pri;
685 itimo = lk->lk_timo;
686 } else {
687 iwmesg = lwa->iwmesg;
688 ipri = lwa->ipri;
689 itimo = lwa->itimo;
690 }
691
692 /*
693 * As far as we have been unable to acquire the
694 * shared lock and the shared waiters flag is set,
695 * we will sleep.
696 */
697 #ifdef KDTRACE_HOOKS
698 sleep_time -= lockstat_nsecs(&lk->lock_object);
699 #endif
700 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
701 SQ_SHARED_QUEUE);
702 #ifdef KDTRACE_HOOKS
703 sleep_time += lockstat_nsecs(&lk->lock_object);
704 #endif
705 flags &= ~LK_INTERLOCK;
706 if (error) {
707 LOCK_LOG3(lk,
708 "%s: interrupted sleep for %p with %d",
709 __func__, lk, error);
710 break;
711 }
712 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
713 __func__, lk);
714 x = lockmgr_read_value(lk);
715 }
716 if (error == 0) {
717 #ifdef KDTRACE_HOOKS
718 if (sleep_time != 0)
719 LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
720 LOCKSTAT_READER, (x & LK_SHARE) == 0,
721 (x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
722 #endif
723 #ifdef LOCK_PROFILING
724 lockmgr_note_shared_acquire(lk, contested, waittime,
725 file, line, flags);
726 #else
727 lockmgr_note_shared_acquire(lk, 0, 0, file, line,
728 flags);
729 #endif
730 }
731
732 out:
733 lockmgr_exit(flags, ilk);
734 return (error);
735 }
736
737 static bool
lockmgr_xlock_adaptive(struct lock_delay_arg * lda,struct lock * lk,uintptr_t * xp)738 lockmgr_xlock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp)
739 {
740 struct thread *owner;
741 uintptr_t x;
742
743 x = *xp;
744 MPASS(x != LK_UNLOCKED);
745 owner = (struct thread *)LK_HOLDER(x);
746 for (;;) {
747 MPASS(owner != curthread);
748 if (owner == NULL)
749 return (false);
750 if ((x & LK_SHARE) && LK_SHARERS(x) > 0)
751 return (false);
752 if (owner == (struct thread *)LK_KERNPROC)
753 return (false);
754 if (!TD_IS_RUNNING(owner))
755 return (false);
756 if ((x & LK_ALL_WAITERS) != 0)
757 return (false);
758 lock_delay(lda);
759 x = lockmgr_read_value(lk);
760 if (x == LK_UNLOCKED) {
761 *xp = x;
762 return (true);
763 }
764 owner = (struct thread *)LK_HOLDER(x);
765 }
766 }
767
768 static __noinline int
lockmgr_xlock_hard(struct lock * lk,u_int flags,struct lock_object * ilk,const char * file,int line,struct lockmgr_wait * lwa)769 lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
770 const char *file, int line, struct lockmgr_wait *lwa)
771 {
772 struct lock_class *class;
773 uintptr_t tid, x, v;
774 int error = 0;
775 const char *iwmesg;
776 int ipri, itimo;
777
778 #ifdef KDTRACE_HOOKS
779 uint64_t sleep_time = 0;
780 #endif
781 #ifdef LOCK_PROFILING
782 uint64_t waittime = 0;
783 int contested = 0;
784 #endif
785 struct lock_delay_arg lda;
786
787 if (SCHEDULER_STOPPED())
788 goto out;
789
790 tid = (uintptr_t)curthread;
791
792 if (LK_CAN_WITNESS(flags))
793 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
794 LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
795 ilk : NULL);
796
797 /*
798 * If curthread already holds the lock and this one is
799 * allowed to recurse, simply recurse on it.
800 */
801 if (lockmgr_xlocked(lk)) {
802 if ((flags & LK_CANRECURSE) == 0 &&
803 (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
804 /*
805 * If the lock is expected to not panic just
806 * give up and return.
807 */
808 if (LK_TRYOP(flags)) {
809 LOCK_LOG2(lk,
810 "%s: %p fails the try operation",
811 __func__, lk);
812 error = EBUSY;
813 goto out;
814 }
815 if (flags & LK_INTERLOCK) {
816 class = LOCK_CLASS(ilk);
817 class->lc_unlock(ilk);
818 }
819 STACK_PRINT(lk);
820 panic("%s: recursing on non recursive lockmgr %p "
821 "@ %s:%d\n", __func__, lk, file, line);
822 }
823 atomic_set_ptr(&lk->lk_lock, LK_WRITER_RECURSED);
824 lk->lk_recurse++;
825 LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
826 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
827 lk->lk_recurse, file, line);
828 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
829 LK_TRYWIT(flags), file, line);
830 TD_LOCKS_INC(curthread);
831 goto out;
832 }
833
834 x = LK_UNLOCKED;
835 lock_delay_arg_init(&lda, &lockmgr_delay);
836 if (!lk_adaptive)
837 flags &= ~LK_ADAPTIVE;
838 for (;;) {
839 if (x == LK_UNLOCKED) {
840 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x, tid))
841 break;
842 continue;
843 }
844
845 lock_profile_obtain_lock_failed(&lk->lock_object, false,
846 &contested, &waittime);
847
848 if ((flags & (LK_ADAPTIVE | LK_INTERLOCK)) == LK_ADAPTIVE) {
849 if (lockmgr_xlock_adaptive(&lda, lk, &x))
850 continue;
851 }
852 #ifdef HWPMC_HOOKS
853 PMC_SOFT_CALL( , , lock, failed);
854 #endif
855
856 /*
857 * If the lock is expected to not sleep just give up
858 * and return.
859 */
860 if (LK_TRYOP(flags)) {
861 LOCK_LOG2(lk, "%s: %p fails the try operation",
862 __func__, lk);
863 error = EBUSY;
864 break;
865 }
866
867 /*
868 * Acquire the sleepqueue chain lock because we
869 * probabilly will need to manipulate waiters flags.
870 */
871 sleepq_lock(&lk->lock_object);
872 x = lockmgr_read_value(lk);
873 retry_sleepq:
874
875 /*
876 * if the lock has been released while we spun on
877 * the sleepqueue chain lock just try again.
878 */
879 if (x == LK_UNLOCKED) {
880 sleepq_release(&lk->lock_object);
881 continue;
882 }
883
884 /*
885 * The lock can be in the state where there is a
886 * pending queue of waiters, but still no owner.
887 * This happens when the lock is contested and an
888 * owner is going to claim the lock.
889 * If curthread is the one successfully acquiring it
890 * claim lock ownership and return, preserving waiters
891 * flags.
892 */
893 v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
894 if ((x & ~v) == LK_UNLOCKED) {
895 v &= ~LK_EXCLUSIVE_SPINNERS;
896 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
897 tid | v)) {
898 sleepq_release(&lk->lock_object);
899 LOCK_LOG2(lk,
900 "%s: %p claimed by a new writer",
901 __func__, lk);
902 break;
903 }
904 goto retry_sleepq;
905 }
906
907 /*
908 * Try to set the LK_EXCLUSIVE_WAITERS flag. If we
909 * fail, loop back and retry.
910 */
911 if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
912 if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
913 x | LK_EXCLUSIVE_WAITERS)) {
914 goto retry_sleepq;
915 }
916 LOCK_LOG2(lk, "%s: %p set excl waiters flag",
917 __func__, lk);
918 }
919
920 if (lwa == NULL) {
921 iwmesg = lk->lock_object.lo_name;
922 ipri = lk->lk_pri;
923 itimo = lk->lk_timo;
924 } else {
925 iwmesg = lwa->iwmesg;
926 ipri = lwa->ipri;
927 itimo = lwa->itimo;
928 }
929
930 /*
931 * As far as we have been unable to acquire the
932 * exclusive lock and the exclusive waiters flag
933 * is set, we will sleep.
934 */
935 #ifdef KDTRACE_HOOKS
936 sleep_time -= lockstat_nsecs(&lk->lock_object);
937 #endif
938 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
939 SQ_EXCLUSIVE_QUEUE);
940 #ifdef KDTRACE_HOOKS
941 sleep_time += lockstat_nsecs(&lk->lock_object);
942 #endif
943 flags &= ~LK_INTERLOCK;
944 if (error) {
945 LOCK_LOG3(lk,
946 "%s: interrupted sleep for %p with %d",
947 __func__, lk, error);
948 break;
949 }
950 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
951 __func__, lk);
952 x = lockmgr_read_value(lk);
953 }
954 if (error == 0) {
955 #ifdef KDTRACE_HOOKS
956 if (sleep_time != 0)
957 LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
958 LOCKSTAT_WRITER, (x & LK_SHARE) == 0,
959 (x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
960 #endif
961 #ifdef LOCK_PROFILING
962 lockmgr_note_exclusive_acquire(lk, contested, waittime,
963 file, line, flags);
964 #else
965 lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
966 flags);
967 #endif
968 }
969
970 out:
971 lockmgr_exit(flags, ilk);
972 return (error);
973 }
974
975 static __noinline int
lockmgr_upgrade(struct lock * lk,u_int flags,struct lock_object * ilk,const char * file,int line,struct lockmgr_wait * lwa)976 lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
977 const char *file, int line, struct lockmgr_wait *lwa)
978 {
979 uintptr_t tid, v, setv;
980 int error = 0;
981 int op;
982
983 if (SCHEDULER_STOPPED())
984 goto out;
985
986 tid = (uintptr_t)curthread;
987
988 _lockmgr_assert(lk, KA_SLOCKED, file, line);
989
990 op = flags & LK_TYPE_MASK;
991 v = lockmgr_read_value(lk);
992 for (;;) {
993 if (LK_SHARERS(v) > 1) {
994 if (op == LK_TRYUPGRADE) {
995 LOCK_LOG2(lk, "%s: %p failed the nowait upgrade",
996 __func__, lk);
997 error = EBUSY;
998 goto out;
999 }
1000 if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &v,
1001 v - LK_ONE_SHARER)) {
1002 lockmgr_note_shared_release(lk, file, line);
1003 goto out_xlock;
1004 }
1005 continue;
1006 }
1007 MPASS((v & ~LK_ALL_WAITERS) == LK_SHARERS_LOCK(1));
1008
1009 setv = tid;
1010 setv |= (v & LK_ALL_WAITERS);
1011
1012 /*
1013 * Try to switch from one shared lock to an exclusive one.
1014 * We need to preserve waiters flags during the operation.
1015 */
1016 if (atomic_fcmpset_ptr(&lk->lk_lock, &v, setv)) {
1017 LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
1018 line);
1019 WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
1020 LK_TRYWIT(flags), file, line);
1021 LOCKSTAT_RECORD0(lockmgr__upgrade, lk);
1022 TD_SLOCKS_DEC(curthread);
1023 goto out;
1024 }
1025 }
1026
1027 out_xlock:
1028 error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
1029 flags &= ~LK_INTERLOCK;
1030 out:
1031 lockmgr_exit(flags, ilk);
1032 return (error);
1033 }
1034
1035 int
lockmgr_lock_flags(struct lock * lk,u_int flags,struct lock_object * ilk,const char * file,int line)1036 lockmgr_lock_flags(struct lock *lk, u_int flags, struct lock_object *ilk,
1037 const char *file, int line)
1038 {
1039 struct lock_class *class;
1040 uintptr_t x, tid;
1041 u_int op;
1042 bool locked;
1043
1044 if (SCHEDULER_STOPPED())
1045 return (0);
1046
1047 op = flags & LK_TYPE_MASK;
1048 locked = false;
1049 switch (op) {
1050 case LK_SHARED:
1051 if (LK_CAN_WITNESS(flags))
1052 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
1053 file, line, flags & LK_INTERLOCK ? ilk : NULL);
1054 if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
1055 break;
1056 x = lockmgr_read_value(lk);
1057 if (lockmgr_slock_try(lk, &x, flags, true)) {
1058 lockmgr_note_shared_acquire(lk, 0, 0,
1059 file, line, flags);
1060 locked = true;
1061 } else {
1062 return (lockmgr_slock_hard(lk, flags, ilk, file, line,
1063 NULL));
1064 }
1065 break;
1066 case LK_EXCLUSIVE:
1067 if (LK_CAN_WITNESS(flags))
1068 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1069 LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
1070 ilk : NULL);
1071 tid = (uintptr_t)curthread;
1072 if (lockmgr_read_value(lk) == LK_UNLOCKED &&
1073 atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
1074 lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
1075 flags);
1076 locked = true;
1077 } else {
1078 return (lockmgr_xlock_hard(lk, flags, ilk, file, line,
1079 NULL));
1080 }
1081 break;
1082 case LK_UPGRADE:
1083 case LK_TRYUPGRADE:
1084 return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL));
1085 default:
1086 break;
1087 }
1088 if (__predict_true(locked)) {
1089 if (__predict_false(flags & LK_INTERLOCK)) {
1090 class = LOCK_CLASS(ilk);
1091 class->lc_unlock(ilk);
1092 }
1093 return (0);
1094 } else {
1095 return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
1096 LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
1097 }
1098 }
1099
1100 static __noinline int
lockmgr_sunlock_hard(struct lock * lk,uintptr_t x,u_int flags,struct lock_object * ilk,const char * file,int line)1101 lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
1102 const char *file, int line)
1103 {
1104 if (!SCHEDULER_STOPPED())
1105 wakeupshlk(lk, file, line);
1106 lockmgr_exit(flags, ilk);
1107 return (0);
1108 }
1109
1110 static __noinline int
lockmgr_xunlock_hard(struct lock * lk,uintptr_t x,u_int flags,struct lock_object * ilk,const char * file,int line)1111 lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
1112 const char *file, int line)
1113 {
1114 uintptr_t tid, v;
1115 u_int realexslp;
1116 int queue;
1117
1118 if (SCHEDULER_STOPPED())
1119 goto out;
1120
1121 tid = (uintptr_t)curthread;
1122
1123 /*
1124 * As first option, treact the lock as if it has not
1125 * any waiter.
1126 * Fix-up the tid var if the lock has been disowned.
1127 */
1128 if (lockmgr_disowned_v(x))
1129 tid = LK_KERNPROC;
1130
1131 /*
1132 * The lock is held in exclusive mode.
1133 * If the lock is recursed also, then unrecurse it.
1134 */
1135 if (lockmgr_recursed_v(x)) {
1136 LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);
1137 lk->lk_recurse--;
1138 if (lk->lk_recurse == 0)
1139 atomic_clear_ptr(&lk->lk_lock, LK_WRITER_RECURSED);
1140 goto out;
1141 }
1142 if (tid != LK_KERNPROC)
1143 LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk,
1144 LOCKSTAT_WRITER);
1145
1146 if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
1147 goto out;
1148
1149 sleepq_lock(&lk->lock_object);
1150 x = lockmgr_read_value(lk);
1151 v = LK_UNLOCKED;
1152
1153 /*
1154 * If the lock has exclusive waiters, give them
1155 * preference in order to avoid deadlock with
1156 * shared runners up.
1157 * If interruptible sleeps left the exclusive queue
1158 * empty avoid a starvation for the threads sleeping
1159 * on the shared queue by giving them precedence
1160 * and cleaning up the exclusive waiters bit anyway.
1161 * Please note that lk_exslpfail count may be lying
1162 * about the real number of waiters with the
1163 * LK_SLEEPFAIL flag on because they may be used in
1164 * conjunction with interruptible sleeps so
1165 * lk_exslpfail might be considered an 'upper limit'
1166 * bound, including the edge cases.
1167 */
1168 MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1169 realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
1170 if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
1171 if (lk->lk_exslpfail != USHRT_MAX && lk->lk_exslpfail < realexslp) {
1172 lk->lk_exslpfail = 0;
1173 queue = SQ_EXCLUSIVE_QUEUE;
1174 v |= (x & LK_SHARED_WAITERS);
1175 } else {
1176 lk->lk_exslpfail = 0;
1177 LOCK_LOG2(lk,
1178 "%s: %p has only LK_SLEEPFAIL sleepers",
1179 __func__, lk);
1180 LOCK_LOG2(lk,
1181 "%s: %p waking up threads on the exclusive queue",
1182 __func__, lk);
1183 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
1184 SQ_EXCLUSIVE_QUEUE);
1185 queue = SQ_SHARED_QUEUE;
1186 }
1187 } else {
1188 /*
1189 * Exclusive waiters sleeping with LK_SLEEPFAIL
1190 * on and using interruptible sleeps/timeout
1191 * may have left spourious lk_exslpfail counts
1192 * on, so clean it up anyway.
1193 */
1194 lk->lk_exslpfail = 0;
1195 queue = SQ_SHARED_QUEUE;
1196 }
1197
1198 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
1199 __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
1200 "exclusive");
1201 atomic_store_rel_ptr(&lk->lk_lock, v);
1202 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
1203 sleepq_release(&lk->lock_object);
1204
1205 out:
1206 lockmgr_exit(flags, ilk);
1207 return (0);
1208 }
1209
1210 /*
1211 * Lightweight entry points for common operations.
1212 *
1213 * Functionality is similar to sx locks, in that none of the additional lockmgr
1214 * features are supported. To be clear, these are NOT supported:
1215 * 1. shared locking disablement
1216 * 2. returning with an error after sleep
1217 * 3. unlocking the interlock
1218 *
1219 * If in doubt, use lockmgr_lock_flags.
1220 */
1221 int
lockmgr_slock(struct lock * lk,u_int flags,const char * file,int line)1222 lockmgr_slock(struct lock *lk, u_int flags, const char *file, int line)
1223 {
1224 uintptr_t x;
1225
1226 MPASS((flags & LK_TYPE_MASK) == LK_SHARED);
1227 MPASS((flags & LK_INTERLOCK) == 0);
1228 MPASS((lk->lock_object.lo_flags & LK_NOSHARE) == 0);
1229
1230 if (LK_CAN_WITNESS(flags))
1231 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
1232 file, line, NULL);
1233 x = lockmgr_read_value(lk);
1234 if (__predict_true(lockmgr_slock_try(lk, &x, flags, true))) {
1235 lockmgr_note_shared_acquire(lk, 0, 0, file, line, flags);
1236 return (0);
1237 }
1238
1239 return (lockmgr_slock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL));
1240 }
1241
1242 int
lockmgr_xlock(struct lock * lk,u_int flags,const char * file,int line)1243 lockmgr_xlock(struct lock *lk, u_int flags, const char *file, int line)
1244 {
1245 uintptr_t tid;
1246
1247 MPASS((flags & LK_TYPE_MASK) == LK_EXCLUSIVE);
1248 MPASS((flags & LK_INTERLOCK) == 0);
1249
1250 if (LK_CAN_WITNESS(flags))
1251 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1252 LOP_EXCLUSIVE, file, line, NULL);
1253 tid = (uintptr_t)curthread;
1254 if (atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
1255 lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
1256 flags);
1257 return (0);
1258 }
1259
1260 return (lockmgr_xlock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL));
1261 }
1262
1263 int
lockmgr_unlock(struct lock * lk)1264 lockmgr_unlock(struct lock *lk)
1265 {
1266 uintptr_t x, tid;
1267 const char *file;
1268 int line;
1269
1270 file = __FILE__;
1271 line = __LINE__;
1272
1273 _lockmgr_assert(lk, KA_LOCKED, file, line);
1274 x = lockmgr_read_value(lk);
1275 if (__predict_true(x & LK_SHARE) != 0) {
1276 lockmgr_note_shared_release(lk, file, line);
1277 if (lockmgr_sunlock_try(lk, &x)) {
1278 LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
1279 } else {
1280 return (lockmgr_sunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
1281 }
1282 } else {
1283 tid = (uintptr_t)curthread;
1284 lockmgr_note_exclusive_release(lk, file, line);
1285 if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
1286 LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk,LOCKSTAT_WRITER);
1287 } else {
1288 return (lockmgr_xunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
1289 }
1290 }
1291 return (0);
1292 }
1293
1294 int
__lockmgr_args(struct lock * lk,u_int flags,struct lock_object * ilk,const char * wmesg,int pri,int timo,const char * file,int line)1295 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
1296 const char *wmesg, int pri, int timo, const char *file, int line)
1297 {
1298 GIANT_DECLARE;
1299 struct lockmgr_wait lwa;
1300 struct lock_class *class;
1301 const char *iwmesg;
1302 uintptr_t tid, v, x;
1303 u_int op, realexslp;
1304 int error, ipri, itimo, queue;
1305 #ifdef LOCK_PROFILING
1306 uint64_t waittime = 0;
1307 int contested = 0;
1308 #endif
1309
1310 if (SCHEDULER_STOPPED())
1311 return (0);
1312
1313 error = 0;
1314 tid = (uintptr_t)curthread;
1315 op = (flags & LK_TYPE_MASK);
1316 iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
1317 ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
1318 itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
1319
1320 lwa.iwmesg = iwmesg;
1321 lwa.ipri = ipri;
1322 lwa.itimo = itimo;
1323
1324 MPASS((flags & ~LK_TOTAL_MASK) == 0);
1325 KASSERT((op & (op - 1)) == 0,
1326 ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
1327 KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
1328 (op != LK_DOWNGRADE && op != LK_RELEASE),
1329 ("%s: Invalid flags in regard of the operation desired @ %s:%d",
1330 __func__, file, line));
1331 KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
1332 ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
1333 __func__, file, line));
1334 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
1335 ("%s: idle thread %p on lockmgr %p @ %s:%d", __func__, curthread,
1336 lk, file, line));
1337
1338 class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
1339
1340 if (lk->lock_object.lo_flags & LK_NOSHARE) {
1341 switch (op) {
1342 case LK_SHARED:
1343 op = LK_EXCLUSIVE;
1344 break;
1345 case LK_UPGRADE:
1346 case LK_TRYUPGRADE:
1347 case LK_DOWNGRADE:
1348 _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
1349 file, line);
1350 if (flags & LK_INTERLOCK)
1351 class->lc_unlock(ilk);
1352 return (0);
1353 }
1354 }
1355
1356 switch (op) {
1357 case LK_SHARED:
1358 return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
1359 break;
1360 case LK_UPGRADE:
1361 case LK_TRYUPGRADE:
1362 return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
1363 break;
1364 case LK_EXCLUSIVE:
1365 return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
1366 break;
1367 case LK_DOWNGRADE:
1368 _lockmgr_assert(lk, KA_XLOCKED, file, line);
1369 WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
1370
1371 /*
1372 * Panic if the lock is recursed.
1373 */
1374 if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
1375 if (flags & LK_INTERLOCK)
1376 class->lc_unlock(ilk);
1377 panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n",
1378 __func__, iwmesg, file, line);
1379 }
1380 TD_SLOCKS_INC(curthread);
1381
1382 /*
1383 * In order to preserve waiters flags, just spin.
1384 */
1385 for (;;) {
1386 x = lockmgr_read_value(lk);
1387 MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1388 x &= LK_ALL_WAITERS;
1389 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1390 LK_SHARERS_LOCK(1) | x))
1391 break;
1392 cpu_spinwait();
1393 }
1394 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
1395 LOCKSTAT_RECORD0(lockmgr__downgrade, lk);
1396 break;
1397 case LK_RELEASE:
1398 _lockmgr_assert(lk, KA_LOCKED, file, line);
1399 x = lockmgr_read_value(lk);
1400
1401 if (__predict_true(x & LK_SHARE) != 0) {
1402 lockmgr_note_shared_release(lk, file, line);
1403 return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
1404 } else {
1405 lockmgr_note_exclusive_release(lk, file, line);
1406 return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
1407 }
1408 break;
1409 case LK_DRAIN:
1410 if (LK_CAN_WITNESS(flags))
1411 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1412 LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
1413 ilk : NULL);
1414
1415 /*
1416 * Trying to drain a lock we already own will result in a
1417 * deadlock.
1418 */
1419 if (lockmgr_xlocked(lk)) {
1420 if (flags & LK_INTERLOCK)
1421 class->lc_unlock(ilk);
1422 panic("%s: draining %s with the lock held @ %s:%d\n",
1423 __func__, iwmesg, file, line);
1424 }
1425
1426 for (;;) {
1427 if (lk->lk_lock == LK_UNLOCKED &&
1428 atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
1429 break;
1430
1431 #ifdef HWPMC_HOOKS
1432 PMC_SOFT_CALL( , , lock, failed);
1433 #endif
1434 lock_profile_obtain_lock_failed(&lk->lock_object, false,
1435 &contested, &waittime);
1436
1437 /*
1438 * If the lock is expected to not sleep just give up
1439 * and return.
1440 */
1441 if (LK_TRYOP(flags)) {
1442 LOCK_LOG2(lk, "%s: %p fails the try operation",
1443 __func__, lk);
1444 error = EBUSY;
1445 break;
1446 }
1447
1448 /*
1449 * Acquire the sleepqueue chain lock because we
1450 * probabilly will need to manipulate waiters flags.
1451 */
1452 sleepq_lock(&lk->lock_object);
1453 x = lockmgr_read_value(lk);
1454
1455 /*
1456 * if the lock has been released while we spun on
1457 * the sleepqueue chain lock just try again.
1458 */
1459 if (x == LK_UNLOCKED) {
1460 sleepq_release(&lk->lock_object);
1461 continue;
1462 }
1463
1464 v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
1465 if ((x & ~v) == LK_UNLOCKED) {
1466 v = (x & ~LK_EXCLUSIVE_SPINNERS);
1467
1468 /*
1469 * If interruptible sleeps left the exclusive
1470 * queue empty avoid a starvation for the
1471 * threads sleeping on the shared queue by
1472 * giving them precedence and cleaning up the
1473 * exclusive waiters bit anyway.
1474 * Please note that lk_exslpfail count may be
1475 * lying about the real number of waiters with
1476 * the LK_SLEEPFAIL flag on because they may
1477 * be used in conjunction with interruptible
1478 * sleeps so lk_exslpfail might be considered
1479 * an 'upper limit' bound, including the edge
1480 * cases.
1481 */
1482 if (v & LK_EXCLUSIVE_WAITERS) {
1483 queue = SQ_EXCLUSIVE_QUEUE;
1484 v &= ~LK_EXCLUSIVE_WAITERS;
1485 } else {
1486 /*
1487 * Exclusive waiters sleeping with
1488 * LK_SLEEPFAIL on and using
1489 * interruptible sleeps/timeout may
1490 * have left spourious lk_exslpfail
1491 * counts on, so clean it up anyway.
1492 */
1493 MPASS(v & LK_SHARED_WAITERS);
1494 lk->lk_exslpfail = 0;
1495 queue = SQ_SHARED_QUEUE;
1496 v &= ~LK_SHARED_WAITERS;
1497 }
1498 if (queue == SQ_EXCLUSIVE_QUEUE) {
1499 realexslp =
1500 sleepq_sleepcnt(&lk->lock_object,
1501 SQ_EXCLUSIVE_QUEUE);
1502 if (lk->lk_exslpfail >= realexslp) {
1503 lk->lk_exslpfail = 0;
1504 queue = SQ_SHARED_QUEUE;
1505 v &= ~LK_SHARED_WAITERS;
1506 if (realexslp != 0) {
1507 LOCK_LOG2(lk,
1508 "%s: %p has only LK_SLEEPFAIL sleepers",
1509 __func__, lk);
1510 LOCK_LOG2(lk,
1511 "%s: %p waking up threads on the exclusive queue",
1512 __func__, lk);
1513 sleepq_broadcast(
1514 &lk->lock_object,
1515 SLEEPQ_LK, 0,
1516 SQ_EXCLUSIVE_QUEUE);
1517 }
1518 } else
1519 lk->lk_exslpfail = 0;
1520 }
1521 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
1522 sleepq_release(&lk->lock_object);
1523 continue;
1524 }
1525 LOCK_LOG3(lk,
1526 "%s: %p waking up all threads on the %s queue",
1527 __func__, lk, queue == SQ_SHARED_QUEUE ?
1528 "shared" : "exclusive");
1529 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
1530 queue);
1531
1532 /*
1533 * If shared waiters have been woken up we need
1534 * to wait for one of them to acquire the lock
1535 * before to set the exclusive waiters in
1536 * order to avoid a deadlock.
1537 */
1538 if (queue == SQ_SHARED_QUEUE) {
1539 for (v = lk->lk_lock;
1540 (v & LK_SHARE) && !LK_SHARERS(v);
1541 v = lk->lk_lock)
1542 cpu_spinwait();
1543 }
1544 }
1545
1546 /*
1547 * Try to set the LK_EXCLUSIVE_WAITERS flag. If we
1548 * fail, loop back and retry.
1549 */
1550 if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
1551 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
1552 x | LK_EXCLUSIVE_WAITERS)) {
1553 sleepq_release(&lk->lock_object);
1554 continue;
1555 }
1556 LOCK_LOG2(lk, "%s: %p set drain waiters flag",
1557 __func__, lk);
1558 }
1559
1560 /*
1561 * As far as we have been unable to acquire the
1562 * exclusive lock and the exclusive waiters flag
1563 * is set, we will sleep.
1564 */
1565 if (flags & LK_INTERLOCK) {
1566 class->lc_unlock(ilk);
1567 flags &= ~LK_INTERLOCK;
1568 }
1569 GIANT_SAVE();
1570 sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
1571 SQ_EXCLUSIVE_QUEUE);
1572 sleepq_wait(&lk->lock_object, ipri & PRIMASK);
1573 GIANT_RESTORE();
1574 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
1575 __func__, lk);
1576 }
1577
1578 if (error == 0) {
1579 lock_profile_obtain_lock_success(&lk->lock_object,
1580 false, contested, waittime, file, line);
1581 LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
1582 lk->lk_recurse, file, line);
1583 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
1584 LK_TRYWIT(flags), file, line);
1585 TD_LOCKS_INC(curthread);
1586 STACK_SAVE(lk);
1587 }
1588 break;
1589 default:
1590 if (flags & LK_INTERLOCK)
1591 class->lc_unlock(ilk);
1592 panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
1593 }
1594
1595 if (flags & LK_INTERLOCK)
1596 class->lc_unlock(ilk);
1597
1598 return (error);
1599 }
1600
1601 void
_lockmgr_disown(struct lock * lk,const char * file,int line)1602 _lockmgr_disown(struct lock *lk, const char *file, int line)
1603 {
1604 uintptr_t tid, x;
1605
1606 if (SCHEDULER_STOPPED())
1607 return;
1608
1609 tid = (uintptr_t)curthread;
1610 _lockmgr_assert(lk, KA_XLOCKED, file, line);
1611
1612 /*
1613 * Panic if the lock is recursed.
1614 */
1615 if (lockmgr_xlocked(lk) && lockmgr_recursed(lk))
1616 panic("%s: disown a recursed lockmgr @ %s:%d\n",
1617 __func__, file, line);
1618
1619 /*
1620 * If the owner is already LK_KERNPROC just skip the whole operation.
1621 */
1622 if (LK_HOLDER(lk->lk_lock) != tid)
1623 return;
1624 lock_profile_release_lock(&lk->lock_object, false);
1625 LOCKSTAT_RECORD1(lockmgr__disown, lk, LOCKSTAT_WRITER);
1626 LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
1627 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
1628 TD_LOCKS_DEC(curthread);
1629 STACK_SAVE(lk);
1630
1631 /*
1632 * In order to preserve waiters flags, just spin.
1633 */
1634 for (;;) {
1635 x = lockmgr_read_value(lk);
1636 MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1637 x &= LK_ALL_WAITERS;
1638 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1639 LK_KERNPROC | x))
1640 return;
1641 cpu_spinwait();
1642 }
1643 }
1644
1645 void
lockmgr_printinfo(const struct lock * lk)1646 lockmgr_printinfo(const struct lock *lk)
1647 {
1648 struct thread *td;
1649 uintptr_t x;
1650
1651 if (lk->lk_lock == LK_UNLOCKED)
1652 printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
1653 else if (lk->lk_lock & LK_SHARE)
1654 printf("lock type %s: SHARED (count %ju)\n",
1655 lk->lock_object.lo_name,
1656 (uintmax_t)LK_SHARERS(lk->lk_lock));
1657 else {
1658 td = lockmgr_xholder(lk);
1659 if (td == (struct thread *)LK_KERNPROC)
1660 printf("lock type %s: EXCL by KERNPROC\n",
1661 lk->lock_object.lo_name);
1662 else
1663 printf("lock type %s: EXCL by thread %p "
1664 "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
1665 td, td->td_proc->p_pid, td->td_proc->p_comm,
1666 td->td_tid);
1667 }
1668
1669 x = lk->lk_lock;
1670 if (x & LK_EXCLUSIVE_WAITERS)
1671 printf(" with exclusive waiters pending\n");
1672 if (x & LK_SHARED_WAITERS)
1673 printf(" with shared waiters pending\n");
1674 if (x & LK_EXCLUSIVE_SPINNERS)
1675 printf(" with exclusive spinners pending\n");
1676
1677 STACK_PRINT(lk);
1678 }
1679
1680 int
lockstatus(const struct lock * lk)1681 lockstatus(const struct lock *lk)
1682 {
1683 uintptr_t v, x;
1684 int ret;
1685
1686 ret = LK_SHARED;
1687 x = lockmgr_read_value(lk);
1688 v = LK_HOLDER(x);
1689
1690 if ((x & LK_SHARE) == 0) {
1691 if (v == (uintptr_t)curthread || v == LK_KERNPROC)
1692 ret = LK_EXCLUSIVE;
1693 else
1694 ret = LK_EXCLOTHER;
1695 } else if (x == LK_UNLOCKED)
1696 ret = 0;
1697
1698 return (ret);
1699 }
1700
1701 #ifdef INVARIANT_SUPPORT
1702
1703 FEATURE(invariant_support,
1704 "Support for modules compiled with INVARIANTS option");
1705
1706 #ifndef INVARIANTS
1707 #undef _lockmgr_assert
1708 #endif
1709
1710 void
_lockmgr_assert(const struct lock * lk,int what,const char * file,int line)1711 _lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
1712 {
1713 int slocked = 0;
1714
1715 if (SCHEDULER_STOPPED())
1716 return;
1717 switch (what) {
1718 case KA_SLOCKED:
1719 case KA_SLOCKED | KA_NOTRECURSED:
1720 case KA_SLOCKED | KA_RECURSED:
1721 slocked = 1;
1722 case KA_LOCKED:
1723 case KA_LOCKED | KA_NOTRECURSED:
1724 case KA_LOCKED | KA_RECURSED:
1725 #ifdef WITNESS
1726
1727 /*
1728 * We cannot trust WITNESS if the lock is held in exclusive
1729 * mode and a call to lockmgr_disown() happened.
1730 * Workaround this skipping the check if the lock is held in
1731 * exclusive mode even for the KA_LOCKED case.
1732 */
1733 if (slocked || (lk->lk_lock & LK_SHARE)) {
1734 witness_assert(&lk->lock_object, what, file, line);
1735 break;
1736 }
1737 #endif
1738 if (lk->lk_lock == LK_UNLOCKED ||
1739 ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
1740 (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
1741 panic("Lock %s not %slocked @ %s:%d\n",
1742 lk->lock_object.lo_name, slocked ? "share" : "",
1743 file, line);
1744
1745 if ((lk->lk_lock & LK_SHARE) == 0) {
1746 if (lockmgr_recursed(lk)) {
1747 if (what & KA_NOTRECURSED)
1748 panic("Lock %s recursed @ %s:%d\n",
1749 lk->lock_object.lo_name, file,
1750 line);
1751 } else if (what & KA_RECURSED)
1752 panic("Lock %s not recursed @ %s:%d\n",
1753 lk->lock_object.lo_name, file, line);
1754 }
1755 break;
1756 case KA_XLOCKED:
1757 case KA_XLOCKED | KA_NOTRECURSED:
1758 case KA_XLOCKED | KA_RECURSED:
1759 if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
1760 panic("Lock %s not exclusively locked @ %s:%d\n",
1761 lk->lock_object.lo_name, file, line);
1762 if (lockmgr_recursed(lk)) {
1763 if (what & KA_NOTRECURSED)
1764 panic("Lock %s recursed @ %s:%d\n",
1765 lk->lock_object.lo_name, file, line);
1766 } else if (what & KA_RECURSED)
1767 panic("Lock %s not recursed @ %s:%d\n",
1768 lk->lock_object.lo_name, file, line);
1769 break;
1770 case KA_UNLOCKED:
1771 if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
1772 panic("Lock %s exclusively locked @ %s:%d\n",
1773 lk->lock_object.lo_name, file, line);
1774 break;
1775 default:
1776 panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
1777 line);
1778 }
1779 }
1780 #endif
1781
1782 #ifdef DDB
1783 int
lockmgr_chain(struct thread * td,struct thread ** ownerp)1784 lockmgr_chain(struct thread *td, struct thread **ownerp)
1785 {
1786 const struct lock *lk;
1787
1788 lk = td->td_wchan;
1789
1790 if (!TD_ON_SLEEPQ(td) || sleepq_type(td->td_wchan) != SLEEPQ_LK ||
1791 LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
1792 return (0);
1793 db_printf("blocked on lock %p (%s) \"%s\" ", &lk->lock_object,
1794 lock_class_lockmgr.lc_name, lk->lock_object.lo_name);
1795 if (lk->lk_lock & LK_SHARE)
1796 db_printf("SHARED (count %ju)\n",
1797 (uintmax_t)LK_SHARERS(lk->lk_lock));
1798 else
1799 db_printf("EXCL\n");
1800 *ownerp = lockmgr_xholder(lk);
1801
1802 return (1);
1803 }
1804
1805 static void
db_show_lockmgr(const struct lock_object * lock)1806 db_show_lockmgr(const struct lock_object *lock)
1807 {
1808 struct thread *td;
1809 const struct lock *lk;
1810
1811 lk = (const struct lock *)lock;
1812
1813 db_printf(" state: ");
1814 if (lk->lk_lock == LK_UNLOCKED)
1815 db_printf("UNLOCKED\n");
1816 else if (lk->lk_lock & LK_SHARE)
1817 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
1818 else {
1819 td = lockmgr_xholder(lk);
1820 if (td == (struct thread *)LK_KERNPROC)
1821 db_printf("XLOCK: LK_KERNPROC\n");
1822 else
1823 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1824 td->td_tid, td->td_proc->p_pid,
1825 td->td_proc->p_comm);
1826 if (lockmgr_recursed(lk))
1827 db_printf(" recursed: %d\n", lk->lk_recurse);
1828 }
1829 db_printf(" waiters: ");
1830 switch (lk->lk_lock & LK_ALL_WAITERS) {
1831 case LK_SHARED_WAITERS:
1832 db_printf("shared\n");
1833 break;
1834 case LK_EXCLUSIVE_WAITERS:
1835 db_printf("exclusive\n");
1836 break;
1837 case LK_ALL_WAITERS:
1838 db_printf("shared and exclusive\n");
1839 break;
1840 default:
1841 db_printf("none\n");
1842 }
1843 db_printf(" spinners: ");
1844 if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
1845 db_printf("exclusive\n");
1846 else
1847 db_printf("none\n");
1848 }
1849 #endif
1850