xref: /freebsd/sys/kern/subr_witness.c (revision f9218d3d4fd34f082473b3a021c6d4d109fb47cf)
1 /*-
2  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  * 3. Berkeley Software Design Inc's name may not be used to endorse or
13  *    promote products derived from this software without specific prior
14  *    written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  *	from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
29  *	and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30  * $FreeBSD$
31  */
32 
33 /*
34  * Implementation of the `witness' lock verifier.  Originally implemented for
35  * mutexes in BSD/OS.  Extended to handle generic lock objects and lock
36  * classes in FreeBSD.
37  */
38 
39 /*
40  *	Main Entry: witness
41  *	Pronunciation: 'wit-n&s
42  *	Function: noun
43  *	Etymology: Middle English witnesse, from Old English witnes knowledge,
44  *	    testimony, witness, from 2wit
45  *	Date: before 12th century
46  *	1 : attestation of a fact or event : TESTIMONY
47  *	2 : one that gives evidence; specifically : one who testifies in
48  *	    a cause or before a judicial tribunal
49  *	3 : one asked to be present at a transaction so as to be able to
50  *	    testify to its having taken place
51  *	4 : one who has personal knowledge of something
52  *	5 a : something serving as evidence or proof : SIGN
53  *	  b : public affirmation by word or example of usually
54  *	      religious faith or conviction <the heroic witness to divine
55  *	      life -- Pilot>
56  *	6 capitalized : a member of the Jehovah's Witnesses
57  */
58 
59 #include "opt_ddb.h"
60 #include "opt_witness.h"
61 
62 #include <sys/param.h>
63 #include <sys/bus.h>
64 #include <sys/kernel.h>
65 #include <sys/ktr.h>
66 #include <sys/lock.h>
67 #include <sys/malloc.h>
68 #include <sys/mutex.h>
69 #include <sys/proc.h>
70 #include <sys/sysctl.h>
71 #include <sys/systm.h>
72 
73 #include <ddb/ddb.h>
74 
75 /* Define this to check for blessed mutexes */
76 #undef BLESSING
77 
78 #define WITNESS_COUNT 200
79 #define WITNESS_CHILDCOUNT (WITNESS_COUNT * 4)
80 /*
81  * XXX: This is somewhat bogus, as we assume here that at most 1024 threads
82  * will hold LOCK_NCHILDREN * 2 locks.  We handle failure ok, and we should
83  * probably be safe for the most part, but it's still a SWAG.
84  */
85 #define LOCK_CHILDCOUNT (MAXCPU + 1024) * 2
86 
87 #define	WITNESS_NCHILDREN 6
88 
89 struct witness_child_list_entry;
90 
91 struct witness {
92 	const	char *w_name;
93 	struct	lock_class *w_class;
94 	STAILQ_ENTRY(witness) w_list;		/* List of all witnesses. */
95 	STAILQ_ENTRY(witness) w_typelist;	/* Witnesses of a type. */
96 	struct	witness_child_list_entry *w_children;	/* Great evilness... */
97 	const	char *w_file;
98 	int	w_line;
99 	u_int	w_level;
100 	u_int	w_refcount;
101 	u_char	w_Giant_squawked:1;
102 	u_char	w_other_squawked:1;
103 	u_char	w_same_squawked:1;
104 };
105 
106 struct witness_child_list_entry {
107 	struct	witness_child_list_entry *wcl_next;
108 	struct	witness *wcl_children[WITNESS_NCHILDREN];
109 	u_int	wcl_count;
110 };
111 
112 STAILQ_HEAD(witness_list, witness);
113 
114 #ifdef BLESSING
115 struct witness_blessed {
116 	const	char *b_lock1;
117 	const	char *b_lock2;
118 };
119 #endif
120 
121 struct witness_order_list_entry {
122 	const	char *w_name;
123 	struct	lock_class *w_class;
124 };
125 
126 static struct	witness *enroll(const char *description,
127 				struct lock_class *lock_class);
128 static int	itismychild(struct witness *parent, struct witness *child);
129 static void	removechild(struct witness *parent, struct witness *child);
130 static int	isitmychild(struct witness *parent, struct witness *child);
131 static int	isitmydescendant(struct witness *parent, struct witness *child);
132 #ifdef BLESSING
133 static int	blessed(struct witness *, struct witness *);
134 #endif
135 static void	witness_displaydescendants(void(*)(const char *fmt, ...),
136 					   struct witness *);
137 static void	witness_leveldescendents(struct witness *parent, int level);
138 static void	witness_levelall(void);
139 static struct	witness *witness_get(void);
140 static void	witness_free(struct witness *m);
141 static struct	witness_child_list_entry *witness_child_get(void);
142 static void	witness_child_free(struct witness_child_list_entry *wcl);
143 static struct	lock_list_entry *witness_lock_list_get(void);
144 static void	witness_lock_list_free(struct lock_list_entry *lle);
145 static struct	lock_instance *find_instance(struct lock_list_entry *lock_list,
146 					     struct lock_object *lock);
147 #if defined(DDB)
148 static void	witness_display_list(void(*prnt)(const char *fmt, ...),
149 				     struct witness_list *list);
150 static void	witness_display(void(*)(const char *fmt, ...));
151 #endif
152 
153 MALLOC_DEFINE(M_WITNESS, "witness", "witness structure");
154 
155 static int witness_watch = 1;
156 TUNABLE_INT("debug.witness_watch", &witness_watch);
157 SYSCTL_INT(_debug, OID_AUTO, witness_watch, CTLFLAG_RD, &witness_watch, 0, "");
158 
159 #ifdef DDB
160 /*
161  * When DDB is enabled and witness_ddb is set to 1, it will cause the system to
162  * drop into kdebug() when:
163  *	- a lock heirarchy violation occurs
164  *	- locks are held when going to sleep.
165  */
166 #ifdef WITNESS_DDB
167 int	witness_ddb = 1;
168 #else
169 int	witness_ddb = 0;
170 #endif
171 TUNABLE_INT("debug.witness_ddb", &witness_ddb);
172 SYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, "");
173 
174 /*
175  * When DDB is enabled and witness_trace is set to 1, it will cause the system
176  * to print a stack trace:
177  *	- a lock heirarchy violation occurs
178  *	- locks are held when going to sleep.
179  */
180 int	witness_trace = 1;
181 TUNABLE_INT("debug.witness_trace", &witness_trace);
182 SYSCTL_INT(_debug, OID_AUTO, witness_trace, CTLFLAG_RW, &witness_trace, 0, "");
183 #endif /* DDB */
184 
185 #ifdef WITNESS_SKIPSPIN
186 int	witness_skipspin = 1;
187 #else
188 int	witness_skipspin = 0;
189 #endif
190 TUNABLE_INT("debug.witness_skipspin", &witness_skipspin);
191 SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
192     "");
193 
194 static struct mtx w_mtx;
195 static struct witness_list w_free = STAILQ_HEAD_INITIALIZER(w_free);
196 static struct witness_list w_all = STAILQ_HEAD_INITIALIZER(w_all);
197 static struct witness_list w_spin = STAILQ_HEAD_INITIALIZER(w_spin);
198 static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep);
199 static struct witness_child_list_entry *w_child_free = NULL;
200 static struct lock_list_entry *w_lock_list_free = NULL;
201 static int witness_dead;	/* fatal error, probably no memory */
202 
203 static struct witness w_data[WITNESS_COUNT];
204 static struct witness_child_list_entry w_childdata[WITNESS_CHILDCOUNT];
205 static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT];
206 
207 static struct witness_order_list_entry order_lists[] = {
208 	{ "Giant", &lock_class_mtx_sleep },
209 	{ "proctree", &lock_class_sx },
210 	{ "allproc", &lock_class_sx },
211 	{ "filedesc structure", &lock_class_mtx_sleep },
212 	{ "pipe mutex", &lock_class_mtx_sleep },
213 	{ "sigio lock", &lock_class_mtx_sleep },
214 	{ "process group", &lock_class_mtx_sleep },
215 	{ "process lock", &lock_class_mtx_sleep },
216 	{ "session", &lock_class_mtx_sleep },
217 	{ "uidinfo hash", &lock_class_mtx_sleep },
218 	{ "uidinfo struct", &lock_class_mtx_sleep },
219 	{ NULL, NULL },
220 	/*
221 	 * spin locks
222 	 */
223 #ifdef SMP
224 	{ "ap boot", &lock_class_mtx_spin },
225 #ifdef __i386__
226 	{ "com", &lock_class_mtx_spin },
227 #endif
228 #endif
229 	{ "sio", &lock_class_mtx_spin },
230 #ifdef __i386__
231 	{ "cy", &lock_class_mtx_spin },
232 #endif
233 	{ "sabtty", &lock_class_mtx_spin },
234 	{ "zstty", &lock_class_mtx_spin },
235 	{ "ng_node", &lock_class_mtx_spin },
236 	{ "ng_worklist", &lock_class_mtx_spin },
237 	{ "ithread table lock", &lock_class_mtx_spin },
238 	{ "sched lock", &lock_class_mtx_spin },
239 	{ "callout", &lock_class_mtx_spin },
240 	/*
241 	 * leaf locks
242 	 */
243 	{ "allpmaps", &lock_class_mtx_spin },
244 	{ "vm page queue free mutex", &lock_class_mtx_spin },
245 	{ "icu", &lock_class_mtx_spin },
246 #ifdef SMP
247 	{ "smp rendezvous", &lock_class_mtx_spin },
248 #if defined(__i386__) && defined(APIC_IO)
249 	{ "tlb", &lock_class_mtx_spin },
250 #endif
251 #ifdef __sparc64__
252 	{ "ipi", &lock_class_mtx_spin },
253 #endif
254 #endif
255 	{ "clk", &lock_class_mtx_spin },
256 	{ "mutex profiling lock", &lock_class_mtx_spin },
257 	{ "kse zombie lock", &lock_class_mtx_spin },
258 	{ "ALD Queue", &lock_class_mtx_spin },
259 #ifdef __ia64__
260 	{ "MCA spin lock", &lock_class_mtx_spin },
261 #endif
262 #ifdef __i386__
263 	{ "pcicfg", &lock_class_mtx_spin },
264 #endif
265 	{ NULL, NULL },
266 	{ NULL, NULL }
267 };
268 
269 #ifdef BLESSING
270 /*
271  * Pairs of locks which have been blessed
272  * Don't complain about order problems with blessed locks
273  */
274 static struct witness_blessed blessed_list[] = {
275 };
276 static int blessed_count =
277 	sizeof(blessed_list) / sizeof(struct witness_blessed);
278 #endif
279 
280 /*
281  * List of all locks in the system.
282  */
283 TAILQ_HEAD(, lock_object) all_locks = TAILQ_HEAD_INITIALIZER(all_locks);
284 
285 static struct mtx all_mtx = {
286 	{ &lock_class_mtx_sleep,	/* mtx_object.lo_class */
287 	  "All locks list",		/* mtx_object.lo_name */
288 	  "All locks list",		/* mtx_object.lo_type */
289 	  LO_INITIALIZED,		/* mtx_object.lo_flags */
290 	  { NULL, NULL },		/* mtx_object.lo_list */
291 	  NULL },			/* mtx_object.lo_witness */
292 	MTX_UNOWNED, 0,			/* mtx_lock, mtx_recurse */
293 	TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
294 	{ NULL, NULL }			/* mtx_contested */
295 };
296 
297 /*
298  * This global is set to 0 once it becomes safe to use the witness code.
299  */
300 static int witness_cold = 1;
301 
302 /*
303  * Global variables for book keeping.
304  */
305 static int lock_cur_cnt;
306 static int lock_max_cnt;
307 
308 /*
309  * The WITNESS-enabled diagnostic code.
310  */
311 static void
312 witness_initialize(void *dummy __unused)
313 {
314 	struct lock_object *lock;
315 	struct witness_order_list_entry *order;
316 	struct witness *w, *w1;
317 	int i;
318 
319 	/*
320 	 * We have to release Giant before initializing its witness
321 	 * structure so that WITNESS doesn't get confused.
322 	 */
323 	mtx_unlock(&Giant);
324 	mtx_assert(&Giant, MA_NOTOWNED);
325 
326 	CTR1(KTR_WITNESS, "%s: initializing witness", __func__);
327 	TAILQ_INSERT_HEAD(&all_locks, &all_mtx.mtx_object, lo_list);
328 	mtx_init(&w_mtx, "witness lock", NULL, MTX_SPIN | MTX_QUIET |
329 	    MTX_NOWITNESS);
330 	for (i = 0; i < WITNESS_COUNT; i++)
331 		witness_free(&w_data[i]);
332 	for (i = 0; i < WITNESS_CHILDCOUNT; i++)
333 		witness_child_free(&w_childdata[i]);
334 	for (i = 0; i < LOCK_CHILDCOUNT; i++)
335 		witness_lock_list_free(&w_locklistdata[i]);
336 
337 	/* First add in all the specified order lists. */
338 	for (order = order_lists; order->w_name != NULL; order++) {
339 		w = enroll(order->w_name, order->w_class);
340 		if (w == NULL)
341 			continue;
342 		w->w_file = "order list";
343 		for (order++; order->w_name != NULL; order++) {
344 			w1 = enroll(order->w_name, order->w_class);
345 			if (w1 == NULL)
346 				continue;
347 			w1->w_file = "order list";
348 			itismychild(w, w1);
349 			w = w1;
350 		}
351 	}
352 
353 	/* Iterate through all locks and add them to witness. */
354 	mtx_lock(&all_mtx);
355 	TAILQ_FOREACH(lock, &all_locks, lo_list) {
356 		if (lock->lo_flags & LO_WITNESS)
357 			lock->lo_witness = enroll(lock->lo_type,
358 			    lock->lo_class);
359 		else
360 			lock->lo_witness = NULL;
361 	}
362 	mtx_unlock(&all_mtx);
363 
364 	/* Mark the witness code as being ready for use. */
365 	atomic_store_rel_int(&witness_cold, 0);
366 
367 	mtx_lock(&Giant);
368 }
369 SYSINIT(witness_init, SI_SUB_WITNESS, SI_ORDER_FIRST, witness_initialize, NULL)
370 
371 void
372 witness_init(struct lock_object *lock)
373 {
374 	struct lock_class *class;
375 
376 	class = lock->lo_class;
377 	if (lock->lo_flags & LO_INITIALIZED)
378 		panic("%s: lock (%s) %s is already initialized", __func__,
379 		    class->lc_name, lock->lo_name);
380 	if ((lock->lo_flags & LO_RECURSABLE) != 0 &&
381 	    (class->lc_flags & LC_RECURSABLE) == 0)
382 		panic("%s: lock (%s) %s can not be recursable", __func__,
383 		    class->lc_name, lock->lo_name);
384 	if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
385 	    (class->lc_flags & LC_SLEEPABLE) == 0)
386 		panic("%s: lock (%s) %s can not be sleepable", __func__,
387 		    class->lc_name, lock->lo_name);
388 	if ((lock->lo_flags & LO_UPGRADABLE) != 0 &&
389 	    (class->lc_flags & LC_UPGRADABLE) == 0)
390 		panic("%s: lock (%s) %s can not be upgradable", __func__,
391 		    class->lc_name, lock->lo_name);
392 
393 	mtx_lock(&all_mtx);
394 	TAILQ_INSERT_TAIL(&all_locks, lock, lo_list);
395 	lock->lo_flags |= LO_INITIALIZED;
396 	lock_cur_cnt++;
397 	if (lock_cur_cnt > lock_max_cnt)
398 		lock_max_cnt = lock_cur_cnt;
399 	mtx_unlock(&all_mtx);
400 	if (!witness_cold && !witness_dead && panicstr == NULL &&
401 	    (lock->lo_flags & LO_WITNESS) != 0)
402 		lock->lo_witness = enroll(lock->lo_type, class);
403 	else
404 		lock->lo_witness = NULL;
405 }
406 
407 void
408 witness_destroy(struct lock_object *lock)
409 {
410 	struct witness *w;
411 
412 	if (witness_cold)
413 		panic("lock (%s) %s destroyed while witness_cold",
414 		    lock->lo_class->lc_name, lock->lo_name);
415 	if ((lock->lo_flags & LO_INITIALIZED) == 0)
416 		panic("%s: lock (%s) %s is not initialized", __func__,
417 		    lock->lo_class->lc_name, lock->lo_name);
418 
419 	/* XXX: need to verify that no one holds the lock */
420 	w = lock->lo_witness;
421 	if (w != NULL) {
422 		mtx_lock_spin(&w_mtx);
423 		MPASS(w->w_refcount > 0);
424 		w->w_refcount--;
425 		mtx_unlock_spin(&w_mtx);
426 	}
427 
428 	mtx_lock(&all_mtx);
429 	lock_cur_cnt--;
430 	TAILQ_REMOVE(&all_locks, lock, lo_list);
431 	lock->lo_flags &= ~LO_INITIALIZED;
432 	mtx_unlock(&all_mtx);
433 }
434 
435 #if defined(DDB)
436 static void
437 witness_display_list(void(*prnt)(const char *fmt, ...),
438 		     struct witness_list *list)
439 {
440 	struct witness *w, *w1;
441 	int found;
442 
443 	STAILQ_FOREACH(w, list, w_typelist) {
444 		if (w->w_file == NULL)
445 			continue;
446 		found = 0;
447 		STAILQ_FOREACH(w1, list, w_typelist) {
448 			if (isitmychild(w1, w)) {
449 				found++;
450 				break;
451 			}
452 		}
453 		if (found)
454 			continue;
455 		/*
456 		 * This lock has no anscestors, display its descendants.
457 		 */
458 		witness_displaydescendants(prnt, w);
459 	}
460 }
461 
462 static void
463 witness_display(void(*prnt)(const char *fmt, ...))
464 {
465 	struct witness *w;
466 
467 	KASSERT(!witness_cold, ("%s: witness_cold", __func__));
468 	witness_levelall();
469 
470 	/*
471 	 * First, handle sleep locks which have been acquired at least
472 	 * once.
473 	 */
474 	prnt("Sleep locks:\n");
475 	witness_display_list(prnt, &w_sleep);
476 
477 	/*
478 	 * Now do spin locks which have been acquired at least once.
479 	 */
480 	prnt("\nSpin locks:\n");
481 	witness_display_list(prnt, &w_spin);
482 
483 	/*
484 	 * Finally, any locks which have not been acquired yet.
485 	 */
486 	prnt("\nLocks which were never acquired:\n");
487 	STAILQ_FOREACH(w, &w_all, w_list) {
488 		if (w->w_file != NULL || w->w_refcount == 0)
489 			continue;
490 		prnt("%s\n", w->w_name);
491 	}
492 }
493 #endif
494 
495 void
496 witness_lock(struct lock_object *lock, int flags, const char *file, int line)
497 {
498 	struct lock_list_entry **lock_list, *lle;
499 	struct lock_instance *lock1, *lock2;
500 	struct lock_class *class;
501 	struct witness *w, *w1;
502 	struct thread *td;
503 	int i, j;
504 #ifdef DDB
505 	int go_into_ddb = 0;
506 #endif /* DDB */
507 
508 	if (witness_cold || witness_dead || lock->lo_witness == NULL ||
509 	    panicstr != NULL)
510 		return;
511 	w = lock->lo_witness;
512 	class = lock->lo_class;
513 	td = curthread;
514 
515 	if (class->lc_flags & LC_SLEEPLOCK) {
516 		/*
517 		 * Since spin locks include a critical section, this check
518 		 * impliclty enforces a lock order of all sleep locks before
519 		 * all spin locks.
520 		 */
521 		if (td->td_critnest != 0 && (flags & LOP_TRYLOCK) == 0)
522 			panic("blockable sleep lock (%s) %s @ %s:%d",
523 			    class->lc_name, lock->lo_name, file, line);
524 		lock_list = &td->td_sleeplocks;
525 	} else
526 		lock_list = PCPU_PTR(spinlocks);
527 
528 	/*
529 	 * Try locks do not block if they fail to acquire the lock, thus
530 	 * there is no danger of deadlocks or of switching while holding a
531 	 * spin lock if we acquire a lock via a try operation.
532 	 */
533 	if (flags & LOP_TRYLOCK)
534 		goto out;
535 
536 	/*
537 	 * Is this the first lock acquired?  If so, then no order checking
538 	 * is needed.
539 	 */
540 	if (*lock_list == NULL)
541 		goto out;
542 
543 	/*
544 	 * Check to see if we are recursing on a lock we already own.
545 	 */
546 	lock1 = find_instance(*lock_list, lock);
547 	if (lock1 != NULL) {
548 		if ((lock1->li_flags & LI_EXCLUSIVE) != 0 &&
549 		    (flags & LOP_EXCLUSIVE) == 0) {
550 			printf("shared lock of (%s) %s @ %s:%d\n",
551 			    class->lc_name, lock->lo_name, file, line);
552 			printf("while exclusively locked from %s:%d\n",
553 			    lock1->li_file, lock1->li_line);
554 			panic("share->excl");
555 		}
556 		if ((lock1->li_flags & LI_EXCLUSIVE) == 0 &&
557 		    (flags & LOP_EXCLUSIVE) != 0) {
558 			printf("exclusive lock of (%s) %s @ %s:%d\n",
559 			    class->lc_name, lock->lo_name, file, line);
560 			printf("while share locked from %s:%d\n",
561 			    lock1->li_file, lock1->li_line);
562 			panic("excl->share");
563 		}
564 		lock1->li_flags++;
565 		if ((lock->lo_flags & LO_RECURSABLE) == 0) {
566 			printf(
567 			"recursed on non-recursive lock (%s) %s @ %s:%d\n",
568 			    class->lc_name, lock->lo_name, file, line);
569 			printf("first acquired @ %s:%d\n", lock1->li_file,
570 			    lock1->li_line);
571 			panic("recurse");
572 		}
573 		CTR4(KTR_WITNESS, "%s: pid %d recursed on %s r=%d", __func__,
574 		    td->td_proc->p_pid, lock->lo_name,
575 		    lock1->li_flags & LI_RECURSEMASK);
576 		lock1->li_file = file;
577 		lock1->li_line = line;
578 		return;
579 	}
580 
581 	/*
582 	 * Check for duplicate locks of the same type.  Note that we only
583 	 * have to check for this on the last lock we just acquired.  Any
584 	 * other cases will be caught as lock order violations.
585 	 */
586 	lock1 = &(*lock_list)->ll_children[(*lock_list)->ll_count - 1];
587 	w1 = lock1->li_lock->lo_witness;
588 	if (w1 == w) {
589 		if (w->w_same_squawked || (lock->lo_flags & LO_DUPOK))
590 			goto out;
591 		w->w_same_squawked = 1;
592 		printf("acquiring duplicate lock of same type: \"%s\"\n",
593 			lock->lo_type);
594 		printf(" 1st %s @ %s:%d\n", lock1->li_lock->lo_name,
595 		    lock1->li_file, lock1->li_line);
596 		printf(" 2nd %s @ %s:%d\n", lock->lo_name, file, line);
597 #ifdef DDB
598 		go_into_ddb = 1;
599 #endif /* DDB */
600 		goto out;
601 	}
602 	MPASS(!mtx_owned(&w_mtx));
603 	mtx_lock_spin(&w_mtx);
604 	/*
605 	 * If we have a known higher number just say ok
606 	 */
607 	if (witness_watch > 1 && w->w_level > w1->w_level) {
608 		mtx_unlock_spin(&w_mtx);
609 		goto out;
610 	}
611 	if (isitmydescendant(w1, w)) {
612 		mtx_unlock_spin(&w_mtx);
613 		goto out;
614 	}
615 	for (j = 0, lle = *lock_list; lle != NULL; lle = lle->ll_next) {
616 		for (i = lle->ll_count - 1; i >= 0; i--, j++) {
617 
618 			MPASS(j < WITNESS_COUNT);
619 			lock1 = &lle->ll_children[i];
620 			w1 = lock1->li_lock->lo_witness;
621 
622 			/*
623 			 * If this lock doesn't undergo witness checking,
624 			 * then skip it.
625 			 */
626 			if (w1 == NULL) {
627 				KASSERT((lock1->li_lock->lo_flags & LO_WITNESS) == 0,
628 				    ("lock missing witness structure"));
629 				continue;
630 			}
631 			/*
632 			 * If we are locking Giant and we slept with this
633 			 * lock, then skip it.
634 			 */
635 			if ((lock1->li_flags & LI_SLEPT) != 0 &&
636 			    lock == &Giant.mtx_object)
637 				continue;
638 			/*
639 			 * If we are locking a sleepable lock and this lock
640 			 * isn't sleepable and isn't Giant, we want to treat
641 			 * it as a lock order violation to enfore a general
642 			 * lock order of sleepable locks before non-sleepable
643 			 * locks.  Thus, we only bother checking the lock
644 			 * order hierarchy if we pass the initial test.
645 			 */
646 			if (!((lock->lo_flags & LO_SLEEPABLE) != 0 &&
647 			    ((lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0 &&
648 			    lock1->li_lock != &Giant.mtx_object)) &&
649 			    !isitmydescendant(w, w1))
650 				continue;
651 			/*
652 			 * We have a lock order violation, check to see if it
653 			 * is allowed or has already been yelled about.
654 			 */
655 			mtx_unlock_spin(&w_mtx);
656 #ifdef BLESSING
657 			if (blessed(w, w1))
658 				goto out;
659 #endif
660 			if (lock1->li_lock == &Giant.mtx_object) {
661 				if (w1->w_Giant_squawked)
662 					goto out;
663 				else
664 					w1->w_Giant_squawked = 1;
665 			} else {
666 				if (w1->w_other_squawked)
667 					goto out;
668 				else
669 					w1->w_other_squawked = 1;
670 			}
671 			/*
672 			 * Ok, yell about it.
673 			 */
674 			printf("lock order reversal\n");
675 			/*
676 			 * Try to locate an earlier lock with
677 			 * witness w in our list.
678 			 */
679 			do {
680 				lock2 = &lle->ll_children[i];
681 				MPASS(lock2->li_lock != NULL);
682 				if (lock2->li_lock->lo_witness == w)
683 					break;
684 				i--;
685 				if (i == 0 && lle->ll_next != NULL) {
686 					lle = lle->ll_next;
687 					i = lle->ll_count - 1;
688 					MPASS(i >= 0 && i < LOCK_NCHILDREN);
689 				}
690 			} while (i >= 0);
691 			if (i < 0) {
692 				printf(" 1st %p %s (%s) @ %s:%d\n",
693 				    lock1->li_lock, lock1->li_lock->lo_name,
694 				    lock1->li_lock->lo_type, lock1->li_file,
695 				    lock1->li_line);
696 				printf(" 2nd %p %s (%s) @ %s:%d\n", lock,
697 				    lock->lo_name, lock->lo_type, file, line);
698 			} else {
699 				printf(" 1st %p %s (%s) @ %s:%d\n",
700 				    lock2->li_lock, lock2->li_lock->lo_name,
701 				    lock2->li_lock->lo_type, lock2->li_file,
702 				    lock2->li_line);
703 				printf(" 2nd %p %s (%s) @ %s:%d\n",
704 				    lock1->li_lock, lock1->li_lock->lo_name,
705 				    lock1->li_lock->lo_type, lock1->li_file,
706 				    lock1->li_line);
707 				printf(" 3rd %p %s (%s) @ %s:%d\n", lock,
708 				    lock->lo_name, lock->lo_type, file, line);
709 			}
710 #ifdef DDB
711 			go_into_ddb = 1;
712 #endif /* DDB */
713 			goto out;
714 		}
715 	}
716 	lock1 = &(*lock_list)->ll_children[(*lock_list)->ll_count - 1];
717 	/*
718 	 * Don't build a new relationship if we are locking Giant just
719 	 * after waking up and the previous lock in the list was acquired
720 	 * prior to blocking.
721 	 */
722 	if (lock == &Giant.mtx_object && (lock1->li_flags & LI_SLEPT) != 0)
723 		mtx_unlock_spin(&w_mtx);
724 	else {
725 		CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__,
726 		    lock->lo_type, lock1->li_lock->lo_type);
727 		if (!itismychild(lock1->li_lock->lo_witness, w))
728 			mtx_unlock_spin(&w_mtx);
729 	}
730 
731 out:
732 #ifdef DDB
733 	if (go_into_ddb) {
734 		if (witness_trace)
735 			backtrace();
736 		if (witness_ddb)
737 			Debugger(__func__);
738 	}
739 #endif /* DDB */
740 	w->w_file = file;
741 	w->w_line = line;
742 
743 	lle = *lock_list;
744 	if (lle == NULL || lle->ll_count == LOCK_NCHILDREN) {
745 		lle = witness_lock_list_get();
746 		if (lle == NULL)
747 			return;
748 		lle->ll_next = *lock_list;
749 		CTR3(KTR_WITNESS, "%s: pid %d added lle %p", __func__,
750 		    td->td_proc->p_pid, lle);
751 		*lock_list = lle;
752 	}
753 	lock1 = &lle->ll_children[lle->ll_count++];
754 	lock1->li_lock = lock;
755 	lock1->li_line = line;
756 	lock1->li_file = file;
757 	if ((flags & LOP_EXCLUSIVE) != 0)
758 		lock1->li_flags = LI_EXCLUSIVE;
759 	else
760 		lock1->li_flags = 0;
761 	CTR4(KTR_WITNESS, "%s: pid %d added %s as lle[%d]", __func__,
762 	    td->td_proc->p_pid, lock->lo_name, lle->ll_count - 1);
763 }
764 
765 void
766 witness_upgrade(struct lock_object *lock, int flags, const char *file, int line)
767 {
768 	struct lock_instance *instance;
769 	struct lock_class *class;
770 
771 	KASSERT(!witness_cold, ("%s: witness_cold", __func__));
772 	if (lock->lo_witness == NULL || witness_dead || panicstr != NULL)
773 		return;
774 	class = lock->lo_class;
775 	if ((lock->lo_flags & LO_UPGRADABLE) == 0)
776 		panic("upgrade of non-upgradable lock (%s) %s @ %s:%d",
777 		    class->lc_name, lock->lo_name, file, line);
778 	if ((flags & LOP_TRYLOCK) == 0)
779 		panic("non-try upgrade of lock (%s) %s @ %s:%d", class->lc_name,
780 		    lock->lo_name, file, line);
781 	if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
782 		panic("upgrade of non-sleep lock (%s) %s @ %s:%d",
783 		    class->lc_name, lock->lo_name, file, line);
784 	instance = find_instance(curthread->td_sleeplocks, lock);
785 	if (instance == NULL)
786 		panic("upgrade of unlocked lock (%s) %s @ %s:%d",
787 		    class->lc_name, lock->lo_name, file, line);
788 	if ((instance->li_flags & LI_EXCLUSIVE) != 0)
789 		panic("upgrade of exclusive lock (%s) %s @ %s:%d",
790 		    class->lc_name, lock->lo_name, file, line);
791 	if ((instance->li_flags & LI_RECURSEMASK) != 0)
792 		panic("upgrade of recursed lock (%s) %s r=%d @ %s:%d",
793 		    class->lc_name, lock->lo_name,
794 		    instance->li_flags & LI_RECURSEMASK, file, line);
795 	instance->li_flags |= LI_EXCLUSIVE;
796 }
797 
798 void
799 witness_downgrade(struct lock_object *lock, int flags, const char *file,
800     int line)
801 {
802 	struct lock_instance *instance;
803 	struct lock_class *class;
804 
805 	KASSERT(!witness_cold, ("%s: witness_cold", __func__));
806 	if (lock->lo_witness == NULL || witness_dead || panicstr != NULL)
807 		return;
808 	class = lock->lo_class;
809 	if ((lock->lo_flags & LO_UPGRADABLE) == 0)
810 		panic("downgrade of non-upgradable lock (%s) %s @ %s:%d",
811 		    class->lc_name, lock->lo_name, file, line);
812 	if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
813 		panic("downgrade of non-sleep lock (%s) %s @ %s:%d",
814 		    class->lc_name, lock->lo_name, file, line);
815 	instance = find_instance(curthread->td_sleeplocks, lock);
816 	if (instance == NULL)
817 		panic("downgrade of unlocked lock (%s) %s @ %s:%d",
818 		    class->lc_name, lock->lo_name, file, line);
819 	if ((instance->li_flags & LI_EXCLUSIVE) == 0)
820 		panic("downgrade of shared lock (%s) %s @ %s:%d",
821 		    class->lc_name, lock->lo_name, file, line);
822 	if ((instance->li_flags & LI_RECURSEMASK) != 0)
823 		panic("downgrade of recursed lock (%s) %s r=%d @ %s:%d",
824 		    class->lc_name, lock->lo_name,
825 		    instance->li_flags & LI_RECURSEMASK, file, line);
826 	instance->li_flags &= ~LI_EXCLUSIVE;
827 }
828 
829 void
830 witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
831 {
832 	struct lock_list_entry **lock_list, *lle;
833 	struct lock_instance *instance;
834 	struct lock_class *class;
835 	struct thread *td;
836 	register_t s;
837 	int i, j;
838 
839 	if (witness_cold || witness_dead || lock->lo_witness == NULL ||
840 	    panicstr != NULL)
841 		return;
842 	td = curthread;
843 	class = lock->lo_class;
844 	if (class->lc_flags & LC_SLEEPLOCK)
845 		lock_list = &td->td_sleeplocks;
846 	else
847 		lock_list = PCPU_PTR(spinlocks);
848 	for (; *lock_list != NULL; lock_list = &(*lock_list)->ll_next)
849 		for (i = 0; i < (*lock_list)->ll_count; i++) {
850 			instance = &(*lock_list)->ll_children[i];
851 			if (instance->li_lock == lock) {
852 				if ((instance->li_flags & LI_EXCLUSIVE) != 0 &&
853 				    (flags & LOP_EXCLUSIVE) == 0) {
854 					printf(
855 					"shared unlock of (%s) %s @ %s:%d\n",
856 					    class->lc_name, lock->lo_name,
857 					    file, line);
858 					printf(
859 					"while exclusively locked from %s:%d\n",
860 					    instance->li_file,
861 					    instance->li_line);
862 					panic("excl->ushare");
863 				}
864 				if ((instance->li_flags & LI_EXCLUSIVE) == 0 &&
865 				    (flags & LOP_EXCLUSIVE) != 0) {
866 					printf(
867 					"exclusive unlock of (%s) %s @ %s:%d\n",
868 					    class->lc_name, lock->lo_name,
869 					    file, line);
870 					printf(
871 					"while share locked from %s:%d\n",
872 					    instance->li_file,
873 					    instance->li_line);
874 					panic("share->uexcl");
875 				}
876 				/* If we are recursed, unrecurse. */
877 				if ((instance->li_flags & LI_RECURSEMASK) > 0) {
878 					CTR4(KTR_WITNESS,
879 				    "%s: pid %d unrecursed on %s r=%d", __func__,
880 					    td->td_proc->p_pid,
881 					    instance->li_lock->lo_name,
882 					    instance->li_flags);
883 					instance->li_flags--;
884 					return;
885 				}
886 				s = intr_disable();
887 				CTR4(KTR_WITNESS,
888 				    "%s: pid %d removed %s from lle[%d]", __func__,
889 				    td->td_proc->p_pid,
890 				    instance->li_lock->lo_name,
891 				    (*lock_list)->ll_count - 1);
892 				for (j = i; j < (*lock_list)->ll_count - 1; j++)
893 					(*lock_list)->ll_children[j] =
894 					    (*lock_list)->ll_children[j + 1];
895 				(*lock_list)->ll_count--;
896 				intr_restore(s);
897 				if ((*lock_list)->ll_count == 0) {
898 					lle = *lock_list;
899 					*lock_list = lle->ll_next;
900 					CTR3(KTR_WITNESS,
901 					    "%s: pid %d removed lle %p", __func__,
902 					    td->td_proc->p_pid, lle);
903 					witness_lock_list_free(lle);
904 				}
905 				return;
906 			}
907 		}
908 	panic("lock (%s) %s not locked @ %s:%d", class->lc_name, lock->lo_name,
909 	    file, line);
910 }
911 
912 /*
913  * Warn if any held locks are not sleepable.  Note that Giant and the lock
914  * passed in are both special cases since they are both released during the
915  * sleep process and aren't actually held while the thread is asleep.
916  */
917 int
918 witness_sleep(int check_only, struct lock_object *lock, const char *file,
919 	      int line)
920 {
921 	struct lock_list_entry **lock_list, *lle;
922 	struct lock_instance *lock1;
923 	struct thread *td;
924 	int i, n;
925 
926 	if (witness_cold || witness_dead || panicstr != NULL)
927 		return (0);
928 	n = 0;
929 	td = curthread;
930 	lock_list = &td->td_sleeplocks;
931 again:
932 	for (lle = *lock_list; lle != NULL; lle = lle->ll_next)
933 		for (i = lle->ll_count - 1; i >= 0; i--) {
934 			lock1 = &lle->ll_children[i];
935 			if (lock1->li_lock == lock ||
936 			    lock1->li_lock == &Giant.mtx_object)
937 				continue;
938 			if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) != 0) {
939 				if (check_only == 0) {
940 					CTR3(KTR_WITNESS,
941 				    "pid %d: sleeping with lock (%s) %s held",
942 					    td->td_proc->p_pid,
943 					    lock1->li_lock->lo_class->lc_name,
944 					    lock1->li_lock->lo_name);
945 					lock1->li_flags |= LI_SLEPT;
946 				}
947 				continue;
948 			}
949 			n++;
950 			printf("%s:%d: %s with \"%s\" locked from %s:%d\n",
951 			    file, line, check_only ? "could sleep" : "sleeping",
952 			    lock1->li_lock->lo_name, lock1->li_file,
953 			    lock1->li_line);
954 		}
955 	if (lock_list == &td->td_sleeplocks && PCPU_GET(spinlocks) != NULL) {
956 		/*
957 		 * Since we already hold a spinlock preemption is
958 		 * already blocked.
959 		 */
960 		lock_list = PCPU_PTR(spinlocks);
961 		goto again;
962 	}
963 #ifdef DDB
964 	if (witness_ddb && n)
965 		Debugger(__func__);
966 #endif /* DDB */
967 	return (n);
968 }
969 
970 const char *
971 witness_file(struct lock_object *lock)
972 {
973 	struct witness *w;
974 
975 	if (witness_cold || witness_dead || lock->lo_witness == NULL)
976 		return ("?");
977 	w = lock->lo_witness;
978 	return (w->w_file);
979 }
980 
981 int
982 witness_line(struct lock_object *lock)
983 {
984 	struct witness *w;
985 
986 	if (witness_cold || witness_dead || lock->lo_witness == NULL)
987 		return (0);
988 	w = lock->lo_witness;
989 	return (w->w_line);
990 }
991 
992 static struct witness *
993 enroll(const char *description, struct lock_class *lock_class)
994 {
995 	struct witness *w;
996 
997 	if (!witness_watch || witness_dead || panicstr != NULL)
998 		return (NULL);
999 	if ((lock_class->lc_flags & LC_SPINLOCK) && witness_skipspin)
1000 		return (NULL);
1001 	mtx_lock_spin(&w_mtx);
1002 	STAILQ_FOREACH(w, &w_all, w_list) {
1003 		if (w->w_name == description || (w->w_refcount > 0 &&
1004 		    strcmp(description, w->w_name) == 0)) {
1005 			w->w_refcount++;
1006 			mtx_unlock_spin(&w_mtx);
1007 			if (lock_class != w->w_class)
1008 				panic(
1009 				"lock (%s) %s does not match earlier (%s) lock",
1010 				    description, lock_class->lc_name,
1011 				    w->w_class->lc_name);
1012 			return (w);
1013 		}
1014 	}
1015 	/*
1016 	 * This isn't quite right, as witness_cold is still 0 while we
1017 	 * enroll all the locks initialized before witness_initialize().
1018 	 */
1019 	if ((lock_class->lc_flags & LC_SPINLOCK) && !witness_cold) {
1020 		mtx_unlock_spin(&w_mtx);
1021 		panic("spin lock %s not in order list", description);
1022 	}
1023 	if ((w = witness_get()) == NULL)
1024 		return (NULL);
1025 	w->w_name = description;
1026 	w->w_class = lock_class;
1027 	w->w_refcount = 1;
1028 	STAILQ_INSERT_HEAD(&w_all, w, w_list);
1029 	if (lock_class->lc_flags & LC_SPINLOCK)
1030 		STAILQ_INSERT_HEAD(&w_spin, w, w_typelist);
1031 	else if (lock_class->lc_flags & LC_SLEEPLOCK)
1032 		STAILQ_INSERT_HEAD(&w_sleep, w, w_typelist);
1033 	else {
1034 		mtx_unlock_spin(&w_mtx);
1035 		panic("lock class %s is not sleep or spin",
1036 		    lock_class->lc_name);
1037 	}
1038 	mtx_unlock_spin(&w_mtx);
1039 	return (w);
1040 }
1041 
1042 static int
1043 itismychild(struct witness *parent, struct witness *child)
1044 {
1045 	static int recursed;
1046 	struct witness_child_list_entry **wcl;
1047 	struct witness_list *list;
1048 
1049 	MPASS(child != NULL && parent != NULL);
1050 	if ((parent->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) !=
1051 	    (child->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)))
1052 		panic(
1053 		"%s: parent (%s) and child (%s) are not the same lock type",
1054 		    __func__, parent->w_class->lc_name,
1055 		    child->w_class->lc_name);
1056 
1057 	/*
1058 	 * Insert "child" after "parent"
1059 	 */
1060 	wcl = &parent->w_children;
1061 	while (*wcl != NULL && (*wcl)->wcl_count == WITNESS_NCHILDREN)
1062 		wcl = &(*wcl)->wcl_next;
1063 	if (*wcl == NULL) {
1064 		*wcl = witness_child_get();
1065 		if (*wcl == NULL)
1066 			return (1);
1067 	}
1068 	(*wcl)->wcl_children[(*wcl)->wcl_count++] = child;
1069 
1070 	/*
1071 	 * Now prune whole tree.  We look for cases where a lock is now
1072 	 * both a descendant and a direct child of a given lock.  In that
1073 	 * case, we want to remove the direct child link from the tree.
1074 	 */
1075 	if (recursed)
1076 		return (0);
1077 	recursed = 1;
1078 	if (parent->w_class->lc_flags & LC_SLEEPLOCK)
1079 		list = &w_sleep;
1080 	else
1081 		list = &w_spin;
1082 	STAILQ_FOREACH(child, list, w_typelist) {
1083 		STAILQ_FOREACH(parent, list, w_typelist) {
1084 			if (!isitmychild(parent, child))
1085 				continue;
1086 			removechild(parent, child);
1087 			if (isitmydescendant(parent, child))
1088 				continue;
1089 			itismychild(parent, child);
1090 		}
1091 	}
1092 	recursed = 0;
1093 	witness_levelall();
1094 	return (0);
1095 }
1096 
1097 static void
1098 removechild(struct witness *parent, struct witness *child)
1099 {
1100 	struct witness_child_list_entry **wcl, *wcl1;
1101 	int i;
1102 
1103 	for (wcl = &parent->w_children; *wcl != NULL; wcl = &(*wcl)->wcl_next)
1104 		for (i = 0; i < (*wcl)->wcl_count; i++)
1105 			if ((*wcl)->wcl_children[i] == child)
1106 				goto found;
1107 	return;
1108 found:
1109 	(*wcl)->wcl_count--;
1110 	if ((*wcl)->wcl_count > i)
1111 		(*wcl)->wcl_children[i] =
1112 		    (*wcl)->wcl_children[(*wcl)->wcl_count];
1113 	MPASS((*wcl)->wcl_children[i] != NULL);
1114 	if ((*wcl)->wcl_count != 0)
1115 		return;
1116 	wcl1 = *wcl;
1117 	*wcl = wcl1->wcl_next;
1118 	witness_child_free(wcl1);
1119 }
1120 
1121 static int
1122 isitmychild(struct witness *parent, struct witness *child)
1123 {
1124 	struct witness_child_list_entry *wcl;
1125 	int i;
1126 
1127 	for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
1128 		for (i = 0; i < wcl->wcl_count; i++) {
1129 			if (wcl->wcl_children[i] == child)
1130 				return (1);
1131 		}
1132 	}
1133 	return (0);
1134 }
1135 
1136 static int
1137 isitmydescendant(struct witness *parent, struct witness *child)
1138 {
1139 	struct witness_child_list_entry *wcl;
1140 	int i, j;
1141 
1142 	if (isitmychild(parent, child))
1143 		return (1);
1144 	j = 0;
1145 	for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
1146 		MPASS(j < 1000);
1147 		for (i = 0; i < wcl->wcl_count; i++) {
1148 			if (isitmydescendant(wcl->wcl_children[i], child))
1149 				return (1);
1150 		}
1151 		j++;
1152 	}
1153 	return (0);
1154 }
1155 
1156 static void
1157 witness_levelall (void)
1158 {
1159 	struct witness_list *list;
1160 	struct witness *w, *w1;
1161 
1162 	/*
1163 	 * First clear all levels.
1164 	 */
1165 	STAILQ_FOREACH(w, &w_all, w_list) {
1166 		w->w_level = 0;
1167 	}
1168 
1169 	/*
1170 	 * Look for locks with no parent and level all their descendants.
1171 	 */
1172 	STAILQ_FOREACH(w, &w_all, w_list) {
1173 		/*
1174 		 * This is just an optimization, technically we could get
1175 		 * away just walking the all list each time.
1176 		 */
1177 		if (w->w_class->lc_flags & LC_SLEEPLOCK)
1178 			list = &w_sleep;
1179 		else
1180 			list = &w_spin;
1181 		STAILQ_FOREACH(w1, list, w_typelist) {
1182 			if (isitmychild(w1, w))
1183 				goto skip;
1184 		}
1185 		witness_leveldescendents(w, 0);
1186 	skip:
1187 		;	/* silence GCC 3.x */
1188 	}
1189 }
1190 
1191 static void
1192 witness_leveldescendents(struct witness *parent, int level)
1193 {
1194 	struct witness_child_list_entry *wcl;
1195 	int i;
1196 
1197 	if (parent->w_level < level)
1198 		parent->w_level = level;
1199 	level++;
1200 	for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
1201 		for (i = 0; i < wcl->wcl_count; i++)
1202 			witness_leveldescendents(wcl->wcl_children[i], level);
1203 }
1204 
1205 static void
1206 witness_displaydescendants(void(*prnt)(const char *fmt, ...),
1207 			   struct witness *parent)
1208 {
1209 	struct witness_child_list_entry *wcl;
1210 	int i, level;
1211 
1212 	level = parent->w_level;
1213 	prnt("%-2d", level);
1214 	for (i = 0; i < level; i++)
1215 		prnt(" ");
1216 	if (parent->w_refcount > 0) {
1217 		prnt("%s", parent->w_name);
1218 		if (parent->w_file != NULL)
1219 			prnt(" -- last acquired @ %s:%d\n", parent->w_file,
1220 			    parent->w_line);
1221 	} else
1222 		prnt("(dead)\n");
1223 	for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
1224 		for (i = 0; i < wcl->wcl_count; i++)
1225 			    witness_displaydescendants(prnt,
1226 				wcl->wcl_children[i]);
1227 }
1228 
1229 #ifdef BLESSING
1230 static int
1231 blessed(struct witness *w1, struct witness *w2)
1232 {
1233 	int i;
1234 	struct witness_blessed *b;
1235 
1236 	for (i = 0; i < blessed_count; i++) {
1237 		b = &blessed_list[i];
1238 		if (strcmp(w1->w_name, b->b_lock1) == 0) {
1239 			if (strcmp(w2->w_name, b->b_lock2) == 0)
1240 				return (1);
1241 			continue;
1242 		}
1243 		if (strcmp(w1->w_name, b->b_lock2) == 0)
1244 			if (strcmp(w2->w_name, b->b_lock1) == 0)
1245 				return (1);
1246 	}
1247 	return (0);
1248 }
1249 #endif
1250 
1251 static struct witness *
1252 witness_get(void)
1253 {
1254 	struct witness *w;
1255 
1256 	if (witness_dead) {
1257 		mtx_unlock_spin(&w_mtx);
1258 		return (NULL);
1259 	}
1260 	if (STAILQ_EMPTY(&w_free)) {
1261 		witness_dead = 1;
1262 		mtx_unlock_spin(&w_mtx);
1263 		printf("%s: witness exhausted\n", __func__);
1264 		return (NULL);
1265 	}
1266 	w = STAILQ_FIRST(&w_free);
1267 	STAILQ_REMOVE_HEAD(&w_free, w_list);
1268 	bzero(w, sizeof(*w));
1269 	return (w);
1270 }
1271 
1272 static void
1273 witness_free(struct witness *w)
1274 {
1275 
1276 	STAILQ_INSERT_HEAD(&w_free, w, w_list);
1277 }
1278 
1279 static struct witness_child_list_entry *
1280 witness_child_get(void)
1281 {
1282 	struct witness_child_list_entry *wcl;
1283 
1284 	if (witness_dead) {
1285 		mtx_unlock_spin(&w_mtx);
1286 		return (NULL);
1287 	}
1288 	wcl = w_child_free;
1289 	if (wcl == NULL) {
1290 		witness_dead = 1;
1291 		mtx_unlock_spin(&w_mtx);
1292 		printf("%s: witness exhausted\n", __func__);
1293 		return (NULL);
1294 	}
1295 	w_child_free = wcl->wcl_next;
1296 	bzero(wcl, sizeof(*wcl));
1297 	return (wcl);
1298 }
1299 
1300 static void
1301 witness_child_free(struct witness_child_list_entry *wcl)
1302 {
1303 
1304 	wcl->wcl_next = w_child_free;
1305 	w_child_free = wcl;
1306 }
1307 
1308 static struct lock_list_entry *
1309 witness_lock_list_get(void)
1310 {
1311 	struct lock_list_entry *lle;
1312 
1313 	if (witness_dead)
1314 		return (NULL);
1315 	mtx_lock_spin(&w_mtx);
1316 	lle = w_lock_list_free;
1317 	if (lle == NULL) {
1318 		witness_dead = 1;
1319 		mtx_unlock_spin(&w_mtx);
1320 		printf("%s: witness exhausted\n", __func__);
1321 		return (NULL);
1322 	}
1323 	w_lock_list_free = lle->ll_next;
1324 	mtx_unlock_spin(&w_mtx);
1325 	bzero(lle, sizeof(*lle));
1326 	return (lle);
1327 }
1328 
1329 static void
1330 witness_lock_list_free(struct lock_list_entry *lle)
1331 {
1332 
1333 	mtx_lock_spin(&w_mtx);
1334 	lle->ll_next = w_lock_list_free;
1335 	w_lock_list_free = lle;
1336 	mtx_unlock_spin(&w_mtx);
1337 }
1338 
1339 static struct lock_instance *
1340 find_instance(struct lock_list_entry *lock_list, struct lock_object *lock)
1341 {
1342 	struct lock_list_entry *lle;
1343 	struct lock_instance *instance;
1344 	int i;
1345 
1346 	for (lle = lock_list; lle != NULL; lle = lle->ll_next)
1347 		for (i = lle->ll_count - 1; i >= 0; i--) {
1348 			instance = &lle->ll_children[i];
1349 			if (instance->li_lock == lock)
1350 				return (instance);
1351 		}
1352 	return (NULL);
1353 }
1354 
1355 int
1356 witness_list_locks(struct lock_list_entry **lock_list)
1357 {
1358 	struct lock_list_entry *lle;
1359 	struct lock_instance *instance;
1360 	struct lock_object *lock;
1361 	int i, nheld;
1362 
1363 	nheld = 0;
1364 	for (lle = *lock_list; lle != NULL; lle = lle->ll_next)
1365 		for (i = lle->ll_count - 1; i >= 0; i--) {
1366 			instance = &lle->ll_children[i];
1367 			lock = instance->li_lock;
1368 			printf("%s %s %s",
1369 			    (instance->li_flags & LI_EXCLUSIVE) != 0 ?
1370 			    "exclusive" : "shared",
1371 			    lock->lo_class->lc_name, lock->lo_name);
1372 			if (lock->lo_type != lock->lo_name)
1373 				printf(" (%s)", lock->lo_type);
1374 			printf(" r = %d (%p) locked @ %s:%d\n",
1375 			    instance->li_flags & LI_RECURSEMASK, lock,
1376 			    instance->li_file, instance->li_line);
1377 			nheld++;
1378 		}
1379 	return (nheld);
1380 }
1381 
1382 /*
1383  * Calling this on td != curthread is bad unless we are in ddb.
1384  */
1385 int
1386 witness_list(struct thread *td)
1387 {
1388 	int nheld;
1389 
1390 	KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1391 #ifdef DDB
1392 	KASSERT(td == curthread || db_active,
1393 	    ("%s: td != curthread and we aren't in the debugger", __func__));
1394 	if (!db_active && witness_dead)
1395 		return (0);
1396 #else
1397 	KASSERT(td == curthread, ("%s: p != curthread", __func__));
1398 	if (witness_dead)
1399 		return (0);
1400 #endif
1401 	nheld = witness_list_locks(&td->td_sleeplocks);
1402 
1403 	/*
1404 	 * We only handle spinlocks if td == curthread.  This is somewhat broken
1405 	 * if td is currently executing on some other CPU and holds spin locks
1406 	 * as we won't display those locks.  If we had a MI way of getting
1407 	 * the per-cpu data for a given cpu then we could use
1408 	 * td->td_kse->ke_oncpu to get the list of spinlocks for this thread
1409 	 * and "fix" this.
1410 	 *
1411 	 * That still wouldn't really fix this unless we locked sched_lock
1412 	 * or stopped the other CPU to make sure it wasn't changing the list
1413 	 * out from under us.  It is probably best to just not try to handle
1414 	 * threads on other CPU's for now.
1415 	 */
1416 	if (td == curthread && PCPU_GET(spinlocks) != NULL)
1417 		nheld += witness_list_locks(PCPU_PTR(spinlocks));
1418 
1419 	return (nheld);
1420 }
1421 
1422 void
1423 witness_save(struct lock_object *lock, const char **filep, int *linep)
1424 {
1425 	struct lock_instance *instance;
1426 
1427 	KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1428 	if (lock->lo_witness == NULL || witness_dead || panicstr != NULL)
1429 		return;
1430 	if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
1431 		panic("%s: lock (%s) %s is not a sleep lock", __func__,
1432 		    lock->lo_class->lc_name, lock->lo_name);
1433 	instance = find_instance(curthread->td_sleeplocks, lock);
1434 	if (instance == NULL)
1435 		panic("%s: lock (%s) %s not locked", __func__,
1436 		    lock->lo_class->lc_name, lock->lo_name);
1437 	*filep = instance->li_file;
1438 	*linep = instance->li_line;
1439 }
1440 
1441 void
1442 witness_restore(struct lock_object *lock, const char *file, int line)
1443 {
1444 	struct lock_instance *instance;
1445 
1446 	KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1447 	if (lock->lo_witness == NULL || witness_dead || panicstr != NULL)
1448 		return;
1449 	if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
1450 		panic("%s: lock (%s) %s is not a sleep lock", __func__,
1451 		    lock->lo_class->lc_name, lock->lo_name);
1452 	instance = find_instance(curthread->td_sleeplocks, lock);
1453 	if (instance == NULL)
1454 		panic("%s: lock (%s) %s not locked", __func__,
1455 		    lock->lo_class->lc_name, lock->lo_name);
1456 	lock->lo_witness->w_file = file;
1457 	lock->lo_witness->w_line = line;
1458 	instance->li_file = file;
1459 	instance->li_line = line;
1460 }
1461 
1462 void
1463 witness_assert(struct lock_object *lock, int flags, const char *file, int line)
1464 {
1465 #ifdef INVARIANT_SUPPORT
1466 	struct lock_instance *instance;
1467 
1468 	if (lock->lo_witness == NULL || witness_dead || panicstr != NULL)
1469 		return;
1470 	if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) != 0)
1471 		instance = find_instance(curthread->td_sleeplocks, lock);
1472 	else if ((lock->lo_class->lc_flags & LC_SPINLOCK) != 0)
1473 		instance = find_instance(PCPU_GET(spinlocks), lock);
1474 	else {
1475 		panic("Lock (%s) %s is not sleep or spin!",
1476 		    lock->lo_class->lc_name, lock->lo_name);
1477 		return;
1478 	}
1479 	switch (flags) {
1480 	case LA_UNLOCKED:
1481 		if (instance != NULL)
1482 			panic("Lock (%s) %s locked @ %s:%d.",
1483 			    lock->lo_class->lc_name, lock->lo_name, file, line);
1484 		break;
1485 	case LA_LOCKED:
1486 	case LA_LOCKED | LA_RECURSED:
1487 	case LA_LOCKED | LA_NOTRECURSED:
1488 	case LA_SLOCKED:
1489 	case LA_SLOCKED | LA_RECURSED:
1490 	case LA_SLOCKED | LA_NOTRECURSED:
1491 	case LA_XLOCKED:
1492 	case LA_XLOCKED | LA_RECURSED:
1493 	case LA_XLOCKED | LA_NOTRECURSED:
1494 		if (instance == NULL) {
1495 			panic("Lock (%s) %s not locked @ %s:%d.",
1496 			    lock->lo_class->lc_name, lock->lo_name, file, line);
1497 			break;
1498 		}
1499 		if ((flags & LA_XLOCKED) != 0 &&
1500 		    (instance->li_flags & LI_EXCLUSIVE) == 0)
1501 			panic("Lock (%s) %s not exclusively locked @ %s:%d.",
1502 			    lock->lo_class->lc_name, lock->lo_name, file, line);
1503 		if ((flags & LA_SLOCKED) != 0 &&
1504 		    (instance->li_flags & LI_EXCLUSIVE) != 0)
1505 			panic("Lock (%s) %s exclusively locked @ %s:%d.",
1506 			    lock->lo_class->lc_name, lock->lo_name, file, line);
1507 		if ((flags & LA_RECURSED) != 0 &&
1508 		    (instance->li_flags & LI_RECURSEMASK) == 0)
1509 			panic("Lock (%s) %s not recursed @ %s:%d.",
1510 			    lock->lo_class->lc_name, lock->lo_name, file, line);
1511 		if ((flags & LA_NOTRECURSED) != 0 &&
1512 		    (instance->li_flags & LI_RECURSEMASK) != 0)
1513 			panic("Lock (%s) %s recursed @ %s:%d.",
1514 			    lock->lo_class->lc_name, lock->lo_name, file, line);
1515 		break;
1516 	default:
1517 		panic("Invalid lock assertion at %s:%d.", file, line);
1518 
1519 	}
1520 #endif	/* INVARIANT_SUPPORT */
1521 }
1522 
1523 #ifdef DDB
1524 
1525 DB_SHOW_COMMAND(locks, db_witness_list)
1526 {
1527 	struct thread *td;
1528 	pid_t pid;
1529 	struct proc *p;
1530 
1531 	if (have_addr) {
1532 		pid = (addr % 16) + ((addr >> 4) % 16) * 10 +
1533 		    ((addr >> 8) % 16) * 100 + ((addr >> 12) % 16) * 1000 +
1534 		    ((addr >> 16) % 16) * 10000;
1535 		/* sx_slock(&allproc_lock); */
1536 		FOREACH_PROC_IN_SYSTEM(p) {
1537 			if (p->p_pid == pid)
1538 				break;
1539 		}
1540 		/* sx_sunlock(&allproc_lock); */
1541 		if (p == NULL) {
1542 			db_printf("pid %d not found\n", pid);
1543 			return;
1544 		}
1545 		FOREACH_THREAD_IN_PROC(p, td) {
1546 			witness_list(td);
1547 		}
1548 	} else {
1549 		td = curthread;
1550 		witness_list(td);
1551 	}
1552 }
1553 
1554 DB_SHOW_COMMAND(witness, db_witness_display)
1555 {
1556 
1557 	witness_display(db_printf);
1558 }
1559 #endif
1560