xref: /freebsd/sys/kern/kern_mutex.c (revision 675be9115aae86ad6b3d877155d4fd7822892105)
1 /*-
2  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  * 3. Berkeley Software Design Inc's name may not be used to endorse or
13  *    promote products derived from this software without specific prior
14  *    written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  *	from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
29  *	and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30  */
31 
32 /*
33  * Machine independent bits of mutex implementation.
34  */
35 
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38 
39 #include "opt_adaptive_mutexes.h"
40 #include "opt_ddb.h"
41 #include "opt_global.h"
42 #include "opt_kdtrace.h"
43 #include "opt_sched.h"
44 
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/bus.h>
48 #include <sys/conf.h>
49 #include <sys/kdb.h>
50 #include <sys/kernel.h>
51 #include <sys/ktr.h>
52 #include <sys/lock.h>
53 #include <sys/malloc.h>
54 #include <sys/mutex.h>
55 #include <sys/proc.h>
56 #include <sys/resourcevar.h>
57 #include <sys/sched.h>
58 #include <sys/sbuf.h>
59 #include <sys/sysctl.h>
60 #include <sys/turnstile.h>
61 #include <sys/vmmeter.h>
62 #include <sys/lock_profile.h>
63 
64 #include <machine/atomic.h>
65 #include <machine/bus.h>
66 #include <machine/cpu.h>
67 
68 #include <ddb/ddb.h>
69 
70 #include <fs/devfs/devfs_int.h>
71 
72 #include <vm/vm.h>
73 #include <vm/vm_extern.h>
74 
75 #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
76 #define	ADAPTIVE_MUTEXES
77 #endif
78 
79 /*
80  * Internal utility macros.
81  */
82 #define mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)
83 
84 #define	mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED)
85 
86 #define	mtx_owner(m)	((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK))
87 
88 static void	assert_mtx(const struct lock_object *lock, int what);
89 #ifdef DDB
90 static void	db_show_mtx(const struct lock_object *lock);
91 #endif
92 static void	lock_mtx(struct lock_object *lock, int how);
93 static void	lock_spin(struct lock_object *lock, int how);
94 #ifdef KDTRACE_HOOKS
95 static int	owner_mtx(const struct lock_object *lock,
96 		    struct thread **owner);
97 #endif
98 static int	unlock_mtx(struct lock_object *lock);
99 static int	unlock_spin(struct lock_object *lock);
100 
101 /*
102  * Lock classes for sleep and spin mutexes.
103  */
104 struct lock_class lock_class_mtx_sleep = {
105 	.lc_name = "sleep mutex",
106 	.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
107 	.lc_assert = assert_mtx,
108 #ifdef DDB
109 	.lc_ddb_show = db_show_mtx,
110 #endif
111 	.lc_lock = lock_mtx,
112 	.lc_unlock = unlock_mtx,
113 #ifdef KDTRACE_HOOKS
114 	.lc_owner = owner_mtx,
115 #endif
116 };
117 struct lock_class lock_class_mtx_spin = {
118 	.lc_name = "spin mutex",
119 	.lc_flags = LC_SPINLOCK | LC_RECURSABLE,
120 	.lc_assert = assert_mtx,
121 #ifdef DDB
122 	.lc_ddb_show = db_show_mtx,
123 #endif
124 	.lc_lock = lock_spin,
125 	.lc_unlock = unlock_spin,
126 #ifdef KDTRACE_HOOKS
127 	.lc_owner = owner_mtx,
128 #endif
129 };
130 
131 /*
132  * System-wide mutexes
133  */
134 struct mtx blocked_lock;
135 struct mtx Giant;
136 
137 void
138 assert_mtx(const struct lock_object *lock, int what)
139 {
140 
141 	mtx_assert((const struct mtx *)lock, what);
142 }
143 
144 void
145 lock_mtx(struct lock_object *lock, int how)
146 {
147 
148 	mtx_lock((struct mtx *)lock);
149 }
150 
151 void
152 lock_spin(struct lock_object *lock, int how)
153 {
154 
155 	panic("spin locks can only use msleep_spin");
156 }
157 
158 int
159 unlock_mtx(struct lock_object *lock)
160 {
161 	struct mtx *m;
162 
163 	m = (struct mtx *)lock;
164 	mtx_assert(m, MA_OWNED | MA_NOTRECURSED);
165 	mtx_unlock(m);
166 	return (0);
167 }
168 
169 int
170 unlock_spin(struct lock_object *lock)
171 {
172 
173 	panic("spin locks can only use msleep_spin");
174 }
175 
176 #ifdef KDTRACE_HOOKS
177 int
178 owner_mtx(const struct lock_object *lock, struct thread **owner)
179 {
180 	const struct mtx *m = (const struct mtx *)lock;
181 
182 	*owner = mtx_owner(m);
183 	return (mtx_unowned(m) == 0);
184 }
185 #endif
186 
187 /*
188  * Function versions of the inlined __mtx_* macros.  These are used by
189  * modules and can also be called from assembly language if needed.
190  */
191 void
192 _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
193 {
194 
195 	if (SCHEDULER_STOPPED())
196 		return;
197 	MPASS(curthread != NULL);
198 	KASSERT(m->mtx_lock != MTX_DESTROYED,
199 	    ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
200 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
201 	    ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
202 	    file, line));
203 	WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
204 	    file, line, NULL);
205 
206 	__mtx_lock(m, curthread, opts, file, line);
207 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
208 	    line);
209 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
210 	curthread->td_locks++;
211 }
212 
213 void
214 _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
215 {
216 
217 	if (SCHEDULER_STOPPED())
218 		return;
219 	MPASS(curthread != NULL);
220 	KASSERT(m->mtx_lock != MTX_DESTROYED,
221 	    ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
222 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
223 	    ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
224 	    file, line));
225 	curthread->td_locks--;
226 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
227 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
228 	    line);
229 	mtx_assert(m, MA_OWNED);
230 
231 	if (m->mtx_recurse == 0)
232 		LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_UNLOCK_RELEASE, m);
233 	__mtx_unlock(m, curthread, opts, file, line);
234 }
235 
236 void
237 _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
238 {
239 
240 	if (SCHEDULER_STOPPED())
241 		return;
242 	MPASS(curthread != NULL);
243 	KASSERT(m->mtx_lock != MTX_DESTROYED,
244 	    ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
245 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
246 	    ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
247 	    m->lock_object.lo_name, file, line));
248 	if (mtx_owned(m))
249 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
250 	    ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n",
251 		    m->lock_object.lo_name, file, line));
252 	WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
253 	    file, line, NULL);
254 	__mtx_lock_spin(m, curthread, opts, file, line);
255 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
256 	    line);
257 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
258 }
259 
260 void
261 _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
262 {
263 
264 	if (SCHEDULER_STOPPED())
265 		return;
266 	MPASS(curthread != NULL);
267 	KASSERT(m->mtx_lock != MTX_DESTROYED,
268 	    ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
269 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
270 	    ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
271 	    m->lock_object.lo_name, file, line));
272 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
273 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
274 	    line);
275 	mtx_assert(m, MA_OWNED);
276 
277 	__mtx_unlock_spin(m);
278 }
279 
280 /*
281  * The important part of mtx_trylock{,_flags}()
282  * Tries to acquire lock `m.'  If this function is called on a mutex that
283  * is already owned, it will recursively acquire the lock.
284  */
285 int
286 mtx_trylock_flags_(struct mtx *m, int opts, const char *file, int line)
287 {
288 #ifdef LOCK_PROFILING
289 	uint64_t waittime = 0;
290 	int contested = 0;
291 #endif
292 	int rval;
293 
294 	if (SCHEDULER_STOPPED())
295 		return (1);
296 
297 	MPASS(curthread != NULL);
298 	KASSERT(m->mtx_lock != MTX_DESTROYED,
299 	    ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
300 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
301 	    ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
302 	    file, line));
303 
304 	if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) {
305 		m->mtx_recurse++;
306 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
307 		rval = 1;
308 	} else
309 		rval = _mtx_obtain_lock(m, (uintptr_t)curthread);
310 
311 	LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
312 	if (rval) {
313 		WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
314 		    file, line);
315 		curthread->td_locks++;
316 		if (m->mtx_recurse == 0)
317 			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE,
318 			    m, contested, waittime, file, line);
319 
320 	}
321 
322 	return (rval);
323 }
324 
325 /*
326  * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
327  *
328  * We call this if the lock is either contested (i.e. we need to go to
329  * sleep waiting for it), or if we need to recurse on it.
330  */
331 void
332 _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
333     int line)
334 {
335 	struct turnstile *ts;
336 	uintptr_t v;
337 #ifdef ADAPTIVE_MUTEXES
338 	volatile struct thread *owner;
339 #endif
340 #ifdef KTR
341 	int cont_logged = 0;
342 #endif
343 #ifdef LOCK_PROFILING
344 	int contested = 0;
345 	uint64_t waittime = 0;
346 #endif
347 #ifdef KDTRACE_HOOKS
348 	uint64_t spin_cnt = 0;
349 	uint64_t sleep_cnt = 0;
350 	int64_t sleep_time = 0;
351 #endif
352 
353 	if (SCHEDULER_STOPPED())
354 		return;
355 
356 	if (mtx_owned(m)) {
357 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
358 	    ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
359 		    m->lock_object.lo_name, file, line));
360 		m->mtx_recurse++;
361 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
362 		if (LOCK_LOG_TEST(&m->lock_object, opts))
363 			CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
364 		return;
365 	}
366 
367 	lock_profile_obtain_lock_failed(&m->lock_object,
368 		    &contested, &waittime);
369 	if (LOCK_LOG_TEST(&m->lock_object, opts))
370 		CTR4(KTR_LOCK,
371 		    "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
372 		    m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
373 
374 	while (!_mtx_obtain_lock(m, tid)) {
375 #ifdef KDTRACE_HOOKS
376 		spin_cnt++;
377 #endif
378 #ifdef ADAPTIVE_MUTEXES
379 		/*
380 		 * If the owner is running on another CPU, spin until the
381 		 * owner stops running or the state of the lock changes.
382 		 */
383 		v = m->mtx_lock;
384 		if (v != MTX_UNOWNED) {
385 			owner = (struct thread *)(v & ~MTX_FLAGMASK);
386 			if (TD_IS_RUNNING(owner)) {
387 				if (LOCK_LOG_TEST(&m->lock_object, 0))
388 					CTR3(KTR_LOCK,
389 					    "%s: spinning on %p held by %p",
390 					    __func__, m, owner);
391 				while (mtx_owner(m) == owner &&
392 				    TD_IS_RUNNING(owner)) {
393 					cpu_spinwait();
394 #ifdef KDTRACE_HOOKS
395 					spin_cnt++;
396 #endif
397 				}
398 				continue;
399 			}
400 		}
401 #endif
402 
403 		ts = turnstile_trywait(&m->lock_object);
404 		v = m->mtx_lock;
405 
406 		/*
407 		 * Check if the lock has been released while spinning for
408 		 * the turnstile chain lock.
409 		 */
410 		if (v == MTX_UNOWNED) {
411 			turnstile_cancel(ts);
412 			continue;
413 		}
414 
415 #ifdef ADAPTIVE_MUTEXES
416 		/*
417 		 * The current lock owner might have started executing
418 		 * on another CPU (or the lock could have changed
419 		 * owners) while we were waiting on the turnstile
420 		 * chain lock.  If so, drop the turnstile lock and try
421 		 * again.
422 		 */
423 		owner = (struct thread *)(v & ~MTX_FLAGMASK);
424 		if (TD_IS_RUNNING(owner)) {
425 			turnstile_cancel(ts);
426 			continue;
427 		}
428 #endif
429 
430 		/*
431 		 * If the mutex isn't already contested and a failure occurs
432 		 * setting the contested bit, the mutex was either released
433 		 * or the state of the MTX_RECURSED bit changed.
434 		 */
435 		if ((v & MTX_CONTESTED) == 0 &&
436 		    !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
437 			turnstile_cancel(ts);
438 			continue;
439 		}
440 
441 		/*
442 		 * We definitely must sleep for this lock.
443 		 */
444 		mtx_assert(m, MA_NOTOWNED);
445 
446 #ifdef KTR
447 		if (!cont_logged) {
448 			CTR6(KTR_CONTENTION,
449 			    "contention: %p at %s:%d wants %s, taken by %s:%d",
450 			    (void *)tid, file, line, m->lock_object.lo_name,
451 			    WITNESS_FILE(&m->lock_object),
452 			    WITNESS_LINE(&m->lock_object));
453 			cont_logged = 1;
454 		}
455 #endif
456 
457 		/*
458 		 * Block on the turnstile.
459 		 */
460 #ifdef KDTRACE_HOOKS
461 		sleep_time -= lockstat_nsecs();
462 #endif
463 		turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
464 #ifdef KDTRACE_HOOKS
465 		sleep_time += lockstat_nsecs();
466 		sleep_cnt++;
467 #endif
468 	}
469 #ifdef KTR
470 	if (cont_logged) {
471 		CTR4(KTR_CONTENTION,
472 		    "contention end: %s acquired by %p at %s:%d",
473 		    m->lock_object.lo_name, (void *)tid, file, line);
474 	}
475 #endif
476 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested,
477 	    waittime, file, line);
478 #ifdef KDTRACE_HOOKS
479 	if (sleep_time)
480 		LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time);
481 
482 	/*
483 	 * Only record the loops spinning and not sleeping.
484 	 */
485 	if (spin_cnt > sleep_cnt)
486 		LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (spin_cnt - sleep_cnt));
487 #endif
488 }
489 
490 static void
491 _mtx_lock_spin_failed(struct mtx *m)
492 {
493 	struct thread *td;
494 
495 	td = mtx_owner(m);
496 
497 	/* If the mutex is unlocked, try again. */
498 	if (td == NULL)
499 		return;
500 
501 	printf( "spin lock %p (%s) held by %p (tid %d) too long\n",
502 	    m, m->lock_object.lo_name, td, td->td_tid);
503 #ifdef WITNESS
504 	witness_display_spinlock(&m->lock_object, td, printf);
505 #endif
506 	panic("spin lock held too long");
507 }
508 
509 #ifdef SMP
510 /*
511  * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
512  *
513  * This is only called if we need to actually spin for the lock. Recursion
514  * is handled inline.
515  */
516 void
517 _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
518     int line)
519 {
520 	int i = 0;
521 #ifdef LOCK_PROFILING
522 	int contested = 0;
523 	uint64_t waittime = 0;
524 #endif
525 
526 	if (SCHEDULER_STOPPED())
527 		return;
528 
529 	if (LOCK_LOG_TEST(&m->lock_object, opts))
530 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
531 
532 	lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
533 	while (!_mtx_obtain_lock(m, tid)) {
534 
535 		/* Give interrupts a chance while we spin. */
536 		spinlock_exit();
537 		while (m->mtx_lock != MTX_UNOWNED) {
538 			if (i++ < 10000000) {
539 				cpu_spinwait();
540 				continue;
541 			}
542 			if (i < 60000000 || kdb_active || panicstr != NULL)
543 				DELAY(1);
544 			else
545 				_mtx_lock_spin_failed(m);
546 			cpu_spinwait();
547 		}
548 		spinlock_enter();
549 	}
550 
551 	if (LOCK_LOG_TEST(&m->lock_object, opts))
552 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
553 
554 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m,
555 	    contested, waittime, (file), (line));
556 	LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, i);
557 }
558 #endif /* SMP */
559 
560 void
561 thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
562 {
563 	struct mtx *m;
564 	uintptr_t tid;
565 	int i;
566 #ifdef LOCK_PROFILING
567 	int contested = 0;
568 	uint64_t waittime = 0;
569 #endif
570 #ifdef KDTRACE_HOOKS
571 	uint64_t spin_cnt = 0;
572 #endif
573 
574 	i = 0;
575 	tid = (uintptr_t)curthread;
576 
577 	if (SCHEDULER_STOPPED())
578 		return;
579 
580 	for (;;) {
581 retry:
582 		spinlock_enter();
583 		m = td->td_lock;
584 		KASSERT(m->mtx_lock != MTX_DESTROYED,
585 		    ("thread_lock() of destroyed mutex @ %s:%d", file, line));
586 		KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
587 		    ("thread_lock() of sleep mutex %s @ %s:%d",
588 		    m->lock_object.lo_name, file, line));
589 		if (mtx_owned(m))
590 			KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
591 	    ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
592 			    m->lock_object.lo_name, file, line));
593 		WITNESS_CHECKORDER(&m->lock_object,
594 		    opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
595 		while (!_mtx_obtain_lock(m, tid)) {
596 #ifdef KDTRACE_HOOKS
597 			spin_cnt++;
598 #endif
599 			if (m->mtx_lock == tid) {
600 				m->mtx_recurse++;
601 				break;
602 			}
603 			lock_profile_obtain_lock_failed(&m->lock_object,
604 			    &contested, &waittime);
605 			/* Give interrupts a chance while we spin. */
606 			spinlock_exit();
607 			while (m->mtx_lock != MTX_UNOWNED) {
608 				if (i++ < 10000000)
609 					cpu_spinwait();
610 				else if (i < 60000000 ||
611 				    kdb_active || panicstr != NULL)
612 					DELAY(1);
613 				else
614 					_mtx_lock_spin_failed(m);
615 				cpu_spinwait();
616 				if (m != td->td_lock)
617 					goto retry;
618 			}
619 			spinlock_enter();
620 		}
621 		if (m == td->td_lock)
622 			break;
623 		__mtx_unlock_spin(m);	/* does spinlock_exit() */
624 #ifdef KDTRACE_HOOKS
625 		spin_cnt++;
626 #endif
627 	}
628 	if (m->mtx_recurse == 0)
629 		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE,
630 		    m, contested, waittime, (file), (line));
631 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
632 	    line);
633 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
634 	LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_cnt);
635 }
636 
637 struct mtx *
638 thread_lock_block(struct thread *td)
639 {
640 	struct mtx *lock;
641 
642 	THREAD_LOCK_ASSERT(td, MA_OWNED);
643 	lock = td->td_lock;
644 	td->td_lock = &blocked_lock;
645 	mtx_unlock_spin(lock);
646 
647 	return (lock);
648 }
649 
650 void
651 thread_lock_unblock(struct thread *td, struct mtx *new)
652 {
653 	mtx_assert(new, MA_OWNED);
654 	MPASS(td->td_lock == &blocked_lock);
655 	atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new);
656 }
657 
658 void
659 thread_lock_set(struct thread *td, struct mtx *new)
660 {
661 	struct mtx *lock;
662 
663 	mtx_assert(new, MA_OWNED);
664 	THREAD_LOCK_ASSERT(td, MA_OWNED);
665 	lock = td->td_lock;
666 	td->td_lock = new;
667 	mtx_unlock_spin(lock);
668 }
669 
670 /*
671  * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
672  *
673  * We are only called here if the lock is recursed or contested (i.e. we
674  * need to wake up a blocked thread).
675  */
676 void
677 _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
678 {
679 	struct turnstile *ts;
680 
681 	if (SCHEDULER_STOPPED())
682 		return;
683 
684 	if (mtx_recursed(m)) {
685 		if (--(m->mtx_recurse) == 0)
686 			atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
687 		if (LOCK_LOG_TEST(&m->lock_object, opts))
688 			CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
689 		return;
690 	}
691 
692 	/*
693 	 * We have to lock the chain before the turnstile so this turnstile
694 	 * can be removed from the hash list if it is empty.
695 	 */
696 	turnstile_chain_lock(&m->lock_object);
697 	ts = turnstile_lookup(&m->lock_object);
698 	if (LOCK_LOG_TEST(&m->lock_object, opts))
699 		CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
700 	MPASS(ts != NULL);
701 	turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
702 	_mtx_release_lock_quick(m);
703 
704 	/*
705 	 * This turnstile is now no longer associated with the mutex.  We can
706 	 * unlock the chain lock so a new turnstile may take it's place.
707 	 */
708 	turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
709 	turnstile_chain_unlock(&m->lock_object);
710 }
711 
712 /*
713  * All the unlocking of MTX_SPIN locks is done inline.
714  * See the __mtx_unlock_spin() macro for the details.
715  */
716 
717 /*
718  * The backing function for the INVARIANTS-enabled mtx_assert()
719  */
720 #ifdef INVARIANT_SUPPORT
721 void
722 _mtx_assert(const struct mtx *m, int what, const char *file, int line)
723 {
724 
725 	if (panicstr != NULL || dumping)
726 		return;
727 	switch (what) {
728 	case MA_OWNED:
729 	case MA_OWNED | MA_RECURSED:
730 	case MA_OWNED | MA_NOTRECURSED:
731 		if (!mtx_owned(m))
732 			panic("mutex %s not owned at %s:%d",
733 			    m->lock_object.lo_name, file, line);
734 		if (mtx_recursed(m)) {
735 			if ((what & MA_NOTRECURSED) != 0)
736 				panic("mutex %s recursed at %s:%d",
737 				    m->lock_object.lo_name, file, line);
738 		} else if ((what & MA_RECURSED) != 0) {
739 			panic("mutex %s unrecursed at %s:%d",
740 			    m->lock_object.lo_name, file, line);
741 		}
742 		break;
743 	case MA_NOTOWNED:
744 		if (mtx_owned(m))
745 			panic("mutex %s owned at %s:%d",
746 			    m->lock_object.lo_name, file, line);
747 		break;
748 	default:
749 		panic("unknown mtx_assert at %s:%d", file, line);
750 	}
751 }
752 #endif
753 
754 /*
755  * The MUTEX_DEBUG-enabled mtx_validate()
756  *
757  * Most of these checks have been moved off into the LO_INITIALIZED flag
758  * maintained by the witness code.
759  */
760 #ifdef MUTEX_DEBUG
761 
762 void	mtx_validate(struct mtx *);
763 
764 void
765 mtx_validate(struct mtx *m)
766 {
767 
768 /*
769  * XXX: When kernacc() does not require Giant we can reenable this check
770  */
771 #ifdef notyet
772 	/*
773 	 * Can't call kernacc() from early init386(), especially when
774 	 * initializing Giant mutex, because some stuff in kernacc()
775 	 * requires Giant itself.
776 	 */
777 	if (!cold)
778 		if (!kernacc((caddr_t)m, sizeof(m),
779 		    VM_PROT_READ | VM_PROT_WRITE))
780 			panic("Can't read and write to mutex %p", m);
781 #endif
782 }
783 #endif
784 
785 /*
786  * General init routine used by the MTX_SYSINIT() macro.
787  */
788 void
789 mtx_sysinit(void *arg)
790 {
791 	struct mtx_args *margs = arg;
792 
793 	mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts);
794 }
795 
796 /*
797  * Mutex initialization routine; initialize lock `m' of type contained in
798  * `opts' with options contained in `opts' and name `name.'  The optional
799  * lock type `type' is used as a general lock category name for use with
800  * witness.
801  */
802 void
803 mtx_init(struct mtx *m, const char *name, const char *type, int opts)
804 {
805 	struct lock_class *class;
806 	int flags;
807 
808 	MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
809 		MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0);
810 	ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock,
811 	    ("%s: mtx_lock not aligned for %s: %p", __func__, name,
812 	    &m->mtx_lock));
813 
814 #ifdef MUTEX_DEBUG
815 	/* Diagnostic and error correction */
816 	mtx_validate(m);
817 #endif
818 
819 	/* Determine lock class and lock flags. */
820 	if (opts & MTX_SPIN)
821 		class = &lock_class_mtx_spin;
822 	else
823 		class = &lock_class_mtx_sleep;
824 	flags = 0;
825 	if (opts & MTX_QUIET)
826 		flags |= LO_QUIET;
827 	if (opts & MTX_RECURSE)
828 		flags |= LO_RECURSABLE;
829 	if ((opts & MTX_NOWITNESS) == 0)
830 		flags |= LO_WITNESS;
831 	if (opts & MTX_DUPOK)
832 		flags |= LO_DUPOK;
833 	if (opts & MTX_NOPROFILE)
834 		flags |= LO_NOPROFILE;
835 
836 	/* Initialize mutex. */
837 	m->mtx_lock = MTX_UNOWNED;
838 	m->mtx_recurse = 0;
839 
840 	lock_init(&m->lock_object, class, name, type, flags);
841 }
842 
843 /*
844  * Remove lock `m' from all_mtx queue.  We don't allow MTX_QUIET to be
845  * passed in as a flag here because if the corresponding mtx_init() was
846  * called with MTX_QUIET set, then it will already be set in the mutex's
847  * flags.
848  */
849 void
850 mtx_destroy(struct mtx *m)
851 {
852 
853 	if (!mtx_owned(m))
854 		MPASS(mtx_unowned(m));
855 	else {
856 		MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
857 
858 		/* Perform the non-mtx related part of mtx_unlock_spin(). */
859 		if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin)
860 			spinlock_exit();
861 		else
862 			curthread->td_locks--;
863 
864 		lock_profile_release_lock(&m->lock_object);
865 		/* Tell witness this isn't locked to make it happy. */
866 		WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__,
867 		    __LINE__);
868 	}
869 
870 	m->mtx_lock = MTX_DESTROYED;
871 	lock_destroy(&m->lock_object);
872 }
873 
874 /*
875  * Intialize the mutex code and system mutexes.  This is called from the MD
876  * startup code prior to mi_startup().  The per-CPU data space needs to be
877  * setup before this is called.
878  */
879 void
880 mutex_init(void)
881 {
882 
883 	/* Setup turnstiles so that sleep mutexes work. */
884 	init_turnstiles();
885 
886 	/*
887 	 * Initialize mutexes.
888 	 */
889 	mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
890 	mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN);
891 	blocked_lock.mtx_lock = 0xdeadc0de;	/* Always blocked. */
892 	mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
893 	mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE);
894 	mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
895 	mtx_lock(&Giant);
896 }
897 
898 #ifdef DDB
899 void
900 db_show_mtx(const struct lock_object *lock)
901 {
902 	struct thread *td;
903 	const struct mtx *m;
904 
905 	m = (const struct mtx *)lock;
906 
907 	db_printf(" flags: {");
908 	if (LOCK_CLASS(lock) == &lock_class_mtx_spin)
909 		db_printf("SPIN");
910 	else
911 		db_printf("DEF");
912 	if (m->lock_object.lo_flags & LO_RECURSABLE)
913 		db_printf(", RECURSE");
914 	if (m->lock_object.lo_flags & LO_DUPOK)
915 		db_printf(", DUPOK");
916 	db_printf("}\n");
917 	db_printf(" state: {");
918 	if (mtx_unowned(m))
919 		db_printf("UNOWNED");
920 	else if (mtx_destroyed(m))
921 		db_printf("DESTROYED");
922 	else {
923 		db_printf("OWNED");
924 		if (m->mtx_lock & MTX_CONTESTED)
925 			db_printf(", CONTESTED");
926 		if (m->mtx_lock & MTX_RECURSED)
927 			db_printf(", RECURSED");
928 	}
929 	db_printf("}\n");
930 	if (!mtx_unowned(m) && !mtx_destroyed(m)) {
931 		td = mtx_owner(m);
932 		db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td,
933 		    td->td_tid, td->td_proc->p_pid, td->td_name);
934 		if (mtx_recursed(m))
935 			db_printf(" recursed: %d\n", m->mtx_recurse);
936 	}
937 }
938 #endif
939