xref: /freebsd/sys/kern/kern_mutex.c (revision ad69e26b692a0947e997d6fb47f9949f5b19d597)
10384fff8SJason Evans /*-
20384fff8SJason Evans  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
30384fff8SJason Evans  *
40384fff8SJason Evans  * Redistribution and use in source and binary forms, with or without
50384fff8SJason Evans  * modification, are permitted provided that the following conditions
60384fff8SJason Evans  * are met:
70384fff8SJason Evans  * 1. Redistributions of source code must retain the above copyright
80384fff8SJason Evans  *    notice, this list of conditions and the following disclaimer.
90384fff8SJason Evans  * 2. Redistributions in binary form must reproduce the above copyright
100384fff8SJason Evans  *    notice, this list of conditions and the following disclaimer in the
110384fff8SJason Evans  *    documentation and/or other materials provided with the distribution.
120384fff8SJason Evans  * 3. Berkeley Software Design Inc's name may not be used to endorse or
130384fff8SJason Evans  *    promote products derived from this software without specific prior
140384fff8SJason Evans  *    written permission.
150384fff8SJason Evans  *
160384fff8SJason Evans  * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
170384fff8SJason Evans  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
180384fff8SJason Evans  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
190384fff8SJason Evans  * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
200384fff8SJason Evans  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
210384fff8SJason Evans  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
220384fff8SJason Evans  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
230384fff8SJason Evans  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
240384fff8SJason Evans  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
250384fff8SJason Evans  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
260384fff8SJason Evans  * SUCH DAMAGE.
270384fff8SJason Evans  *
280384fff8SJason Evans  *	from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
2936412d79SJohn Baldwin  *	and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
300384fff8SJason Evans  */
310384fff8SJason Evans 
320384fff8SJason Evans /*
33ba48b69aSJohn Baldwin  * Machine independent bits of mutex implementation.
340384fff8SJason Evans  */
350384fff8SJason Evans 
36677b542eSDavid E. O'Brien #include <sys/cdefs.h>
37677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
38677b542eSDavid E. O'Brien 
392498cf8cSJohn Baldwin #include "opt_adaptive_mutexes.h"
409c36c934SJohn Baldwin #include "opt_ddb.h"
417c0435b9SKip Macy #include "opt_global.h"
429923b511SScott Long #include "opt_sched.h"
43a5a96a19SJohn Baldwin 
440384fff8SJason Evans #include <sys/param.h>
456c35e809SDag-Erling Smørgrav #include <sys/systm.h>
4636412d79SJohn Baldwin #include <sys/bus.h>
471126349aSPaul Saab #include <sys/conf.h>
482d50560aSMarcel Moolenaar #include <sys/kdb.h>
4936412d79SJohn Baldwin #include <sys/kernel.h>
506c35e809SDag-Erling Smørgrav #include <sys/ktr.h>
5119284646SJohn Baldwin #include <sys/lock.h>
52fb919e4dSMark Murray #include <sys/malloc.h>
5319284646SJohn Baldwin #include <sys/mutex.h>
540384fff8SJason Evans #include <sys/proc.h>
55c4f7a187SJohn Baldwin #include <sys/resourcevar.h>
56b43179fbSJeff Roberson #include <sys/sched.h>
576c35e809SDag-Erling Smørgrav #include <sys/sbuf.h>
58a5a96a19SJohn Baldwin #include <sys/sysctl.h>
59961a7b24SJohn Baldwin #include <sys/turnstile.h>
6036412d79SJohn Baldwin #include <sys/vmmeter.h>
617c0435b9SKip Macy #include <sys/lock_profile.h>
620384fff8SJason Evans 
6336412d79SJohn Baldwin #include <machine/atomic.h>
6436412d79SJohn Baldwin #include <machine/bus.h>
650384fff8SJason Evans #include <machine/cpu.h>
6636412d79SJohn Baldwin 
679c36c934SJohn Baldwin #include <ddb/ddb.h>
689c36c934SJohn Baldwin 
698c4b6380SJohn Baldwin #include <fs/devfs/devfs_int.h>
708c4b6380SJohn Baldwin 
7136412d79SJohn Baldwin #include <vm/vm.h>
7236412d79SJohn Baldwin #include <vm/vm_extern.h>
7336412d79SJohn Baldwin 
74cd6e6e4eSJohn Baldwin #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
75cd6e6e4eSJohn Baldwin #define	ADAPTIVE_MUTEXES
76cd6e6e4eSJohn Baldwin #endif
77cd6e6e4eSJohn Baldwin 
78b9a80acaSStephan Uphoff /*
799ed346baSBosko Milekic  * Internal utility macros.
800cde2e34SJason Evans  */
819ed346baSBosko Milekic #define mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)
820cde2e34SJason Evans 
83c0bfd703SJohn Baldwin #define	mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED)
84c0bfd703SJohn Baldwin 
8549b94bfcSJohn Baldwin #define	mtx_owner(m)	((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK))
869ed346baSBosko Milekic 
87f9721b43SAttilio Rao static void	assert_mtx(struct lock_object *lock, int what);
88d272fe53SJohn Baldwin #ifdef DDB
89d272fe53SJohn Baldwin static void	db_show_mtx(struct lock_object *lock);
90d272fe53SJohn Baldwin #endif
916e21afd4SJohn Baldwin static void	lock_mtx(struct lock_object *lock, int how);
926e21afd4SJohn Baldwin static void	lock_spin(struct lock_object *lock, int how);
936e21afd4SJohn Baldwin static int	unlock_mtx(struct lock_object *lock);
946e21afd4SJohn Baldwin static int	unlock_spin(struct lock_object *lock);
95d272fe53SJohn Baldwin 
960cde2e34SJason Evans /*
9719284646SJohn Baldwin  * Lock classes for sleep and spin mutexes.
980cde2e34SJason Evans  */
9919284646SJohn Baldwin struct lock_class lock_class_mtx_sleep = {
100ae8dde30SJohn Baldwin 	.lc_name = "sleep mutex",
101ae8dde30SJohn Baldwin 	.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
102f9721b43SAttilio Rao 	.lc_assert = assert_mtx,
103d272fe53SJohn Baldwin #ifdef DDB
104ae8dde30SJohn Baldwin 	.lc_ddb_show = db_show_mtx,
105d272fe53SJohn Baldwin #endif
1066e21afd4SJohn Baldwin 	.lc_lock = lock_mtx,
1076e21afd4SJohn Baldwin 	.lc_unlock = unlock_mtx,
10819284646SJohn Baldwin };
10919284646SJohn Baldwin struct lock_class lock_class_mtx_spin = {
110ae8dde30SJohn Baldwin 	.lc_name = "spin mutex",
111ae8dde30SJohn Baldwin 	.lc_flags = LC_SPINLOCK | LC_RECURSABLE,
112f9721b43SAttilio Rao 	.lc_assert = assert_mtx,
113d272fe53SJohn Baldwin #ifdef DDB
114ae8dde30SJohn Baldwin 	.lc_ddb_show = db_show_mtx,
115d272fe53SJohn Baldwin #endif
1166e21afd4SJohn Baldwin 	.lc_lock = lock_spin,
1176e21afd4SJohn Baldwin 	.lc_unlock = unlock_spin,
1188484de75SJohn Baldwin };
1198484de75SJohn Baldwin 
1209ed346baSBosko Milekic /*
121c53c013bSJohn Baldwin  * System-wide mutexes
122c53c013bSJohn Baldwin  */
1232502c107SJeff Roberson struct mtx blocked_lock;
124c53c013bSJohn Baldwin struct mtx Giant;
125c53c013bSJohn Baldwin 
1266e21afd4SJohn Baldwin void
127f9721b43SAttilio Rao assert_mtx(struct lock_object *lock, int what)
128f9721b43SAttilio Rao {
129f9721b43SAttilio Rao 
130f9721b43SAttilio Rao 	mtx_assert((struct mtx *)lock, what);
131f9721b43SAttilio Rao }
132f9721b43SAttilio Rao 
133f9721b43SAttilio Rao void
1346e21afd4SJohn Baldwin lock_mtx(struct lock_object *lock, int how)
1356e21afd4SJohn Baldwin {
1366e21afd4SJohn Baldwin 
1376e21afd4SJohn Baldwin 	mtx_lock((struct mtx *)lock);
1386e21afd4SJohn Baldwin }
1396e21afd4SJohn Baldwin 
1406e21afd4SJohn Baldwin void
1416e21afd4SJohn Baldwin lock_spin(struct lock_object *lock, int how)
1426e21afd4SJohn Baldwin {
1436e21afd4SJohn Baldwin 
1446e21afd4SJohn Baldwin 	panic("spin locks can only use msleep_spin");
1456e21afd4SJohn Baldwin }
1466e21afd4SJohn Baldwin 
1476e21afd4SJohn Baldwin int
1486e21afd4SJohn Baldwin unlock_mtx(struct lock_object *lock)
1496e21afd4SJohn Baldwin {
1506e21afd4SJohn Baldwin 	struct mtx *m;
1516e21afd4SJohn Baldwin 
1526e21afd4SJohn Baldwin 	m = (struct mtx *)lock;
1536e21afd4SJohn Baldwin 	mtx_assert(m, MA_OWNED | MA_NOTRECURSED);
1546e21afd4SJohn Baldwin 	mtx_unlock(m);
1556e21afd4SJohn Baldwin 	return (0);
1566e21afd4SJohn Baldwin }
1576e21afd4SJohn Baldwin 
1586e21afd4SJohn Baldwin int
1596e21afd4SJohn Baldwin unlock_spin(struct lock_object *lock)
1606e21afd4SJohn Baldwin {
1616e21afd4SJohn Baldwin 
1626e21afd4SJohn Baldwin 	panic("spin locks can only use msleep_spin");
1636e21afd4SJohn Baldwin }
1646e21afd4SJohn Baldwin 
1650cde2e34SJason Evans /*
1666283b7d0SJohn Baldwin  * Function versions of the inlined __mtx_* macros.  These are used by
1676283b7d0SJohn Baldwin  * modules and can also be called from assembly language if needed.
1686283b7d0SJohn Baldwin  */
1696283b7d0SJohn Baldwin void
1706283b7d0SJohn Baldwin _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
1716283b7d0SJohn Baldwin {
1726283b7d0SJohn Baldwin 
173dde96c99SJohn Baldwin 	MPASS(curthread != NULL);
174186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
175186abbd7SJohn Baldwin 	    ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
176aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
177aa89d8cdSJohn Baldwin 	    ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
1780d975d63SJohn Baldwin 	    file, line));
179aa89d8cdSJohn Baldwin 	WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
1808d768e76SJohn Baldwin 	    file, line);
1817c0435b9SKip Macy 
182dde96c99SJohn Baldwin 	_get_sleep_lock(m, curthread, opts, file, line);
183aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
184dde96c99SJohn Baldwin 	    line);
185aa89d8cdSJohn Baldwin 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
186764e4d54SJohn Baldwin 	curthread->td_locks++;
1876283b7d0SJohn Baldwin }
1886283b7d0SJohn Baldwin 
1896283b7d0SJohn Baldwin void
1906283b7d0SJohn Baldwin _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
1916283b7d0SJohn Baldwin {
192dde96c99SJohn Baldwin 	MPASS(curthread != NULL);
193186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
194186abbd7SJohn Baldwin 	    ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
195aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
196aa89d8cdSJohn Baldwin 	    ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
1970d975d63SJohn Baldwin 	    file, line));
198764e4d54SJohn Baldwin 	curthread->td_locks--;
199aa89d8cdSJohn Baldwin 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
200aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
2010d975d63SJohn Baldwin 	    line);
20221377ce0SJohn Baldwin 	mtx_assert(m, MA_OWNED);
203c66d7606SKip Macy 
20470fe8436SKip Macy 	if (m->mtx_recurse == 0)
205aa89d8cdSJohn Baldwin 		lock_profile_release_lock(&m->lock_object);
206dde96c99SJohn Baldwin 	_rel_sleep_lock(m, curthread, opts, file, line);
2076283b7d0SJohn Baldwin }
2086283b7d0SJohn Baldwin 
2096283b7d0SJohn Baldwin void
2106283b7d0SJohn Baldwin _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
2116283b7d0SJohn Baldwin {
2126283b7d0SJohn Baldwin 
213dde96c99SJohn Baldwin 	MPASS(curthread != NULL);
214186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
215186abbd7SJohn Baldwin 	    ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
216aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
2170d975d63SJohn Baldwin 	    ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
218aa89d8cdSJohn Baldwin 	    m->lock_object.lo_name, file, line));
219ad69e26bSJohn Baldwin 	if (mtx_owned(m))
220ad69e26bSJohn Baldwin 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
221ad69e26bSJohn Baldwin 	    ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n",
222ad69e26bSJohn Baldwin 		    m->lock_object.lo_name, file, line));
223aa89d8cdSJohn Baldwin 	WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
2248d768e76SJohn Baldwin 	    file, line);
225dde96c99SJohn Baldwin 	_get_spin_lock(m, curthread, opts, file, line);
226aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
227dde96c99SJohn Baldwin 	    line);
228aa89d8cdSJohn Baldwin 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
2296283b7d0SJohn Baldwin }
2306283b7d0SJohn Baldwin 
2316283b7d0SJohn Baldwin void
2326283b7d0SJohn Baldwin _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
2336283b7d0SJohn Baldwin {
234c66d7606SKip Macy 
235dde96c99SJohn Baldwin 	MPASS(curthread != NULL);
236186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
237186abbd7SJohn Baldwin 	    ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
238aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
2390d975d63SJohn Baldwin 	    ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
240aa89d8cdSJohn Baldwin 	    m->lock_object.lo_name, file, line));
241aa89d8cdSJohn Baldwin 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
242aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
243dde96c99SJohn Baldwin 	    line);
2440d975d63SJohn Baldwin 	mtx_assert(m, MA_OWNED);
245c66d7606SKip Macy 
246dde96c99SJohn Baldwin 	_rel_spin_lock(m);
2476283b7d0SJohn Baldwin }
2486283b7d0SJohn Baldwin 
2496283b7d0SJohn Baldwin /*
2509ed346baSBosko Milekic  * The important part of mtx_trylock{,_flags}()
251eac09796SJohn Baldwin  * Tries to acquire lock `m.'  If this function is called on a mutex that
252eac09796SJohn Baldwin  * is already owned, it will recursively acquire the lock.
2530cde2e34SJason Evans  */
2540cde2e34SJason Evans int
2559ed346baSBosko Milekic _mtx_trylock(struct mtx *m, int opts, const char *file, int line)
2560cde2e34SJason Evans {
257fe68a916SKip Macy 	int rval, contested = 0;
2587c0435b9SKip Macy 	uint64_t waittime = 0;
2590cde2e34SJason Evans 
260b40ce416SJulian Elischer 	MPASS(curthread != NULL);
261186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
262186abbd7SJohn Baldwin 	    ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
263aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
264aa89d8cdSJohn Baldwin 	    ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
26583cece6fSJohn Baldwin 	    file, line));
2669ed346baSBosko Milekic 
267aa89d8cdSJohn Baldwin 	if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) {
268eac09796SJohn Baldwin 		m->mtx_recurse++;
269eac09796SJohn Baldwin 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
270eac09796SJohn Baldwin 		rval = 1;
271eac09796SJohn Baldwin 	} else
272122eceefSJohn Baldwin 		rval = _obtain_lock(m, (uintptr_t)curthread);
2739ed346baSBosko Milekic 
274aa89d8cdSJohn Baldwin 	LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
275764e4d54SJohn Baldwin 	if (rval) {
276aa89d8cdSJohn Baldwin 		WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
2772d96f0b1SJohn Baldwin 		    file, line);
278764e4d54SJohn Baldwin 		curthread->td_locks++;
279fe68a916SKip Macy 		if (m->mtx_recurse == 0)
280aa89d8cdSJohn Baldwin 			lock_profile_obtain_lock_success(&m->lock_object, contested,
281fe68a916SKip Macy 			    waittime, file, line);
2827c0435b9SKip Macy 
283764e4d54SJohn Baldwin 	}
2849ed346baSBosko Milekic 
28519284646SJohn Baldwin 	return (rval);
2860cde2e34SJason Evans }
2870cde2e34SJason Evans 
2880cde2e34SJason Evans /*
2899ed346baSBosko Milekic  * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
2909ed346baSBosko Milekic  *
2919ed346baSBosko Milekic  * We call this if the lock is either contested (i.e. we need to go to
2929ed346baSBosko Milekic  * sleep waiting for it), or if we need to recurse on it.
2930cde2e34SJason Evans  */
2940cde2e34SJason Evans void
295122eceefSJohn Baldwin _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
296bdcfcf5bSJohn Baldwin     int line)
29736412d79SJohn Baldwin {
2982502c107SJeff Roberson 	struct turnstile *ts;
299cd6e6e4eSJohn Baldwin #ifdef ADAPTIVE_MUTEXES
30076447e56SJohn Baldwin 	volatile struct thread *owner;
3012498cf8cSJohn Baldwin #endif
30202bd1bcdSIan Dowse #ifdef KTR
30302bd1bcdSIan Dowse 	int cont_logged = 0;
30402bd1bcdSIan Dowse #endif
30570fe8436SKip Macy 	int contested = 0;
30670fe8436SKip Macy 	uint64_t waittime = 0;
3077c0435b9SKip Macy 	uintptr_t v;
30836412d79SJohn Baldwin 
3095fa8dd90SJohn Baldwin 	if (mtx_owned(m)) {
310aa89d8cdSJohn Baldwin 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
311eac09796SJohn Baldwin 	    ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
312aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, file, line));
31336412d79SJohn Baldwin 		m->mtx_recurse++;
31408812b39SBosko Milekic 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
315aa89d8cdSJohn Baldwin 		if (LOCK_LOG_TEST(&m->lock_object, opts))
3165746a1d8SBosko Milekic 			CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
31736412d79SJohn Baldwin 		return;
31836412d79SJohn Baldwin 	}
3199ed346baSBosko Milekic 
32070fe8436SKip Macy 	lock_profile_obtain_lock_failed(&m->lock_object,
32170fe8436SKip Macy 		    &contested, &waittime);
322aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
32315ec816aSJohn Baldwin 		CTR4(KTR_LOCK,
32415ec816aSJohn Baldwin 		    "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
325aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
3261bd0eefbSJohn Baldwin 
327122eceefSJohn Baldwin 	while (!_obtain_lock(m, tid)) {
32849aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES
32949aead8aSAttilio Rao 		/*
33049aead8aSAttilio Rao 		 * If the owner is running on another CPU, spin until the
33149aead8aSAttilio Rao 		 * owner stops running or the state of the lock changes.
33249aead8aSAttilio Rao 		 */
33349aead8aSAttilio Rao 		v = m->mtx_lock;
33449aead8aSAttilio Rao 		if (v != MTX_UNOWNED) {
33549aead8aSAttilio Rao 			owner = (struct thread *)(v & ~MTX_FLAGMASK);
33649aead8aSAttilio Rao 			if (TD_IS_RUNNING(owner)) {
33749aead8aSAttilio Rao 				if (LOCK_LOG_TEST(&m->lock_object, 0))
33849aead8aSAttilio Rao 					CTR3(KTR_LOCK,
33949aead8aSAttilio Rao 					    "%s: spinning on %p held by %p",
34049aead8aSAttilio Rao 					    __func__, m, owner);
34149aead8aSAttilio Rao 				while (mtx_owner(m) == owner &&
34249aead8aSAttilio Rao 				    TD_IS_RUNNING(owner))
34349aead8aSAttilio Rao 					cpu_spinwait();
34449aead8aSAttilio Rao 				continue;
34549aead8aSAttilio Rao 			}
34649aead8aSAttilio Rao 		}
34749aead8aSAttilio Rao #endif
34849aead8aSAttilio Rao 
3492502c107SJeff Roberson 		ts = turnstile_trywait(&m->lock_object);
3505fa8dd90SJohn Baldwin 		v = m->mtx_lock;
3515fa8dd90SJohn Baldwin 
35236412d79SJohn Baldwin 		/*
3539ed346baSBosko Milekic 		 * Check if the lock has been released while spinning for
354961a7b24SJohn Baldwin 		 * the turnstile chain lock.
35536412d79SJohn Baldwin 		 */
3565fa8dd90SJohn Baldwin 		if (v == MTX_UNOWNED) {
3572502c107SJeff Roberson 			turnstile_cancel(ts);
3589f1b87f1SMaxime Henrion 			cpu_spinwait();
35936412d79SJohn Baldwin 			continue;
36036412d79SJohn Baldwin 		}
3619ed346baSBosko Milekic 
362535eb309SJohn Baldwin 		MPASS(v != MTX_CONTESTED);
3639ed346baSBosko Milekic 
36449aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES
36549aead8aSAttilio Rao 		/*
36649aead8aSAttilio Rao 		 * If the current owner of the lock is executing on another
36749aead8aSAttilio Rao 		 * CPU quit the hard path and try to spin.
36849aead8aSAttilio Rao 		 */
36949aead8aSAttilio Rao 		owner = (struct thread *)(v & ~MTX_FLAGMASK);
37049aead8aSAttilio Rao 		if (TD_IS_RUNNING(owner)) {
37149aead8aSAttilio Rao 			turnstile_cancel(ts);
37249aead8aSAttilio Rao 			cpu_spinwait();
37349aead8aSAttilio Rao 			continue;
37449aead8aSAttilio Rao 		}
37549aead8aSAttilio Rao #endif
37649aead8aSAttilio Rao 
37736412d79SJohn Baldwin 		/*
3789ed346baSBosko Milekic 		 * If the mutex isn't already contested and a failure occurs
3799ed346baSBosko Milekic 		 * setting the contested bit, the mutex was either released
3809ed346baSBosko Milekic 		 * or the state of the MTX_RECURSED bit changed.
38136412d79SJohn Baldwin 		 */
38236412d79SJohn Baldwin 		if ((v & MTX_CONTESTED) == 0 &&
383122eceefSJohn Baldwin 		    !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
3842502c107SJeff Roberson 			turnstile_cancel(ts);
3859f1b87f1SMaxime Henrion 			cpu_spinwait();
38636412d79SJohn Baldwin 			continue;
38736412d79SJohn Baldwin 		}
38836412d79SJohn Baldwin 
3899ed346baSBosko Milekic 		/*
3907feefcd6SJohn Baldwin 		 * We definitely must sleep for this lock.
3919ed346baSBosko Milekic 		 */
39236412d79SJohn Baldwin 		mtx_assert(m, MA_NOTOWNED);
39336412d79SJohn Baldwin 
39402bd1bcdSIan Dowse #ifdef KTR
39502bd1bcdSIan Dowse 		if (!cont_logged) {
39602bd1bcdSIan Dowse 			CTR6(KTR_CONTENTION,
39702bd1bcdSIan Dowse 			    "contention: %p at %s:%d wants %s, taken by %s:%d",
398aa89d8cdSJohn Baldwin 			    (void *)tid, file, line, m->lock_object.lo_name,
399aa89d8cdSJohn Baldwin 			    WITNESS_FILE(&m->lock_object),
400aa89d8cdSJohn Baldwin 			    WITNESS_LINE(&m->lock_object));
40102bd1bcdSIan Dowse 			cont_logged = 1;
40202bd1bcdSIan Dowse 		}
40302bd1bcdSIan Dowse #endif
40436412d79SJohn Baldwin 
4059ed346baSBosko Milekic 		/*
406961a7b24SJohn Baldwin 		 * Block on the turnstile.
4079ed346baSBosko Milekic 		 */
4082502c107SJeff Roberson 		turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
40936412d79SJohn Baldwin 	}
41002bd1bcdSIan Dowse #ifdef KTR
41102bd1bcdSIan Dowse 	if (cont_logged) {
41202bd1bcdSIan Dowse 		CTR4(KTR_CONTENTION,
41302bd1bcdSIan Dowse 		    "contention end: %s acquired by %p at %s:%d",
414aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, (void *)tid, file, line);
41502bd1bcdSIan Dowse 	}
41602bd1bcdSIan Dowse #endif
41770fe8436SKip Macy 	lock_profile_obtain_lock_success(&m->lock_object, contested,
418eea4f254SJeff Roberson 	    waittime, file, line);
4199ed346baSBosko Milekic }
4209ed346baSBosko Milekic 
4212502c107SJeff Roberson static void
4222502c107SJeff Roberson _mtx_lock_spin_failed(struct mtx *m)
4232502c107SJeff Roberson {
4242502c107SJeff Roberson 	struct thread *td;
4252502c107SJeff Roberson 
4262502c107SJeff Roberson 	td = mtx_owner(m);
4272502c107SJeff Roberson 
4282502c107SJeff Roberson 	/* If the mutex is unlocked, try again. */
4292502c107SJeff Roberson 	if (td == NULL)
4302502c107SJeff Roberson 		return;
431b95b98b0SKonstantin Belousov 
4322502c107SJeff Roberson 	printf( "spin lock %p (%s) held by %p (tid %d) too long\n",
4332502c107SJeff Roberson 	    m, m->lock_object.lo_name, td, td->td_tid);
4342502c107SJeff Roberson #ifdef WITNESS
4352502c107SJeff Roberson 	witness_display_spinlock(&m->lock_object, td);
4362502c107SJeff Roberson #endif
4372502c107SJeff Roberson 	panic("spin lock held too long");
4382502c107SJeff Roberson }
4392502c107SJeff Roberson 
440b95b98b0SKonstantin Belousov #ifdef SMP
4419ed346baSBosko Milekic /*
4429ed346baSBosko Milekic  * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
4439ed346baSBosko Milekic  *
4449ed346baSBosko Milekic  * This is only called if we need to actually spin for the lock. Recursion
4459ed346baSBosko Milekic  * is handled inline.
4469ed346baSBosko Milekic  */
4479ed346baSBosko Milekic void
448122eceefSJohn Baldwin _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
449bdcfcf5bSJohn Baldwin     int line)
45036412d79SJohn Baldwin {
45170fe8436SKip Macy 	int i = 0, contested = 0;
45270fe8436SKip Macy 	uint64_t waittime = 0;
45336412d79SJohn Baldwin 
454aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
4555746a1d8SBosko Milekic 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
4569ed346baSBosko Milekic 
45770fe8436SKip Macy 	lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
458f781b5a4SJohn Baldwin 	while (!_obtain_lock(m, tid)) {
4599ed346baSBosko Milekic 
4607141f2adSJohn Baldwin 		/* Give interrupts a chance while we spin. */
461c6a37e84SJohn Baldwin 		spinlock_exit();
46236412d79SJohn Baldwin 		while (m->mtx_lock != MTX_UNOWNED) {
463703fc290SJohn Baldwin 			if (i++ < 10000000) {
4649f1b87f1SMaxime Henrion 				cpu_spinwait();
46536412d79SJohn Baldwin 				continue;
466703fc290SJohn Baldwin 			}
4670fa2168bSJohn Baldwin 			if (i < 60000000 || kdb_active || panicstr != NULL)
46836412d79SJohn Baldwin 				DELAY(1);
4692502c107SJeff Roberson 			else
4702502c107SJeff Roberson 				_mtx_lock_spin_failed(m);
4719f1b87f1SMaxime Henrion 			cpu_spinwait();
47236412d79SJohn Baldwin 		}
473c6a37e84SJohn Baldwin 		spinlock_enter();
47436412d79SJohn Baldwin 	}
47536412d79SJohn Baldwin 
476aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
4779ed346baSBosko Milekic 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
4789ed346baSBosko Milekic 
47970fe8436SKip Macy 	lock_profile_obtain_lock_success(&m->lock_object, contested,
48070fe8436SKip Macy 	    waittime, (file), (line));
48136412d79SJohn Baldwin }
48233fb8a38SJohn Baldwin #endif /* SMP */
48336412d79SJohn Baldwin 
4842502c107SJeff Roberson void
4852502c107SJeff Roberson _thread_lock_flags(struct thread *td, int opts, const char *file, int line)
4862502c107SJeff Roberson {
4872502c107SJeff Roberson 	struct mtx *m;
4882502c107SJeff Roberson 	uintptr_t tid;
489773890b9SJeff Roberson 	int i, contested;
490773890b9SJeff Roberson 	uint64_t waittime;
4912502c107SJeff Roberson 
492773890b9SJeff Roberson 	contested = i = 0;
493773890b9SJeff Roberson 	waittime = 0;
4942502c107SJeff Roberson 	tid = (uintptr_t)curthread;
4952502c107SJeff Roberson 	for (;;) {
4962502c107SJeff Roberson retry:
4972502c107SJeff Roberson 		spinlock_enter();
498710eacdcSJeff Roberson 		m = td->td_lock;
49913c85a48SJohn Baldwin 		KASSERT(m->mtx_lock != MTX_DESTROYED,
50013c85a48SJohn Baldwin 		    ("thread_lock() of destroyed mutex @ %s:%d", file, line));
50113c85a48SJohn Baldwin 		KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
50213c85a48SJohn Baldwin 		    ("thread_lock() of sleep mutex %s @ %s:%d",
50313c85a48SJohn Baldwin 		    m->lock_object.lo_name, file, line));
504ad69e26bSJohn Baldwin 		if (mtx_owned(m))
505ad69e26bSJohn Baldwin 			KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
506ad69e26bSJohn Baldwin 	    ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
507ad69e26bSJohn Baldwin 			    m->lock_object.lo_name, file, line));
5082502c107SJeff Roberson 		WITNESS_CHECKORDER(&m->lock_object,
5092502c107SJeff Roberson 		    opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line);
5102502c107SJeff Roberson 		while (!_obtain_lock(m, tid)) {
5112502c107SJeff Roberson 			if (m->mtx_lock == tid) {
5122502c107SJeff Roberson 				m->mtx_recurse++;
5132502c107SJeff Roberson 				break;
5142502c107SJeff Roberson 			}
515eea4f254SJeff Roberson 			lock_profile_obtain_lock_failed(&m->lock_object,
516eea4f254SJeff Roberson 			    &contested, &waittime);
5172502c107SJeff Roberson 			/* Give interrupts a chance while we spin. */
5182502c107SJeff Roberson 			spinlock_exit();
5192502c107SJeff Roberson 			while (m->mtx_lock != MTX_UNOWNED) {
5202502c107SJeff Roberson 				if (i++ < 10000000)
5212502c107SJeff Roberson 					cpu_spinwait();
5222502c107SJeff Roberson 				else if (i < 60000000 ||
5232502c107SJeff Roberson 				    kdb_active || panicstr != NULL)
5242502c107SJeff Roberson 					DELAY(1);
5252502c107SJeff Roberson 				else
5262502c107SJeff Roberson 					_mtx_lock_spin_failed(m);
5272502c107SJeff Roberson 				cpu_spinwait();
5282502c107SJeff Roberson 				if (m != td->td_lock)
5292502c107SJeff Roberson 					goto retry;
5302502c107SJeff Roberson 			}
5312502c107SJeff Roberson 			spinlock_enter();
5322502c107SJeff Roberson 		}
5332502c107SJeff Roberson 		if (m == td->td_lock)
5342502c107SJeff Roberson 			break;
5352502c107SJeff Roberson 		_rel_spin_lock(m);	/* does spinlock_exit() */
5362502c107SJeff Roberson 	}
537eea4f254SJeff Roberson 	if (m->mtx_recurse == 0)
538773890b9SJeff Roberson 		lock_profile_obtain_lock_success(&m->lock_object, contested,
539773890b9SJeff Roberson 		    waittime, (file), (line));
54013c85a48SJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
54113c85a48SJohn Baldwin 	    line);
5422502c107SJeff Roberson 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
5432502c107SJeff Roberson }
5442502c107SJeff Roberson 
5452502c107SJeff Roberson struct mtx *
5462502c107SJeff Roberson thread_lock_block(struct thread *td)
5472502c107SJeff Roberson {
5482502c107SJeff Roberson 	struct mtx *lock;
5492502c107SJeff Roberson 
5502502c107SJeff Roberson 	spinlock_enter();
5512502c107SJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
552710eacdcSJeff Roberson 	lock = td->td_lock;
5532502c107SJeff Roberson 	td->td_lock = &blocked_lock;
5542502c107SJeff Roberson 	mtx_unlock_spin(lock);
5552502c107SJeff Roberson 
5562502c107SJeff Roberson 	return (lock);
5572502c107SJeff Roberson }
5582502c107SJeff Roberson 
5592502c107SJeff Roberson void
5602502c107SJeff Roberson thread_lock_unblock(struct thread *td, struct mtx *new)
5612502c107SJeff Roberson {
5622502c107SJeff Roberson 	mtx_assert(new, MA_OWNED);
5632502c107SJeff Roberson 	MPASS(td->td_lock == &blocked_lock);
56465d32cd8SMatt Jacob 	atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new);
5652502c107SJeff Roberson 	spinlock_exit();
5662502c107SJeff Roberson }
5672502c107SJeff Roberson 
5682502c107SJeff Roberson void
5692502c107SJeff Roberson thread_lock_set(struct thread *td, struct mtx *new)
5702502c107SJeff Roberson {
5712502c107SJeff Roberson 	struct mtx *lock;
5722502c107SJeff Roberson 
5732502c107SJeff Roberson 	mtx_assert(new, MA_OWNED);
5742502c107SJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
575710eacdcSJeff Roberson 	lock = td->td_lock;
5762502c107SJeff Roberson 	td->td_lock = new;
5772502c107SJeff Roberson 	mtx_unlock_spin(lock);
5782502c107SJeff Roberson }
5792502c107SJeff Roberson 
5809ed346baSBosko Milekic /*
5819ed346baSBosko Milekic  * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
5829ed346baSBosko Milekic  *
5839ed346baSBosko Milekic  * We are only called here if the lock is recursed or contested (i.e. we
5849ed346baSBosko Milekic  * need to wake up a blocked thread).
5859ed346baSBosko Milekic  */
58636412d79SJohn Baldwin void
5879ed346baSBosko Milekic _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
58836412d79SJohn Baldwin {
589961a7b24SJohn Baldwin 	struct turnstile *ts;
5909ed346baSBosko Milekic 
59108812b39SBosko Milekic 	if (mtx_recursed(m)) {
59236412d79SJohn Baldwin 		if (--(m->mtx_recurse) == 0)
59308812b39SBosko Milekic 			atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
594aa89d8cdSJohn Baldwin 		if (LOCK_LOG_TEST(&m->lock_object, opts))
5959ed346baSBosko Milekic 			CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
59636412d79SJohn Baldwin 		return;
59736412d79SJohn Baldwin 	}
5989ed346baSBosko Milekic 
5992502c107SJeff Roberson 	/*
6002502c107SJeff Roberson 	 * We have to lock the chain before the turnstile so this turnstile
6012502c107SJeff Roberson 	 * can be removed from the hash list if it is empty.
6022502c107SJeff Roberson 	 */
6032502c107SJeff Roberson 	turnstile_chain_lock(&m->lock_object);
604aa89d8cdSJohn Baldwin 	ts = turnstile_lookup(&m->lock_object);
605aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
6069ed346baSBosko Milekic 		CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
6079ed346baSBosko Milekic 
608961a7b24SJohn Baldwin 	MPASS(ts != NULL);
6097aa4f685SJohn Baldwin 	turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
610535eb309SJohn Baldwin 	_release_lock_quick(m);
6112502c107SJeff Roberson 	/*
6122502c107SJeff Roberson 	 * This turnstile is now no longer associated with the mutex.  We can
6132502c107SJeff Roberson 	 * unlock the chain lock so a new turnstile may take it's place.
6142502c107SJeff Roberson 	 */
6157aa4f685SJohn Baldwin 	turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
6162502c107SJeff Roberson 	turnstile_chain_unlock(&m->lock_object);
6179ed346baSBosko Milekic }
6189ed346baSBosko Milekic 
6199ed346baSBosko Milekic /*
6209ed346baSBosko Milekic  * All the unlocking of MTX_SPIN locks is done inline.
6219ed346baSBosko Milekic  * See the _rel_spin_lock() macro for the details.
6229ed346baSBosko Milekic  */
6239ed346baSBosko Milekic 
6249ed346baSBosko Milekic /*
62515ec816aSJohn Baldwin  * The backing function for the INVARIANTS-enabled mtx_assert()
6269ed346baSBosko Milekic  */
6271103f3b0SJohn Baldwin #ifdef INVARIANT_SUPPORT
6280cde2e34SJason Evans void
62956771ca7SJason Evans _mtx_assert(struct mtx *m, int what, const char *file, int line)
6300cde2e34SJason Evans {
6315cb0fbe4SJohn Baldwin 
6321126349aSPaul Saab 	if (panicstr != NULL || dumping)
6335cb0fbe4SJohn Baldwin 		return;
634a10f4966SJake Burkholder 	switch (what) {
6350cde2e34SJason Evans 	case MA_OWNED:
6360cde2e34SJason Evans 	case MA_OWNED | MA_RECURSED:
6370cde2e34SJason Evans 	case MA_OWNED | MA_NOTRECURSED:
638a10f4966SJake Burkholder 		if (!mtx_owned(m))
6390cde2e34SJason Evans 			panic("mutex %s not owned at %s:%d",
640aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
641a10f4966SJake Burkholder 		if (mtx_recursed(m)) {
642a10f4966SJake Burkholder 			if ((what & MA_NOTRECURSED) != 0)
6430cde2e34SJason Evans 				panic("mutex %s recursed at %s:%d",
644aa89d8cdSJohn Baldwin 				    m->lock_object.lo_name, file, line);
645a10f4966SJake Burkholder 		} else if ((what & MA_RECURSED) != 0) {
6460cde2e34SJason Evans 			panic("mutex %s unrecursed at %s:%d",
647aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
6480cde2e34SJason Evans 		}
6490cde2e34SJason Evans 		break;
6500cde2e34SJason Evans 	case MA_NOTOWNED:
651a10f4966SJake Burkholder 		if (mtx_owned(m))
6520cde2e34SJason Evans 			panic("mutex %s owned at %s:%d",
653aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
6540cde2e34SJason Evans 		break;
6550cde2e34SJason Evans 	default:
65656771ca7SJason Evans 		panic("unknown mtx_assert at %s:%d", file, line);
6570cde2e34SJason Evans 	}
6580cde2e34SJason Evans }
6590cde2e34SJason Evans #endif
6600cde2e34SJason Evans 
6619ed346baSBosko Milekic /*
6629ed346baSBosko Milekic  * The MUTEX_DEBUG-enabled mtx_validate()
66319284646SJohn Baldwin  *
66419284646SJohn Baldwin  * Most of these checks have been moved off into the LO_INITIALIZED flag
66519284646SJohn Baldwin  * maintained by the witness code.
6669ed346baSBosko Milekic  */
66736412d79SJohn Baldwin #ifdef MUTEX_DEBUG
66836412d79SJohn Baldwin 
6694d77a549SAlfred Perlstein void	mtx_validate(struct mtx *);
67036412d79SJohn Baldwin 
67119284646SJohn Baldwin void
67219284646SJohn Baldwin mtx_validate(struct mtx *m)
67336412d79SJohn Baldwin {
67436412d79SJohn Baldwin 
67536412d79SJohn Baldwin /*
676fa669ab7SPoul-Henning Kamp  * XXX: When kernacc() does not require Giant we can reenable this check
677fa669ab7SPoul-Henning Kamp  */
678fa669ab7SPoul-Henning Kamp #ifdef notyet
679fa669ab7SPoul-Henning Kamp 	/*
68076dcbd6fSBosko Milekic 	 * Can't call kernacc() from early init386(), especially when
68176dcbd6fSBosko Milekic 	 * initializing Giant mutex, because some stuff in kernacc()
68276dcbd6fSBosko Milekic 	 * requires Giant itself.
68376dcbd6fSBosko Milekic 	 */
684ab07087eSBosko Milekic 	if (!cold)
685ab07087eSBosko Milekic 		if (!kernacc((caddr_t)m, sizeof(m),
686ab07087eSBosko Milekic 		    VM_PROT_READ | VM_PROT_WRITE))
68719284646SJohn Baldwin 			panic("Can't read and write to mutex %p", m);
68836412d79SJohn Baldwin #endif
68936412d79SJohn Baldwin }
69036412d79SJohn Baldwin #endif
69136412d79SJohn Baldwin 
6929ed346baSBosko Milekic /*
693c27b5699SAndrew R. Reiter  * General init routine used by the MTX_SYSINIT() macro.
694c27b5699SAndrew R. Reiter  */
695c27b5699SAndrew R. Reiter void
696c27b5699SAndrew R. Reiter mtx_sysinit(void *arg)
697c27b5699SAndrew R. Reiter {
698c27b5699SAndrew R. Reiter 	struct mtx_args *margs = arg;
699c27b5699SAndrew R. Reiter 
7000c88508aSJohn Baldwin 	mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts);
701c27b5699SAndrew R. Reiter }
702c27b5699SAndrew R. Reiter 
703c27b5699SAndrew R. Reiter /*
7049ed346baSBosko Milekic  * Mutex initialization routine; initialize lock `m' of type contained in
7050c88508aSJohn Baldwin  * `opts' with options contained in `opts' and name `name.'  The optional
7060c88508aSJohn Baldwin  * lock type `type' is used as a general lock category name for use with
7070c88508aSJohn Baldwin  * witness.
7089ed346baSBosko Milekic  */
70936412d79SJohn Baldwin void
7100c88508aSJohn Baldwin mtx_init(struct mtx *m, const char *name, const char *type, int opts)
71136412d79SJohn Baldwin {
71283a81bcbSJohn Baldwin 	struct lock_class *class;
71383a81bcbSJohn Baldwin 	int flags;
7149ed346baSBosko Milekic 
71519284646SJohn Baldwin 	MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
7167c0435b9SKip Macy 		MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0);
7179ed346baSBosko Milekic 
71836412d79SJohn Baldwin #ifdef MUTEX_DEBUG
7199ed346baSBosko Milekic 	/* Diagnostic and error correction */
72019284646SJohn Baldwin 	mtx_validate(m);
7216936206eSJohn Baldwin #endif
72236412d79SJohn Baldwin 
72383a81bcbSJohn Baldwin 	/* Determine lock class and lock flags. */
72419284646SJohn Baldwin 	if (opts & MTX_SPIN)
72583a81bcbSJohn Baldwin 		class = &lock_class_mtx_spin;
72619284646SJohn Baldwin 	else
72783a81bcbSJohn Baldwin 		class = &lock_class_mtx_sleep;
72883a81bcbSJohn Baldwin 	flags = 0;
72919284646SJohn Baldwin 	if (opts & MTX_QUIET)
73083a81bcbSJohn Baldwin 		flags |= LO_QUIET;
73119284646SJohn Baldwin 	if (opts & MTX_RECURSE)
73283a81bcbSJohn Baldwin 		flags |= LO_RECURSABLE;
73319284646SJohn Baldwin 	if ((opts & MTX_NOWITNESS) == 0)
73483a81bcbSJohn Baldwin 		flags |= LO_WITNESS;
735f22a4b62SJeff Roberson 	if (opts & MTX_DUPOK)
73683a81bcbSJohn Baldwin 		flags |= LO_DUPOK;
7377c0435b9SKip Macy 	if (opts & MTX_NOPROFILE)
7387c0435b9SKip Macy 		flags |= LO_NOPROFILE;
73919284646SJohn Baldwin 
74083a81bcbSJohn Baldwin 	/* Initialize mutex. */
74119284646SJohn Baldwin 	m->mtx_lock = MTX_UNOWNED;
74283a81bcbSJohn Baldwin 	m->mtx_recurse = 0;
7439ed346baSBosko Milekic 
744aa89d8cdSJohn Baldwin 	lock_init(&m->lock_object, class, name, type, flags);
74536412d79SJohn Baldwin }
74636412d79SJohn Baldwin 
7479ed346baSBosko Milekic /*
74819284646SJohn Baldwin  * Remove lock `m' from all_mtx queue.  We don't allow MTX_QUIET to be
74919284646SJohn Baldwin  * passed in as a flag here because if the corresponding mtx_init() was
75019284646SJohn Baldwin  * called with MTX_QUIET set, then it will already be set in the mutex's
75119284646SJohn Baldwin  * flags.
7529ed346baSBosko Milekic  */
75336412d79SJohn Baldwin void
75436412d79SJohn Baldwin mtx_destroy(struct mtx *m)
75536412d79SJohn Baldwin {
75636412d79SJohn Baldwin 
75719284646SJohn Baldwin 	if (!mtx_owned(m))
75819284646SJohn Baldwin 		MPASS(mtx_unowned(m));
75919284646SJohn Baldwin 	else {
76008812b39SBosko Milekic 		MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
7619ed346baSBosko Milekic 
762861a2308SScott Long 		/* Perform the non-mtx related part of mtx_unlock_spin(). */
763aa89d8cdSJohn Baldwin 		if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin)
764861a2308SScott Long 			spinlock_exit();
765764e4d54SJohn Baldwin 		else
766764e4d54SJohn Baldwin 			curthread->td_locks--;
767861a2308SScott Long 
76819284646SJohn Baldwin 		/* Tell witness this isn't locked to make it happy. */
769aa89d8cdSJohn Baldwin 		WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__,
770c86b6ff5SJohn Baldwin 		    __LINE__);
77136412d79SJohn Baldwin 	}
7720384fff8SJason Evans 
773186abbd7SJohn Baldwin 	m->mtx_lock = MTX_DESTROYED;
774aa89d8cdSJohn Baldwin 	lock_destroy(&m->lock_object);
7750384fff8SJason Evans }
776d23f5958SMatthew Dillon 
777d23f5958SMatthew Dillon /*
778c53c013bSJohn Baldwin  * Intialize the mutex code and system mutexes.  This is called from the MD
779c53c013bSJohn Baldwin  * startup code prior to mi_startup().  The per-CPU data space needs to be
780c53c013bSJohn Baldwin  * setup before this is called.
781c53c013bSJohn Baldwin  */
782c53c013bSJohn Baldwin void
783c53c013bSJohn Baldwin mutex_init(void)
784c53c013bSJohn Baldwin {
785c53c013bSJohn Baldwin 
786961a7b24SJohn Baldwin 	/* Setup turnstiles so that sleep mutexes work. */
787961a7b24SJohn Baldwin 	init_turnstiles();
788961a7b24SJohn Baldwin 
789c53c013bSJohn Baldwin 	/*
790c53c013bSJohn Baldwin 	 * Initialize mutexes.
791c53c013bSJohn Baldwin 	 */
7920c88508aSJohn Baldwin 	mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
7932502c107SJeff Roberson 	mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN);
7942502c107SJeff Roberson 	blocked_lock.mtx_lock = 0xdeadc0de;	/* Always blocked. */
7950c88508aSJohn Baldwin 	mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
7962502c107SJeff Roberson 	mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE);
7978c4b6380SJohn Baldwin 	mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
798c53c013bSJohn Baldwin 	mtx_lock(&Giant);
799c53c013bSJohn Baldwin }
800d272fe53SJohn Baldwin 
801d272fe53SJohn Baldwin #ifdef DDB
802d272fe53SJohn Baldwin void
803d272fe53SJohn Baldwin db_show_mtx(struct lock_object *lock)
804d272fe53SJohn Baldwin {
805d272fe53SJohn Baldwin 	struct thread *td;
806d272fe53SJohn Baldwin 	struct mtx *m;
807d272fe53SJohn Baldwin 
808d272fe53SJohn Baldwin 	m = (struct mtx *)lock;
809d272fe53SJohn Baldwin 
810d272fe53SJohn Baldwin 	db_printf(" flags: {");
81183a81bcbSJohn Baldwin 	if (LOCK_CLASS(lock) == &lock_class_mtx_spin)
812d272fe53SJohn Baldwin 		db_printf("SPIN");
813d272fe53SJohn Baldwin 	else
814d272fe53SJohn Baldwin 		db_printf("DEF");
815aa89d8cdSJohn Baldwin 	if (m->lock_object.lo_flags & LO_RECURSABLE)
816d272fe53SJohn Baldwin 		db_printf(", RECURSE");
817aa89d8cdSJohn Baldwin 	if (m->lock_object.lo_flags & LO_DUPOK)
818d272fe53SJohn Baldwin 		db_printf(", DUPOK");
819d272fe53SJohn Baldwin 	db_printf("}\n");
820d272fe53SJohn Baldwin 	db_printf(" state: {");
821d272fe53SJohn Baldwin 	if (mtx_unowned(m))
822d272fe53SJohn Baldwin 		db_printf("UNOWNED");
823c0bfd703SJohn Baldwin 	else if (mtx_destroyed(m))
824c0bfd703SJohn Baldwin 		db_printf("DESTROYED");
825d272fe53SJohn Baldwin 	else {
826d272fe53SJohn Baldwin 		db_printf("OWNED");
827d272fe53SJohn Baldwin 		if (m->mtx_lock & MTX_CONTESTED)
828d272fe53SJohn Baldwin 			db_printf(", CONTESTED");
829d272fe53SJohn Baldwin 		if (m->mtx_lock & MTX_RECURSED)
830d272fe53SJohn Baldwin 			db_printf(", RECURSED");
831d272fe53SJohn Baldwin 	}
832d272fe53SJohn Baldwin 	db_printf("}\n");
833c0bfd703SJohn Baldwin 	if (!mtx_unowned(m) && !mtx_destroyed(m)) {
834d272fe53SJohn Baldwin 		td = mtx_owner(m);
835d272fe53SJohn Baldwin 		db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td,
836431f8906SJulian Elischer 		    td->td_tid, td->td_proc->p_pid, td->td_name);
837d272fe53SJohn Baldwin 		if (mtx_recursed(m))
838d272fe53SJohn Baldwin 			db_printf(" recursed: %d\n", m->mtx_recurse);
839d272fe53SJohn Baldwin 	}
840d272fe53SJohn Baldwin }
841d272fe53SJohn Baldwin #endif
842