xref: /freebsd/sys/kern/kern_mutex.c (revision f61d6c5a5bdbc6d02b095dad836c016094b32cf1)
10384fff8SJason Evans /*-
20384fff8SJason Evans  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
30384fff8SJason Evans  *
40384fff8SJason Evans  * Redistribution and use in source and binary forms, with or without
50384fff8SJason Evans  * modification, are permitted provided that the following conditions
60384fff8SJason Evans  * are met:
70384fff8SJason Evans  * 1. Redistributions of source code must retain the above copyright
80384fff8SJason Evans  *    notice, this list of conditions and the following disclaimer.
90384fff8SJason Evans  * 2. Redistributions in binary form must reproduce the above copyright
100384fff8SJason Evans  *    notice, this list of conditions and the following disclaimer in the
110384fff8SJason Evans  *    documentation and/or other materials provided with the distribution.
120384fff8SJason Evans  * 3. Berkeley Software Design Inc's name may not be used to endorse or
130384fff8SJason Evans  *    promote products derived from this software without specific prior
140384fff8SJason Evans  *    written permission.
150384fff8SJason Evans  *
160384fff8SJason Evans  * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
170384fff8SJason Evans  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
180384fff8SJason Evans  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
190384fff8SJason Evans  * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
200384fff8SJason Evans  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
210384fff8SJason Evans  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
220384fff8SJason Evans  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
230384fff8SJason Evans  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
240384fff8SJason Evans  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
250384fff8SJason Evans  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
260384fff8SJason Evans  * SUCH DAMAGE.
270384fff8SJason Evans  *
280384fff8SJason Evans  *	from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
2936412d79SJohn Baldwin  *	and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
300384fff8SJason Evans  */
310384fff8SJason Evans 
320384fff8SJason Evans /*
33ba48b69aSJohn Baldwin  * Machine independent bits of mutex implementation.
340384fff8SJason Evans  */
350384fff8SJason Evans 
36677b542eSDavid E. O'Brien #include <sys/cdefs.h>
37677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
38677b542eSDavid E. O'Brien 
392498cf8cSJohn Baldwin #include "opt_adaptive_mutexes.h"
409c36c934SJohn Baldwin #include "opt_ddb.h"
41f5f9340bSFabien Thomas #include "opt_hwpmc_hooks.h"
429923b511SScott Long #include "opt_sched.h"
43a5a96a19SJohn Baldwin 
440384fff8SJason Evans #include <sys/param.h>
456c35e809SDag-Erling Smørgrav #include <sys/systm.h>
4636412d79SJohn Baldwin #include <sys/bus.h>
471126349aSPaul Saab #include <sys/conf.h>
482d50560aSMarcel Moolenaar #include <sys/kdb.h>
4936412d79SJohn Baldwin #include <sys/kernel.h>
506c35e809SDag-Erling Smørgrav #include <sys/ktr.h>
5119284646SJohn Baldwin #include <sys/lock.h>
52fb919e4dSMark Murray #include <sys/malloc.h>
5319284646SJohn Baldwin #include <sys/mutex.h>
540384fff8SJason Evans #include <sys/proc.h>
55c4f7a187SJohn Baldwin #include <sys/resourcevar.h>
56b43179fbSJeff Roberson #include <sys/sched.h>
576c35e809SDag-Erling Smørgrav #include <sys/sbuf.h>
58a5a96a19SJohn Baldwin #include <sys/sysctl.h>
59961a7b24SJohn Baldwin #include <sys/turnstile.h>
6036412d79SJohn Baldwin #include <sys/vmmeter.h>
617c0435b9SKip Macy #include <sys/lock_profile.h>
620384fff8SJason Evans 
6336412d79SJohn Baldwin #include <machine/atomic.h>
6436412d79SJohn Baldwin #include <machine/bus.h>
650384fff8SJason Evans #include <machine/cpu.h>
6636412d79SJohn Baldwin 
679c36c934SJohn Baldwin #include <ddb/ddb.h>
689c36c934SJohn Baldwin 
698c4b6380SJohn Baldwin #include <fs/devfs/devfs_int.h>
708c4b6380SJohn Baldwin 
7136412d79SJohn Baldwin #include <vm/vm.h>
7236412d79SJohn Baldwin #include <vm/vm_extern.h>
7336412d79SJohn Baldwin 
74cd6e6e4eSJohn Baldwin #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
75cd6e6e4eSJohn Baldwin #define	ADAPTIVE_MUTEXES
76cd6e6e4eSJohn Baldwin #endif
77cd6e6e4eSJohn Baldwin 
78f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
79f5f9340bSFabien Thomas #include <sys/pmckern.h>
80f5f9340bSFabien Thomas PMC_SOFT_DEFINE( , , lock, failed);
81f5f9340bSFabien Thomas #endif
82f5f9340bSFabien Thomas 
83b9a80acaSStephan Uphoff /*
847f44c618SAttilio Rao  * Return the mutex address when the lock cookie address is provided.
857f44c618SAttilio Rao  * This functionality assumes that struct mtx* have a member named mtx_lock.
867f44c618SAttilio Rao  */
877f44c618SAttilio Rao #define	mtxlock2mtx(c)	(__containerof(c, struct mtx, mtx_lock))
887f44c618SAttilio Rao 
897f44c618SAttilio Rao /*
909ed346baSBosko Milekic  * Internal utility macros.
910cde2e34SJason Evans  */
929ed346baSBosko Milekic #define mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)
930cde2e34SJason Evans 
94c0bfd703SJohn Baldwin #define	mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED)
95c0bfd703SJohn Baldwin 
9649b94bfcSJohn Baldwin #define	mtx_owner(m)	((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK))
979ed346baSBosko Milekic 
98d576deedSPawel Jakub Dawidek static void	assert_mtx(const struct lock_object *lock, int what);
99d272fe53SJohn Baldwin #ifdef DDB
100d576deedSPawel Jakub Dawidek static void	db_show_mtx(const struct lock_object *lock);
101d272fe53SJohn Baldwin #endif
1027faf4d90SDavide Italiano static void	lock_mtx(struct lock_object *lock, uintptr_t how);
1037faf4d90SDavide Italiano static void	lock_spin(struct lock_object *lock, uintptr_t how);
104a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
105d576deedSPawel Jakub Dawidek static int	owner_mtx(const struct lock_object *lock,
106d576deedSPawel Jakub Dawidek 		    struct thread **owner);
107a5aedd68SStacey Son #endif
1087faf4d90SDavide Italiano static uintptr_t unlock_mtx(struct lock_object *lock);
1097faf4d90SDavide Italiano static uintptr_t unlock_spin(struct lock_object *lock);
110d272fe53SJohn Baldwin 
1110cde2e34SJason Evans /*
11219284646SJohn Baldwin  * Lock classes for sleep and spin mutexes.
1130cde2e34SJason Evans  */
11419284646SJohn Baldwin struct lock_class lock_class_mtx_sleep = {
115ae8dde30SJohn Baldwin 	.lc_name = "sleep mutex",
116ae8dde30SJohn Baldwin 	.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
117f9721b43SAttilio Rao 	.lc_assert = assert_mtx,
118d272fe53SJohn Baldwin #ifdef DDB
119ae8dde30SJohn Baldwin 	.lc_ddb_show = db_show_mtx,
120d272fe53SJohn Baldwin #endif
1216e21afd4SJohn Baldwin 	.lc_lock = lock_mtx,
1226e21afd4SJohn Baldwin 	.lc_unlock = unlock_mtx,
123a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
124a5aedd68SStacey Son 	.lc_owner = owner_mtx,
125a5aedd68SStacey Son #endif
12619284646SJohn Baldwin };
12719284646SJohn Baldwin struct lock_class lock_class_mtx_spin = {
128ae8dde30SJohn Baldwin 	.lc_name = "spin mutex",
129ae8dde30SJohn Baldwin 	.lc_flags = LC_SPINLOCK | LC_RECURSABLE,
130f9721b43SAttilio Rao 	.lc_assert = assert_mtx,
131d272fe53SJohn Baldwin #ifdef DDB
132ae8dde30SJohn Baldwin 	.lc_ddb_show = db_show_mtx,
133d272fe53SJohn Baldwin #endif
1346e21afd4SJohn Baldwin 	.lc_lock = lock_spin,
1356e21afd4SJohn Baldwin 	.lc_unlock = unlock_spin,
136a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
137a5aedd68SStacey Son 	.lc_owner = owner_mtx,
138a5aedd68SStacey Son #endif
1398484de75SJohn Baldwin };
1408484de75SJohn Baldwin 
1419ed346baSBosko Milekic /*
142c53c013bSJohn Baldwin  * System-wide mutexes
143c53c013bSJohn Baldwin  */
1442502c107SJeff Roberson struct mtx blocked_lock;
145c53c013bSJohn Baldwin struct mtx Giant;
146c53c013bSJohn Baldwin 
14767784314SPoul-Henning Kamp void
148d576deedSPawel Jakub Dawidek assert_mtx(const struct lock_object *lock, int what)
149f9721b43SAttilio Rao {
150f9721b43SAttilio Rao 
151d576deedSPawel Jakub Dawidek 	mtx_assert((const struct mtx *)lock, what);
152f9721b43SAttilio Rao }
153f9721b43SAttilio Rao 
15467784314SPoul-Henning Kamp void
1557faf4d90SDavide Italiano lock_mtx(struct lock_object *lock, uintptr_t how)
1566e21afd4SJohn Baldwin {
1576e21afd4SJohn Baldwin 
1586e21afd4SJohn Baldwin 	mtx_lock((struct mtx *)lock);
1596e21afd4SJohn Baldwin }
1606e21afd4SJohn Baldwin 
16167784314SPoul-Henning Kamp void
1627faf4d90SDavide Italiano lock_spin(struct lock_object *lock, uintptr_t how)
1636e21afd4SJohn Baldwin {
1646e21afd4SJohn Baldwin 
1656e21afd4SJohn Baldwin 	panic("spin locks can only use msleep_spin");
1666e21afd4SJohn Baldwin }
1676e21afd4SJohn Baldwin 
1687faf4d90SDavide Italiano uintptr_t
1696e21afd4SJohn Baldwin unlock_mtx(struct lock_object *lock)
1706e21afd4SJohn Baldwin {
1716e21afd4SJohn Baldwin 	struct mtx *m;
1726e21afd4SJohn Baldwin 
1736e21afd4SJohn Baldwin 	m = (struct mtx *)lock;
1746e21afd4SJohn Baldwin 	mtx_assert(m, MA_OWNED | MA_NOTRECURSED);
1756e21afd4SJohn Baldwin 	mtx_unlock(m);
1766e21afd4SJohn Baldwin 	return (0);
1776e21afd4SJohn Baldwin }
1786e21afd4SJohn Baldwin 
1797faf4d90SDavide Italiano uintptr_t
1806e21afd4SJohn Baldwin unlock_spin(struct lock_object *lock)
1816e21afd4SJohn Baldwin {
1826e21afd4SJohn Baldwin 
1836e21afd4SJohn Baldwin 	panic("spin locks can only use msleep_spin");
1846e21afd4SJohn Baldwin }
1856e21afd4SJohn Baldwin 
186a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
187a5aedd68SStacey Son int
188d576deedSPawel Jakub Dawidek owner_mtx(const struct lock_object *lock, struct thread **owner)
189a5aedd68SStacey Son {
190d576deedSPawel Jakub Dawidek 	const struct mtx *m = (const struct mtx *)lock;
191a5aedd68SStacey Son 
192a5aedd68SStacey Son 	*owner = mtx_owner(m);
193a5aedd68SStacey Son 	return (mtx_unowned(m) == 0);
194a5aedd68SStacey Son }
195a5aedd68SStacey Son #endif
196a5aedd68SStacey Son 
1970cde2e34SJason Evans /*
1986283b7d0SJohn Baldwin  * Function versions of the inlined __mtx_* macros.  These are used by
1996283b7d0SJohn Baldwin  * modules and can also be called from assembly language if needed.
2006283b7d0SJohn Baldwin  */
2016283b7d0SJohn Baldwin void
2027f44c618SAttilio Rao __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
2036283b7d0SJohn Baldwin {
2047f44c618SAttilio Rao 	struct mtx *m;
2056283b7d0SJohn Baldwin 
20635370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
20735370593SAndriy Gapon 		return;
2087f44c618SAttilio Rao 
2097f44c618SAttilio Rao 	m = mtxlock2mtx(c);
2107f44c618SAttilio Rao 
211cd2fe4e6SAttilio Rao 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
212e3ae0dfeSAttilio Rao 	    ("mtx_lock() by idle thread %p on sleep mutex %s @ %s:%d",
213e3ae0dfeSAttilio Rao 	    curthread, m->lock_object.lo_name, file, line));
214186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
215186abbd7SJohn Baldwin 	    ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
216aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
217aa89d8cdSJohn Baldwin 	    ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
2180d975d63SJohn Baldwin 	    file, line));
219ac6b769bSAttilio Rao 	WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) |
220ac6b769bSAttilio Rao 	    LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
2217c0435b9SKip Macy 
222961135eaSJohn Baldwin 	__mtx_lock(m, curthread, opts, file, line);
223aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
224dde96c99SJohn Baldwin 	    line);
225ac6b769bSAttilio Rao 	WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE,
226ac6b769bSAttilio Rao 	    file, line);
227ce1c953eSMark Johnston 	TD_LOCKS_INC(curthread);
2286283b7d0SJohn Baldwin }
2296283b7d0SJohn Baldwin 
2306283b7d0SJohn Baldwin void
2317f44c618SAttilio Rao __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
2326283b7d0SJohn Baldwin {
2337f44c618SAttilio Rao 	struct mtx *m;
23435370593SAndriy Gapon 
23535370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
23635370593SAndriy Gapon 		return;
2377f44c618SAttilio Rao 
2387f44c618SAttilio Rao 	m = mtxlock2mtx(c);
2397f44c618SAttilio Rao 
240186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
241186abbd7SJohn Baldwin 	    ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
242aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
243aa89d8cdSJohn Baldwin 	    ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
2440d975d63SJohn Baldwin 	    file, line));
245aa89d8cdSJohn Baldwin 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
246aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
2470d975d63SJohn Baldwin 	    line);
24821377ce0SJohn Baldwin 	mtx_assert(m, MA_OWNED);
249c66d7606SKip Macy 
250961135eaSJohn Baldwin 	__mtx_unlock(m, curthread, opts, file, line);
251ce1c953eSMark Johnston 	TD_LOCKS_DEC(curthread);
2526283b7d0SJohn Baldwin }
2536283b7d0SJohn Baldwin 
2546283b7d0SJohn Baldwin void
2557f44c618SAttilio Rao __mtx_lock_spin_flags(volatile uintptr_t *c, int opts, const char *file,
2567f44c618SAttilio Rao     int line)
2576283b7d0SJohn Baldwin {
2587f44c618SAttilio Rao 	struct mtx *m;
2596283b7d0SJohn Baldwin 
26035370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
26135370593SAndriy Gapon 		return;
2627f44c618SAttilio Rao 
2637f44c618SAttilio Rao 	m = mtxlock2mtx(c);
2647f44c618SAttilio Rao 
265186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
266186abbd7SJohn Baldwin 	    ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
267aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
2680d975d63SJohn Baldwin 	    ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
269aa89d8cdSJohn Baldwin 	    m->lock_object.lo_name, file, line));
270ad69e26bSJohn Baldwin 	if (mtx_owned(m))
271ac6b769bSAttilio Rao 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
272ac6b769bSAttilio Rao 		    (opts & MTX_RECURSE) != 0,
273ad69e26bSJohn Baldwin 	    ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n",
274ad69e26bSJohn Baldwin 		    m->lock_object.lo_name, file, line));
275ac6b769bSAttilio Rao 	opts &= ~MTX_RECURSE;
276aa89d8cdSJohn Baldwin 	WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
27741313430SJohn Baldwin 	    file, line, NULL);
278961135eaSJohn Baldwin 	__mtx_lock_spin(m, curthread, opts, file, line);
279aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
280dde96c99SJohn Baldwin 	    line);
281aa89d8cdSJohn Baldwin 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
2826283b7d0SJohn Baldwin }
2836283b7d0SJohn Baldwin 
2846283b7d0SJohn Baldwin void
2857f44c618SAttilio Rao __mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, const char *file,
2867f44c618SAttilio Rao     int line)
2876283b7d0SJohn Baldwin {
2887f44c618SAttilio Rao 	struct mtx *m;
289c66d7606SKip Macy 
29035370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
29135370593SAndriy Gapon 		return;
2927f44c618SAttilio Rao 
2937f44c618SAttilio Rao 	m = mtxlock2mtx(c);
2947f44c618SAttilio Rao 
295186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
296186abbd7SJohn Baldwin 	    ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
297aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
2980d975d63SJohn Baldwin 	    ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
299aa89d8cdSJohn Baldwin 	    m->lock_object.lo_name, file, line));
300aa89d8cdSJohn Baldwin 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
301aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
302dde96c99SJohn Baldwin 	    line);
3030d975d63SJohn Baldwin 	mtx_assert(m, MA_OWNED);
304c66d7606SKip Macy 
305961135eaSJohn Baldwin 	__mtx_unlock_spin(m);
3066283b7d0SJohn Baldwin }
3076283b7d0SJohn Baldwin 
3086283b7d0SJohn Baldwin /*
3099ed346baSBosko Milekic  * The important part of mtx_trylock{,_flags}()
310eac09796SJohn Baldwin  * Tries to acquire lock `m.'  If this function is called on a mutex that
311eac09796SJohn Baldwin  * is already owned, it will recursively acquire the lock.
3120cde2e34SJason Evans  */
3130cde2e34SJason Evans int
3147f44c618SAttilio Rao _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
3150cde2e34SJason Evans {
3167f44c618SAttilio Rao 	struct mtx *m;
3171723a064SJeff Roberson #ifdef LOCK_PROFILING
3187c0435b9SKip Macy 	uint64_t waittime = 0;
3191723a064SJeff Roberson 	int contested = 0;
3201723a064SJeff Roberson #endif
3211723a064SJeff Roberson 	int rval;
3220cde2e34SJason Evans 
32335370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
32435370593SAndriy Gapon 		return (1);
32535370593SAndriy Gapon 
3267f44c618SAttilio Rao 	m = mtxlock2mtx(c);
3277f44c618SAttilio Rao 
328cd2fe4e6SAttilio Rao 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
329e3ae0dfeSAttilio Rao 	    ("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d",
330e3ae0dfeSAttilio Rao 	    curthread, m->lock_object.lo_name, file, line));
331186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
332186abbd7SJohn Baldwin 	    ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
333aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
334aa89d8cdSJohn Baldwin 	    ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
33583cece6fSJohn Baldwin 	    file, line));
3369ed346baSBosko Milekic 
337ac6b769bSAttilio Rao 	if (mtx_owned(m) && ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
338ac6b769bSAttilio Rao 	    (opts & MTX_RECURSE) != 0)) {
339eac09796SJohn Baldwin 		m->mtx_recurse++;
340eac09796SJohn Baldwin 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
341eac09796SJohn Baldwin 		rval = 1;
342eac09796SJohn Baldwin 	} else
343961135eaSJohn Baldwin 		rval = _mtx_obtain_lock(m, (uintptr_t)curthread);
344ac6b769bSAttilio Rao 	opts &= ~MTX_RECURSE;
3459ed346baSBosko Milekic 
346aa89d8cdSJohn Baldwin 	LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
347764e4d54SJohn Baldwin 	if (rval) {
348aa89d8cdSJohn Baldwin 		WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
3492d96f0b1SJohn Baldwin 		    file, line);
350ce1c953eSMark Johnston 		TD_LOCKS_INC(curthread);
351fe68a916SKip Macy 		if (m->mtx_recurse == 0)
35232cd0147SMark Johnston 			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,
353a5aedd68SStacey Son 			    m, contested, waittime, file, line);
3547c0435b9SKip Macy 
355764e4d54SJohn Baldwin 	}
3569ed346baSBosko Milekic 
35719284646SJohn Baldwin 	return (rval);
3580cde2e34SJason Evans }
3590cde2e34SJason Evans 
3600cde2e34SJason Evans /*
3617f44c618SAttilio Rao  * __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
3629ed346baSBosko Milekic  *
3639ed346baSBosko Milekic  * We call this if the lock is either contested (i.e. we need to go to
3649ed346baSBosko Milekic  * sleep waiting for it), or if we need to recurse on it.
3650cde2e34SJason Evans  */
3660cde2e34SJason Evans void
3677f44c618SAttilio Rao __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
3687f44c618SAttilio Rao     const char *file, int line)
36936412d79SJohn Baldwin {
3707f44c618SAttilio Rao 	struct mtx *m;
3712502c107SJeff Roberson 	struct turnstile *ts;
3721723a064SJeff Roberson 	uintptr_t v;
373cd6e6e4eSJohn Baldwin #ifdef ADAPTIVE_MUTEXES
37476447e56SJohn Baldwin 	volatile struct thread *owner;
3752498cf8cSJohn Baldwin #endif
37602bd1bcdSIan Dowse #ifdef KTR
37702bd1bcdSIan Dowse 	int cont_logged = 0;
37802bd1bcdSIan Dowse #endif
3791723a064SJeff Roberson #ifdef LOCK_PROFILING
38070fe8436SKip Macy 	int contested = 0;
38170fe8436SKip Macy 	uint64_t waittime = 0;
3821723a064SJeff Roberson #endif
383a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
384a5aedd68SStacey Son 	uint64_t spin_cnt = 0;
385a5aedd68SStacey Son 	uint64_t sleep_cnt = 0;
386a5aedd68SStacey Son 	int64_t sleep_time = 0;
387076dd8ebSAndriy Gapon 	int64_t all_time = 0;
388a5aedd68SStacey Son #endif
38936412d79SJohn Baldwin 
39035370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
39135370593SAndriy Gapon 		return;
39235370593SAndriy Gapon 
3937f44c618SAttilio Rao 	m = mtxlock2mtx(c);
3947f44c618SAttilio Rao 
3955fa8dd90SJohn Baldwin 	if (mtx_owned(m)) {
396ac6b769bSAttilio Rao 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
397ac6b769bSAttilio Rao 		    (opts & MTX_RECURSE) != 0,
398eac09796SJohn Baldwin 	    ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
399aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, file, line));
400ac6b769bSAttilio Rao 		opts &= ~MTX_RECURSE;
40136412d79SJohn Baldwin 		m->mtx_recurse++;
40208812b39SBosko Milekic 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
403aa89d8cdSJohn Baldwin 		if (LOCK_LOG_TEST(&m->lock_object, opts))
4045746a1d8SBosko Milekic 			CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
40536412d79SJohn Baldwin 		return;
40636412d79SJohn Baldwin 	}
407ac6b769bSAttilio Rao 	opts &= ~MTX_RECURSE;
4089ed346baSBosko Milekic 
409f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
410f5f9340bSFabien Thomas 	PMC_SOFT_CALL( , , lock, failed);
411f5f9340bSFabien Thomas #endif
41270fe8436SKip Macy 	lock_profile_obtain_lock_failed(&m->lock_object,
41370fe8436SKip Macy 		    &contested, &waittime);
414aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
41515ec816aSJohn Baldwin 		CTR4(KTR_LOCK,
41615ec816aSJohn Baldwin 		    "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
417aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
418076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
419e2b25737SMark Johnston 	all_time -= lockstat_nsecs(&m->lock_object);
420076dd8ebSAndriy Gapon #endif
4211bd0eefbSJohn Baldwin 
422fc4f686dSMateusz Guzik 	for (;;) {
423fc4f686dSMateusz Guzik 		if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid))
424fc4f686dSMateusz Guzik 			break;
425a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
426a5aedd68SStacey Son 		spin_cnt++;
427a5aedd68SStacey Son #endif
42849aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES
42949aead8aSAttilio Rao 		/*
43049aead8aSAttilio Rao 		 * If the owner is running on another CPU, spin until the
43149aead8aSAttilio Rao 		 * owner stops running or the state of the lock changes.
43249aead8aSAttilio Rao 		 */
43349aead8aSAttilio Rao 		v = m->mtx_lock;
43449aead8aSAttilio Rao 		if (v != MTX_UNOWNED) {
43549aead8aSAttilio Rao 			owner = (struct thread *)(v & ~MTX_FLAGMASK);
43649aead8aSAttilio Rao 			if (TD_IS_RUNNING(owner)) {
43749aead8aSAttilio Rao 				if (LOCK_LOG_TEST(&m->lock_object, 0))
43849aead8aSAttilio Rao 					CTR3(KTR_LOCK,
43949aead8aSAttilio Rao 					    "%s: spinning on %p held by %p",
44049aead8aSAttilio Rao 					    __func__, m, owner);
4412cba8dd3SJohn Baldwin 				KTR_STATE1(KTR_SCHED, "thread",
4422cba8dd3SJohn Baldwin 				    sched_tdname((struct thread *)tid),
4432cba8dd3SJohn Baldwin 				    "spinning", "lockname:\"%s\"",
4442cba8dd3SJohn Baldwin 				    m->lock_object.lo_name);
44549aead8aSAttilio Rao 				while (mtx_owner(m) == owner &&
446a5aedd68SStacey Son 				    TD_IS_RUNNING(owner)) {
44749aead8aSAttilio Rao 					cpu_spinwait();
448a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
449a5aedd68SStacey Son 					spin_cnt++;
450a5aedd68SStacey Son #endif
451a5aedd68SStacey Son 				}
4522cba8dd3SJohn Baldwin 				KTR_STATE0(KTR_SCHED, "thread",
4532cba8dd3SJohn Baldwin 				    sched_tdname((struct thread *)tid),
4542cba8dd3SJohn Baldwin 				    "running");
45549aead8aSAttilio Rao 				continue;
45649aead8aSAttilio Rao 			}
45749aead8aSAttilio Rao 		}
45849aead8aSAttilio Rao #endif
45949aead8aSAttilio Rao 
4602502c107SJeff Roberson 		ts = turnstile_trywait(&m->lock_object);
4615fa8dd90SJohn Baldwin 		v = m->mtx_lock;
4625fa8dd90SJohn Baldwin 
46336412d79SJohn Baldwin 		/*
4649ed346baSBosko Milekic 		 * Check if the lock has been released while spinning for
465961a7b24SJohn Baldwin 		 * the turnstile chain lock.
46636412d79SJohn Baldwin 		 */
4675fa8dd90SJohn Baldwin 		if (v == MTX_UNOWNED) {
4682502c107SJeff Roberson 			turnstile_cancel(ts);
46936412d79SJohn Baldwin 			continue;
47036412d79SJohn Baldwin 		}
4719ed346baSBosko Milekic 
47249aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES
47349aead8aSAttilio Rao 		/*
474fa29f023SJohn Baldwin 		 * The current lock owner might have started executing
475fa29f023SJohn Baldwin 		 * on another CPU (or the lock could have changed
476fa29f023SJohn Baldwin 		 * owners) while we were waiting on the turnstile
477fa29f023SJohn Baldwin 		 * chain lock.  If so, drop the turnstile lock and try
478fa29f023SJohn Baldwin 		 * again.
47949aead8aSAttilio Rao 		 */
48049aead8aSAttilio Rao 		owner = (struct thread *)(v & ~MTX_FLAGMASK);
48149aead8aSAttilio Rao 		if (TD_IS_RUNNING(owner)) {
48249aead8aSAttilio Rao 			turnstile_cancel(ts);
48349aead8aSAttilio Rao 			continue;
48449aead8aSAttilio Rao 		}
48549aead8aSAttilio Rao #endif
48649aead8aSAttilio Rao 
48736412d79SJohn Baldwin 		/*
4889ed346baSBosko Milekic 		 * If the mutex isn't already contested and a failure occurs
4899ed346baSBosko Milekic 		 * setting the contested bit, the mutex was either released
4909ed346baSBosko Milekic 		 * or the state of the MTX_RECURSED bit changed.
49136412d79SJohn Baldwin 		 */
49236412d79SJohn Baldwin 		if ((v & MTX_CONTESTED) == 0 &&
493122eceefSJohn Baldwin 		    !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
4942502c107SJeff Roberson 			turnstile_cancel(ts);
49536412d79SJohn Baldwin 			continue;
49636412d79SJohn Baldwin 		}
49736412d79SJohn Baldwin 
4989ed346baSBosko Milekic 		/*
4997feefcd6SJohn Baldwin 		 * We definitely must sleep for this lock.
5009ed346baSBosko Milekic 		 */
50136412d79SJohn Baldwin 		mtx_assert(m, MA_NOTOWNED);
50236412d79SJohn Baldwin 
50302bd1bcdSIan Dowse #ifdef KTR
50402bd1bcdSIan Dowse 		if (!cont_logged) {
50502bd1bcdSIan Dowse 			CTR6(KTR_CONTENTION,
50602bd1bcdSIan Dowse 			    "contention: %p at %s:%d wants %s, taken by %s:%d",
507aa89d8cdSJohn Baldwin 			    (void *)tid, file, line, m->lock_object.lo_name,
508aa89d8cdSJohn Baldwin 			    WITNESS_FILE(&m->lock_object),
509aa89d8cdSJohn Baldwin 			    WITNESS_LINE(&m->lock_object));
51002bd1bcdSIan Dowse 			cont_logged = 1;
51102bd1bcdSIan Dowse 		}
51202bd1bcdSIan Dowse #endif
51336412d79SJohn Baldwin 
5149ed346baSBosko Milekic 		/*
515961a7b24SJohn Baldwin 		 * Block on the turnstile.
5169ed346baSBosko Milekic 		 */
517a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
518e2b25737SMark Johnston 		sleep_time -= lockstat_nsecs(&m->lock_object);
519a5aedd68SStacey Son #endif
5202502c107SJeff Roberson 		turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
521a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
522e2b25737SMark Johnston 		sleep_time += lockstat_nsecs(&m->lock_object);
523a5aedd68SStacey Son 		sleep_cnt++;
524a5aedd68SStacey Son #endif
52536412d79SJohn Baldwin 	}
526076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
527e2b25737SMark Johnston 	all_time += lockstat_nsecs(&m->lock_object);
528076dd8ebSAndriy Gapon #endif
52902bd1bcdSIan Dowse #ifdef KTR
53002bd1bcdSIan Dowse 	if (cont_logged) {
53102bd1bcdSIan Dowse 		CTR4(KTR_CONTENTION,
53202bd1bcdSIan Dowse 		    "contention end: %s acquired by %p at %s:%d",
533aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, (void *)tid, file, line);
53402bd1bcdSIan Dowse 	}
53502bd1bcdSIan Dowse #endif
53632cd0147SMark Johnston 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested,
537eea4f254SJeff Roberson 	    waittime, file, line);
538a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
539a5aedd68SStacey Son 	if (sleep_time)
54032cd0147SMark Johnston 		LOCKSTAT_RECORD1(adaptive__block, m, sleep_time);
541a5aedd68SStacey Son 
542a5aedd68SStacey Son 	/*
543a5aedd68SStacey Son 	 * Only record the loops spinning and not sleeping.
544a5aedd68SStacey Son 	 */
545a5aedd68SStacey Son 	if (spin_cnt > sleep_cnt)
54632cd0147SMark Johnston 		LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time);
547a5aedd68SStacey Son #endif
5489ed346baSBosko Milekic }
5499ed346baSBosko Milekic 
5502502c107SJeff Roberson static void
5512502c107SJeff Roberson _mtx_lock_spin_failed(struct mtx *m)
5522502c107SJeff Roberson {
5532502c107SJeff Roberson 	struct thread *td;
5542502c107SJeff Roberson 
5552502c107SJeff Roberson 	td = mtx_owner(m);
5562502c107SJeff Roberson 
5572502c107SJeff Roberson 	/* If the mutex is unlocked, try again. */
5582502c107SJeff Roberson 	if (td == NULL)
5592502c107SJeff Roberson 		return;
560b95b98b0SKonstantin Belousov 
5612502c107SJeff Roberson 	printf( "spin lock %p (%s) held by %p (tid %d) too long\n",
5622502c107SJeff Roberson 	    m, m->lock_object.lo_name, td, td->td_tid);
5632502c107SJeff Roberson #ifdef WITNESS
56498332c8cSAttilio Rao 	witness_display_spinlock(&m->lock_object, td, printf);
5652502c107SJeff Roberson #endif
5662502c107SJeff Roberson 	panic("spin lock held too long");
5672502c107SJeff Roberson }
5682502c107SJeff Roberson 
569b95b98b0SKonstantin Belousov #ifdef SMP
5709ed346baSBosko Milekic /*
5717f44c618SAttilio Rao  * _mtx_lock_spin_cookie: the tougher part of acquiring an MTX_SPIN lock.
5729ed346baSBosko Milekic  *
5739ed346baSBosko Milekic  * This is only called if we need to actually spin for the lock. Recursion
5749ed346baSBosko Milekic  * is handled inline.
5759ed346baSBosko Milekic  */
5769ed346baSBosko Milekic void
5777f44c618SAttilio Rao _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts,
5787f44c618SAttilio Rao     const char *file, int line)
57936412d79SJohn Baldwin {
5807f44c618SAttilio Rao 	struct mtx *m;
5811723a064SJeff Roberson 	int i = 0;
5821723a064SJeff Roberson #ifdef LOCK_PROFILING
5831723a064SJeff Roberson 	int contested = 0;
58470fe8436SKip Macy 	uint64_t waittime = 0;
5851723a064SJeff Roberson #endif
586076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
587076dd8ebSAndriy Gapon 	int64_t spin_time = 0;
588076dd8ebSAndriy Gapon #endif
58936412d79SJohn Baldwin 
59035370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
59135370593SAndriy Gapon 		return;
59235370593SAndriy Gapon 
5937f44c618SAttilio Rao 	m = mtxlock2mtx(c);
5947f44c618SAttilio Rao 
595aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
5965746a1d8SBosko Milekic 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
5972cba8dd3SJohn Baldwin 	KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
5982cba8dd3SJohn Baldwin 	    "spinning", "lockname:\"%s\"", m->lock_object.lo_name);
5999ed346baSBosko Milekic 
600f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
601f5f9340bSFabien Thomas 	PMC_SOFT_CALL( , , lock, failed);
602f5f9340bSFabien Thomas #endif
60370fe8436SKip Macy 	lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
604076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
605e2b25737SMark Johnston 	spin_time -= lockstat_nsecs(&m->lock_object);
606076dd8ebSAndriy Gapon #endif
607fc4f686dSMateusz Guzik 	for (;;) {
608fc4f686dSMateusz Guzik 		if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid))
609fc4f686dSMateusz Guzik 			break;
6107141f2adSJohn Baldwin 		/* Give interrupts a chance while we spin. */
611c6a37e84SJohn Baldwin 		spinlock_exit();
61236412d79SJohn Baldwin 		while (m->mtx_lock != MTX_UNOWNED) {
613703fc290SJohn Baldwin 			if (i++ < 10000000) {
6149f1b87f1SMaxime Henrion 				cpu_spinwait();
61536412d79SJohn Baldwin 				continue;
616703fc290SJohn Baldwin 			}
6170fa2168bSJohn Baldwin 			if (i < 60000000 || kdb_active || panicstr != NULL)
61836412d79SJohn Baldwin 				DELAY(1);
6192502c107SJeff Roberson 			else
6202502c107SJeff Roberson 				_mtx_lock_spin_failed(m);
6219f1b87f1SMaxime Henrion 			cpu_spinwait();
62236412d79SJohn Baldwin 		}
623c6a37e84SJohn Baldwin 		spinlock_enter();
62436412d79SJohn Baldwin 	}
625076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
626e2b25737SMark Johnston 	spin_time += lockstat_nsecs(&m->lock_object);
627076dd8ebSAndriy Gapon #endif
62836412d79SJohn Baldwin 
629aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
6309ed346baSBosko Milekic 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
6312cba8dd3SJohn Baldwin 	KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
6322cba8dd3SJohn Baldwin 	    "running");
6339ed346baSBosko Milekic 
634c6d48c87SMark Johnston #ifdef KDTRACE_HOOKS
63532cd0147SMark Johnston 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m,
63632cd0147SMark Johnston 	    contested, waittime, file, line);
637e2b25737SMark Johnston 	if (spin_time != 0)
63832cd0147SMark Johnston 		LOCKSTAT_RECORD1(spin__spin, m, spin_time);
639c6d48c87SMark Johnston #endif
64036412d79SJohn Baldwin }
64133fb8a38SJohn Baldwin #endif /* SMP */
64236412d79SJohn Baldwin 
6432502c107SJeff Roberson void
644ccdf2333SAttilio Rao thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
6452502c107SJeff Roberson {
6462502c107SJeff Roberson 	struct mtx *m;
6472502c107SJeff Roberson 	uintptr_t tid;
6481723a064SJeff Roberson 	int i;
6491723a064SJeff Roberson #ifdef LOCK_PROFILING
6501723a064SJeff Roberson 	int contested = 0;
6511723a064SJeff Roberson 	uint64_t waittime = 0;
6521723a064SJeff Roberson #endif
653a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
654076dd8ebSAndriy Gapon 	int64_t spin_time = 0;
655a5aedd68SStacey Son #endif
6562502c107SJeff Roberson 
6571723a064SJeff Roberson 	i = 0;
6582502c107SJeff Roberson 	tid = (uintptr_t)curthread;
65935370593SAndriy Gapon 
660*f61d6c5aSMark Johnston 	if (SCHEDULER_STOPPED()) {
661*f61d6c5aSMark Johnston 		/*
662*f61d6c5aSMark Johnston 		 * Ensure that spinlock sections are balanced even when the
663*f61d6c5aSMark Johnston 		 * scheduler is stopped, since we may otherwise inadvertently
664*f61d6c5aSMark Johnston 		 * re-enable interrupts while dumping core.
665*f61d6c5aSMark Johnston 		 */
666*f61d6c5aSMark Johnston 		spinlock_enter();
66735370593SAndriy Gapon 		return;
668*f61d6c5aSMark Johnston 	}
66935370593SAndriy Gapon 
670076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
671e2b25737SMark Johnston 	spin_time -= lockstat_nsecs(&td->td_lock->lock_object);
672076dd8ebSAndriy Gapon #endif
6732502c107SJeff Roberson 	for (;;) {
6742502c107SJeff Roberson retry:
6752502c107SJeff Roberson 		spinlock_enter();
676710eacdcSJeff Roberson 		m = td->td_lock;
67713c85a48SJohn Baldwin 		KASSERT(m->mtx_lock != MTX_DESTROYED,
67813c85a48SJohn Baldwin 		    ("thread_lock() of destroyed mutex @ %s:%d", file, line));
67913c85a48SJohn Baldwin 		KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
68013c85a48SJohn Baldwin 		    ("thread_lock() of sleep mutex %s @ %s:%d",
68113c85a48SJohn Baldwin 		    m->lock_object.lo_name, file, line));
682ad69e26bSJohn Baldwin 		if (mtx_owned(m))
683ad69e26bSJohn Baldwin 			KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
684ad69e26bSJohn Baldwin 	    ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
685ad69e26bSJohn Baldwin 			    m->lock_object.lo_name, file, line));
6862502c107SJeff Roberson 		WITNESS_CHECKORDER(&m->lock_object,
68741313430SJohn Baldwin 		    opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
688fc4f686dSMateusz Guzik 		for (;;) {
689fc4f686dSMateusz Guzik 			if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid))
690fc4f686dSMateusz Guzik 				break;
6912502c107SJeff Roberson 			if (m->mtx_lock == tid) {
6922502c107SJeff Roberson 				m->mtx_recurse++;
6932502c107SJeff Roberson 				break;
6942502c107SJeff Roberson 			}
695f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
696f5f9340bSFabien Thomas 			PMC_SOFT_CALL( , , lock, failed);
697f5f9340bSFabien Thomas #endif
698eea4f254SJeff Roberson 			lock_profile_obtain_lock_failed(&m->lock_object,
699eea4f254SJeff Roberson 			    &contested, &waittime);
7002502c107SJeff Roberson 			/* Give interrupts a chance while we spin. */
7012502c107SJeff Roberson 			spinlock_exit();
7022502c107SJeff Roberson 			while (m->mtx_lock != MTX_UNOWNED) {
7032502c107SJeff Roberson 				if (i++ < 10000000)
7042502c107SJeff Roberson 					cpu_spinwait();
7052502c107SJeff Roberson 				else if (i < 60000000 ||
7062502c107SJeff Roberson 				    kdb_active || panicstr != NULL)
7072502c107SJeff Roberson 					DELAY(1);
7082502c107SJeff Roberson 				else
7092502c107SJeff Roberson 					_mtx_lock_spin_failed(m);
7102502c107SJeff Roberson 				cpu_spinwait();
7112502c107SJeff Roberson 				if (m != td->td_lock)
7122502c107SJeff Roberson 					goto retry;
7132502c107SJeff Roberson 			}
7142502c107SJeff Roberson 			spinlock_enter();
7152502c107SJeff Roberson 		}
7162502c107SJeff Roberson 		if (m == td->td_lock)
7172502c107SJeff Roberson 			break;
718961135eaSJohn Baldwin 		__mtx_unlock_spin(m);	/* does spinlock_exit() */
7192502c107SJeff Roberson 	}
720076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
721e2b25737SMark Johnston 	spin_time += lockstat_nsecs(&m->lock_object);
722076dd8ebSAndriy Gapon #endif
723eea4f254SJeff Roberson 	if (m->mtx_recurse == 0)
72432cd0147SMark Johnston 		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m,
72532cd0147SMark Johnston 		    contested, waittime, file, line);
72613c85a48SJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
72713c85a48SJohn Baldwin 	    line);
7282502c107SJeff Roberson 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
7295002e195SMark Johnston #ifdef KDTRACE_HOOKS
730156fbc14SMark Johnston 	if (spin_time != 0)
73132cd0147SMark Johnston 		LOCKSTAT_RECORD1(thread__spin, m, spin_time);
7325002e195SMark Johnston #endif
7332502c107SJeff Roberson }
7342502c107SJeff Roberson 
7352502c107SJeff Roberson struct mtx *
7362502c107SJeff Roberson thread_lock_block(struct thread *td)
7372502c107SJeff Roberson {
7382502c107SJeff Roberson 	struct mtx *lock;
7392502c107SJeff Roberson 
7402502c107SJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
741710eacdcSJeff Roberson 	lock = td->td_lock;
7422502c107SJeff Roberson 	td->td_lock = &blocked_lock;
7432502c107SJeff Roberson 	mtx_unlock_spin(lock);
7442502c107SJeff Roberson 
7452502c107SJeff Roberson 	return (lock);
7462502c107SJeff Roberson }
7472502c107SJeff Roberson 
7482502c107SJeff Roberson void
7492502c107SJeff Roberson thread_lock_unblock(struct thread *td, struct mtx *new)
7502502c107SJeff Roberson {
7512502c107SJeff Roberson 	mtx_assert(new, MA_OWNED);
7522502c107SJeff Roberson 	MPASS(td->td_lock == &blocked_lock);
75365d32cd8SMatt Jacob 	atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new);
7542502c107SJeff Roberson }
7552502c107SJeff Roberson 
7562502c107SJeff Roberson void
7572502c107SJeff Roberson thread_lock_set(struct thread *td, struct mtx *new)
7582502c107SJeff Roberson {
7592502c107SJeff Roberson 	struct mtx *lock;
7602502c107SJeff Roberson 
7612502c107SJeff Roberson 	mtx_assert(new, MA_OWNED);
7622502c107SJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
763710eacdcSJeff Roberson 	lock = td->td_lock;
7642502c107SJeff Roberson 	td->td_lock = new;
7652502c107SJeff Roberson 	mtx_unlock_spin(lock);
7662502c107SJeff Roberson }
7672502c107SJeff Roberson 
7689ed346baSBosko Milekic /*
7697f44c618SAttilio Rao  * __mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
7709ed346baSBosko Milekic  *
7719ed346baSBosko Milekic  * We are only called here if the lock is recursed or contested (i.e. we
7729ed346baSBosko Milekic  * need to wake up a blocked thread).
7739ed346baSBosko Milekic  */
77436412d79SJohn Baldwin void
7757f44c618SAttilio Rao __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line)
77636412d79SJohn Baldwin {
7777f44c618SAttilio Rao 	struct mtx *m;
778961a7b24SJohn Baldwin 	struct turnstile *ts;
7799ed346baSBosko Milekic 
78035370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
78135370593SAndriy Gapon 		return;
78235370593SAndriy Gapon 
7837f44c618SAttilio Rao 	m = mtxlock2mtx(c);
7847f44c618SAttilio Rao 
78508812b39SBosko Milekic 	if (mtx_recursed(m)) {
78636412d79SJohn Baldwin 		if (--(m->mtx_recurse) == 0)
78708812b39SBosko Milekic 			atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
788aa89d8cdSJohn Baldwin 		if (LOCK_LOG_TEST(&m->lock_object, opts))
7899ed346baSBosko Milekic 			CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
79036412d79SJohn Baldwin 		return;
79136412d79SJohn Baldwin 	}
7929ed346baSBosko Milekic 
7932502c107SJeff Roberson 	/*
7942502c107SJeff Roberson 	 * We have to lock the chain before the turnstile so this turnstile
7952502c107SJeff Roberson 	 * can be removed from the hash list if it is empty.
7962502c107SJeff Roberson 	 */
7972502c107SJeff Roberson 	turnstile_chain_lock(&m->lock_object);
798aa89d8cdSJohn Baldwin 	ts = turnstile_lookup(&m->lock_object);
799aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
8009ed346baSBosko Milekic 		CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
801961a7b24SJohn Baldwin 	MPASS(ts != NULL);
8027aa4f685SJohn Baldwin 	turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
803961135eaSJohn Baldwin 	_mtx_release_lock_quick(m);
804bf9c6c31SJohn Baldwin 
8052502c107SJeff Roberson 	/*
8062502c107SJeff Roberson 	 * This turnstile is now no longer associated with the mutex.  We can
8072502c107SJeff Roberson 	 * unlock the chain lock so a new turnstile may take it's place.
8082502c107SJeff Roberson 	 */
8097aa4f685SJohn Baldwin 	turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
8102502c107SJeff Roberson 	turnstile_chain_unlock(&m->lock_object);
8119ed346baSBosko Milekic }
8129ed346baSBosko Milekic 
8139ed346baSBosko Milekic /*
8149ed346baSBosko Milekic  * All the unlocking of MTX_SPIN locks is done inline.
815961135eaSJohn Baldwin  * See the __mtx_unlock_spin() macro for the details.
8169ed346baSBosko Milekic  */
8179ed346baSBosko Milekic 
8189ed346baSBosko Milekic /*
81915ec816aSJohn Baldwin  * The backing function for the INVARIANTS-enabled mtx_assert()
8209ed346baSBosko Milekic  */
8211103f3b0SJohn Baldwin #ifdef INVARIANT_SUPPORT
8220cde2e34SJason Evans void
8237f44c618SAttilio Rao __mtx_assert(const volatile uintptr_t *c, int what, const char *file, int line)
8240cde2e34SJason Evans {
8257f44c618SAttilio Rao 	const struct mtx *m;
8265cb0fbe4SJohn Baldwin 
8271126349aSPaul Saab 	if (panicstr != NULL || dumping)
8285cb0fbe4SJohn Baldwin 		return;
8297f44c618SAttilio Rao 
8307f44c618SAttilio Rao 	m = mtxlock2mtx(c);
8317f44c618SAttilio Rao 
832a10f4966SJake Burkholder 	switch (what) {
8330cde2e34SJason Evans 	case MA_OWNED:
8340cde2e34SJason Evans 	case MA_OWNED | MA_RECURSED:
8350cde2e34SJason Evans 	case MA_OWNED | MA_NOTRECURSED:
836a10f4966SJake Burkholder 		if (!mtx_owned(m))
8370cde2e34SJason Evans 			panic("mutex %s not owned at %s:%d",
838aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
839a10f4966SJake Burkholder 		if (mtx_recursed(m)) {
840a10f4966SJake Burkholder 			if ((what & MA_NOTRECURSED) != 0)
8410cde2e34SJason Evans 				panic("mutex %s recursed at %s:%d",
842aa89d8cdSJohn Baldwin 				    m->lock_object.lo_name, file, line);
843a10f4966SJake Burkholder 		} else if ((what & MA_RECURSED) != 0) {
8440cde2e34SJason Evans 			panic("mutex %s unrecursed at %s:%d",
845aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
8460cde2e34SJason Evans 		}
8470cde2e34SJason Evans 		break;
8480cde2e34SJason Evans 	case MA_NOTOWNED:
849a10f4966SJake Burkholder 		if (mtx_owned(m))
8500cde2e34SJason Evans 			panic("mutex %s owned at %s:%d",
851aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
8520cde2e34SJason Evans 		break;
8530cde2e34SJason Evans 	default:
85456771ca7SJason Evans 		panic("unknown mtx_assert at %s:%d", file, line);
8550cde2e34SJason Evans 	}
8560cde2e34SJason Evans }
8570cde2e34SJason Evans #endif
8580cde2e34SJason Evans 
8599ed346baSBosko Milekic /*
860c27b5699SAndrew R. Reiter  * General init routine used by the MTX_SYSINIT() macro.
861c27b5699SAndrew R. Reiter  */
862c27b5699SAndrew R. Reiter void
863c27b5699SAndrew R. Reiter mtx_sysinit(void *arg)
864c27b5699SAndrew R. Reiter {
865c27b5699SAndrew R. Reiter 	struct mtx_args *margs = arg;
866c27b5699SAndrew R. Reiter 
8677f44c618SAttilio Rao 	mtx_init((struct mtx *)margs->ma_mtx, margs->ma_desc, NULL,
8687f44c618SAttilio Rao 	    margs->ma_opts);
869c27b5699SAndrew R. Reiter }
870c27b5699SAndrew R. Reiter 
871c27b5699SAndrew R. Reiter /*
8729ed346baSBosko Milekic  * Mutex initialization routine; initialize lock `m' of type contained in
8730c88508aSJohn Baldwin  * `opts' with options contained in `opts' and name `name.'  The optional
8740c88508aSJohn Baldwin  * lock type `type' is used as a general lock category name for use with
8750c88508aSJohn Baldwin  * witness.
8769ed346baSBosko Milekic  */
87736412d79SJohn Baldwin void
8787f44c618SAttilio Rao _mtx_init(volatile uintptr_t *c, const char *name, const char *type, int opts)
87936412d79SJohn Baldwin {
8807f44c618SAttilio Rao 	struct mtx *m;
88183a81bcbSJohn Baldwin 	struct lock_class *class;
88283a81bcbSJohn Baldwin 	int flags;
8839ed346baSBosko Milekic 
8847f44c618SAttilio Rao 	m = mtxlock2mtx(c);
8857f44c618SAttilio Rao 
88619284646SJohn Baldwin 	MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
887fd07ddcfSDmitry Chagin 	    MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE | MTX_NEW)) == 0);
888353998acSAttilio Rao 	ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock,
889353998acSAttilio Rao 	    ("%s: mtx_lock not aligned for %s: %p", __func__, name,
890353998acSAttilio Rao 	    &m->mtx_lock));
8919ed346baSBosko Milekic 
89283a81bcbSJohn Baldwin 	/* Determine lock class and lock flags. */
89319284646SJohn Baldwin 	if (opts & MTX_SPIN)
89483a81bcbSJohn Baldwin 		class = &lock_class_mtx_spin;
89519284646SJohn Baldwin 	else
89683a81bcbSJohn Baldwin 		class = &lock_class_mtx_sleep;
89783a81bcbSJohn Baldwin 	flags = 0;
89819284646SJohn Baldwin 	if (opts & MTX_QUIET)
89983a81bcbSJohn Baldwin 		flags |= LO_QUIET;
90019284646SJohn Baldwin 	if (opts & MTX_RECURSE)
90183a81bcbSJohn Baldwin 		flags |= LO_RECURSABLE;
90219284646SJohn Baldwin 	if ((opts & MTX_NOWITNESS) == 0)
90383a81bcbSJohn Baldwin 		flags |= LO_WITNESS;
904f22a4b62SJeff Roberson 	if (opts & MTX_DUPOK)
90583a81bcbSJohn Baldwin 		flags |= LO_DUPOK;
9067c0435b9SKip Macy 	if (opts & MTX_NOPROFILE)
9077c0435b9SKip Macy 		flags |= LO_NOPROFILE;
908fd07ddcfSDmitry Chagin 	if (opts & MTX_NEW)
909fd07ddcfSDmitry Chagin 		flags |= LO_NEW;
91019284646SJohn Baldwin 
91183a81bcbSJohn Baldwin 	/* Initialize mutex. */
912b5fb43e5SJohn Baldwin 	lock_init(&m->lock_object, class, name, type, flags);
913b5fb43e5SJohn Baldwin 
91419284646SJohn Baldwin 	m->mtx_lock = MTX_UNOWNED;
91583a81bcbSJohn Baldwin 	m->mtx_recurse = 0;
91636412d79SJohn Baldwin }
91736412d79SJohn Baldwin 
9189ed346baSBosko Milekic /*
91919284646SJohn Baldwin  * Remove lock `m' from all_mtx queue.  We don't allow MTX_QUIET to be
92019284646SJohn Baldwin  * passed in as a flag here because if the corresponding mtx_init() was
92119284646SJohn Baldwin  * called with MTX_QUIET set, then it will already be set in the mutex's
92219284646SJohn Baldwin  * flags.
9239ed346baSBosko Milekic  */
92436412d79SJohn Baldwin void
9257f44c618SAttilio Rao _mtx_destroy(volatile uintptr_t *c)
92636412d79SJohn Baldwin {
9277f44c618SAttilio Rao 	struct mtx *m;
9287f44c618SAttilio Rao 
9297f44c618SAttilio Rao 	m = mtxlock2mtx(c);
93036412d79SJohn Baldwin 
93119284646SJohn Baldwin 	if (!mtx_owned(m))
93219284646SJohn Baldwin 		MPASS(mtx_unowned(m));
93319284646SJohn Baldwin 	else {
93408812b39SBosko Milekic 		MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
9359ed346baSBosko Milekic 
936861a2308SScott Long 		/* Perform the non-mtx related part of mtx_unlock_spin(). */
937aa89d8cdSJohn Baldwin 		if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin)
938861a2308SScott Long 			spinlock_exit();
939764e4d54SJohn Baldwin 		else
940ce1c953eSMark Johnston 			TD_LOCKS_DEC(curthread);
941861a2308SScott Long 
942d3df4af3SJeff Roberson 		lock_profile_release_lock(&m->lock_object);
94319284646SJohn Baldwin 		/* Tell witness this isn't locked to make it happy. */
944aa89d8cdSJohn Baldwin 		WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__,
945c86b6ff5SJohn Baldwin 		    __LINE__);
94636412d79SJohn Baldwin 	}
9470384fff8SJason Evans 
948186abbd7SJohn Baldwin 	m->mtx_lock = MTX_DESTROYED;
949aa89d8cdSJohn Baldwin 	lock_destroy(&m->lock_object);
9500384fff8SJason Evans }
951d23f5958SMatthew Dillon 
952d23f5958SMatthew Dillon /*
953c53c013bSJohn Baldwin  * Intialize the mutex code and system mutexes.  This is called from the MD
954c53c013bSJohn Baldwin  * startup code prior to mi_startup().  The per-CPU data space needs to be
955c53c013bSJohn Baldwin  * setup before this is called.
956c53c013bSJohn Baldwin  */
957c53c013bSJohn Baldwin void
958c53c013bSJohn Baldwin mutex_init(void)
959c53c013bSJohn Baldwin {
960c53c013bSJohn Baldwin 
961961a7b24SJohn Baldwin 	/* Setup turnstiles so that sleep mutexes work. */
962961a7b24SJohn Baldwin 	init_turnstiles();
963961a7b24SJohn Baldwin 
964c53c013bSJohn Baldwin 	/*
965c53c013bSJohn Baldwin 	 * Initialize mutexes.
966c53c013bSJohn Baldwin 	 */
9670c88508aSJohn Baldwin 	mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
9682502c107SJeff Roberson 	mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN);
9692502c107SJeff Roberson 	blocked_lock.mtx_lock = 0xdeadc0de;	/* Always blocked. */
9700c88508aSJohn Baldwin 	mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
9716afb32fcSKonstantin Belousov 	mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN);
9725c7bebf9SKonstantin Belousov 	mtx_init(&proc0.p_statmtx, "pstatl", NULL, MTX_SPIN);
9735c7bebf9SKonstantin Belousov 	mtx_init(&proc0.p_itimmtx, "pitiml", NULL, MTX_SPIN);
9745c7bebf9SKonstantin Belousov 	mtx_init(&proc0.p_profmtx, "pprofl", NULL, MTX_SPIN);
9758c4b6380SJohn Baldwin 	mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
976c53c013bSJohn Baldwin 	mtx_lock(&Giant);
977c53c013bSJohn Baldwin }
978d272fe53SJohn Baldwin 
979d272fe53SJohn Baldwin #ifdef DDB
980d272fe53SJohn Baldwin void
981d576deedSPawel Jakub Dawidek db_show_mtx(const struct lock_object *lock)
982d272fe53SJohn Baldwin {
983d272fe53SJohn Baldwin 	struct thread *td;
984d576deedSPawel Jakub Dawidek 	const struct mtx *m;
985d272fe53SJohn Baldwin 
986d576deedSPawel Jakub Dawidek 	m = (const struct mtx *)lock;
987d272fe53SJohn Baldwin 
988d272fe53SJohn Baldwin 	db_printf(" flags: {");
98983a81bcbSJohn Baldwin 	if (LOCK_CLASS(lock) == &lock_class_mtx_spin)
990d272fe53SJohn Baldwin 		db_printf("SPIN");
991d272fe53SJohn Baldwin 	else
992d272fe53SJohn Baldwin 		db_printf("DEF");
993aa89d8cdSJohn Baldwin 	if (m->lock_object.lo_flags & LO_RECURSABLE)
994d272fe53SJohn Baldwin 		db_printf(", RECURSE");
995aa89d8cdSJohn Baldwin 	if (m->lock_object.lo_flags & LO_DUPOK)
996d272fe53SJohn Baldwin 		db_printf(", DUPOK");
997d272fe53SJohn Baldwin 	db_printf("}\n");
998d272fe53SJohn Baldwin 	db_printf(" state: {");
999d272fe53SJohn Baldwin 	if (mtx_unowned(m))
1000d272fe53SJohn Baldwin 		db_printf("UNOWNED");
1001c0bfd703SJohn Baldwin 	else if (mtx_destroyed(m))
1002c0bfd703SJohn Baldwin 		db_printf("DESTROYED");
1003d272fe53SJohn Baldwin 	else {
1004d272fe53SJohn Baldwin 		db_printf("OWNED");
1005d272fe53SJohn Baldwin 		if (m->mtx_lock & MTX_CONTESTED)
1006d272fe53SJohn Baldwin 			db_printf(", CONTESTED");
1007d272fe53SJohn Baldwin 		if (m->mtx_lock & MTX_RECURSED)
1008d272fe53SJohn Baldwin 			db_printf(", RECURSED");
1009d272fe53SJohn Baldwin 	}
1010d272fe53SJohn Baldwin 	db_printf("}\n");
1011c0bfd703SJohn Baldwin 	if (!mtx_unowned(m) && !mtx_destroyed(m)) {
1012d272fe53SJohn Baldwin 		td = mtx_owner(m);
1013d272fe53SJohn Baldwin 		db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td,
1014431f8906SJulian Elischer 		    td->td_tid, td->td_proc->p_pid, td->td_name);
1015d272fe53SJohn Baldwin 		if (mtx_recursed(m))
1016d272fe53SJohn Baldwin 			db_printf(" recursed: %d\n", m->mtx_recurse);
1017d272fe53SJohn Baldwin 	}
1018d272fe53SJohn Baldwin }
1019d272fe53SJohn Baldwin #endif
1020