xref: /freebsd/sys/kern/kern_mutex.c (revision 5e5ad162ad03792dcad6b018e8f8339cfba685e7)
10384fff8SJason Evans /*-
20384fff8SJason Evans  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
30384fff8SJason Evans  *
40384fff8SJason Evans  * Redistribution and use in source and binary forms, with or without
50384fff8SJason Evans  * modification, are permitted provided that the following conditions
60384fff8SJason Evans  * are met:
70384fff8SJason Evans  * 1. Redistributions of source code must retain the above copyright
80384fff8SJason Evans  *    notice, this list of conditions and the following disclaimer.
90384fff8SJason Evans  * 2. Redistributions in binary form must reproduce the above copyright
100384fff8SJason Evans  *    notice, this list of conditions and the following disclaimer in the
110384fff8SJason Evans  *    documentation and/or other materials provided with the distribution.
120384fff8SJason Evans  * 3. Berkeley Software Design Inc's name may not be used to endorse or
130384fff8SJason Evans  *    promote products derived from this software without specific prior
140384fff8SJason Evans  *    written permission.
150384fff8SJason Evans  *
160384fff8SJason Evans  * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
170384fff8SJason Evans  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
180384fff8SJason Evans  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
190384fff8SJason Evans  * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
200384fff8SJason Evans  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
210384fff8SJason Evans  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
220384fff8SJason Evans  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
230384fff8SJason Evans  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
240384fff8SJason Evans  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
250384fff8SJason Evans  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
260384fff8SJason Evans  * SUCH DAMAGE.
270384fff8SJason Evans  *
280384fff8SJason Evans  *	from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
2936412d79SJohn Baldwin  *	and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
300384fff8SJason Evans  */
310384fff8SJason Evans 
320384fff8SJason Evans /*
33ba48b69aSJohn Baldwin  * Machine independent bits of mutex implementation.
340384fff8SJason Evans  */
350384fff8SJason Evans 
36677b542eSDavid E. O'Brien #include <sys/cdefs.h>
37677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
38677b542eSDavid E. O'Brien 
392498cf8cSJohn Baldwin #include "opt_adaptive_mutexes.h"
409c36c934SJohn Baldwin #include "opt_ddb.h"
41f5f9340bSFabien Thomas #include "opt_hwpmc_hooks.h"
429923b511SScott Long #include "opt_sched.h"
43a5a96a19SJohn Baldwin 
440384fff8SJason Evans #include <sys/param.h>
456c35e809SDag-Erling Smørgrav #include <sys/systm.h>
4636412d79SJohn Baldwin #include <sys/bus.h>
471126349aSPaul Saab #include <sys/conf.h>
482d50560aSMarcel Moolenaar #include <sys/kdb.h>
4936412d79SJohn Baldwin #include <sys/kernel.h>
506c35e809SDag-Erling Smørgrav #include <sys/ktr.h>
5119284646SJohn Baldwin #include <sys/lock.h>
52fb919e4dSMark Murray #include <sys/malloc.h>
5319284646SJohn Baldwin #include <sys/mutex.h>
540384fff8SJason Evans #include <sys/proc.h>
55c4f7a187SJohn Baldwin #include <sys/resourcevar.h>
56b43179fbSJeff Roberson #include <sys/sched.h>
576c35e809SDag-Erling Smørgrav #include <sys/sbuf.h>
581ada9041SMateusz Guzik #include <sys/smp.h>
59a5a96a19SJohn Baldwin #include <sys/sysctl.h>
60961a7b24SJohn Baldwin #include <sys/turnstile.h>
6136412d79SJohn Baldwin #include <sys/vmmeter.h>
627c0435b9SKip Macy #include <sys/lock_profile.h>
630384fff8SJason Evans 
6436412d79SJohn Baldwin #include <machine/atomic.h>
6536412d79SJohn Baldwin #include <machine/bus.h>
660384fff8SJason Evans #include <machine/cpu.h>
6736412d79SJohn Baldwin 
689c36c934SJohn Baldwin #include <ddb/ddb.h>
699c36c934SJohn Baldwin 
708c4b6380SJohn Baldwin #include <fs/devfs/devfs_int.h>
718c4b6380SJohn Baldwin 
7236412d79SJohn Baldwin #include <vm/vm.h>
7336412d79SJohn Baldwin #include <vm/vm_extern.h>
7436412d79SJohn Baldwin 
75cd6e6e4eSJohn Baldwin #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
76cd6e6e4eSJohn Baldwin #define	ADAPTIVE_MUTEXES
77cd6e6e4eSJohn Baldwin #endif
78cd6e6e4eSJohn Baldwin 
79f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
80f5f9340bSFabien Thomas #include <sys/pmckern.h>
81f5f9340bSFabien Thomas PMC_SOFT_DEFINE( , , lock, failed);
82f5f9340bSFabien Thomas #endif
83f5f9340bSFabien Thomas 
84b9a80acaSStephan Uphoff /*
857f44c618SAttilio Rao  * Return the mutex address when the lock cookie address is provided.
867f44c618SAttilio Rao  * This functionality assumes that struct mtx* have a member named mtx_lock.
877f44c618SAttilio Rao  */
887f44c618SAttilio Rao #define	mtxlock2mtx(c)	(__containerof(c, struct mtx, mtx_lock))
897f44c618SAttilio Rao 
907f44c618SAttilio Rao /*
919ed346baSBosko Milekic  * Internal utility macros.
920cde2e34SJason Evans  */
939ed346baSBosko Milekic #define mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)
940cde2e34SJason Evans 
95c0bfd703SJohn Baldwin #define	mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED)
96c0bfd703SJohn Baldwin 
97d576deedSPawel Jakub Dawidek static void	assert_mtx(const struct lock_object *lock, int what);
98d272fe53SJohn Baldwin #ifdef DDB
99d576deedSPawel Jakub Dawidek static void	db_show_mtx(const struct lock_object *lock);
100d272fe53SJohn Baldwin #endif
1017faf4d90SDavide Italiano static void	lock_mtx(struct lock_object *lock, uintptr_t how);
1027faf4d90SDavide Italiano static void	lock_spin(struct lock_object *lock, uintptr_t how);
103a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
104d576deedSPawel Jakub Dawidek static int	owner_mtx(const struct lock_object *lock,
105d576deedSPawel Jakub Dawidek 		    struct thread **owner);
106a5aedd68SStacey Son #endif
1077faf4d90SDavide Italiano static uintptr_t unlock_mtx(struct lock_object *lock);
1087faf4d90SDavide Italiano static uintptr_t unlock_spin(struct lock_object *lock);
109d272fe53SJohn Baldwin 
1100cde2e34SJason Evans /*
11119284646SJohn Baldwin  * Lock classes for sleep and spin mutexes.
1120cde2e34SJason Evans  */
11319284646SJohn Baldwin struct lock_class lock_class_mtx_sleep = {
114ae8dde30SJohn Baldwin 	.lc_name = "sleep mutex",
115ae8dde30SJohn Baldwin 	.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
116f9721b43SAttilio Rao 	.lc_assert = assert_mtx,
117d272fe53SJohn Baldwin #ifdef DDB
118ae8dde30SJohn Baldwin 	.lc_ddb_show = db_show_mtx,
119d272fe53SJohn Baldwin #endif
1206e21afd4SJohn Baldwin 	.lc_lock = lock_mtx,
1216e21afd4SJohn Baldwin 	.lc_unlock = unlock_mtx,
122a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
123a5aedd68SStacey Son 	.lc_owner = owner_mtx,
124a5aedd68SStacey Son #endif
12519284646SJohn Baldwin };
12619284646SJohn Baldwin struct lock_class lock_class_mtx_spin = {
127ae8dde30SJohn Baldwin 	.lc_name = "spin mutex",
128ae8dde30SJohn Baldwin 	.lc_flags = LC_SPINLOCK | LC_RECURSABLE,
129f9721b43SAttilio Rao 	.lc_assert = assert_mtx,
130d272fe53SJohn Baldwin #ifdef DDB
131ae8dde30SJohn Baldwin 	.lc_ddb_show = db_show_mtx,
132d272fe53SJohn Baldwin #endif
1336e21afd4SJohn Baldwin 	.lc_lock = lock_spin,
1346e21afd4SJohn Baldwin 	.lc_unlock = unlock_spin,
135a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
136a5aedd68SStacey Son 	.lc_owner = owner_mtx,
137a5aedd68SStacey Son #endif
1388484de75SJohn Baldwin };
1398484de75SJohn Baldwin 
1401ada9041SMateusz Guzik #ifdef ADAPTIVE_MUTEXES
1411ada9041SMateusz Guzik static SYSCTL_NODE(_debug, OID_AUTO, mtx, CTLFLAG_RD, NULL, "mtx debugging");
1421ada9041SMateusz Guzik 
1431ada9041SMateusz Guzik static struct lock_delay_config mtx_delay = {
1441ada9041SMateusz Guzik 	.initial	= 1000,
1451ada9041SMateusz Guzik 	.step		= 500,
1461ada9041SMateusz Guzik 	.min		= 100,
1471ada9041SMateusz Guzik 	.max		= 5000,
1481ada9041SMateusz Guzik };
1491ada9041SMateusz Guzik 
1501ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_initial, CTLFLAG_RW, &mtx_delay.initial,
1511ada9041SMateusz Guzik     0, "");
1521ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_delay.step,
1531ada9041SMateusz Guzik     0, "");
1541ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_delay.min,
1551ada9041SMateusz Guzik     0, "");
1561ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_delay.max,
1571ada9041SMateusz Guzik     0, "");
1581ada9041SMateusz Guzik 
1591ada9041SMateusz Guzik static void
1601ada9041SMateusz Guzik mtx_delay_sysinit(void *dummy)
1611ada9041SMateusz Guzik {
1621ada9041SMateusz Guzik 
1631ada9041SMateusz Guzik 	mtx_delay.initial = mp_ncpus * 25;
1641ada9041SMateusz Guzik 	mtx_delay.step = (mp_ncpus * 25) / 2;
1651ada9041SMateusz Guzik 	mtx_delay.min = mp_ncpus * 5;
1661ada9041SMateusz Guzik 	mtx_delay.max = mp_ncpus * 25 * 10;
1671ada9041SMateusz Guzik }
1681ada9041SMateusz Guzik LOCK_DELAY_SYSINIT(mtx_delay_sysinit);
1691ada9041SMateusz Guzik #endif
1701ada9041SMateusz Guzik 
171a0d45f0fSMateusz Guzik static SYSCTL_NODE(_debug, OID_AUTO, mtx_spin, CTLFLAG_RD, NULL,
172a0d45f0fSMateusz Guzik     "mtx spin debugging");
173a0d45f0fSMateusz Guzik 
174a0d45f0fSMateusz Guzik static struct lock_delay_config mtx_spin_delay = {
175a0d45f0fSMateusz Guzik 	.initial        = 1000,
176a0d45f0fSMateusz Guzik 	.step           = 500,
177a0d45f0fSMateusz Guzik 	.min            = 100,
178a0d45f0fSMateusz Guzik 	.max            = 5000,
179a0d45f0fSMateusz Guzik };
180a0d45f0fSMateusz Guzik 
181a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_initial, CTLFLAG_RW,
182a0d45f0fSMateusz Guzik     &mtx_spin_delay.initial, 0, "");
183a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_spin_delay.step,
184a0d45f0fSMateusz Guzik     0, "");
185a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_spin_delay.min,
186a0d45f0fSMateusz Guzik     0, "");
187a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_spin_delay.max,
188a0d45f0fSMateusz Guzik     0, "");
189a0d45f0fSMateusz Guzik 
190a0d45f0fSMateusz Guzik static void
191a0d45f0fSMateusz Guzik mtx_spin_delay_sysinit(void *dummy)
192a0d45f0fSMateusz Guzik {
193a0d45f0fSMateusz Guzik 
194a0d45f0fSMateusz Guzik 	mtx_spin_delay.initial = mp_ncpus * 25;
195a0d45f0fSMateusz Guzik 	mtx_spin_delay.step = (mp_ncpus * 25) / 2;
196a0d45f0fSMateusz Guzik 	mtx_spin_delay.min = mp_ncpus * 5;
197a0d45f0fSMateusz Guzik 	mtx_spin_delay.max = mp_ncpus * 25 * 10;
198a0d45f0fSMateusz Guzik }
199a0d45f0fSMateusz Guzik LOCK_DELAY_SYSINIT(mtx_spin_delay_sysinit);
200a0d45f0fSMateusz Guzik 
2019ed346baSBosko Milekic /*
202c53c013bSJohn Baldwin  * System-wide mutexes
203c53c013bSJohn Baldwin  */
2042502c107SJeff Roberson struct mtx blocked_lock;
205c53c013bSJohn Baldwin struct mtx Giant;
206c53c013bSJohn Baldwin 
20767784314SPoul-Henning Kamp void
208d576deedSPawel Jakub Dawidek assert_mtx(const struct lock_object *lock, int what)
209f9721b43SAttilio Rao {
210f9721b43SAttilio Rao 
211d576deedSPawel Jakub Dawidek 	mtx_assert((const struct mtx *)lock, what);
212f9721b43SAttilio Rao }
213f9721b43SAttilio Rao 
21467784314SPoul-Henning Kamp void
2157faf4d90SDavide Italiano lock_mtx(struct lock_object *lock, uintptr_t how)
2166e21afd4SJohn Baldwin {
2176e21afd4SJohn Baldwin 
2186e21afd4SJohn Baldwin 	mtx_lock((struct mtx *)lock);
2196e21afd4SJohn Baldwin }
2206e21afd4SJohn Baldwin 
22167784314SPoul-Henning Kamp void
2227faf4d90SDavide Italiano lock_spin(struct lock_object *lock, uintptr_t how)
2236e21afd4SJohn Baldwin {
2246e21afd4SJohn Baldwin 
2256e21afd4SJohn Baldwin 	panic("spin locks can only use msleep_spin");
2266e21afd4SJohn Baldwin }
2276e21afd4SJohn Baldwin 
2287faf4d90SDavide Italiano uintptr_t
2296e21afd4SJohn Baldwin unlock_mtx(struct lock_object *lock)
2306e21afd4SJohn Baldwin {
2316e21afd4SJohn Baldwin 	struct mtx *m;
2326e21afd4SJohn Baldwin 
2336e21afd4SJohn Baldwin 	m = (struct mtx *)lock;
2346e21afd4SJohn Baldwin 	mtx_assert(m, MA_OWNED | MA_NOTRECURSED);
2356e21afd4SJohn Baldwin 	mtx_unlock(m);
2366e21afd4SJohn Baldwin 	return (0);
2376e21afd4SJohn Baldwin }
2386e21afd4SJohn Baldwin 
2397faf4d90SDavide Italiano uintptr_t
2406e21afd4SJohn Baldwin unlock_spin(struct lock_object *lock)
2416e21afd4SJohn Baldwin {
2426e21afd4SJohn Baldwin 
2436e21afd4SJohn Baldwin 	panic("spin locks can only use msleep_spin");
2446e21afd4SJohn Baldwin }
2456e21afd4SJohn Baldwin 
246a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
247a5aedd68SStacey Son int
248d576deedSPawel Jakub Dawidek owner_mtx(const struct lock_object *lock, struct thread **owner)
249a5aedd68SStacey Son {
25002315a67SMark Johnston 	const struct mtx *m;
25102315a67SMark Johnston 	uintptr_t x;
252a5aedd68SStacey Son 
25302315a67SMark Johnston 	m = (const struct mtx *)lock;
25402315a67SMark Johnston 	x = m->mtx_lock;
25502315a67SMark Johnston 	*owner = (struct thread *)(x & ~MTX_FLAGMASK);
25602315a67SMark Johnston 	return (x != MTX_UNOWNED);
257a5aedd68SStacey Son }
258a5aedd68SStacey Son #endif
259a5aedd68SStacey Son 
2600cde2e34SJason Evans /*
2616283b7d0SJohn Baldwin  * Function versions of the inlined __mtx_* macros.  These are used by
2626283b7d0SJohn Baldwin  * modules and can also be called from assembly language if needed.
2636283b7d0SJohn Baldwin  */
2646283b7d0SJohn Baldwin void
2657f44c618SAttilio Rao __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
2666283b7d0SJohn Baldwin {
2677f44c618SAttilio Rao 	struct mtx *m;
2686283b7d0SJohn Baldwin 
26935370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
27035370593SAndriy Gapon 		return;
2717f44c618SAttilio Rao 
2727f44c618SAttilio Rao 	m = mtxlock2mtx(c);
2737f44c618SAttilio Rao 
274cd2fe4e6SAttilio Rao 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
275e3ae0dfeSAttilio Rao 	    ("mtx_lock() by idle thread %p on sleep mutex %s @ %s:%d",
276e3ae0dfeSAttilio Rao 	    curthread, m->lock_object.lo_name, file, line));
277186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
278186abbd7SJohn Baldwin 	    ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
279aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
280aa89d8cdSJohn Baldwin 	    ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
2810d975d63SJohn Baldwin 	    file, line));
282ac6b769bSAttilio Rao 	WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) |
283ac6b769bSAttilio Rao 	    LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
2847c0435b9SKip Macy 
285961135eaSJohn Baldwin 	__mtx_lock(m, curthread, opts, file, line);
286aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
287dde96c99SJohn Baldwin 	    line);
288ac6b769bSAttilio Rao 	WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE,
289ac6b769bSAttilio Rao 	    file, line);
290ce1c953eSMark Johnston 	TD_LOCKS_INC(curthread);
2916283b7d0SJohn Baldwin }
2926283b7d0SJohn Baldwin 
2936283b7d0SJohn Baldwin void
2947f44c618SAttilio Rao __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
2956283b7d0SJohn Baldwin {
2967f44c618SAttilio Rao 	struct mtx *m;
29735370593SAndriy Gapon 
29835370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
29935370593SAndriy Gapon 		return;
3007f44c618SAttilio Rao 
3017f44c618SAttilio Rao 	m = mtxlock2mtx(c);
3027f44c618SAttilio Rao 
303186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
304186abbd7SJohn Baldwin 	    ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
305aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
306aa89d8cdSJohn Baldwin 	    ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
3070d975d63SJohn Baldwin 	    file, line));
308aa89d8cdSJohn Baldwin 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
309aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
3100d975d63SJohn Baldwin 	    line);
31121377ce0SJohn Baldwin 	mtx_assert(m, MA_OWNED);
312c66d7606SKip Macy 
313961135eaSJohn Baldwin 	__mtx_unlock(m, curthread, opts, file, line);
314ce1c953eSMark Johnston 	TD_LOCKS_DEC(curthread);
3156283b7d0SJohn Baldwin }
3166283b7d0SJohn Baldwin 
3176283b7d0SJohn Baldwin void
3187f44c618SAttilio Rao __mtx_lock_spin_flags(volatile uintptr_t *c, int opts, const char *file,
3197f44c618SAttilio Rao     int line)
3206283b7d0SJohn Baldwin {
3217f44c618SAttilio Rao 	struct mtx *m;
3226283b7d0SJohn Baldwin 
32335370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
32435370593SAndriy Gapon 		return;
3257f44c618SAttilio Rao 
3267f44c618SAttilio Rao 	m = mtxlock2mtx(c);
3277f44c618SAttilio Rao 
328186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
329186abbd7SJohn Baldwin 	    ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
330aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
3310d975d63SJohn Baldwin 	    ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
332aa89d8cdSJohn Baldwin 	    m->lock_object.lo_name, file, line));
333ad69e26bSJohn Baldwin 	if (mtx_owned(m))
334ac6b769bSAttilio Rao 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
335ac6b769bSAttilio Rao 		    (opts & MTX_RECURSE) != 0,
336ad69e26bSJohn Baldwin 	    ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n",
337ad69e26bSJohn Baldwin 		    m->lock_object.lo_name, file, line));
338ac6b769bSAttilio Rao 	opts &= ~MTX_RECURSE;
339aa89d8cdSJohn Baldwin 	WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
34041313430SJohn Baldwin 	    file, line, NULL);
341961135eaSJohn Baldwin 	__mtx_lock_spin(m, curthread, opts, file, line);
342aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
343dde96c99SJohn Baldwin 	    line);
344aa89d8cdSJohn Baldwin 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
3456283b7d0SJohn Baldwin }
3466283b7d0SJohn Baldwin 
34790b581f2SKonstantin Belousov int
34890b581f2SKonstantin Belousov __mtx_trylock_spin_flags(volatile uintptr_t *c, int opts, const char *file,
34990b581f2SKonstantin Belousov     int line)
35090b581f2SKonstantin Belousov {
35190b581f2SKonstantin Belousov 	struct mtx *m;
35290b581f2SKonstantin Belousov 
35390b581f2SKonstantin Belousov 	if (SCHEDULER_STOPPED())
35490b581f2SKonstantin Belousov 		return (1);
35590b581f2SKonstantin Belousov 
35690b581f2SKonstantin Belousov 	m = mtxlock2mtx(c);
35790b581f2SKonstantin Belousov 
35890b581f2SKonstantin Belousov 	KASSERT(m->mtx_lock != MTX_DESTROYED,
35990b581f2SKonstantin Belousov 	    ("mtx_trylock_spin() of destroyed mutex @ %s:%d", file, line));
36090b581f2SKonstantin Belousov 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
36190b581f2SKonstantin Belousov 	    ("mtx_trylock_spin() of sleep mutex %s @ %s:%d",
36290b581f2SKonstantin Belousov 	    m->lock_object.lo_name, file, line));
36390b581f2SKonstantin Belousov 	KASSERT((opts & MTX_RECURSE) == 0,
36490b581f2SKonstantin Belousov 	    ("mtx_trylock_spin: unsupp. opt MTX_RECURSE on mutex %s @ %s:%d\n",
36590b581f2SKonstantin Belousov 	    m->lock_object.lo_name, file, line));
36690b581f2SKonstantin Belousov 	if (__mtx_trylock_spin(m, curthread, opts, file, line)) {
36790b581f2SKonstantin Belousov 		LOCK_LOG_TRY("LOCK", &m->lock_object, opts, 1, file, line);
36890b581f2SKonstantin Belousov 		WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
36990b581f2SKonstantin Belousov 		return (1);
37090b581f2SKonstantin Belousov 	}
37190b581f2SKonstantin Belousov 	LOCK_LOG_TRY("LOCK", &m->lock_object, opts, 0, file, line);
37290b581f2SKonstantin Belousov 	return (0);
37390b581f2SKonstantin Belousov }
37490b581f2SKonstantin Belousov 
3756283b7d0SJohn Baldwin void
3767f44c618SAttilio Rao __mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, const char *file,
3777f44c618SAttilio Rao     int line)
3786283b7d0SJohn Baldwin {
3797f44c618SAttilio Rao 	struct mtx *m;
380c66d7606SKip Macy 
38135370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
38235370593SAndriy Gapon 		return;
3837f44c618SAttilio Rao 
3847f44c618SAttilio Rao 	m = mtxlock2mtx(c);
3857f44c618SAttilio Rao 
386186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
387186abbd7SJohn Baldwin 	    ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
388aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
3890d975d63SJohn Baldwin 	    ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
390aa89d8cdSJohn Baldwin 	    m->lock_object.lo_name, file, line));
391aa89d8cdSJohn Baldwin 	WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
392aa89d8cdSJohn Baldwin 	LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
393dde96c99SJohn Baldwin 	    line);
3940d975d63SJohn Baldwin 	mtx_assert(m, MA_OWNED);
395c66d7606SKip Macy 
396961135eaSJohn Baldwin 	__mtx_unlock_spin(m);
3976283b7d0SJohn Baldwin }
3986283b7d0SJohn Baldwin 
3996283b7d0SJohn Baldwin /*
4009ed346baSBosko Milekic  * The important part of mtx_trylock{,_flags}()
401eac09796SJohn Baldwin  * Tries to acquire lock `m.'  If this function is called on a mutex that
402eac09796SJohn Baldwin  * is already owned, it will recursively acquire the lock.
4030cde2e34SJason Evans  */
4040cde2e34SJason Evans int
4057f44c618SAttilio Rao _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
4060cde2e34SJason Evans {
4077f44c618SAttilio Rao 	struct mtx *m;
4081723a064SJeff Roberson #ifdef LOCK_PROFILING
4097c0435b9SKip Macy 	uint64_t waittime = 0;
4101723a064SJeff Roberson 	int contested = 0;
4111723a064SJeff Roberson #endif
4121723a064SJeff Roberson 	int rval;
4130cde2e34SJason Evans 
41435370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
41535370593SAndriy Gapon 		return (1);
41635370593SAndriy Gapon 
4177f44c618SAttilio Rao 	m = mtxlock2mtx(c);
4187f44c618SAttilio Rao 
419cd2fe4e6SAttilio Rao 	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
420e3ae0dfeSAttilio Rao 	    ("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d",
421e3ae0dfeSAttilio Rao 	    curthread, m->lock_object.lo_name, file, line));
422186abbd7SJohn Baldwin 	KASSERT(m->mtx_lock != MTX_DESTROYED,
423186abbd7SJohn Baldwin 	    ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
424aa89d8cdSJohn Baldwin 	KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
425aa89d8cdSJohn Baldwin 	    ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
42683cece6fSJohn Baldwin 	    file, line));
4279ed346baSBosko Milekic 
428ac6b769bSAttilio Rao 	if (mtx_owned(m) && ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
429ac6b769bSAttilio Rao 	    (opts & MTX_RECURSE) != 0)) {
430eac09796SJohn Baldwin 		m->mtx_recurse++;
431eac09796SJohn Baldwin 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
432eac09796SJohn Baldwin 		rval = 1;
433eac09796SJohn Baldwin 	} else
434961135eaSJohn Baldwin 		rval = _mtx_obtain_lock(m, (uintptr_t)curthread);
435ac6b769bSAttilio Rao 	opts &= ~MTX_RECURSE;
4369ed346baSBosko Milekic 
437aa89d8cdSJohn Baldwin 	LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
438764e4d54SJohn Baldwin 	if (rval) {
439aa89d8cdSJohn Baldwin 		WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
4402d96f0b1SJohn Baldwin 		    file, line);
441ce1c953eSMark Johnston 		TD_LOCKS_INC(curthread);
442fe68a916SKip Macy 		if (m->mtx_recurse == 0)
44332cd0147SMark Johnston 			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,
444a5aedd68SStacey Son 			    m, contested, waittime, file, line);
4457c0435b9SKip Macy 
446764e4d54SJohn Baldwin 	}
4479ed346baSBosko Milekic 
44819284646SJohn Baldwin 	return (rval);
4490cde2e34SJason Evans }
4500cde2e34SJason Evans 
4510cde2e34SJason Evans /*
4527f44c618SAttilio Rao  * __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
4539ed346baSBosko Milekic  *
4549ed346baSBosko Milekic  * We call this if the lock is either contested (i.e. we need to go to
4559ed346baSBosko Milekic  * sleep waiting for it), or if we need to recurse on it.
4560cde2e34SJason Evans  */
4570cde2e34SJason Evans void
4587f44c618SAttilio Rao __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
4597f44c618SAttilio Rao     const char *file, int line)
46036412d79SJohn Baldwin {
4617f44c618SAttilio Rao 	struct mtx *m;
4622502c107SJeff Roberson 	struct turnstile *ts;
4631723a064SJeff Roberson 	uintptr_t v;
464cd6e6e4eSJohn Baldwin #ifdef ADAPTIVE_MUTEXES
46576447e56SJohn Baldwin 	volatile struct thread *owner;
4662498cf8cSJohn Baldwin #endif
46702bd1bcdSIan Dowse #ifdef KTR
46802bd1bcdSIan Dowse 	int cont_logged = 0;
46902bd1bcdSIan Dowse #endif
4701723a064SJeff Roberson #ifdef LOCK_PROFILING
47170fe8436SKip Macy 	int contested = 0;
47270fe8436SKip Macy 	uint64_t waittime = 0;
4731723a064SJeff Roberson #endif
4741ada9041SMateusz Guzik #if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS)
4751ada9041SMateusz Guzik 	struct lock_delay_arg lda;
4761ada9041SMateusz Guzik #endif
477a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
47861852185SMateusz Guzik 	u_int sleep_cnt = 0;
479a5aedd68SStacey Son 	int64_t sleep_time = 0;
480076dd8ebSAndriy Gapon 	int64_t all_time = 0;
481a5aedd68SStacey Son #endif
48236412d79SJohn Baldwin 
48335370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
48435370593SAndriy Gapon 		return;
48535370593SAndriy Gapon 
486fa5000a4SMateusz Guzik #if defined(ADAPTIVE_MUTEXES)
4871ada9041SMateusz Guzik 	lock_delay_arg_init(&lda, &mtx_delay);
488fa5000a4SMateusz Guzik #elif defined(KDTRACE_HOOKS)
489fa5000a4SMateusz Guzik 	lock_delay_arg_init(&lda, NULL);
4901ada9041SMateusz Guzik #endif
4917f44c618SAttilio Rao 	m = mtxlock2mtx(c);
4922604eb9eSMateusz Guzik 	v = MTX_READ_VALUE(m);
4937f44c618SAttilio Rao 
4942604eb9eSMateusz Guzik 	if (__predict_false(lv_mtx_owner(v) == (struct thread *)tid)) {
495ac6b769bSAttilio Rao 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
496ac6b769bSAttilio Rao 		    (opts & MTX_RECURSE) != 0,
497eac09796SJohn Baldwin 	    ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
498aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, file, line));
499ac6b769bSAttilio Rao 		opts &= ~MTX_RECURSE;
50036412d79SJohn Baldwin 		m->mtx_recurse++;
50108812b39SBosko Milekic 		atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
502aa89d8cdSJohn Baldwin 		if (LOCK_LOG_TEST(&m->lock_object, opts))
5035746a1d8SBosko Milekic 			CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
50436412d79SJohn Baldwin 		return;
50536412d79SJohn Baldwin 	}
506ac6b769bSAttilio Rao 	opts &= ~MTX_RECURSE;
5079ed346baSBosko Milekic 
508f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
509f5f9340bSFabien Thomas 	PMC_SOFT_CALL( , , lock, failed);
510f5f9340bSFabien Thomas #endif
51170fe8436SKip Macy 	lock_profile_obtain_lock_failed(&m->lock_object,
51270fe8436SKip Macy 		    &contested, &waittime);
513aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
51415ec816aSJohn Baldwin 		CTR4(KTR_LOCK,
51515ec816aSJohn Baldwin 		    "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
516aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
517076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
518e2b25737SMark Johnston 	all_time -= lockstat_nsecs(&m->lock_object);
519076dd8ebSAndriy Gapon #endif
5201bd0eefbSJohn Baldwin 
521fc4f686dSMateusz Guzik 	for (;;) {
5222604eb9eSMateusz Guzik 		if (v == MTX_UNOWNED) {
5232604eb9eSMateusz Guzik 			if (_mtx_obtain_lock(m, tid))
524fc4f686dSMateusz Guzik 				break;
5252604eb9eSMateusz Guzik 			v = MTX_READ_VALUE(m);
5262604eb9eSMateusz Guzik 			continue;
5272604eb9eSMateusz Guzik 		}
528a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
5291ada9041SMateusz Guzik 		lda.spin_cnt++;
530a5aedd68SStacey Son #endif
53149aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES
53249aead8aSAttilio Rao 		/*
53349aead8aSAttilio Rao 		 * If the owner is running on another CPU, spin until the
53449aead8aSAttilio Rao 		 * owner stops running or the state of the lock changes.
53549aead8aSAttilio Rao 		 */
5362604eb9eSMateusz Guzik 		owner = lv_mtx_owner(v);
53749aead8aSAttilio Rao 		if (TD_IS_RUNNING(owner)) {
53849aead8aSAttilio Rao 			if (LOCK_LOG_TEST(&m->lock_object, 0))
53949aead8aSAttilio Rao 				CTR3(KTR_LOCK,
54049aead8aSAttilio Rao 				    "%s: spinning on %p held by %p",
54149aead8aSAttilio Rao 				    __func__, m, owner);
5422cba8dd3SJohn Baldwin 			KTR_STATE1(KTR_SCHED, "thread",
5432cba8dd3SJohn Baldwin 			    sched_tdname((struct thread *)tid),
5442cba8dd3SJohn Baldwin 			    "spinning", "lockname:\"%s\"",
5452cba8dd3SJohn Baldwin 			    m->lock_object.lo_name);
5462604eb9eSMateusz Guzik 			do {
5471ada9041SMateusz Guzik 				lock_delay(&lda);
5482604eb9eSMateusz Guzik 				v = m->mtx_lock;
5492604eb9eSMateusz Guzik 				owner = lv_mtx_owner(v);
5502604eb9eSMateusz Guzik 			} while (v != MTX_UNOWNED && TD_IS_RUNNING(owner));
5512cba8dd3SJohn Baldwin 			KTR_STATE0(KTR_SCHED, "thread",
5522cba8dd3SJohn Baldwin 			    sched_tdname((struct thread *)tid),
5532cba8dd3SJohn Baldwin 			    "running");
55449aead8aSAttilio Rao 			continue;
55549aead8aSAttilio Rao 		}
55649aead8aSAttilio Rao #endif
55749aead8aSAttilio Rao 
5582502c107SJeff Roberson 		ts = turnstile_trywait(&m->lock_object);
5592604eb9eSMateusz Guzik 		v = MTX_READ_VALUE(m);
5605fa8dd90SJohn Baldwin 
56136412d79SJohn Baldwin 		/*
5629ed346baSBosko Milekic 		 * Check if the lock has been released while spinning for
563961a7b24SJohn Baldwin 		 * the turnstile chain lock.
56436412d79SJohn Baldwin 		 */
5655fa8dd90SJohn Baldwin 		if (v == MTX_UNOWNED) {
5662502c107SJeff Roberson 			turnstile_cancel(ts);
56736412d79SJohn Baldwin 			continue;
56836412d79SJohn Baldwin 		}
5699ed346baSBosko Milekic 
57049aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES
57149aead8aSAttilio Rao 		/*
572fa29f023SJohn Baldwin 		 * The current lock owner might have started executing
573fa29f023SJohn Baldwin 		 * on another CPU (or the lock could have changed
574fa29f023SJohn Baldwin 		 * owners) while we were waiting on the turnstile
575fa29f023SJohn Baldwin 		 * chain lock.  If so, drop the turnstile lock and try
576fa29f023SJohn Baldwin 		 * again.
57749aead8aSAttilio Rao 		 */
5782604eb9eSMateusz Guzik 		owner = lv_mtx_owner(v);
57949aead8aSAttilio Rao 		if (TD_IS_RUNNING(owner)) {
58049aead8aSAttilio Rao 			turnstile_cancel(ts);
58149aead8aSAttilio Rao 			continue;
58249aead8aSAttilio Rao 		}
58349aead8aSAttilio Rao #endif
58449aead8aSAttilio Rao 
58536412d79SJohn Baldwin 		/*
5869ed346baSBosko Milekic 		 * If the mutex isn't already contested and a failure occurs
5879ed346baSBosko Milekic 		 * setting the contested bit, the mutex was either released
5889ed346baSBosko Milekic 		 * or the state of the MTX_RECURSED bit changed.
58936412d79SJohn Baldwin 		 */
59036412d79SJohn Baldwin 		if ((v & MTX_CONTESTED) == 0 &&
591122eceefSJohn Baldwin 		    !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
5922502c107SJeff Roberson 			turnstile_cancel(ts);
5932604eb9eSMateusz Guzik 			v = MTX_READ_VALUE(m);
59436412d79SJohn Baldwin 			continue;
59536412d79SJohn Baldwin 		}
59636412d79SJohn Baldwin 
5979ed346baSBosko Milekic 		/*
5987feefcd6SJohn Baldwin 		 * We definitely must sleep for this lock.
5999ed346baSBosko Milekic 		 */
60036412d79SJohn Baldwin 		mtx_assert(m, MA_NOTOWNED);
60136412d79SJohn Baldwin 
60202bd1bcdSIan Dowse #ifdef KTR
60302bd1bcdSIan Dowse 		if (!cont_logged) {
60402bd1bcdSIan Dowse 			CTR6(KTR_CONTENTION,
60502bd1bcdSIan Dowse 			    "contention: %p at %s:%d wants %s, taken by %s:%d",
606aa89d8cdSJohn Baldwin 			    (void *)tid, file, line, m->lock_object.lo_name,
607aa89d8cdSJohn Baldwin 			    WITNESS_FILE(&m->lock_object),
608aa89d8cdSJohn Baldwin 			    WITNESS_LINE(&m->lock_object));
60902bd1bcdSIan Dowse 			cont_logged = 1;
61002bd1bcdSIan Dowse 		}
61102bd1bcdSIan Dowse #endif
61236412d79SJohn Baldwin 
6139ed346baSBosko Milekic 		/*
614961a7b24SJohn Baldwin 		 * Block on the turnstile.
6159ed346baSBosko Milekic 		 */
616a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
617e2b25737SMark Johnston 		sleep_time -= lockstat_nsecs(&m->lock_object);
618a5aedd68SStacey Son #endif
6192502c107SJeff Roberson 		turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
620a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
621e2b25737SMark Johnston 		sleep_time += lockstat_nsecs(&m->lock_object);
622a5aedd68SStacey Son 		sleep_cnt++;
623a5aedd68SStacey Son #endif
6242604eb9eSMateusz Guzik 		v = MTX_READ_VALUE(m);
62536412d79SJohn Baldwin 	}
626076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
627e2b25737SMark Johnston 	all_time += lockstat_nsecs(&m->lock_object);
628076dd8ebSAndriy Gapon #endif
62902bd1bcdSIan Dowse #ifdef KTR
63002bd1bcdSIan Dowse 	if (cont_logged) {
63102bd1bcdSIan Dowse 		CTR4(KTR_CONTENTION,
63202bd1bcdSIan Dowse 		    "contention end: %s acquired by %p at %s:%d",
633aa89d8cdSJohn Baldwin 		    m->lock_object.lo_name, (void *)tid, file, line);
63402bd1bcdSIan Dowse 	}
63502bd1bcdSIan Dowse #endif
63632cd0147SMark Johnston 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested,
637eea4f254SJeff Roberson 	    waittime, file, line);
638a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
639a5aedd68SStacey Son 	if (sleep_time)
64032cd0147SMark Johnston 		LOCKSTAT_RECORD1(adaptive__block, m, sleep_time);
641a5aedd68SStacey Son 
642a5aedd68SStacey Son 	/*
643a5aedd68SStacey Son 	 * Only record the loops spinning and not sleeping.
644a5aedd68SStacey Son 	 */
6451ada9041SMateusz Guzik 	if (lda.spin_cnt > sleep_cnt)
64632cd0147SMark Johnston 		LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time);
647a5aedd68SStacey Son #endif
6489ed346baSBosko Milekic }
6499ed346baSBosko Milekic 
6502502c107SJeff Roberson static void
6512502c107SJeff Roberson _mtx_lock_spin_failed(struct mtx *m)
6522502c107SJeff Roberson {
6532502c107SJeff Roberson 	struct thread *td;
6542502c107SJeff Roberson 
6552502c107SJeff Roberson 	td = mtx_owner(m);
6562502c107SJeff Roberson 
6572502c107SJeff Roberson 	/* If the mutex is unlocked, try again. */
6582502c107SJeff Roberson 	if (td == NULL)
6592502c107SJeff Roberson 		return;
660b95b98b0SKonstantin Belousov 
6612502c107SJeff Roberson 	printf( "spin lock %p (%s) held by %p (tid %d) too long\n",
6622502c107SJeff Roberson 	    m, m->lock_object.lo_name, td, td->td_tid);
6632502c107SJeff Roberson #ifdef WITNESS
66498332c8cSAttilio Rao 	witness_display_spinlock(&m->lock_object, td, printf);
6652502c107SJeff Roberson #endif
6662502c107SJeff Roberson 	panic("spin lock held too long");
6672502c107SJeff Roberson }
6682502c107SJeff Roberson 
669b95b98b0SKonstantin Belousov #ifdef SMP
6709ed346baSBosko Milekic /*
6717f44c618SAttilio Rao  * _mtx_lock_spin_cookie: the tougher part of acquiring an MTX_SPIN lock.
6729ed346baSBosko Milekic  *
6739ed346baSBosko Milekic  * This is only called if we need to actually spin for the lock. Recursion
6749ed346baSBosko Milekic  * is handled inline.
6759ed346baSBosko Milekic  */
6769ed346baSBosko Milekic void
6777f44c618SAttilio Rao _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts,
6787f44c618SAttilio Rao     const char *file, int line)
67936412d79SJohn Baldwin {
6807f44c618SAttilio Rao 	struct mtx *m;
681a0d45f0fSMateusz Guzik 	struct lock_delay_arg lda;
6822604eb9eSMateusz Guzik 	uintptr_t v;
6831723a064SJeff Roberson #ifdef LOCK_PROFILING
6841723a064SJeff Roberson 	int contested = 0;
68570fe8436SKip Macy 	uint64_t waittime = 0;
6861723a064SJeff Roberson #endif
687076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
688076dd8ebSAndriy Gapon 	int64_t spin_time = 0;
689076dd8ebSAndriy Gapon #endif
69036412d79SJohn Baldwin 
69135370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
69235370593SAndriy Gapon 		return;
69335370593SAndriy Gapon 
694a0d45f0fSMateusz Guzik 	lock_delay_arg_init(&lda, &mtx_spin_delay);
6957f44c618SAttilio Rao 	m = mtxlock2mtx(c);
6967f44c618SAttilio Rao 
697aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
6985746a1d8SBosko Milekic 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
6992cba8dd3SJohn Baldwin 	KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
7002cba8dd3SJohn Baldwin 	    "spinning", "lockname:\"%s\"", m->lock_object.lo_name);
7019ed346baSBosko Milekic 
702f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
703f5f9340bSFabien Thomas 	PMC_SOFT_CALL( , , lock, failed);
704f5f9340bSFabien Thomas #endif
70570fe8436SKip Macy 	lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
706076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
707e2b25737SMark Johnston 	spin_time -= lockstat_nsecs(&m->lock_object);
708076dd8ebSAndriy Gapon #endif
7092604eb9eSMateusz Guzik 	v = MTX_READ_VALUE(m);
710fc4f686dSMateusz Guzik 	for (;;) {
7112604eb9eSMateusz Guzik 		if (v == MTX_UNOWNED) {
7122604eb9eSMateusz Guzik 			if (_mtx_obtain_lock(m, tid))
713fc4f686dSMateusz Guzik 				break;
7142604eb9eSMateusz Guzik 			v = MTX_READ_VALUE(m);
71536412d79SJohn Baldwin 			continue;
716703fc290SJohn Baldwin 		}
7172604eb9eSMateusz Guzik 		/* Give interrupts a chance while we spin. */
7182604eb9eSMateusz Guzik 		spinlock_exit();
7192604eb9eSMateusz Guzik 		do {
7202604eb9eSMateusz Guzik 			if (lda.spin_cnt < 10000000) {
7212604eb9eSMateusz Guzik 				lock_delay(&lda);
7222604eb9eSMateusz Guzik 			} else {
723a0d45f0fSMateusz Guzik 				lda.spin_cnt++;
724a0d45f0fSMateusz Guzik 				if (lda.spin_cnt < 60000000 || kdb_active ||
725a0d45f0fSMateusz Guzik 				    panicstr != NULL)
72636412d79SJohn Baldwin 					DELAY(1);
7272502c107SJeff Roberson 				else
7282502c107SJeff Roberson 					_mtx_lock_spin_failed(m);
7299f1b87f1SMaxime Henrion 				cpu_spinwait();
73036412d79SJohn Baldwin 			}
7312604eb9eSMateusz Guzik 			v = MTX_READ_VALUE(m);
7322604eb9eSMateusz Guzik 		} while (v != MTX_UNOWNED);
733c6a37e84SJohn Baldwin 		spinlock_enter();
73436412d79SJohn Baldwin 	}
735076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
736e2b25737SMark Johnston 	spin_time += lockstat_nsecs(&m->lock_object);
737076dd8ebSAndriy Gapon #endif
73836412d79SJohn Baldwin 
739aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
7409ed346baSBosko Milekic 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
7412cba8dd3SJohn Baldwin 	KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
7422cba8dd3SJohn Baldwin 	    "running");
7439ed346baSBosko Milekic 
744c6d48c87SMark Johnston #ifdef KDTRACE_HOOKS
74532cd0147SMark Johnston 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m,
74632cd0147SMark Johnston 	    contested, waittime, file, line);
747e2b25737SMark Johnston 	if (spin_time != 0)
74832cd0147SMark Johnston 		LOCKSTAT_RECORD1(spin__spin, m, spin_time);
749c6d48c87SMark Johnston #endif
75036412d79SJohn Baldwin }
75133fb8a38SJohn Baldwin #endif /* SMP */
75236412d79SJohn Baldwin 
7532502c107SJeff Roberson void
754ccdf2333SAttilio Rao thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
7552502c107SJeff Roberson {
7562502c107SJeff Roberson 	struct mtx *m;
757*5e5ad162SMateusz Guzik 	uintptr_t tid, v;
758a0d45f0fSMateusz Guzik 	struct lock_delay_arg lda;
7591723a064SJeff Roberson #ifdef LOCK_PROFILING
7601723a064SJeff Roberson 	int contested = 0;
7611723a064SJeff Roberson 	uint64_t waittime = 0;
7621723a064SJeff Roberson #endif
763a5aedd68SStacey Son #ifdef KDTRACE_HOOKS
764076dd8ebSAndriy Gapon 	int64_t spin_time = 0;
765a5aedd68SStacey Son #endif
7662502c107SJeff Roberson 
7672502c107SJeff Roberson 	tid = (uintptr_t)curthread;
76835370593SAndriy Gapon 
769f61d6c5aSMark Johnston 	if (SCHEDULER_STOPPED()) {
770f61d6c5aSMark Johnston 		/*
771f61d6c5aSMark Johnston 		 * Ensure that spinlock sections are balanced even when the
772f61d6c5aSMark Johnston 		 * scheduler is stopped, since we may otherwise inadvertently
773f61d6c5aSMark Johnston 		 * re-enable interrupts while dumping core.
774f61d6c5aSMark Johnston 		 */
775f61d6c5aSMark Johnston 		spinlock_enter();
77635370593SAndriy Gapon 		return;
777f61d6c5aSMark Johnston 	}
77835370593SAndriy Gapon 
779a0d45f0fSMateusz Guzik 	lock_delay_arg_init(&lda, &mtx_spin_delay);
780a0d45f0fSMateusz Guzik 
781076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
782e2b25737SMark Johnston 	spin_time -= lockstat_nsecs(&td->td_lock->lock_object);
783076dd8ebSAndriy Gapon #endif
7842502c107SJeff Roberson 	for (;;) {
7852502c107SJeff Roberson retry:
7862502c107SJeff Roberson 		spinlock_enter();
787710eacdcSJeff Roberson 		m = td->td_lock;
78813c85a48SJohn Baldwin 		KASSERT(m->mtx_lock != MTX_DESTROYED,
78913c85a48SJohn Baldwin 		    ("thread_lock() of destroyed mutex @ %s:%d", file, line));
79013c85a48SJohn Baldwin 		KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
79113c85a48SJohn Baldwin 		    ("thread_lock() of sleep mutex %s @ %s:%d",
79213c85a48SJohn Baldwin 		    m->lock_object.lo_name, file, line));
793ad69e26bSJohn Baldwin 		if (mtx_owned(m))
794ad69e26bSJohn Baldwin 			KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
795ad69e26bSJohn Baldwin 	    ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
796ad69e26bSJohn Baldwin 			    m->lock_object.lo_name, file, line));
7972502c107SJeff Roberson 		WITNESS_CHECKORDER(&m->lock_object,
79841313430SJohn Baldwin 		    opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
799*5e5ad162SMateusz Guzik 		v = MTX_READ_VALUE(m);
800fc4f686dSMateusz Guzik 		for (;;) {
801*5e5ad162SMateusz Guzik 			if (v == MTX_UNOWNED) {
802*5e5ad162SMateusz Guzik 				if (_mtx_obtain_lock(m, tid))
803fc4f686dSMateusz Guzik 					break;
804*5e5ad162SMateusz Guzik 				v = MTX_READ_VALUE(m);
805*5e5ad162SMateusz Guzik 				continue;
806*5e5ad162SMateusz Guzik 			}
807*5e5ad162SMateusz Guzik 			if (v == tid) {
8082502c107SJeff Roberson 				m->mtx_recurse++;
8092502c107SJeff Roberson 				break;
8102502c107SJeff Roberson 			}
811f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS
812f5f9340bSFabien Thomas 			PMC_SOFT_CALL( , , lock, failed);
813f5f9340bSFabien Thomas #endif
814eea4f254SJeff Roberson 			lock_profile_obtain_lock_failed(&m->lock_object,
815eea4f254SJeff Roberson 			    &contested, &waittime);
8162502c107SJeff Roberson 			/* Give interrupts a chance while we spin. */
8172502c107SJeff Roberson 			spinlock_exit();
818*5e5ad162SMateusz Guzik 			do {
819a0d45f0fSMateusz Guzik 				if (lda.spin_cnt < 10000000) {
820a0d45f0fSMateusz Guzik 					lock_delay(&lda);
821a0d45f0fSMateusz Guzik 				} else {
822a0d45f0fSMateusz Guzik 					lda.spin_cnt++;
823a0d45f0fSMateusz Guzik 					if (lda.spin_cnt < 60000000 ||
8242502c107SJeff Roberson 					    kdb_active || panicstr != NULL)
8252502c107SJeff Roberson 						DELAY(1);
8262502c107SJeff Roberson 					else
8272502c107SJeff Roberson 						_mtx_lock_spin_failed(m);
8282502c107SJeff Roberson 					cpu_spinwait();
829a0d45f0fSMateusz Guzik 				}
8302502c107SJeff Roberson 				if (m != td->td_lock)
8312502c107SJeff Roberson 					goto retry;
832*5e5ad162SMateusz Guzik 				v = MTX_READ_VALUE(m);
833*5e5ad162SMateusz Guzik 			} while (v != MTX_UNOWNED);
8342502c107SJeff Roberson 			spinlock_enter();
8352502c107SJeff Roberson 		}
8362502c107SJeff Roberson 		if (m == td->td_lock)
8372502c107SJeff Roberson 			break;
838961135eaSJohn Baldwin 		__mtx_unlock_spin(m);	/* does spinlock_exit() */
8392502c107SJeff Roberson 	}
840076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS
841e2b25737SMark Johnston 	spin_time += lockstat_nsecs(&m->lock_object);
842076dd8ebSAndriy Gapon #endif
843eea4f254SJeff Roberson 	if (m->mtx_recurse == 0)
84432cd0147SMark Johnston 		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m,
84532cd0147SMark Johnston 		    contested, waittime, file, line);
84613c85a48SJohn Baldwin 	LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
84713c85a48SJohn Baldwin 	    line);
8482502c107SJeff Roberson 	WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
8495002e195SMark Johnston #ifdef KDTRACE_HOOKS
850156fbc14SMark Johnston 	if (spin_time != 0)
85132cd0147SMark Johnston 		LOCKSTAT_RECORD1(thread__spin, m, spin_time);
8525002e195SMark Johnston #endif
8532502c107SJeff Roberson }
8542502c107SJeff Roberson 
8552502c107SJeff Roberson struct mtx *
8562502c107SJeff Roberson thread_lock_block(struct thread *td)
8572502c107SJeff Roberson {
8582502c107SJeff Roberson 	struct mtx *lock;
8592502c107SJeff Roberson 
8602502c107SJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
861710eacdcSJeff Roberson 	lock = td->td_lock;
8622502c107SJeff Roberson 	td->td_lock = &blocked_lock;
8632502c107SJeff Roberson 	mtx_unlock_spin(lock);
8642502c107SJeff Roberson 
8652502c107SJeff Roberson 	return (lock);
8662502c107SJeff Roberson }
8672502c107SJeff Roberson 
8682502c107SJeff Roberson void
8692502c107SJeff Roberson thread_lock_unblock(struct thread *td, struct mtx *new)
8702502c107SJeff Roberson {
8712502c107SJeff Roberson 	mtx_assert(new, MA_OWNED);
8722502c107SJeff Roberson 	MPASS(td->td_lock == &blocked_lock);
87365d32cd8SMatt Jacob 	atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new);
8742502c107SJeff Roberson }
8752502c107SJeff Roberson 
8762502c107SJeff Roberson void
8772502c107SJeff Roberson thread_lock_set(struct thread *td, struct mtx *new)
8782502c107SJeff Roberson {
8792502c107SJeff Roberson 	struct mtx *lock;
8802502c107SJeff Roberson 
8812502c107SJeff Roberson 	mtx_assert(new, MA_OWNED);
8822502c107SJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
883710eacdcSJeff Roberson 	lock = td->td_lock;
8842502c107SJeff Roberson 	td->td_lock = new;
8852502c107SJeff Roberson 	mtx_unlock_spin(lock);
8862502c107SJeff Roberson }
8872502c107SJeff Roberson 
8889ed346baSBosko Milekic /*
8897f44c618SAttilio Rao  * __mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
8909ed346baSBosko Milekic  *
8919ed346baSBosko Milekic  * We are only called here if the lock is recursed or contested (i.e. we
8929ed346baSBosko Milekic  * need to wake up a blocked thread).
8939ed346baSBosko Milekic  */
89436412d79SJohn Baldwin void
8957f44c618SAttilio Rao __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line)
89636412d79SJohn Baldwin {
8977f44c618SAttilio Rao 	struct mtx *m;
898961a7b24SJohn Baldwin 	struct turnstile *ts;
8999ed346baSBosko Milekic 
90035370593SAndriy Gapon 	if (SCHEDULER_STOPPED())
90135370593SAndriy Gapon 		return;
90235370593SAndriy Gapon 
9037f44c618SAttilio Rao 	m = mtxlock2mtx(c);
9047f44c618SAttilio Rao 
90508812b39SBosko Milekic 	if (mtx_recursed(m)) {
90636412d79SJohn Baldwin 		if (--(m->mtx_recurse) == 0)
90708812b39SBosko Milekic 			atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
908aa89d8cdSJohn Baldwin 		if (LOCK_LOG_TEST(&m->lock_object, opts))
9099ed346baSBosko Milekic 			CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
91036412d79SJohn Baldwin 		return;
91136412d79SJohn Baldwin 	}
9129ed346baSBosko Milekic 
9132502c107SJeff Roberson 	/*
9142502c107SJeff Roberson 	 * We have to lock the chain before the turnstile so this turnstile
9152502c107SJeff Roberson 	 * can be removed from the hash list if it is empty.
9162502c107SJeff Roberson 	 */
9172502c107SJeff Roberson 	turnstile_chain_lock(&m->lock_object);
918aa89d8cdSJohn Baldwin 	ts = turnstile_lookup(&m->lock_object);
919aa89d8cdSJohn Baldwin 	if (LOCK_LOG_TEST(&m->lock_object, opts))
9209ed346baSBosko Milekic 		CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
921961a7b24SJohn Baldwin 	MPASS(ts != NULL);
9227aa4f685SJohn Baldwin 	turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
923961135eaSJohn Baldwin 	_mtx_release_lock_quick(m);
924bf9c6c31SJohn Baldwin 
9252502c107SJeff Roberson 	/*
9262502c107SJeff Roberson 	 * This turnstile is now no longer associated with the mutex.  We can
9272502c107SJeff Roberson 	 * unlock the chain lock so a new turnstile may take it's place.
9282502c107SJeff Roberson 	 */
9297aa4f685SJohn Baldwin 	turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
9302502c107SJeff Roberson 	turnstile_chain_unlock(&m->lock_object);
9319ed346baSBosko Milekic }
9329ed346baSBosko Milekic 
9339ed346baSBosko Milekic /*
9349ed346baSBosko Milekic  * All the unlocking of MTX_SPIN locks is done inline.
935961135eaSJohn Baldwin  * See the __mtx_unlock_spin() macro for the details.
9369ed346baSBosko Milekic  */
9379ed346baSBosko Milekic 
9389ed346baSBosko Milekic /*
93915ec816aSJohn Baldwin  * The backing function for the INVARIANTS-enabled mtx_assert()
9409ed346baSBosko Milekic  */
9411103f3b0SJohn Baldwin #ifdef INVARIANT_SUPPORT
9420cde2e34SJason Evans void
9437f44c618SAttilio Rao __mtx_assert(const volatile uintptr_t *c, int what, const char *file, int line)
9440cde2e34SJason Evans {
9457f44c618SAttilio Rao 	const struct mtx *m;
9465cb0fbe4SJohn Baldwin 
947310ab671SEric van Gyzen 	if (panicstr != NULL || dumping || SCHEDULER_STOPPED())
9485cb0fbe4SJohn Baldwin 		return;
9497f44c618SAttilio Rao 
9507f44c618SAttilio Rao 	m = mtxlock2mtx(c);
9517f44c618SAttilio Rao 
952a10f4966SJake Burkholder 	switch (what) {
9530cde2e34SJason Evans 	case MA_OWNED:
9540cde2e34SJason Evans 	case MA_OWNED | MA_RECURSED:
9550cde2e34SJason Evans 	case MA_OWNED | MA_NOTRECURSED:
956a10f4966SJake Burkholder 		if (!mtx_owned(m))
9570cde2e34SJason Evans 			panic("mutex %s not owned at %s:%d",
958aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
959a10f4966SJake Burkholder 		if (mtx_recursed(m)) {
960a10f4966SJake Burkholder 			if ((what & MA_NOTRECURSED) != 0)
9610cde2e34SJason Evans 				panic("mutex %s recursed at %s:%d",
962aa89d8cdSJohn Baldwin 				    m->lock_object.lo_name, file, line);
963a10f4966SJake Burkholder 		} else if ((what & MA_RECURSED) != 0) {
9640cde2e34SJason Evans 			panic("mutex %s unrecursed at %s:%d",
965aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
9660cde2e34SJason Evans 		}
9670cde2e34SJason Evans 		break;
9680cde2e34SJason Evans 	case MA_NOTOWNED:
969a10f4966SJake Burkholder 		if (mtx_owned(m))
9700cde2e34SJason Evans 			panic("mutex %s owned at %s:%d",
971aa89d8cdSJohn Baldwin 			    m->lock_object.lo_name, file, line);
9720cde2e34SJason Evans 		break;
9730cde2e34SJason Evans 	default:
97456771ca7SJason Evans 		panic("unknown mtx_assert at %s:%d", file, line);
9750cde2e34SJason Evans 	}
9760cde2e34SJason Evans }
9770cde2e34SJason Evans #endif
9780cde2e34SJason Evans 
9799ed346baSBosko Milekic /*
980c27b5699SAndrew R. Reiter  * General init routine used by the MTX_SYSINIT() macro.
981c27b5699SAndrew R. Reiter  */
982c27b5699SAndrew R. Reiter void
983c27b5699SAndrew R. Reiter mtx_sysinit(void *arg)
984c27b5699SAndrew R. Reiter {
985c27b5699SAndrew R. Reiter 	struct mtx_args *margs = arg;
986c27b5699SAndrew R. Reiter 
9877f44c618SAttilio Rao 	mtx_init((struct mtx *)margs->ma_mtx, margs->ma_desc, NULL,
9887f44c618SAttilio Rao 	    margs->ma_opts);
989c27b5699SAndrew R. Reiter }
990c27b5699SAndrew R. Reiter 
991c27b5699SAndrew R. Reiter /*
9929ed346baSBosko Milekic  * Mutex initialization routine; initialize lock `m' of type contained in
9930c88508aSJohn Baldwin  * `opts' with options contained in `opts' and name `name.'  The optional
9940c88508aSJohn Baldwin  * lock type `type' is used as a general lock category name for use with
9950c88508aSJohn Baldwin  * witness.
9969ed346baSBosko Milekic  */
99736412d79SJohn Baldwin void
9987f44c618SAttilio Rao _mtx_init(volatile uintptr_t *c, const char *name, const char *type, int opts)
99936412d79SJohn Baldwin {
10007f44c618SAttilio Rao 	struct mtx *m;
100183a81bcbSJohn Baldwin 	struct lock_class *class;
100283a81bcbSJohn Baldwin 	int flags;
10039ed346baSBosko Milekic 
10047f44c618SAttilio Rao 	m = mtxlock2mtx(c);
10057f44c618SAttilio Rao 
100619284646SJohn Baldwin 	MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
1007fd07ddcfSDmitry Chagin 	    MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE | MTX_NEW)) == 0);
1008353998acSAttilio Rao 	ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock,
1009353998acSAttilio Rao 	    ("%s: mtx_lock not aligned for %s: %p", __func__, name,
1010353998acSAttilio Rao 	    &m->mtx_lock));
10119ed346baSBosko Milekic 
101283a81bcbSJohn Baldwin 	/* Determine lock class and lock flags. */
101319284646SJohn Baldwin 	if (opts & MTX_SPIN)
101483a81bcbSJohn Baldwin 		class = &lock_class_mtx_spin;
101519284646SJohn Baldwin 	else
101683a81bcbSJohn Baldwin 		class = &lock_class_mtx_sleep;
101783a81bcbSJohn Baldwin 	flags = 0;
101819284646SJohn Baldwin 	if (opts & MTX_QUIET)
101983a81bcbSJohn Baldwin 		flags |= LO_QUIET;
102019284646SJohn Baldwin 	if (opts & MTX_RECURSE)
102183a81bcbSJohn Baldwin 		flags |= LO_RECURSABLE;
102219284646SJohn Baldwin 	if ((opts & MTX_NOWITNESS) == 0)
102383a81bcbSJohn Baldwin 		flags |= LO_WITNESS;
1024f22a4b62SJeff Roberson 	if (opts & MTX_DUPOK)
102583a81bcbSJohn Baldwin 		flags |= LO_DUPOK;
10267c0435b9SKip Macy 	if (opts & MTX_NOPROFILE)
10277c0435b9SKip Macy 		flags |= LO_NOPROFILE;
1028fd07ddcfSDmitry Chagin 	if (opts & MTX_NEW)
1029fd07ddcfSDmitry Chagin 		flags |= LO_NEW;
103019284646SJohn Baldwin 
103183a81bcbSJohn Baldwin 	/* Initialize mutex. */
1032b5fb43e5SJohn Baldwin 	lock_init(&m->lock_object, class, name, type, flags);
1033b5fb43e5SJohn Baldwin 
103419284646SJohn Baldwin 	m->mtx_lock = MTX_UNOWNED;
103583a81bcbSJohn Baldwin 	m->mtx_recurse = 0;
103636412d79SJohn Baldwin }
103736412d79SJohn Baldwin 
10389ed346baSBosko Milekic /*
103919284646SJohn Baldwin  * Remove lock `m' from all_mtx queue.  We don't allow MTX_QUIET to be
104019284646SJohn Baldwin  * passed in as a flag here because if the corresponding mtx_init() was
104119284646SJohn Baldwin  * called with MTX_QUIET set, then it will already be set in the mutex's
104219284646SJohn Baldwin  * flags.
10439ed346baSBosko Milekic  */
104436412d79SJohn Baldwin void
10457f44c618SAttilio Rao _mtx_destroy(volatile uintptr_t *c)
104636412d79SJohn Baldwin {
10477f44c618SAttilio Rao 	struct mtx *m;
10487f44c618SAttilio Rao 
10497f44c618SAttilio Rao 	m = mtxlock2mtx(c);
105036412d79SJohn Baldwin 
105119284646SJohn Baldwin 	if (!mtx_owned(m))
105219284646SJohn Baldwin 		MPASS(mtx_unowned(m));
105319284646SJohn Baldwin 	else {
105408812b39SBosko Milekic 		MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
10559ed346baSBosko Milekic 
1056861a2308SScott Long 		/* Perform the non-mtx related part of mtx_unlock_spin(). */
1057aa89d8cdSJohn Baldwin 		if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin)
1058861a2308SScott Long 			spinlock_exit();
1059764e4d54SJohn Baldwin 		else
1060ce1c953eSMark Johnston 			TD_LOCKS_DEC(curthread);
1061861a2308SScott Long 
1062d3df4af3SJeff Roberson 		lock_profile_release_lock(&m->lock_object);
106319284646SJohn Baldwin 		/* Tell witness this isn't locked to make it happy. */
1064aa89d8cdSJohn Baldwin 		WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__,
1065c86b6ff5SJohn Baldwin 		    __LINE__);
106636412d79SJohn Baldwin 	}
10670384fff8SJason Evans 
1068186abbd7SJohn Baldwin 	m->mtx_lock = MTX_DESTROYED;
1069aa89d8cdSJohn Baldwin 	lock_destroy(&m->lock_object);
10700384fff8SJason Evans }
1071d23f5958SMatthew Dillon 
1072d23f5958SMatthew Dillon /*
1073c53c013bSJohn Baldwin  * Intialize the mutex code and system mutexes.  This is called from the MD
1074c53c013bSJohn Baldwin  * startup code prior to mi_startup().  The per-CPU data space needs to be
1075c53c013bSJohn Baldwin  * setup before this is called.
1076c53c013bSJohn Baldwin  */
1077c53c013bSJohn Baldwin void
1078c53c013bSJohn Baldwin mutex_init(void)
1079c53c013bSJohn Baldwin {
1080c53c013bSJohn Baldwin 
1081961a7b24SJohn Baldwin 	/* Setup turnstiles so that sleep mutexes work. */
1082961a7b24SJohn Baldwin 	init_turnstiles();
1083961a7b24SJohn Baldwin 
1084c53c013bSJohn Baldwin 	/*
1085c53c013bSJohn Baldwin 	 * Initialize mutexes.
1086c53c013bSJohn Baldwin 	 */
10870c88508aSJohn Baldwin 	mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
10882502c107SJeff Roberson 	mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN);
10892502c107SJeff Roberson 	blocked_lock.mtx_lock = 0xdeadc0de;	/* Always blocked. */
10900c88508aSJohn Baldwin 	mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
10916afb32fcSKonstantin Belousov 	mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN);
10925c7bebf9SKonstantin Belousov 	mtx_init(&proc0.p_statmtx, "pstatl", NULL, MTX_SPIN);
10935c7bebf9SKonstantin Belousov 	mtx_init(&proc0.p_itimmtx, "pitiml", NULL, MTX_SPIN);
10945c7bebf9SKonstantin Belousov 	mtx_init(&proc0.p_profmtx, "pprofl", NULL, MTX_SPIN);
10958c4b6380SJohn Baldwin 	mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
1096c53c013bSJohn Baldwin 	mtx_lock(&Giant);
1097c53c013bSJohn Baldwin }
1098d272fe53SJohn Baldwin 
1099d272fe53SJohn Baldwin #ifdef DDB
1100d272fe53SJohn Baldwin void
1101d576deedSPawel Jakub Dawidek db_show_mtx(const struct lock_object *lock)
1102d272fe53SJohn Baldwin {
1103d272fe53SJohn Baldwin 	struct thread *td;
1104d576deedSPawel Jakub Dawidek 	const struct mtx *m;
1105d272fe53SJohn Baldwin 
1106d576deedSPawel Jakub Dawidek 	m = (const struct mtx *)lock;
1107d272fe53SJohn Baldwin 
1108d272fe53SJohn Baldwin 	db_printf(" flags: {");
110983a81bcbSJohn Baldwin 	if (LOCK_CLASS(lock) == &lock_class_mtx_spin)
1110d272fe53SJohn Baldwin 		db_printf("SPIN");
1111d272fe53SJohn Baldwin 	else
1112d272fe53SJohn Baldwin 		db_printf("DEF");
1113aa89d8cdSJohn Baldwin 	if (m->lock_object.lo_flags & LO_RECURSABLE)
1114d272fe53SJohn Baldwin 		db_printf(", RECURSE");
1115aa89d8cdSJohn Baldwin 	if (m->lock_object.lo_flags & LO_DUPOK)
1116d272fe53SJohn Baldwin 		db_printf(", DUPOK");
1117d272fe53SJohn Baldwin 	db_printf("}\n");
1118d272fe53SJohn Baldwin 	db_printf(" state: {");
1119d272fe53SJohn Baldwin 	if (mtx_unowned(m))
1120d272fe53SJohn Baldwin 		db_printf("UNOWNED");
1121c0bfd703SJohn Baldwin 	else if (mtx_destroyed(m))
1122c0bfd703SJohn Baldwin 		db_printf("DESTROYED");
1123d272fe53SJohn Baldwin 	else {
1124d272fe53SJohn Baldwin 		db_printf("OWNED");
1125d272fe53SJohn Baldwin 		if (m->mtx_lock & MTX_CONTESTED)
1126d272fe53SJohn Baldwin 			db_printf(", CONTESTED");
1127d272fe53SJohn Baldwin 		if (m->mtx_lock & MTX_RECURSED)
1128d272fe53SJohn Baldwin 			db_printf(", RECURSED");
1129d272fe53SJohn Baldwin 	}
1130d272fe53SJohn Baldwin 	db_printf("}\n");
1131c0bfd703SJohn Baldwin 	if (!mtx_unowned(m) && !mtx_destroyed(m)) {
1132d272fe53SJohn Baldwin 		td = mtx_owner(m);
1133d272fe53SJohn Baldwin 		db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td,
1134431f8906SJulian Elischer 		    td->td_tid, td->td_proc->p_pid, td->td_name);
1135d272fe53SJohn Baldwin 		if (mtx_recursed(m))
1136d272fe53SJohn Baldwin 			db_printf(" recursed: %d\n", m->mtx_recurse);
1137d272fe53SJohn Baldwin 	}
1138d272fe53SJohn Baldwin }
1139d272fe53SJohn Baldwin #endif
1140