10384fff8SJason Evans /*- 20384fff8SJason Evans * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 30384fff8SJason Evans * 40384fff8SJason Evans * Redistribution and use in source and binary forms, with or without 50384fff8SJason Evans * modification, are permitted provided that the following conditions 60384fff8SJason Evans * are met: 70384fff8SJason Evans * 1. Redistributions of source code must retain the above copyright 80384fff8SJason Evans * notice, this list of conditions and the following disclaimer. 90384fff8SJason Evans * 2. Redistributions in binary form must reproduce the above copyright 100384fff8SJason Evans * notice, this list of conditions and the following disclaimer in the 110384fff8SJason Evans * documentation and/or other materials provided with the distribution. 120384fff8SJason Evans * 3. Berkeley Software Design Inc's name may not be used to endorse or 130384fff8SJason Evans * promote products derived from this software without specific prior 140384fff8SJason Evans * written permission. 150384fff8SJason Evans * 160384fff8SJason Evans * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 170384fff8SJason Evans * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 180384fff8SJason Evans * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 190384fff8SJason Evans * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 200384fff8SJason Evans * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 210384fff8SJason Evans * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 220384fff8SJason Evans * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 230384fff8SJason Evans * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 240384fff8SJason Evans * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 250384fff8SJason Evans * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 260384fff8SJason Evans * SUCH DAMAGE. 270384fff8SJason Evans * 280384fff8SJason Evans * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2936412d79SJohn Baldwin * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 300384fff8SJason Evans */ 310384fff8SJason Evans 320384fff8SJason Evans /* 33ba48b69aSJohn Baldwin * Machine independent bits of mutex implementation. 340384fff8SJason Evans */ 350384fff8SJason Evans 36677b542eSDavid E. O'Brien #include <sys/cdefs.h> 37677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 38677b542eSDavid E. O'Brien 392498cf8cSJohn Baldwin #include "opt_adaptive_mutexes.h" 409c36c934SJohn Baldwin #include "opt_ddb.h" 41f5f9340bSFabien Thomas #include "opt_hwpmc_hooks.h" 429923b511SScott Long #include "opt_sched.h" 43a5a96a19SJohn Baldwin 440384fff8SJason Evans #include <sys/param.h> 456c35e809SDag-Erling Smørgrav #include <sys/systm.h> 4636412d79SJohn Baldwin #include <sys/bus.h> 471126349aSPaul Saab #include <sys/conf.h> 482d50560aSMarcel Moolenaar #include <sys/kdb.h> 4936412d79SJohn Baldwin #include <sys/kernel.h> 506c35e809SDag-Erling Smørgrav #include <sys/ktr.h> 5119284646SJohn Baldwin #include <sys/lock.h> 52fb919e4dSMark Murray #include <sys/malloc.h> 5319284646SJohn Baldwin #include <sys/mutex.h> 540384fff8SJason Evans #include <sys/proc.h> 55c4f7a187SJohn Baldwin #include <sys/resourcevar.h> 56b43179fbSJeff Roberson #include <sys/sched.h> 576c35e809SDag-Erling Smørgrav #include <sys/sbuf.h> 581ada9041SMateusz Guzik #include <sys/smp.h> 59a5a96a19SJohn Baldwin #include <sys/sysctl.h> 60961a7b24SJohn Baldwin #include <sys/turnstile.h> 6136412d79SJohn Baldwin #include <sys/vmmeter.h> 627c0435b9SKip Macy #include <sys/lock_profile.h> 630384fff8SJason Evans 6436412d79SJohn Baldwin #include <machine/atomic.h> 6536412d79SJohn Baldwin #include <machine/bus.h> 660384fff8SJason Evans #include <machine/cpu.h> 6736412d79SJohn Baldwin 689c36c934SJohn Baldwin #include <ddb/ddb.h> 699c36c934SJohn Baldwin 708c4b6380SJohn Baldwin #include <fs/devfs/devfs_int.h> 718c4b6380SJohn Baldwin 7236412d79SJohn Baldwin #include <vm/vm.h> 7336412d79SJohn Baldwin #include <vm/vm_extern.h> 7436412d79SJohn Baldwin 75cd6e6e4eSJohn Baldwin #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 76cd6e6e4eSJohn Baldwin #define ADAPTIVE_MUTEXES 77cd6e6e4eSJohn Baldwin #endif 78cd6e6e4eSJohn Baldwin 79f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 80f5f9340bSFabien Thomas #include <sys/pmckern.h> 81f5f9340bSFabien Thomas PMC_SOFT_DEFINE( , , lock, failed); 82f5f9340bSFabien Thomas #endif 83f5f9340bSFabien Thomas 84b9a80acaSStephan Uphoff /* 857f44c618SAttilio Rao * Return the mutex address when the lock cookie address is provided. 867f44c618SAttilio Rao * This functionality assumes that struct mtx* have a member named mtx_lock. 877f44c618SAttilio Rao */ 887f44c618SAttilio Rao #define mtxlock2mtx(c) (__containerof(c, struct mtx, mtx_lock)) 897f44c618SAttilio Rao 907f44c618SAttilio Rao /* 919ed346baSBosko Milekic * Internal utility macros. 920cde2e34SJason Evans */ 939ed346baSBosko Milekic #define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 940cde2e34SJason Evans 95c0bfd703SJohn Baldwin #define mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED) 96c0bfd703SJohn Baldwin 97d576deedSPawel Jakub Dawidek static void assert_mtx(const struct lock_object *lock, int what); 98d272fe53SJohn Baldwin #ifdef DDB 99d576deedSPawel Jakub Dawidek static void db_show_mtx(const struct lock_object *lock); 100d272fe53SJohn Baldwin #endif 1017faf4d90SDavide Italiano static void lock_mtx(struct lock_object *lock, uintptr_t how); 1027faf4d90SDavide Italiano static void lock_spin(struct lock_object *lock, uintptr_t how); 103a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 104d576deedSPawel Jakub Dawidek static int owner_mtx(const struct lock_object *lock, 105d576deedSPawel Jakub Dawidek struct thread **owner); 106a5aedd68SStacey Son #endif 1077faf4d90SDavide Italiano static uintptr_t unlock_mtx(struct lock_object *lock); 1087faf4d90SDavide Italiano static uintptr_t unlock_spin(struct lock_object *lock); 109d272fe53SJohn Baldwin 1100cde2e34SJason Evans /* 11119284646SJohn Baldwin * Lock classes for sleep and spin mutexes. 1120cde2e34SJason Evans */ 11319284646SJohn Baldwin struct lock_class lock_class_mtx_sleep = { 114ae8dde30SJohn Baldwin .lc_name = "sleep mutex", 115ae8dde30SJohn Baldwin .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 116f9721b43SAttilio Rao .lc_assert = assert_mtx, 117d272fe53SJohn Baldwin #ifdef DDB 118ae8dde30SJohn Baldwin .lc_ddb_show = db_show_mtx, 119d272fe53SJohn Baldwin #endif 1206e21afd4SJohn Baldwin .lc_lock = lock_mtx, 1216e21afd4SJohn Baldwin .lc_unlock = unlock_mtx, 122a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 123a5aedd68SStacey Son .lc_owner = owner_mtx, 124a5aedd68SStacey Son #endif 12519284646SJohn Baldwin }; 12619284646SJohn Baldwin struct lock_class lock_class_mtx_spin = { 127ae8dde30SJohn Baldwin .lc_name = "spin mutex", 128ae8dde30SJohn Baldwin .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 129f9721b43SAttilio Rao .lc_assert = assert_mtx, 130d272fe53SJohn Baldwin #ifdef DDB 131ae8dde30SJohn Baldwin .lc_ddb_show = db_show_mtx, 132d272fe53SJohn Baldwin #endif 1336e21afd4SJohn Baldwin .lc_lock = lock_spin, 1346e21afd4SJohn Baldwin .lc_unlock = unlock_spin, 135a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 136a5aedd68SStacey Son .lc_owner = owner_mtx, 137a5aedd68SStacey Son #endif 1388484de75SJohn Baldwin }; 1398484de75SJohn Baldwin 1401ada9041SMateusz Guzik #ifdef ADAPTIVE_MUTEXES 1411ada9041SMateusz Guzik static SYSCTL_NODE(_debug, OID_AUTO, mtx, CTLFLAG_RD, NULL, "mtx debugging"); 1421ada9041SMateusz Guzik 1431ada9041SMateusz Guzik static struct lock_delay_config mtx_delay = { 1441ada9041SMateusz Guzik .initial = 1000, 1451ada9041SMateusz Guzik .step = 500, 1461ada9041SMateusz Guzik .min = 100, 1471ada9041SMateusz Guzik .max = 5000, 1481ada9041SMateusz Guzik }; 1491ada9041SMateusz Guzik 1501ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_initial, CTLFLAG_RW, &mtx_delay.initial, 1511ada9041SMateusz Guzik 0, ""); 1521ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_delay.step, 1531ada9041SMateusz Guzik 0, ""); 1541ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_delay.min, 1551ada9041SMateusz Guzik 0, ""); 1561ada9041SMateusz Guzik SYSCTL_INT(_debug_mtx, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_delay.max, 1571ada9041SMateusz Guzik 0, ""); 1581ada9041SMateusz Guzik 1591ada9041SMateusz Guzik static void 1601ada9041SMateusz Guzik mtx_delay_sysinit(void *dummy) 1611ada9041SMateusz Guzik { 1621ada9041SMateusz Guzik 1631ada9041SMateusz Guzik mtx_delay.initial = mp_ncpus * 25; 1641ada9041SMateusz Guzik mtx_delay.step = (mp_ncpus * 25) / 2; 1651ada9041SMateusz Guzik mtx_delay.min = mp_ncpus * 5; 1661ada9041SMateusz Guzik mtx_delay.max = mp_ncpus * 25 * 10; 1671ada9041SMateusz Guzik } 1681ada9041SMateusz Guzik LOCK_DELAY_SYSINIT(mtx_delay_sysinit); 1691ada9041SMateusz Guzik #endif 1701ada9041SMateusz Guzik 171a0d45f0fSMateusz Guzik static SYSCTL_NODE(_debug, OID_AUTO, mtx_spin, CTLFLAG_RD, NULL, 172a0d45f0fSMateusz Guzik "mtx spin debugging"); 173a0d45f0fSMateusz Guzik 174a0d45f0fSMateusz Guzik static struct lock_delay_config mtx_spin_delay = { 175a0d45f0fSMateusz Guzik .initial = 1000, 176a0d45f0fSMateusz Guzik .step = 500, 177a0d45f0fSMateusz Guzik .min = 100, 178a0d45f0fSMateusz Guzik .max = 5000, 179a0d45f0fSMateusz Guzik }; 180a0d45f0fSMateusz Guzik 181a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_initial, CTLFLAG_RW, 182a0d45f0fSMateusz Guzik &mtx_spin_delay.initial, 0, ""); 183a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_spin_delay.step, 184a0d45f0fSMateusz Guzik 0, ""); 185a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_spin_delay.min, 186a0d45f0fSMateusz Guzik 0, ""); 187a0d45f0fSMateusz Guzik SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_spin_delay.max, 188a0d45f0fSMateusz Guzik 0, ""); 189a0d45f0fSMateusz Guzik 190a0d45f0fSMateusz Guzik static void 191a0d45f0fSMateusz Guzik mtx_spin_delay_sysinit(void *dummy) 192a0d45f0fSMateusz Guzik { 193a0d45f0fSMateusz Guzik 194a0d45f0fSMateusz Guzik mtx_spin_delay.initial = mp_ncpus * 25; 195a0d45f0fSMateusz Guzik mtx_spin_delay.step = (mp_ncpus * 25) / 2; 196a0d45f0fSMateusz Guzik mtx_spin_delay.min = mp_ncpus * 5; 197a0d45f0fSMateusz Guzik mtx_spin_delay.max = mp_ncpus * 25 * 10; 198a0d45f0fSMateusz Guzik } 199a0d45f0fSMateusz Guzik LOCK_DELAY_SYSINIT(mtx_spin_delay_sysinit); 200a0d45f0fSMateusz Guzik 2019ed346baSBosko Milekic /* 202c53c013bSJohn Baldwin * System-wide mutexes 203c53c013bSJohn Baldwin */ 2042502c107SJeff Roberson struct mtx blocked_lock; 205c53c013bSJohn Baldwin struct mtx Giant; 206c53c013bSJohn Baldwin 20767784314SPoul-Henning Kamp void 208d576deedSPawel Jakub Dawidek assert_mtx(const struct lock_object *lock, int what) 209f9721b43SAttilio Rao { 210f9721b43SAttilio Rao 211d576deedSPawel Jakub Dawidek mtx_assert((const struct mtx *)lock, what); 212f9721b43SAttilio Rao } 213f9721b43SAttilio Rao 21467784314SPoul-Henning Kamp void 2157faf4d90SDavide Italiano lock_mtx(struct lock_object *lock, uintptr_t how) 2166e21afd4SJohn Baldwin { 2176e21afd4SJohn Baldwin 2186e21afd4SJohn Baldwin mtx_lock((struct mtx *)lock); 2196e21afd4SJohn Baldwin } 2206e21afd4SJohn Baldwin 22167784314SPoul-Henning Kamp void 2227faf4d90SDavide Italiano lock_spin(struct lock_object *lock, uintptr_t how) 2236e21afd4SJohn Baldwin { 2246e21afd4SJohn Baldwin 2256e21afd4SJohn Baldwin panic("spin locks can only use msleep_spin"); 2266e21afd4SJohn Baldwin } 2276e21afd4SJohn Baldwin 2287faf4d90SDavide Italiano uintptr_t 2296e21afd4SJohn Baldwin unlock_mtx(struct lock_object *lock) 2306e21afd4SJohn Baldwin { 2316e21afd4SJohn Baldwin struct mtx *m; 2326e21afd4SJohn Baldwin 2336e21afd4SJohn Baldwin m = (struct mtx *)lock; 2346e21afd4SJohn Baldwin mtx_assert(m, MA_OWNED | MA_NOTRECURSED); 2356e21afd4SJohn Baldwin mtx_unlock(m); 2366e21afd4SJohn Baldwin return (0); 2376e21afd4SJohn Baldwin } 2386e21afd4SJohn Baldwin 2397faf4d90SDavide Italiano uintptr_t 2406e21afd4SJohn Baldwin unlock_spin(struct lock_object *lock) 2416e21afd4SJohn Baldwin { 2426e21afd4SJohn Baldwin 2436e21afd4SJohn Baldwin panic("spin locks can only use msleep_spin"); 2446e21afd4SJohn Baldwin } 2456e21afd4SJohn Baldwin 246a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 247a5aedd68SStacey Son int 248d576deedSPawel Jakub Dawidek owner_mtx(const struct lock_object *lock, struct thread **owner) 249a5aedd68SStacey Son { 25002315a67SMark Johnston const struct mtx *m; 25102315a67SMark Johnston uintptr_t x; 252a5aedd68SStacey Son 25302315a67SMark Johnston m = (const struct mtx *)lock; 25402315a67SMark Johnston x = m->mtx_lock; 25502315a67SMark Johnston *owner = (struct thread *)(x & ~MTX_FLAGMASK); 25602315a67SMark Johnston return (x != MTX_UNOWNED); 257a5aedd68SStacey Son } 258a5aedd68SStacey Son #endif 259a5aedd68SStacey Son 2600cde2e34SJason Evans /* 2616283b7d0SJohn Baldwin * Function versions of the inlined __mtx_* macros. These are used by 2626283b7d0SJohn Baldwin * modules and can also be called from assembly language if needed. 2636283b7d0SJohn Baldwin */ 2646283b7d0SJohn Baldwin void 2657f44c618SAttilio Rao __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line) 2666283b7d0SJohn Baldwin { 2677f44c618SAttilio Rao struct mtx *m; 2686283b7d0SJohn Baldwin 26935370593SAndriy Gapon if (SCHEDULER_STOPPED()) 27035370593SAndriy Gapon return; 2717f44c618SAttilio Rao 2727f44c618SAttilio Rao m = mtxlock2mtx(c); 2737f44c618SAttilio Rao 274cd2fe4e6SAttilio Rao KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 275e3ae0dfeSAttilio Rao ("mtx_lock() by idle thread %p on sleep mutex %s @ %s:%d", 276e3ae0dfeSAttilio Rao curthread, m->lock_object.lo_name, file, line)); 277186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 278186abbd7SJohn Baldwin ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 279aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 280aa89d8cdSJohn Baldwin ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 2810d975d63SJohn Baldwin file, line)); 282ac6b769bSAttilio Rao WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) | 283ac6b769bSAttilio Rao LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 2847c0435b9SKip Macy 285961135eaSJohn Baldwin __mtx_lock(m, curthread, opts, file, line); 286aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 287dde96c99SJohn Baldwin line); 288ac6b769bSAttilio Rao WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE, 289ac6b769bSAttilio Rao file, line); 290ce1c953eSMark Johnston TD_LOCKS_INC(curthread); 2916283b7d0SJohn Baldwin } 2926283b7d0SJohn Baldwin 2936283b7d0SJohn Baldwin void 2947f44c618SAttilio Rao __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line) 2956283b7d0SJohn Baldwin { 2967f44c618SAttilio Rao struct mtx *m; 29735370593SAndriy Gapon 29835370593SAndriy Gapon if (SCHEDULER_STOPPED()) 29935370593SAndriy Gapon return; 3007f44c618SAttilio Rao 3017f44c618SAttilio Rao m = mtxlock2mtx(c); 3027f44c618SAttilio Rao 303186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 304186abbd7SJohn Baldwin ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 305aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 306aa89d8cdSJohn Baldwin ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 3070d975d63SJohn Baldwin file, line)); 308aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 309aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 3100d975d63SJohn Baldwin line); 31121377ce0SJohn Baldwin mtx_assert(m, MA_OWNED); 312c66d7606SKip Macy 313961135eaSJohn Baldwin __mtx_unlock(m, curthread, opts, file, line); 314ce1c953eSMark Johnston TD_LOCKS_DEC(curthread); 3156283b7d0SJohn Baldwin } 3166283b7d0SJohn Baldwin 3176283b7d0SJohn Baldwin void 3187f44c618SAttilio Rao __mtx_lock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 3197f44c618SAttilio Rao int line) 3206283b7d0SJohn Baldwin { 3217f44c618SAttilio Rao struct mtx *m; 3226283b7d0SJohn Baldwin 32335370593SAndriy Gapon if (SCHEDULER_STOPPED()) 32435370593SAndriy Gapon return; 3257f44c618SAttilio Rao 3267f44c618SAttilio Rao m = mtxlock2mtx(c); 3277f44c618SAttilio Rao 328186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 329186abbd7SJohn Baldwin ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 330aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 3310d975d63SJohn Baldwin ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 332aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 333ad69e26bSJohn Baldwin if (mtx_owned(m)) 334ac6b769bSAttilio Rao KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 335ac6b769bSAttilio Rao (opts & MTX_RECURSE) != 0, 336ad69e26bSJohn Baldwin ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n", 337ad69e26bSJohn Baldwin m->lock_object.lo_name, file, line)); 338ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 339aa89d8cdSJohn Baldwin WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 34041313430SJohn Baldwin file, line, NULL); 341961135eaSJohn Baldwin __mtx_lock_spin(m, curthread, opts, file, line); 342aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 343dde96c99SJohn Baldwin line); 344aa89d8cdSJohn Baldwin WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 3456283b7d0SJohn Baldwin } 3466283b7d0SJohn Baldwin 34790b581f2SKonstantin Belousov int 34890b581f2SKonstantin Belousov __mtx_trylock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 34990b581f2SKonstantin Belousov int line) 35090b581f2SKonstantin Belousov { 35190b581f2SKonstantin Belousov struct mtx *m; 35290b581f2SKonstantin Belousov 35390b581f2SKonstantin Belousov if (SCHEDULER_STOPPED()) 35490b581f2SKonstantin Belousov return (1); 35590b581f2SKonstantin Belousov 35690b581f2SKonstantin Belousov m = mtxlock2mtx(c); 35790b581f2SKonstantin Belousov 35890b581f2SKonstantin Belousov KASSERT(m->mtx_lock != MTX_DESTROYED, 35990b581f2SKonstantin Belousov ("mtx_trylock_spin() of destroyed mutex @ %s:%d", file, line)); 36090b581f2SKonstantin Belousov KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 36190b581f2SKonstantin Belousov ("mtx_trylock_spin() of sleep mutex %s @ %s:%d", 36290b581f2SKonstantin Belousov m->lock_object.lo_name, file, line)); 36390b581f2SKonstantin Belousov KASSERT((opts & MTX_RECURSE) == 0, 36490b581f2SKonstantin Belousov ("mtx_trylock_spin: unsupp. opt MTX_RECURSE on mutex %s @ %s:%d\n", 36590b581f2SKonstantin Belousov m->lock_object.lo_name, file, line)); 36690b581f2SKonstantin Belousov if (__mtx_trylock_spin(m, curthread, opts, file, line)) { 36790b581f2SKonstantin Belousov LOCK_LOG_TRY("LOCK", &m->lock_object, opts, 1, file, line); 36890b581f2SKonstantin Belousov WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 36990b581f2SKonstantin Belousov return (1); 37090b581f2SKonstantin Belousov } 37190b581f2SKonstantin Belousov LOCK_LOG_TRY("LOCK", &m->lock_object, opts, 0, file, line); 37290b581f2SKonstantin Belousov return (0); 37390b581f2SKonstantin Belousov } 37490b581f2SKonstantin Belousov 3756283b7d0SJohn Baldwin void 3767f44c618SAttilio Rao __mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 3777f44c618SAttilio Rao int line) 3786283b7d0SJohn Baldwin { 3797f44c618SAttilio Rao struct mtx *m; 380c66d7606SKip Macy 38135370593SAndriy Gapon if (SCHEDULER_STOPPED()) 38235370593SAndriy Gapon return; 3837f44c618SAttilio Rao 3847f44c618SAttilio Rao m = mtxlock2mtx(c); 3857f44c618SAttilio Rao 386186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 387186abbd7SJohn Baldwin ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 388aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 3890d975d63SJohn Baldwin ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 390aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 391aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 392aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 393dde96c99SJohn Baldwin line); 3940d975d63SJohn Baldwin mtx_assert(m, MA_OWNED); 395c66d7606SKip Macy 396961135eaSJohn Baldwin __mtx_unlock_spin(m); 3976283b7d0SJohn Baldwin } 3986283b7d0SJohn Baldwin 3996283b7d0SJohn Baldwin /* 4009ed346baSBosko Milekic * The important part of mtx_trylock{,_flags}() 401eac09796SJohn Baldwin * Tries to acquire lock `m.' If this function is called on a mutex that 402eac09796SJohn Baldwin * is already owned, it will recursively acquire the lock. 4030cde2e34SJason Evans */ 4040cde2e34SJason Evans int 4057f44c618SAttilio Rao _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line) 4060cde2e34SJason Evans { 4077f44c618SAttilio Rao struct mtx *m; 4081723a064SJeff Roberson #ifdef LOCK_PROFILING 4097c0435b9SKip Macy uint64_t waittime = 0; 4101723a064SJeff Roberson int contested = 0; 4111723a064SJeff Roberson #endif 4121723a064SJeff Roberson int rval; 4130cde2e34SJason Evans 41435370593SAndriy Gapon if (SCHEDULER_STOPPED()) 41535370593SAndriy Gapon return (1); 41635370593SAndriy Gapon 4177f44c618SAttilio Rao m = mtxlock2mtx(c); 4187f44c618SAttilio Rao 419cd2fe4e6SAttilio Rao KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 420e3ae0dfeSAttilio Rao ("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d", 421e3ae0dfeSAttilio Rao curthread, m->lock_object.lo_name, file, line)); 422186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 423186abbd7SJohn Baldwin ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 424aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 425aa89d8cdSJohn Baldwin ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 42683cece6fSJohn Baldwin file, line)); 4279ed346baSBosko Milekic 428ac6b769bSAttilio Rao if (mtx_owned(m) && ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 429ac6b769bSAttilio Rao (opts & MTX_RECURSE) != 0)) { 430eac09796SJohn Baldwin m->mtx_recurse++; 431eac09796SJohn Baldwin atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 432eac09796SJohn Baldwin rval = 1; 433eac09796SJohn Baldwin } else 434961135eaSJohn Baldwin rval = _mtx_obtain_lock(m, (uintptr_t)curthread); 435ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 4369ed346baSBosko Milekic 437aa89d8cdSJohn Baldwin LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line); 438764e4d54SJohn Baldwin if (rval) { 439aa89d8cdSJohn Baldwin WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 4402d96f0b1SJohn Baldwin file, line); 441ce1c953eSMark Johnston TD_LOCKS_INC(curthread); 442fe68a916SKip Macy if (m->mtx_recurse == 0) 44332cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, 444a5aedd68SStacey Son m, contested, waittime, file, line); 4457c0435b9SKip Macy 446764e4d54SJohn Baldwin } 4479ed346baSBosko Milekic 44819284646SJohn Baldwin return (rval); 4490cde2e34SJason Evans } 4500cde2e34SJason Evans 4510cde2e34SJason Evans /* 4527f44c618SAttilio Rao * __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 4539ed346baSBosko Milekic * 4549ed346baSBosko Milekic * We call this if the lock is either contested (i.e. we need to go to 4559ed346baSBosko Milekic * sleep waiting for it), or if we need to recurse on it. 4560cde2e34SJason Evans */ 4570cde2e34SJason Evans void 4587f44c618SAttilio Rao __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts, 4597f44c618SAttilio Rao const char *file, int line) 46036412d79SJohn Baldwin { 4617f44c618SAttilio Rao struct mtx *m; 4622502c107SJeff Roberson struct turnstile *ts; 4631723a064SJeff Roberson uintptr_t v; 464cd6e6e4eSJohn Baldwin #ifdef ADAPTIVE_MUTEXES 46576447e56SJohn Baldwin volatile struct thread *owner; 4662498cf8cSJohn Baldwin #endif 46702bd1bcdSIan Dowse #ifdef KTR 46802bd1bcdSIan Dowse int cont_logged = 0; 46902bd1bcdSIan Dowse #endif 4701723a064SJeff Roberson #ifdef LOCK_PROFILING 47170fe8436SKip Macy int contested = 0; 47270fe8436SKip Macy uint64_t waittime = 0; 4731723a064SJeff Roberson #endif 4741ada9041SMateusz Guzik #if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS) 4751ada9041SMateusz Guzik struct lock_delay_arg lda; 4761ada9041SMateusz Guzik #endif 477a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 47861852185SMateusz Guzik u_int sleep_cnt = 0; 479a5aedd68SStacey Son int64_t sleep_time = 0; 480076dd8ebSAndriy Gapon int64_t all_time = 0; 481a5aedd68SStacey Son #endif 48236412d79SJohn Baldwin 48335370593SAndriy Gapon if (SCHEDULER_STOPPED()) 48435370593SAndriy Gapon return; 48535370593SAndriy Gapon 486fa5000a4SMateusz Guzik #if defined(ADAPTIVE_MUTEXES) 4871ada9041SMateusz Guzik lock_delay_arg_init(&lda, &mtx_delay); 488fa5000a4SMateusz Guzik #elif defined(KDTRACE_HOOKS) 489fa5000a4SMateusz Guzik lock_delay_arg_init(&lda, NULL); 4901ada9041SMateusz Guzik #endif 4917f44c618SAttilio Rao m = mtxlock2mtx(c); 4922604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 4937f44c618SAttilio Rao 4942604eb9eSMateusz Guzik if (__predict_false(lv_mtx_owner(v) == (struct thread *)tid)) { 495ac6b769bSAttilio Rao KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 496ac6b769bSAttilio Rao (opts & MTX_RECURSE) != 0, 497eac09796SJohn Baldwin ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 498aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 499ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 50036412d79SJohn Baldwin m->mtx_recurse++; 50108812b39SBosko Milekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 502aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 5035746a1d8SBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 50436412d79SJohn Baldwin return; 50536412d79SJohn Baldwin } 506ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 5079ed346baSBosko Milekic 508f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 509f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 510f5f9340bSFabien Thomas #endif 51170fe8436SKip Macy lock_profile_obtain_lock_failed(&m->lock_object, 51270fe8436SKip Macy &contested, &waittime); 513aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 51415ec816aSJohn Baldwin CTR4(KTR_LOCK, 51515ec816aSJohn Baldwin "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 516aa89d8cdSJohn Baldwin m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 517076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 518e2b25737SMark Johnston all_time -= lockstat_nsecs(&m->lock_object); 519076dd8ebSAndriy Gapon #endif 5201bd0eefbSJohn Baldwin 521fc4f686dSMateusz Guzik for (;;) { 5222604eb9eSMateusz Guzik if (v == MTX_UNOWNED) { 5232604eb9eSMateusz Guzik if (_mtx_obtain_lock(m, tid)) 524fc4f686dSMateusz Guzik break; 5252604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 5262604eb9eSMateusz Guzik continue; 5272604eb9eSMateusz Guzik } 528a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 5291ada9041SMateusz Guzik lda.spin_cnt++; 530a5aedd68SStacey Son #endif 53149aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES 53249aead8aSAttilio Rao /* 53349aead8aSAttilio Rao * If the owner is running on another CPU, spin until the 53449aead8aSAttilio Rao * owner stops running or the state of the lock changes. 53549aead8aSAttilio Rao */ 5362604eb9eSMateusz Guzik owner = lv_mtx_owner(v); 53749aead8aSAttilio Rao if (TD_IS_RUNNING(owner)) { 53849aead8aSAttilio Rao if (LOCK_LOG_TEST(&m->lock_object, 0)) 53949aead8aSAttilio Rao CTR3(KTR_LOCK, 54049aead8aSAttilio Rao "%s: spinning on %p held by %p", 54149aead8aSAttilio Rao __func__, m, owner); 5422cba8dd3SJohn Baldwin KTR_STATE1(KTR_SCHED, "thread", 5432cba8dd3SJohn Baldwin sched_tdname((struct thread *)tid), 5442cba8dd3SJohn Baldwin "spinning", "lockname:\"%s\"", 5452cba8dd3SJohn Baldwin m->lock_object.lo_name); 5462604eb9eSMateusz Guzik do { 5471ada9041SMateusz Guzik lock_delay(&lda); 5482604eb9eSMateusz Guzik v = m->mtx_lock; 5492604eb9eSMateusz Guzik owner = lv_mtx_owner(v); 5502604eb9eSMateusz Guzik } while (v != MTX_UNOWNED && TD_IS_RUNNING(owner)); 5512cba8dd3SJohn Baldwin KTR_STATE0(KTR_SCHED, "thread", 5522cba8dd3SJohn Baldwin sched_tdname((struct thread *)tid), 5532cba8dd3SJohn Baldwin "running"); 55449aead8aSAttilio Rao continue; 55549aead8aSAttilio Rao } 55649aead8aSAttilio Rao #endif 55749aead8aSAttilio Rao 5582502c107SJeff Roberson ts = turnstile_trywait(&m->lock_object); 5592604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 5605fa8dd90SJohn Baldwin 56136412d79SJohn Baldwin /* 5629ed346baSBosko Milekic * Check if the lock has been released while spinning for 563961a7b24SJohn Baldwin * the turnstile chain lock. 56436412d79SJohn Baldwin */ 5655fa8dd90SJohn Baldwin if (v == MTX_UNOWNED) { 5662502c107SJeff Roberson turnstile_cancel(ts); 56736412d79SJohn Baldwin continue; 56836412d79SJohn Baldwin } 5699ed346baSBosko Milekic 57049aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES 57149aead8aSAttilio Rao /* 572fa29f023SJohn Baldwin * The current lock owner might have started executing 573fa29f023SJohn Baldwin * on another CPU (or the lock could have changed 574fa29f023SJohn Baldwin * owners) while we were waiting on the turnstile 575fa29f023SJohn Baldwin * chain lock. If so, drop the turnstile lock and try 576fa29f023SJohn Baldwin * again. 57749aead8aSAttilio Rao */ 5782604eb9eSMateusz Guzik owner = lv_mtx_owner(v); 57949aead8aSAttilio Rao if (TD_IS_RUNNING(owner)) { 58049aead8aSAttilio Rao turnstile_cancel(ts); 58149aead8aSAttilio Rao continue; 58249aead8aSAttilio Rao } 58349aead8aSAttilio Rao #endif 58449aead8aSAttilio Rao 58536412d79SJohn Baldwin /* 5869ed346baSBosko Milekic * If the mutex isn't already contested and a failure occurs 5879ed346baSBosko Milekic * setting the contested bit, the mutex was either released 5889ed346baSBosko Milekic * or the state of the MTX_RECURSED bit changed. 58936412d79SJohn Baldwin */ 59036412d79SJohn Baldwin if ((v & MTX_CONTESTED) == 0 && 591122eceefSJohn Baldwin !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 5922502c107SJeff Roberson turnstile_cancel(ts); 5932604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 59436412d79SJohn Baldwin continue; 59536412d79SJohn Baldwin } 59636412d79SJohn Baldwin 5979ed346baSBosko Milekic /* 5987feefcd6SJohn Baldwin * We definitely must sleep for this lock. 5999ed346baSBosko Milekic */ 60036412d79SJohn Baldwin mtx_assert(m, MA_NOTOWNED); 60136412d79SJohn Baldwin 60202bd1bcdSIan Dowse #ifdef KTR 60302bd1bcdSIan Dowse if (!cont_logged) { 60402bd1bcdSIan Dowse CTR6(KTR_CONTENTION, 60502bd1bcdSIan Dowse "contention: %p at %s:%d wants %s, taken by %s:%d", 606aa89d8cdSJohn Baldwin (void *)tid, file, line, m->lock_object.lo_name, 607aa89d8cdSJohn Baldwin WITNESS_FILE(&m->lock_object), 608aa89d8cdSJohn Baldwin WITNESS_LINE(&m->lock_object)); 60902bd1bcdSIan Dowse cont_logged = 1; 61002bd1bcdSIan Dowse } 61102bd1bcdSIan Dowse #endif 61236412d79SJohn Baldwin 6139ed346baSBosko Milekic /* 614961a7b24SJohn Baldwin * Block on the turnstile. 6159ed346baSBosko Milekic */ 616a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 617e2b25737SMark Johnston sleep_time -= lockstat_nsecs(&m->lock_object); 618a5aedd68SStacey Son #endif 6192502c107SJeff Roberson turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 620a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 621e2b25737SMark Johnston sleep_time += lockstat_nsecs(&m->lock_object); 622a5aedd68SStacey Son sleep_cnt++; 623a5aedd68SStacey Son #endif 6242604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 62536412d79SJohn Baldwin } 626076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 627e2b25737SMark Johnston all_time += lockstat_nsecs(&m->lock_object); 628076dd8ebSAndriy Gapon #endif 62902bd1bcdSIan Dowse #ifdef KTR 63002bd1bcdSIan Dowse if (cont_logged) { 63102bd1bcdSIan Dowse CTR4(KTR_CONTENTION, 63202bd1bcdSIan Dowse "contention end: %s acquired by %p at %s:%d", 633aa89d8cdSJohn Baldwin m->lock_object.lo_name, (void *)tid, file, line); 63402bd1bcdSIan Dowse } 63502bd1bcdSIan Dowse #endif 63632cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested, 637eea4f254SJeff Roberson waittime, file, line); 638a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 639a5aedd68SStacey Son if (sleep_time) 64032cd0147SMark Johnston LOCKSTAT_RECORD1(adaptive__block, m, sleep_time); 641a5aedd68SStacey Son 642a5aedd68SStacey Son /* 643a5aedd68SStacey Son * Only record the loops spinning and not sleeping. 644a5aedd68SStacey Son */ 6451ada9041SMateusz Guzik if (lda.spin_cnt > sleep_cnt) 64632cd0147SMark Johnston LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time); 647a5aedd68SStacey Son #endif 6489ed346baSBosko Milekic } 6499ed346baSBosko Milekic 6502502c107SJeff Roberson static void 6512502c107SJeff Roberson _mtx_lock_spin_failed(struct mtx *m) 6522502c107SJeff Roberson { 6532502c107SJeff Roberson struct thread *td; 6542502c107SJeff Roberson 6552502c107SJeff Roberson td = mtx_owner(m); 6562502c107SJeff Roberson 6572502c107SJeff Roberson /* If the mutex is unlocked, try again. */ 6582502c107SJeff Roberson if (td == NULL) 6592502c107SJeff Roberson return; 660b95b98b0SKonstantin Belousov 6612502c107SJeff Roberson printf( "spin lock %p (%s) held by %p (tid %d) too long\n", 6622502c107SJeff Roberson m, m->lock_object.lo_name, td, td->td_tid); 6632502c107SJeff Roberson #ifdef WITNESS 66498332c8cSAttilio Rao witness_display_spinlock(&m->lock_object, td, printf); 6652502c107SJeff Roberson #endif 6662502c107SJeff Roberson panic("spin lock held too long"); 6672502c107SJeff Roberson } 6682502c107SJeff Roberson 669b95b98b0SKonstantin Belousov #ifdef SMP 6709ed346baSBosko Milekic /* 6717f44c618SAttilio Rao * _mtx_lock_spin_cookie: the tougher part of acquiring an MTX_SPIN lock. 6729ed346baSBosko Milekic * 6739ed346baSBosko Milekic * This is only called if we need to actually spin for the lock. Recursion 6749ed346baSBosko Milekic * is handled inline. 6759ed346baSBosko Milekic */ 6769ed346baSBosko Milekic void 6777f44c618SAttilio Rao _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts, 6787f44c618SAttilio Rao const char *file, int line) 67936412d79SJohn Baldwin { 6807f44c618SAttilio Rao struct mtx *m; 681a0d45f0fSMateusz Guzik struct lock_delay_arg lda; 6822604eb9eSMateusz Guzik uintptr_t v; 6831723a064SJeff Roberson #ifdef LOCK_PROFILING 6841723a064SJeff Roberson int contested = 0; 68570fe8436SKip Macy uint64_t waittime = 0; 6861723a064SJeff Roberson #endif 687076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 688076dd8ebSAndriy Gapon int64_t spin_time = 0; 689076dd8ebSAndriy Gapon #endif 69036412d79SJohn Baldwin 69135370593SAndriy Gapon if (SCHEDULER_STOPPED()) 69235370593SAndriy Gapon return; 69335370593SAndriy Gapon 694a0d45f0fSMateusz Guzik lock_delay_arg_init(&lda, &mtx_spin_delay); 6957f44c618SAttilio Rao m = mtxlock2mtx(c); 6967f44c618SAttilio Rao 697aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 6985746a1d8SBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 6992cba8dd3SJohn Baldwin KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 7002cba8dd3SJohn Baldwin "spinning", "lockname:\"%s\"", m->lock_object.lo_name); 7019ed346baSBosko Milekic 702f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 703f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 704f5f9340bSFabien Thomas #endif 70570fe8436SKip Macy lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 706076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 707e2b25737SMark Johnston spin_time -= lockstat_nsecs(&m->lock_object); 708076dd8ebSAndriy Gapon #endif 7092604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 710fc4f686dSMateusz Guzik for (;;) { 7112604eb9eSMateusz Guzik if (v == MTX_UNOWNED) { 7122604eb9eSMateusz Guzik if (_mtx_obtain_lock(m, tid)) 713fc4f686dSMateusz Guzik break; 7142604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 71536412d79SJohn Baldwin continue; 716703fc290SJohn Baldwin } 7172604eb9eSMateusz Guzik /* Give interrupts a chance while we spin. */ 7182604eb9eSMateusz Guzik spinlock_exit(); 7192604eb9eSMateusz Guzik do { 7202604eb9eSMateusz Guzik if (lda.spin_cnt < 10000000) { 7212604eb9eSMateusz Guzik lock_delay(&lda); 7222604eb9eSMateusz Guzik } else { 723a0d45f0fSMateusz Guzik lda.spin_cnt++; 724a0d45f0fSMateusz Guzik if (lda.spin_cnt < 60000000 || kdb_active || 725a0d45f0fSMateusz Guzik panicstr != NULL) 72636412d79SJohn Baldwin DELAY(1); 7272502c107SJeff Roberson else 7282502c107SJeff Roberson _mtx_lock_spin_failed(m); 7299f1b87f1SMaxime Henrion cpu_spinwait(); 73036412d79SJohn Baldwin } 7312604eb9eSMateusz Guzik v = MTX_READ_VALUE(m); 7322604eb9eSMateusz Guzik } while (v != MTX_UNOWNED); 733c6a37e84SJohn Baldwin spinlock_enter(); 73436412d79SJohn Baldwin } 735076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 736e2b25737SMark Johnston spin_time += lockstat_nsecs(&m->lock_object); 737076dd8ebSAndriy Gapon #endif 73836412d79SJohn Baldwin 739aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 7409ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 7412cba8dd3SJohn Baldwin KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 7422cba8dd3SJohn Baldwin "running"); 7439ed346baSBosko Milekic 744c6d48c87SMark Johnston #ifdef KDTRACE_HOOKS 74532cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m, 74632cd0147SMark Johnston contested, waittime, file, line); 747e2b25737SMark Johnston if (spin_time != 0) 74832cd0147SMark Johnston LOCKSTAT_RECORD1(spin__spin, m, spin_time); 749c6d48c87SMark Johnston #endif 75036412d79SJohn Baldwin } 75133fb8a38SJohn Baldwin #endif /* SMP */ 75236412d79SJohn Baldwin 7532502c107SJeff Roberson void 754ccdf2333SAttilio Rao thread_lock_flags_(struct thread *td, int opts, const char *file, int line) 7552502c107SJeff Roberson { 7562502c107SJeff Roberson struct mtx *m; 757*5e5ad162SMateusz Guzik uintptr_t tid, v; 758a0d45f0fSMateusz Guzik struct lock_delay_arg lda; 7591723a064SJeff Roberson #ifdef LOCK_PROFILING 7601723a064SJeff Roberson int contested = 0; 7611723a064SJeff Roberson uint64_t waittime = 0; 7621723a064SJeff Roberson #endif 763a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 764076dd8ebSAndriy Gapon int64_t spin_time = 0; 765a5aedd68SStacey Son #endif 7662502c107SJeff Roberson 7672502c107SJeff Roberson tid = (uintptr_t)curthread; 76835370593SAndriy Gapon 769f61d6c5aSMark Johnston if (SCHEDULER_STOPPED()) { 770f61d6c5aSMark Johnston /* 771f61d6c5aSMark Johnston * Ensure that spinlock sections are balanced even when the 772f61d6c5aSMark Johnston * scheduler is stopped, since we may otherwise inadvertently 773f61d6c5aSMark Johnston * re-enable interrupts while dumping core. 774f61d6c5aSMark Johnston */ 775f61d6c5aSMark Johnston spinlock_enter(); 77635370593SAndriy Gapon return; 777f61d6c5aSMark Johnston } 77835370593SAndriy Gapon 779a0d45f0fSMateusz Guzik lock_delay_arg_init(&lda, &mtx_spin_delay); 780a0d45f0fSMateusz Guzik 781076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 782e2b25737SMark Johnston spin_time -= lockstat_nsecs(&td->td_lock->lock_object); 783076dd8ebSAndriy Gapon #endif 7842502c107SJeff Roberson for (;;) { 7852502c107SJeff Roberson retry: 7862502c107SJeff Roberson spinlock_enter(); 787710eacdcSJeff Roberson m = td->td_lock; 78813c85a48SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 78913c85a48SJohn Baldwin ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 79013c85a48SJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 79113c85a48SJohn Baldwin ("thread_lock() of sleep mutex %s @ %s:%d", 79213c85a48SJohn Baldwin m->lock_object.lo_name, file, line)); 793ad69e26bSJohn Baldwin if (mtx_owned(m)) 794ad69e26bSJohn Baldwin KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 795ad69e26bSJohn Baldwin ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", 796ad69e26bSJohn Baldwin m->lock_object.lo_name, file, line)); 7972502c107SJeff Roberson WITNESS_CHECKORDER(&m->lock_object, 79841313430SJohn Baldwin opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 799*5e5ad162SMateusz Guzik v = MTX_READ_VALUE(m); 800fc4f686dSMateusz Guzik for (;;) { 801*5e5ad162SMateusz Guzik if (v == MTX_UNOWNED) { 802*5e5ad162SMateusz Guzik if (_mtx_obtain_lock(m, tid)) 803fc4f686dSMateusz Guzik break; 804*5e5ad162SMateusz Guzik v = MTX_READ_VALUE(m); 805*5e5ad162SMateusz Guzik continue; 806*5e5ad162SMateusz Guzik } 807*5e5ad162SMateusz Guzik if (v == tid) { 8082502c107SJeff Roberson m->mtx_recurse++; 8092502c107SJeff Roberson break; 8102502c107SJeff Roberson } 811f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 812f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 813f5f9340bSFabien Thomas #endif 814eea4f254SJeff Roberson lock_profile_obtain_lock_failed(&m->lock_object, 815eea4f254SJeff Roberson &contested, &waittime); 8162502c107SJeff Roberson /* Give interrupts a chance while we spin. */ 8172502c107SJeff Roberson spinlock_exit(); 818*5e5ad162SMateusz Guzik do { 819a0d45f0fSMateusz Guzik if (lda.spin_cnt < 10000000) { 820a0d45f0fSMateusz Guzik lock_delay(&lda); 821a0d45f0fSMateusz Guzik } else { 822a0d45f0fSMateusz Guzik lda.spin_cnt++; 823a0d45f0fSMateusz Guzik if (lda.spin_cnt < 60000000 || 8242502c107SJeff Roberson kdb_active || panicstr != NULL) 8252502c107SJeff Roberson DELAY(1); 8262502c107SJeff Roberson else 8272502c107SJeff Roberson _mtx_lock_spin_failed(m); 8282502c107SJeff Roberson cpu_spinwait(); 829a0d45f0fSMateusz Guzik } 8302502c107SJeff Roberson if (m != td->td_lock) 8312502c107SJeff Roberson goto retry; 832*5e5ad162SMateusz Guzik v = MTX_READ_VALUE(m); 833*5e5ad162SMateusz Guzik } while (v != MTX_UNOWNED); 8342502c107SJeff Roberson spinlock_enter(); 8352502c107SJeff Roberson } 8362502c107SJeff Roberson if (m == td->td_lock) 8372502c107SJeff Roberson break; 838961135eaSJohn Baldwin __mtx_unlock_spin(m); /* does spinlock_exit() */ 8392502c107SJeff Roberson } 840076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 841e2b25737SMark Johnston spin_time += lockstat_nsecs(&m->lock_object); 842076dd8ebSAndriy Gapon #endif 843eea4f254SJeff Roberson if (m->mtx_recurse == 0) 84432cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m, 84532cd0147SMark Johnston contested, waittime, file, line); 84613c85a48SJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 84713c85a48SJohn Baldwin line); 8482502c107SJeff Roberson WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 8495002e195SMark Johnston #ifdef KDTRACE_HOOKS 850156fbc14SMark Johnston if (spin_time != 0) 85132cd0147SMark Johnston LOCKSTAT_RECORD1(thread__spin, m, spin_time); 8525002e195SMark Johnston #endif 8532502c107SJeff Roberson } 8542502c107SJeff Roberson 8552502c107SJeff Roberson struct mtx * 8562502c107SJeff Roberson thread_lock_block(struct thread *td) 8572502c107SJeff Roberson { 8582502c107SJeff Roberson struct mtx *lock; 8592502c107SJeff Roberson 8602502c107SJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 861710eacdcSJeff Roberson lock = td->td_lock; 8622502c107SJeff Roberson td->td_lock = &blocked_lock; 8632502c107SJeff Roberson mtx_unlock_spin(lock); 8642502c107SJeff Roberson 8652502c107SJeff Roberson return (lock); 8662502c107SJeff Roberson } 8672502c107SJeff Roberson 8682502c107SJeff Roberson void 8692502c107SJeff Roberson thread_lock_unblock(struct thread *td, struct mtx *new) 8702502c107SJeff Roberson { 8712502c107SJeff Roberson mtx_assert(new, MA_OWNED); 8722502c107SJeff Roberson MPASS(td->td_lock == &blocked_lock); 87365d32cd8SMatt Jacob atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new); 8742502c107SJeff Roberson } 8752502c107SJeff Roberson 8762502c107SJeff Roberson void 8772502c107SJeff Roberson thread_lock_set(struct thread *td, struct mtx *new) 8782502c107SJeff Roberson { 8792502c107SJeff Roberson struct mtx *lock; 8802502c107SJeff Roberson 8812502c107SJeff Roberson mtx_assert(new, MA_OWNED); 8822502c107SJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 883710eacdcSJeff Roberson lock = td->td_lock; 8842502c107SJeff Roberson td->td_lock = new; 8852502c107SJeff Roberson mtx_unlock_spin(lock); 8862502c107SJeff Roberson } 8872502c107SJeff Roberson 8889ed346baSBosko Milekic /* 8897f44c618SAttilio Rao * __mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 8909ed346baSBosko Milekic * 8919ed346baSBosko Milekic * We are only called here if the lock is recursed or contested (i.e. we 8929ed346baSBosko Milekic * need to wake up a blocked thread). 8939ed346baSBosko Milekic */ 89436412d79SJohn Baldwin void 8957f44c618SAttilio Rao __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line) 89636412d79SJohn Baldwin { 8977f44c618SAttilio Rao struct mtx *m; 898961a7b24SJohn Baldwin struct turnstile *ts; 8999ed346baSBosko Milekic 90035370593SAndriy Gapon if (SCHEDULER_STOPPED()) 90135370593SAndriy Gapon return; 90235370593SAndriy Gapon 9037f44c618SAttilio Rao m = mtxlock2mtx(c); 9047f44c618SAttilio Rao 90508812b39SBosko Milekic if (mtx_recursed(m)) { 90636412d79SJohn Baldwin if (--(m->mtx_recurse) == 0) 90708812b39SBosko Milekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 908aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 9099ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 91036412d79SJohn Baldwin return; 91136412d79SJohn Baldwin } 9129ed346baSBosko Milekic 9132502c107SJeff Roberson /* 9142502c107SJeff Roberson * We have to lock the chain before the turnstile so this turnstile 9152502c107SJeff Roberson * can be removed from the hash list if it is empty. 9162502c107SJeff Roberson */ 9172502c107SJeff Roberson turnstile_chain_lock(&m->lock_object); 918aa89d8cdSJohn Baldwin ts = turnstile_lookup(&m->lock_object); 919aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 9209ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 921961a7b24SJohn Baldwin MPASS(ts != NULL); 9227aa4f685SJohn Baldwin turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 923961135eaSJohn Baldwin _mtx_release_lock_quick(m); 924bf9c6c31SJohn Baldwin 9252502c107SJeff Roberson /* 9262502c107SJeff Roberson * This turnstile is now no longer associated with the mutex. We can 9272502c107SJeff Roberson * unlock the chain lock so a new turnstile may take it's place. 9282502c107SJeff Roberson */ 9297aa4f685SJohn Baldwin turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 9302502c107SJeff Roberson turnstile_chain_unlock(&m->lock_object); 9319ed346baSBosko Milekic } 9329ed346baSBosko Milekic 9339ed346baSBosko Milekic /* 9349ed346baSBosko Milekic * All the unlocking of MTX_SPIN locks is done inline. 935961135eaSJohn Baldwin * See the __mtx_unlock_spin() macro for the details. 9369ed346baSBosko Milekic */ 9379ed346baSBosko Milekic 9389ed346baSBosko Milekic /* 93915ec816aSJohn Baldwin * The backing function for the INVARIANTS-enabled mtx_assert() 9409ed346baSBosko Milekic */ 9411103f3b0SJohn Baldwin #ifdef INVARIANT_SUPPORT 9420cde2e34SJason Evans void 9437f44c618SAttilio Rao __mtx_assert(const volatile uintptr_t *c, int what, const char *file, int line) 9440cde2e34SJason Evans { 9457f44c618SAttilio Rao const struct mtx *m; 9465cb0fbe4SJohn Baldwin 947310ab671SEric van Gyzen if (panicstr != NULL || dumping || SCHEDULER_STOPPED()) 9485cb0fbe4SJohn Baldwin return; 9497f44c618SAttilio Rao 9507f44c618SAttilio Rao m = mtxlock2mtx(c); 9517f44c618SAttilio Rao 952a10f4966SJake Burkholder switch (what) { 9530cde2e34SJason Evans case MA_OWNED: 9540cde2e34SJason Evans case MA_OWNED | MA_RECURSED: 9550cde2e34SJason Evans case MA_OWNED | MA_NOTRECURSED: 956a10f4966SJake Burkholder if (!mtx_owned(m)) 9570cde2e34SJason Evans panic("mutex %s not owned at %s:%d", 958aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 959a10f4966SJake Burkholder if (mtx_recursed(m)) { 960a10f4966SJake Burkholder if ((what & MA_NOTRECURSED) != 0) 9610cde2e34SJason Evans panic("mutex %s recursed at %s:%d", 962aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 963a10f4966SJake Burkholder } else if ((what & MA_RECURSED) != 0) { 9640cde2e34SJason Evans panic("mutex %s unrecursed at %s:%d", 965aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 9660cde2e34SJason Evans } 9670cde2e34SJason Evans break; 9680cde2e34SJason Evans case MA_NOTOWNED: 969a10f4966SJake Burkholder if (mtx_owned(m)) 9700cde2e34SJason Evans panic("mutex %s owned at %s:%d", 971aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 9720cde2e34SJason Evans break; 9730cde2e34SJason Evans default: 97456771ca7SJason Evans panic("unknown mtx_assert at %s:%d", file, line); 9750cde2e34SJason Evans } 9760cde2e34SJason Evans } 9770cde2e34SJason Evans #endif 9780cde2e34SJason Evans 9799ed346baSBosko Milekic /* 980c27b5699SAndrew R. Reiter * General init routine used by the MTX_SYSINIT() macro. 981c27b5699SAndrew R. Reiter */ 982c27b5699SAndrew R. Reiter void 983c27b5699SAndrew R. Reiter mtx_sysinit(void *arg) 984c27b5699SAndrew R. Reiter { 985c27b5699SAndrew R. Reiter struct mtx_args *margs = arg; 986c27b5699SAndrew R. Reiter 9877f44c618SAttilio Rao mtx_init((struct mtx *)margs->ma_mtx, margs->ma_desc, NULL, 9887f44c618SAttilio Rao margs->ma_opts); 989c27b5699SAndrew R. Reiter } 990c27b5699SAndrew R. Reiter 991c27b5699SAndrew R. Reiter /* 9929ed346baSBosko Milekic * Mutex initialization routine; initialize lock `m' of type contained in 9930c88508aSJohn Baldwin * `opts' with options contained in `opts' and name `name.' The optional 9940c88508aSJohn Baldwin * lock type `type' is used as a general lock category name for use with 9950c88508aSJohn Baldwin * witness. 9969ed346baSBosko Milekic */ 99736412d79SJohn Baldwin void 9987f44c618SAttilio Rao _mtx_init(volatile uintptr_t *c, const char *name, const char *type, int opts) 99936412d79SJohn Baldwin { 10007f44c618SAttilio Rao struct mtx *m; 100183a81bcbSJohn Baldwin struct lock_class *class; 100283a81bcbSJohn Baldwin int flags; 10039ed346baSBosko Milekic 10047f44c618SAttilio Rao m = mtxlock2mtx(c); 10057f44c618SAttilio Rao 100619284646SJohn Baldwin MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 1007fd07ddcfSDmitry Chagin MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE | MTX_NEW)) == 0); 1008353998acSAttilio Rao ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock, 1009353998acSAttilio Rao ("%s: mtx_lock not aligned for %s: %p", __func__, name, 1010353998acSAttilio Rao &m->mtx_lock)); 10119ed346baSBosko Milekic 101283a81bcbSJohn Baldwin /* Determine lock class and lock flags. */ 101319284646SJohn Baldwin if (opts & MTX_SPIN) 101483a81bcbSJohn Baldwin class = &lock_class_mtx_spin; 101519284646SJohn Baldwin else 101683a81bcbSJohn Baldwin class = &lock_class_mtx_sleep; 101783a81bcbSJohn Baldwin flags = 0; 101819284646SJohn Baldwin if (opts & MTX_QUIET) 101983a81bcbSJohn Baldwin flags |= LO_QUIET; 102019284646SJohn Baldwin if (opts & MTX_RECURSE) 102183a81bcbSJohn Baldwin flags |= LO_RECURSABLE; 102219284646SJohn Baldwin if ((opts & MTX_NOWITNESS) == 0) 102383a81bcbSJohn Baldwin flags |= LO_WITNESS; 1024f22a4b62SJeff Roberson if (opts & MTX_DUPOK) 102583a81bcbSJohn Baldwin flags |= LO_DUPOK; 10267c0435b9SKip Macy if (opts & MTX_NOPROFILE) 10277c0435b9SKip Macy flags |= LO_NOPROFILE; 1028fd07ddcfSDmitry Chagin if (opts & MTX_NEW) 1029fd07ddcfSDmitry Chagin flags |= LO_NEW; 103019284646SJohn Baldwin 103183a81bcbSJohn Baldwin /* Initialize mutex. */ 1032b5fb43e5SJohn Baldwin lock_init(&m->lock_object, class, name, type, flags); 1033b5fb43e5SJohn Baldwin 103419284646SJohn Baldwin m->mtx_lock = MTX_UNOWNED; 103583a81bcbSJohn Baldwin m->mtx_recurse = 0; 103636412d79SJohn Baldwin } 103736412d79SJohn Baldwin 10389ed346baSBosko Milekic /* 103919284646SJohn Baldwin * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 104019284646SJohn Baldwin * passed in as a flag here because if the corresponding mtx_init() was 104119284646SJohn Baldwin * called with MTX_QUIET set, then it will already be set in the mutex's 104219284646SJohn Baldwin * flags. 10439ed346baSBosko Milekic */ 104436412d79SJohn Baldwin void 10457f44c618SAttilio Rao _mtx_destroy(volatile uintptr_t *c) 104636412d79SJohn Baldwin { 10477f44c618SAttilio Rao struct mtx *m; 10487f44c618SAttilio Rao 10497f44c618SAttilio Rao m = mtxlock2mtx(c); 105036412d79SJohn Baldwin 105119284646SJohn Baldwin if (!mtx_owned(m)) 105219284646SJohn Baldwin MPASS(mtx_unowned(m)); 105319284646SJohn Baldwin else { 105408812b39SBosko Milekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 10559ed346baSBosko Milekic 1056861a2308SScott Long /* Perform the non-mtx related part of mtx_unlock_spin(). */ 1057aa89d8cdSJohn Baldwin if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin) 1058861a2308SScott Long spinlock_exit(); 1059764e4d54SJohn Baldwin else 1060ce1c953eSMark Johnston TD_LOCKS_DEC(curthread); 1061861a2308SScott Long 1062d3df4af3SJeff Roberson lock_profile_release_lock(&m->lock_object); 106319284646SJohn Baldwin /* Tell witness this isn't locked to make it happy. */ 1064aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__, 1065c86b6ff5SJohn Baldwin __LINE__); 106636412d79SJohn Baldwin } 10670384fff8SJason Evans 1068186abbd7SJohn Baldwin m->mtx_lock = MTX_DESTROYED; 1069aa89d8cdSJohn Baldwin lock_destroy(&m->lock_object); 10700384fff8SJason Evans } 1071d23f5958SMatthew Dillon 1072d23f5958SMatthew Dillon /* 1073c53c013bSJohn Baldwin * Intialize the mutex code and system mutexes. This is called from the MD 1074c53c013bSJohn Baldwin * startup code prior to mi_startup(). The per-CPU data space needs to be 1075c53c013bSJohn Baldwin * setup before this is called. 1076c53c013bSJohn Baldwin */ 1077c53c013bSJohn Baldwin void 1078c53c013bSJohn Baldwin mutex_init(void) 1079c53c013bSJohn Baldwin { 1080c53c013bSJohn Baldwin 1081961a7b24SJohn Baldwin /* Setup turnstiles so that sleep mutexes work. */ 1082961a7b24SJohn Baldwin init_turnstiles(); 1083961a7b24SJohn Baldwin 1084c53c013bSJohn Baldwin /* 1085c53c013bSJohn Baldwin * Initialize mutexes. 1086c53c013bSJohn Baldwin */ 10870c88508aSJohn Baldwin mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 10882502c107SJeff Roberson mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN); 10892502c107SJeff Roberson blocked_lock.mtx_lock = 0xdeadc0de; /* Always blocked. */ 10900c88508aSJohn Baldwin mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 10916afb32fcSKonstantin Belousov mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN); 10925c7bebf9SKonstantin Belousov mtx_init(&proc0.p_statmtx, "pstatl", NULL, MTX_SPIN); 10935c7bebf9SKonstantin Belousov mtx_init(&proc0.p_itimmtx, "pitiml", NULL, MTX_SPIN); 10945c7bebf9SKonstantin Belousov mtx_init(&proc0.p_profmtx, "pprofl", NULL, MTX_SPIN); 10958c4b6380SJohn Baldwin mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 1096c53c013bSJohn Baldwin mtx_lock(&Giant); 1097c53c013bSJohn Baldwin } 1098d272fe53SJohn Baldwin 1099d272fe53SJohn Baldwin #ifdef DDB 1100d272fe53SJohn Baldwin void 1101d576deedSPawel Jakub Dawidek db_show_mtx(const struct lock_object *lock) 1102d272fe53SJohn Baldwin { 1103d272fe53SJohn Baldwin struct thread *td; 1104d576deedSPawel Jakub Dawidek const struct mtx *m; 1105d272fe53SJohn Baldwin 1106d576deedSPawel Jakub Dawidek m = (const struct mtx *)lock; 1107d272fe53SJohn Baldwin 1108d272fe53SJohn Baldwin db_printf(" flags: {"); 110983a81bcbSJohn Baldwin if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 1110d272fe53SJohn Baldwin db_printf("SPIN"); 1111d272fe53SJohn Baldwin else 1112d272fe53SJohn Baldwin db_printf("DEF"); 1113aa89d8cdSJohn Baldwin if (m->lock_object.lo_flags & LO_RECURSABLE) 1114d272fe53SJohn Baldwin db_printf(", RECURSE"); 1115aa89d8cdSJohn Baldwin if (m->lock_object.lo_flags & LO_DUPOK) 1116d272fe53SJohn Baldwin db_printf(", DUPOK"); 1117d272fe53SJohn Baldwin db_printf("}\n"); 1118d272fe53SJohn Baldwin db_printf(" state: {"); 1119d272fe53SJohn Baldwin if (mtx_unowned(m)) 1120d272fe53SJohn Baldwin db_printf("UNOWNED"); 1121c0bfd703SJohn Baldwin else if (mtx_destroyed(m)) 1122c0bfd703SJohn Baldwin db_printf("DESTROYED"); 1123d272fe53SJohn Baldwin else { 1124d272fe53SJohn Baldwin db_printf("OWNED"); 1125d272fe53SJohn Baldwin if (m->mtx_lock & MTX_CONTESTED) 1126d272fe53SJohn Baldwin db_printf(", CONTESTED"); 1127d272fe53SJohn Baldwin if (m->mtx_lock & MTX_RECURSED) 1128d272fe53SJohn Baldwin db_printf(", RECURSED"); 1129d272fe53SJohn Baldwin } 1130d272fe53SJohn Baldwin db_printf("}\n"); 1131c0bfd703SJohn Baldwin if (!mtx_unowned(m) && !mtx_destroyed(m)) { 1132d272fe53SJohn Baldwin td = mtx_owner(m); 1133d272fe53SJohn Baldwin db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 1134431f8906SJulian Elischer td->td_tid, td->td_proc->p_pid, td->td_name); 1135d272fe53SJohn Baldwin if (mtx_recursed(m)) 1136d272fe53SJohn Baldwin db_printf(" recursed: %d\n", m->mtx_recurse); 1137d272fe53SJohn Baldwin } 1138d272fe53SJohn Baldwin } 1139d272fe53SJohn Baldwin #endif 1140