10384fff8SJason Evans /*- 20384fff8SJason Evans * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 30384fff8SJason Evans * 40384fff8SJason Evans * Redistribution and use in source and binary forms, with or without 50384fff8SJason Evans * modification, are permitted provided that the following conditions 60384fff8SJason Evans * are met: 70384fff8SJason Evans * 1. Redistributions of source code must retain the above copyright 80384fff8SJason Evans * notice, this list of conditions and the following disclaimer. 90384fff8SJason Evans * 2. Redistributions in binary form must reproduce the above copyright 100384fff8SJason Evans * notice, this list of conditions and the following disclaimer in the 110384fff8SJason Evans * documentation and/or other materials provided with the distribution. 120384fff8SJason Evans * 3. Berkeley Software Design Inc's name may not be used to endorse or 130384fff8SJason Evans * promote products derived from this software without specific prior 140384fff8SJason Evans * written permission. 150384fff8SJason Evans * 160384fff8SJason Evans * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 170384fff8SJason Evans * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 180384fff8SJason Evans * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 190384fff8SJason Evans * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 200384fff8SJason Evans * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 210384fff8SJason Evans * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 220384fff8SJason Evans * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 230384fff8SJason Evans * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 240384fff8SJason Evans * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 250384fff8SJason Evans * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 260384fff8SJason Evans * SUCH DAMAGE. 270384fff8SJason Evans * 280384fff8SJason Evans * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2936412d79SJohn Baldwin * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 300384fff8SJason Evans */ 310384fff8SJason Evans 320384fff8SJason Evans /* 33ba48b69aSJohn Baldwin * Machine independent bits of mutex implementation. 340384fff8SJason Evans */ 350384fff8SJason Evans 36677b542eSDavid E. O'Brien #include <sys/cdefs.h> 37677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 38677b542eSDavid E. O'Brien 392498cf8cSJohn Baldwin #include "opt_adaptive_mutexes.h" 409c36c934SJohn Baldwin #include "opt_ddb.h" 417c0435b9SKip Macy #include "opt_global.h" 429923b511SScott Long #include "opt_sched.h" 43a5a96a19SJohn Baldwin 440384fff8SJason Evans #include <sys/param.h> 456c35e809SDag-Erling Smørgrav #include <sys/systm.h> 4636412d79SJohn Baldwin #include <sys/bus.h> 471126349aSPaul Saab #include <sys/conf.h> 482d50560aSMarcel Moolenaar #include <sys/kdb.h> 4936412d79SJohn Baldwin #include <sys/kernel.h> 506c35e809SDag-Erling Smørgrav #include <sys/ktr.h> 5119284646SJohn Baldwin #include <sys/lock.h> 52fb919e4dSMark Murray #include <sys/malloc.h> 5319284646SJohn Baldwin #include <sys/mutex.h> 540384fff8SJason Evans #include <sys/proc.h> 55c4f7a187SJohn Baldwin #include <sys/resourcevar.h> 56b43179fbSJeff Roberson #include <sys/sched.h> 576c35e809SDag-Erling Smørgrav #include <sys/sbuf.h> 58a5a96a19SJohn Baldwin #include <sys/sysctl.h> 59961a7b24SJohn Baldwin #include <sys/turnstile.h> 6036412d79SJohn Baldwin #include <sys/vmmeter.h> 617c0435b9SKip Macy #include <sys/lock_profile.h> 620384fff8SJason Evans 6336412d79SJohn Baldwin #include <machine/atomic.h> 6436412d79SJohn Baldwin #include <machine/bus.h> 650384fff8SJason Evans #include <machine/cpu.h> 6636412d79SJohn Baldwin 679c36c934SJohn Baldwin #include <ddb/ddb.h> 689c36c934SJohn Baldwin 698c4b6380SJohn Baldwin #include <fs/devfs/devfs_int.h> 708c4b6380SJohn Baldwin 7136412d79SJohn Baldwin #include <vm/vm.h> 7236412d79SJohn Baldwin #include <vm/vm_extern.h> 7336412d79SJohn Baldwin 74cd6e6e4eSJohn Baldwin #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 75cd6e6e4eSJohn Baldwin #define ADAPTIVE_MUTEXES 76cd6e6e4eSJohn Baldwin #endif 77cd6e6e4eSJohn Baldwin 78b9a80acaSStephan Uphoff /* 799ed346baSBosko Milekic * Internal utility macros. 800cde2e34SJason Evans */ 819ed346baSBosko Milekic #define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 820cde2e34SJason Evans 83c0bfd703SJohn Baldwin #define mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED) 84c0bfd703SJohn Baldwin 8549b94bfcSJohn Baldwin #define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) 869ed346baSBosko Milekic 87f9721b43SAttilio Rao static void assert_mtx(struct lock_object *lock, int what); 88d272fe53SJohn Baldwin #ifdef DDB 89d272fe53SJohn Baldwin static void db_show_mtx(struct lock_object *lock); 90d272fe53SJohn Baldwin #endif 916e21afd4SJohn Baldwin static void lock_mtx(struct lock_object *lock, int how); 926e21afd4SJohn Baldwin static void lock_spin(struct lock_object *lock, int how); 936e21afd4SJohn Baldwin static int unlock_mtx(struct lock_object *lock); 946e21afd4SJohn Baldwin static int unlock_spin(struct lock_object *lock); 95d272fe53SJohn Baldwin 960cde2e34SJason Evans /* 9719284646SJohn Baldwin * Lock classes for sleep and spin mutexes. 980cde2e34SJason Evans */ 9919284646SJohn Baldwin struct lock_class lock_class_mtx_sleep = { 100ae8dde30SJohn Baldwin .lc_name = "sleep mutex", 101ae8dde30SJohn Baldwin .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 102f9721b43SAttilio Rao .lc_assert = assert_mtx, 103d272fe53SJohn Baldwin #ifdef DDB 104ae8dde30SJohn Baldwin .lc_ddb_show = db_show_mtx, 105d272fe53SJohn Baldwin #endif 1066e21afd4SJohn Baldwin .lc_lock = lock_mtx, 1076e21afd4SJohn Baldwin .lc_unlock = unlock_mtx, 10819284646SJohn Baldwin }; 10919284646SJohn Baldwin struct lock_class lock_class_mtx_spin = { 110ae8dde30SJohn Baldwin .lc_name = "spin mutex", 111ae8dde30SJohn Baldwin .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 112f9721b43SAttilio Rao .lc_assert = assert_mtx, 113d272fe53SJohn Baldwin #ifdef DDB 114ae8dde30SJohn Baldwin .lc_ddb_show = db_show_mtx, 115d272fe53SJohn Baldwin #endif 1166e21afd4SJohn Baldwin .lc_lock = lock_spin, 1176e21afd4SJohn Baldwin .lc_unlock = unlock_spin, 1188484de75SJohn Baldwin }; 1198484de75SJohn Baldwin 1209ed346baSBosko Milekic /* 121c53c013bSJohn Baldwin * System-wide mutexes 122c53c013bSJohn Baldwin */ 1232502c107SJeff Roberson struct mtx blocked_lock; 124c53c013bSJohn Baldwin struct mtx Giant; 125c53c013bSJohn Baldwin 1266e21afd4SJohn Baldwin void 127f9721b43SAttilio Rao assert_mtx(struct lock_object *lock, int what) 128f9721b43SAttilio Rao { 129f9721b43SAttilio Rao 130f9721b43SAttilio Rao mtx_assert((struct mtx *)lock, what); 131f9721b43SAttilio Rao } 132f9721b43SAttilio Rao 133f9721b43SAttilio Rao void 1346e21afd4SJohn Baldwin lock_mtx(struct lock_object *lock, int how) 1356e21afd4SJohn Baldwin { 1366e21afd4SJohn Baldwin 1376e21afd4SJohn Baldwin mtx_lock((struct mtx *)lock); 1386e21afd4SJohn Baldwin } 1396e21afd4SJohn Baldwin 1406e21afd4SJohn Baldwin void 1416e21afd4SJohn Baldwin lock_spin(struct lock_object *lock, int how) 1426e21afd4SJohn Baldwin { 1436e21afd4SJohn Baldwin 1446e21afd4SJohn Baldwin panic("spin locks can only use msleep_spin"); 1456e21afd4SJohn Baldwin } 1466e21afd4SJohn Baldwin 1476e21afd4SJohn Baldwin int 1486e21afd4SJohn Baldwin unlock_mtx(struct lock_object *lock) 1496e21afd4SJohn Baldwin { 1506e21afd4SJohn Baldwin struct mtx *m; 1516e21afd4SJohn Baldwin 1526e21afd4SJohn Baldwin m = (struct mtx *)lock; 1536e21afd4SJohn Baldwin mtx_assert(m, MA_OWNED | MA_NOTRECURSED); 1546e21afd4SJohn Baldwin mtx_unlock(m); 1556e21afd4SJohn Baldwin return (0); 1566e21afd4SJohn Baldwin } 1576e21afd4SJohn Baldwin 1586e21afd4SJohn Baldwin int 1596e21afd4SJohn Baldwin unlock_spin(struct lock_object *lock) 1606e21afd4SJohn Baldwin { 1616e21afd4SJohn Baldwin 1626e21afd4SJohn Baldwin panic("spin locks can only use msleep_spin"); 1636e21afd4SJohn Baldwin } 1646e21afd4SJohn Baldwin 1650cde2e34SJason Evans /* 1666283b7d0SJohn Baldwin * Function versions of the inlined __mtx_* macros. These are used by 1676283b7d0SJohn Baldwin * modules and can also be called from assembly language if needed. 1686283b7d0SJohn Baldwin */ 1696283b7d0SJohn Baldwin void 1706283b7d0SJohn Baldwin _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 1716283b7d0SJohn Baldwin { 1726283b7d0SJohn Baldwin 173dde96c99SJohn Baldwin MPASS(curthread != NULL); 174186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 175186abbd7SJohn Baldwin ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 176aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 177aa89d8cdSJohn Baldwin ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 1780d975d63SJohn Baldwin file, line)); 179aa89d8cdSJohn Baldwin WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 1808d768e76SJohn Baldwin file, line); 1817c0435b9SKip Macy 182dde96c99SJohn Baldwin _get_sleep_lock(m, curthread, opts, file, line); 183aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 184dde96c99SJohn Baldwin line); 185aa89d8cdSJohn Baldwin WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 186764e4d54SJohn Baldwin curthread->td_locks++; 1876283b7d0SJohn Baldwin } 1886283b7d0SJohn Baldwin 1896283b7d0SJohn Baldwin void 1906283b7d0SJohn Baldwin _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 1916283b7d0SJohn Baldwin { 192dde96c99SJohn Baldwin MPASS(curthread != NULL); 193186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 194186abbd7SJohn Baldwin ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 195aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 196aa89d8cdSJohn Baldwin ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 1970d975d63SJohn Baldwin file, line)); 198764e4d54SJohn Baldwin curthread->td_locks--; 199aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 200aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 2010d975d63SJohn Baldwin line); 20221377ce0SJohn Baldwin mtx_assert(m, MA_OWNED); 203c66d7606SKip Macy 20470fe8436SKip Macy if (m->mtx_recurse == 0) 205aa89d8cdSJohn Baldwin lock_profile_release_lock(&m->lock_object); 206dde96c99SJohn Baldwin _rel_sleep_lock(m, curthread, opts, file, line); 2076283b7d0SJohn Baldwin } 2086283b7d0SJohn Baldwin 2096283b7d0SJohn Baldwin void 2106283b7d0SJohn Baldwin _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 2116283b7d0SJohn Baldwin { 2126283b7d0SJohn Baldwin 213dde96c99SJohn Baldwin MPASS(curthread != NULL); 214186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 215186abbd7SJohn Baldwin ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 216aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 2170d975d63SJohn Baldwin ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 218aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 219ad69e26bSJohn Baldwin if (mtx_owned(m)) 220ad69e26bSJohn Baldwin KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 221ad69e26bSJohn Baldwin ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n", 222ad69e26bSJohn Baldwin m->lock_object.lo_name, file, line)); 223aa89d8cdSJohn Baldwin WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 2248d768e76SJohn Baldwin file, line); 225dde96c99SJohn Baldwin _get_spin_lock(m, curthread, opts, file, line); 226aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 227dde96c99SJohn Baldwin line); 228aa89d8cdSJohn Baldwin WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 2296283b7d0SJohn Baldwin } 2306283b7d0SJohn Baldwin 2316283b7d0SJohn Baldwin void 2326283b7d0SJohn Baldwin _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 2336283b7d0SJohn Baldwin { 234c66d7606SKip Macy 235dde96c99SJohn Baldwin MPASS(curthread != NULL); 236186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 237186abbd7SJohn Baldwin ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 238aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 2390d975d63SJohn Baldwin ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 240aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 241aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 242aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 243dde96c99SJohn Baldwin line); 2440d975d63SJohn Baldwin mtx_assert(m, MA_OWNED); 245c66d7606SKip Macy 246dde96c99SJohn Baldwin _rel_spin_lock(m); 2476283b7d0SJohn Baldwin } 2486283b7d0SJohn Baldwin 2496283b7d0SJohn Baldwin /* 2509ed346baSBosko Milekic * The important part of mtx_trylock{,_flags}() 251eac09796SJohn Baldwin * Tries to acquire lock `m.' If this function is called on a mutex that 252eac09796SJohn Baldwin * is already owned, it will recursively acquire the lock. 2530cde2e34SJason Evans */ 2540cde2e34SJason Evans int 2559ed346baSBosko Milekic _mtx_trylock(struct mtx *m, int opts, const char *file, int line) 2560cde2e34SJason Evans { 257fe68a916SKip Macy int rval, contested = 0; 2587c0435b9SKip Macy uint64_t waittime = 0; 2590cde2e34SJason Evans 260b40ce416SJulian Elischer MPASS(curthread != NULL); 261186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 262186abbd7SJohn Baldwin ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 263aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 264aa89d8cdSJohn Baldwin ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 26583cece6fSJohn Baldwin file, line)); 2669ed346baSBosko Milekic 267aa89d8cdSJohn Baldwin if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) { 268eac09796SJohn Baldwin m->mtx_recurse++; 269eac09796SJohn Baldwin atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 270eac09796SJohn Baldwin rval = 1; 271eac09796SJohn Baldwin } else 272122eceefSJohn Baldwin rval = _obtain_lock(m, (uintptr_t)curthread); 2739ed346baSBosko Milekic 274aa89d8cdSJohn Baldwin LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line); 275764e4d54SJohn Baldwin if (rval) { 276aa89d8cdSJohn Baldwin WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 2772d96f0b1SJohn Baldwin file, line); 278764e4d54SJohn Baldwin curthread->td_locks++; 279fe68a916SKip Macy if (m->mtx_recurse == 0) 280aa89d8cdSJohn Baldwin lock_profile_obtain_lock_success(&m->lock_object, contested, 281fe68a916SKip Macy waittime, file, line); 2827c0435b9SKip Macy 283764e4d54SJohn Baldwin } 2849ed346baSBosko Milekic 28519284646SJohn Baldwin return (rval); 2860cde2e34SJason Evans } 2870cde2e34SJason Evans 2880cde2e34SJason Evans /* 2899ed346baSBosko Milekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 2909ed346baSBosko Milekic * 2919ed346baSBosko Milekic * We call this if the lock is either contested (i.e. we need to go to 2929ed346baSBosko Milekic * sleep waiting for it), or if we need to recurse on it. 2930cde2e34SJason Evans */ 2940cde2e34SJason Evans void 295122eceefSJohn Baldwin _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, 296bdcfcf5bSJohn Baldwin int line) 29736412d79SJohn Baldwin { 2982502c107SJeff Roberson struct turnstile *ts; 299cd6e6e4eSJohn Baldwin #ifdef ADAPTIVE_MUTEXES 30076447e56SJohn Baldwin volatile struct thread *owner; 3012498cf8cSJohn Baldwin #endif 30202bd1bcdSIan Dowse #ifdef KTR 30302bd1bcdSIan Dowse int cont_logged = 0; 30402bd1bcdSIan Dowse #endif 30570fe8436SKip Macy int contested = 0; 30670fe8436SKip Macy uint64_t waittime = 0; 3077c0435b9SKip Macy uintptr_t v; 30836412d79SJohn Baldwin 3095fa8dd90SJohn Baldwin if (mtx_owned(m)) { 310aa89d8cdSJohn Baldwin KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 311eac09796SJohn Baldwin ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 312aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 31336412d79SJohn Baldwin m->mtx_recurse++; 31408812b39SBosko Milekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 315aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 3165746a1d8SBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 31736412d79SJohn Baldwin return; 31836412d79SJohn Baldwin } 3199ed346baSBosko Milekic 32070fe8436SKip Macy lock_profile_obtain_lock_failed(&m->lock_object, 32170fe8436SKip Macy &contested, &waittime); 322aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 32315ec816aSJohn Baldwin CTR4(KTR_LOCK, 32415ec816aSJohn Baldwin "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 325aa89d8cdSJohn Baldwin m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 3261bd0eefbSJohn Baldwin 327122eceefSJohn Baldwin while (!_obtain_lock(m, tid)) { 32849aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES 32949aead8aSAttilio Rao /* 33049aead8aSAttilio Rao * If the owner is running on another CPU, spin until the 33149aead8aSAttilio Rao * owner stops running or the state of the lock changes. 33249aead8aSAttilio Rao */ 33349aead8aSAttilio Rao v = m->mtx_lock; 33449aead8aSAttilio Rao if (v != MTX_UNOWNED) { 33549aead8aSAttilio Rao owner = (struct thread *)(v & ~MTX_FLAGMASK); 33649aead8aSAttilio Rao if (TD_IS_RUNNING(owner)) { 33749aead8aSAttilio Rao if (LOCK_LOG_TEST(&m->lock_object, 0)) 33849aead8aSAttilio Rao CTR3(KTR_LOCK, 33949aead8aSAttilio Rao "%s: spinning on %p held by %p", 34049aead8aSAttilio Rao __func__, m, owner); 34149aead8aSAttilio Rao while (mtx_owner(m) == owner && 34249aead8aSAttilio Rao TD_IS_RUNNING(owner)) 34349aead8aSAttilio Rao cpu_spinwait(); 34449aead8aSAttilio Rao continue; 34549aead8aSAttilio Rao } 34649aead8aSAttilio Rao } 34749aead8aSAttilio Rao #endif 34849aead8aSAttilio Rao 3492502c107SJeff Roberson ts = turnstile_trywait(&m->lock_object); 3505fa8dd90SJohn Baldwin v = m->mtx_lock; 3515fa8dd90SJohn Baldwin 35236412d79SJohn Baldwin /* 3539ed346baSBosko Milekic * Check if the lock has been released while spinning for 354961a7b24SJohn Baldwin * the turnstile chain lock. 35536412d79SJohn Baldwin */ 3565fa8dd90SJohn Baldwin if (v == MTX_UNOWNED) { 3572502c107SJeff Roberson turnstile_cancel(ts); 3589f1b87f1SMaxime Henrion cpu_spinwait(); 35936412d79SJohn Baldwin continue; 36036412d79SJohn Baldwin } 3619ed346baSBosko Milekic 362535eb309SJohn Baldwin MPASS(v != MTX_CONTESTED); 3639ed346baSBosko Milekic 36449aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES 36549aead8aSAttilio Rao /* 36649aead8aSAttilio Rao * If the current owner of the lock is executing on another 36749aead8aSAttilio Rao * CPU quit the hard path and try to spin. 36849aead8aSAttilio Rao */ 36949aead8aSAttilio Rao owner = (struct thread *)(v & ~MTX_FLAGMASK); 37049aead8aSAttilio Rao if (TD_IS_RUNNING(owner)) { 37149aead8aSAttilio Rao turnstile_cancel(ts); 37249aead8aSAttilio Rao cpu_spinwait(); 37349aead8aSAttilio Rao continue; 37449aead8aSAttilio Rao } 37549aead8aSAttilio Rao #endif 37649aead8aSAttilio Rao 37736412d79SJohn Baldwin /* 3789ed346baSBosko Milekic * If the mutex isn't already contested and a failure occurs 3799ed346baSBosko Milekic * setting the contested bit, the mutex was either released 3809ed346baSBosko Milekic * or the state of the MTX_RECURSED bit changed. 38136412d79SJohn Baldwin */ 38236412d79SJohn Baldwin if ((v & MTX_CONTESTED) == 0 && 383122eceefSJohn Baldwin !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 3842502c107SJeff Roberson turnstile_cancel(ts); 3859f1b87f1SMaxime Henrion cpu_spinwait(); 38636412d79SJohn Baldwin continue; 38736412d79SJohn Baldwin } 38836412d79SJohn Baldwin 3899ed346baSBosko Milekic /* 3907feefcd6SJohn Baldwin * We definitely must sleep for this lock. 3919ed346baSBosko Milekic */ 39236412d79SJohn Baldwin mtx_assert(m, MA_NOTOWNED); 39336412d79SJohn Baldwin 39402bd1bcdSIan Dowse #ifdef KTR 39502bd1bcdSIan Dowse if (!cont_logged) { 39602bd1bcdSIan Dowse CTR6(KTR_CONTENTION, 39702bd1bcdSIan Dowse "contention: %p at %s:%d wants %s, taken by %s:%d", 398aa89d8cdSJohn Baldwin (void *)tid, file, line, m->lock_object.lo_name, 399aa89d8cdSJohn Baldwin WITNESS_FILE(&m->lock_object), 400aa89d8cdSJohn Baldwin WITNESS_LINE(&m->lock_object)); 40102bd1bcdSIan Dowse cont_logged = 1; 40202bd1bcdSIan Dowse } 40302bd1bcdSIan Dowse #endif 40436412d79SJohn Baldwin 4059ed346baSBosko Milekic /* 406961a7b24SJohn Baldwin * Block on the turnstile. 4079ed346baSBosko Milekic */ 4082502c107SJeff Roberson turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 40936412d79SJohn Baldwin } 41002bd1bcdSIan Dowse #ifdef KTR 41102bd1bcdSIan Dowse if (cont_logged) { 41202bd1bcdSIan Dowse CTR4(KTR_CONTENTION, 41302bd1bcdSIan Dowse "contention end: %s acquired by %p at %s:%d", 414aa89d8cdSJohn Baldwin m->lock_object.lo_name, (void *)tid, file, line); 41502bd1bcdSIan Dowse } 41602bd1bcdSIan Dowse #endif 41770fe8436SKip Macy lock_profile_obtain_lock_success(&m->lock_object, contested, 418eea4f254SJeff Roberson waittime, file, line); 4199ed346baSBosko Milekic } 4209ed346baSBosko Milekic 4212502c107SJeff Roberson static void 4222502c107SJeff Roberson _mtx_lock_spin_failed(struct mtx *m) 4232502c107SJeff Roberson { 4242502c107SJeff Roberson struct thread *td; 4252502c107SJeff Roberson 4262502c107SJeff Roberson td = mtx_owner(m); 4272502c107SJeff Roberson 4282502c107SJeff Roberson /* If the mutex is unlocked, try again. */ 4292502c107SJeff Roberson if (td == NULL) 4302502c107SJeff Roberson return; 431b95b98b0SKonstantin Belousov 4322502c107SJeff Roberson printf( "spin lock %p (%s) held by %p (tid %d) too long\n", 4332502c107SJeff Roberson m, m->lock_object.lo_name, td, td->td_tid); 4342502c107SJeff Roberson #ifdef WITNESS 4352502c107SJeff Roberson witness_display_spinlock(&m->lock_object, td); 4362502c107SJeff Roberson #endif 4372502c107SJeff Roberson panic("spin lock held too long"); 4382502c107SJeff Roberson } 4392502c107SJeff Roberson 440b95b98b0SKonstantin Belousov #ifdef SMP 4419ed346baSBosko Milekic /* 4429ed346baSBosko Milekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 4439ed346baSBosko Milekic * 4449ed346baSBosko Milekic * This is only called if we need to actually spin for the lock. Recursion 4459ed346baSBosko Milekic * is handled inline. 4469ed346baSBosko Milekic */ 4479ed346baSBosko Milekic void 448122eceefSJohn Baldwin _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, 449bdcfcf5bSJohn Baldwin int line) 45036412d79SJohn Baldwin { 45170fe8436SKip Macy int i = 0, contested = 0; 45270fe8436SKip Macy uint64_t waittime = 0; 45336412d79SJohn Baldwin 454aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 4555746a1d8SBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 4569ed346baSBosko Milekic 45770fe8436SKip Macy lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 458f781b5a4SJohn Baldwin while (!_obtain_lock(m, tid)) { 4599ed346baSBosko Milekic 4607141f2adSJohn Baldwin /* Give interrupts a chance while we spin. */ 461c6a37e84SJohn Baldwin spinlock_exit(); 46236412d79SJohn Baldwin while (m->mtx_lock != MTX_UNOWNED) { 463703fc290SJohn Baldwin if (i++ < 10000000) { 4649f1b87f1SMaxime Henrion cpu_spinwait(); 46536412d79SJohn Baldwin continue; 466703fc290SJohn Baldwin } 4670fa2168bSJohn Baldwin if (i < 60000000 || kdb_active || panicstr != NULL) 46836412d79SJohn Baldwin DELAY(1); 4692502c107SJeff Roberson else 4702502c107SJeff Roberson _mtx_lock_spin_failed(m); 4719f1b87f1SMaxime Henrion cpu_spinwait(); 47236412d79SJohn Baldwin } 473c6a37e84SJohn Baldwin spinlock_enter(); 47436412d79SJohn Baldwin } 47536412d79SJohn Baldwin 476aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 4779ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 4789ed346baSBosko Milekic 47970fe8436SKip Macy lock_profile_obtain_lock_success(&m->lock_object, contested, 48070fe8436SKip Macy waittime, (file), (line)); 48136412d79SJohn Baldwin } 48233fb8a38SJohn Baldwin #endif /* SMP */ 48336412d79SJohn Baldwin 4842502c107SJeff Roberson void 4852502c107SJeff Roberson _thread_lock_flags(struct thread *td, int opts, const char *file, int line) 4862502c107SJeff Roberson { 4872502c107SJeff Roberson struct mtx *m; 4882502c107SJeff Roberson uintptr_t tid; 489773890b9SJeff Roberson int i, contested; 490773890b9SJeff Roberson uint64_t waittime; 4912502c107SJeff Roberson 492773890b9SJeff Roberson contested = i = 0; 493773890b9SJeff Roberson waittime = 0; 4942502c107SJeff Roberson tid = (uintptr_t)curthread; 4952502c107SJeff Roberson for (;;) { 4962502c107SJeff Roberson retry: 4972502c107SJeff Roberson spinlock_enter(); 498710eacdcSJeff Roberson m = td->td_lock; 49913c85a48SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 50013c85a48SJohn Baldwin ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 50113c85a48SJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 50213c85a48SJohn Baldwin ("thread_lock() of sleep mutex %s @ %s:%d", 50313c85a48SJohn Baldwin m->lock_object.lo_name, file, line)); 504ad69e26bSJohn Baldwin if (mtx_owned(m)) 505ad69e26bSJohn Baldwin KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 506ad69e26bSJohn Baldwin ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", 507ad69e26bSJohn Baldwin m->lock_object.lo_name, file, line)); 5082502c107SJeff Roberson WITNESS_CHECKORDER(&m->lock_object, 5092502c107SJeff Roberson opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line); 5102502c107SJeff Roberson while (!_obtain_lock(m, tid)) { 5112502c107SJeff Roberson if (m->mtx_lock == tid) { 5122502c107SJeff Roberson m->mtx_recurse++; 5132502c107SJeff Roberson break; 5142502c107SJeff Roberson } 515eea4f254SJeff Roberson lock_profile_obtain_lock_failed(&m->lock_object, 516eea4f254SJeff Roberson &contested, &waittime); 5172502c107SJeff Roberson /* Give interrupts a chance while we spin. */ 5182502c107SJeff Roberson spinlock_exit(); 5192502c107SJeff Roberson while (m->mtx_lock != MTX_UNOWNED) { 5202502c107SJeff Roberson if (i++ < 10000000) 5212502c107SJeff Roberson cpu_spinwait(); 5222502c107SJeff Roberson else if (i < 60000000 || 5232502c107SJeff Roberson kdb_active || panicstr != NULL) 5242502c107SJeff Roberson DELAY(1); 5252502c107SJeff Roberson else 5262502c107SJeff Roberson _mtx_lock_spin_failed(m); 5272502c107SJeff Roberson cpu_spinwait(); 5282502c107SJeff Roberson if (m != td->td_lock) 5292502c107SJeff Roberson goto retry; 5302502c107SJeff Roberson } 5312502c107SJeff Roberson spinlock_enter(); 5322502c107SJeff Roberson } 5332502c107SJeff Roberson if (m == td->td_lock) 5342502c107SJeff Roberson break; 5352502c107SJeff Roberson _rel_spin_lock(m); /* does spinlock_exit() */ 5362502c107SJeff Roberson } 537eea4f254SJeff Roberson if (m->mtx_recurse == 0) 538773890b9SJeff Roberson lock_profile_obtain_lock_success(&m->lock_object, contested, 539773890b9SJeff Roberson waittime, (file), (line)); 54013c85a48SJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 54113c85a48SJohn Baldwin line); 5422502c107SJeff Roberson WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 5432502c107SJeff Roberson } 5442502c107SJeff Roberson 5452502c107SJeff Roberson struct mtx * 5462502c107SJeff Roberson thread_lock_block(struct thread *td) 5472502c107SJeff Roberson { 5482502c107SJeff Roberson struct mtx *lock; 5492502c107SJeff Roberson 5502502c107SJeff Roberson spinlock_enter(); 5512502c107SJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 552710eacdcSJeff Roberson lock = td->td_lock; 5532502c107SJeff Roberson td->td_lock = &blocked_lock; 5542502c107SJeff Roberson mtx_unlock_spin(lock); 5552502c107SJeff Roberson 5562502c107SJeff Roberson return (lock); 5572502c107SJeff Roberson } 5582502c107SJeff Roberson 5592502c107SJeff Roberson void 5602502c107SJeff Roberson thread_lock_unblock(struct thread *td, struct mtx *new) 5612502c107SJeff Roberson { 5622502c107SJeff Roberson mtx_assert(new, MA_OWNED); 5632502c107SJeff Roberson MPASS(td->td_lock == &blocked_lock); 56465d32cd8SMatt Jacob atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new); 5652502c107SJeff Roberson spinlock_exit(); 5662502c107SJeff Roberson } 5672502c107SJeff Roberson 5682502c107SJeff Roberson void 5692502c107SJeff Roberson thread_lock_set(struct thread *td, struct mtx *new) 5702502c107SJeff Roberson { 5712502c107SJeff Roberson struct mtx *lock; 5722502c107SJeff Roberson 5732502c107SJeff Roberson mtx_assert(new, MA_OWNED); 5742502c107SJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 575710eacdcSJeff Roberson lock = td->td_lock; 5762502c107SJeff Roberson td->td_lock = new; 5772502c107SJeff Roberson mtx_unlock_spin(lock); 5782502c107SJeff Roberson } 5792502c107SJeff Roberson 5809ed346baSBosko Milekic /* 5819ed346baSBosko Milekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 5829ed346baSBosko Milekic * 5839ed346baSBosko Milekic * We are only called here if the lock is recursed or contested (i.e. we 5849ed346baSBosko Milekic * need to wake up a blocked thread). 5859ed346baSBosko Milekic */ 58636412d79SJohn Baldwin void 5879ed346baSBosko Milekic _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 58836412d79SJohn Baldwin { 589961a7b24SJohn Baldwin struct turnstile *ts; 5909ed346baSBosko Milekic 59108812b39SBosko Milekic if (mtx_recursed(m)) { 59236412d79SJohn Baldwin if (--(m->mtx_recurse) == 0) 59308812b39SBosko Milekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 594aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 5959ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 59636412d79SJohn Baldwin return; 59736412d79SJohn Baldwin } 5989ed346baSBosko Milekic 5992502c107SJeff Roberson /* 6002502c107SJeff Roberson * We have to lock the chain before the turnstile so this turnstile 6012502c107SJeff Roberson * can be removed from the hash list if it is empty. 6022502c107SJeff Roberson */ 6032502c107SJeff Roberson turnstile_chain_lock(&m->lock_object); 604aa89d8cdSJohn Baldwin ts = turnstile_lookup(&m->lock_object); 605aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 6069ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 6079ed346baSBosko Milekic 608961a7b24SJohn Baldwin MPASS(ts != NULL); 6097aa4f685SJohn Baldwin turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 610535eb309SJohn Baldwin _release_lock_quick(m); 6112502c107SJeff Roberson /* 6122502c107SJeff Roberson * This turnstile is now no longer associated with the mutex. We can 6132502c107SJeff Roberson * unlock the chain lock so a new turnstile may take it's place. 6142502c107SJeff Roberson */ 6157aa4f685SJohn Baldwin turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 6162502c107SJeff Roberson turnstile_chain_unlock(&m->lock_object); 6179ed346baSBosko Milekic } 6189ed346baSBosko Milekic 6199ed346baSBosko Milekic /* 6209ed346baSBosko Milekic * All the unlocking of MTX_SPIN locks is done inline. 6219ed346baSBosko Milekic * See the _rel_spin_lock() macro for the details. 6229ed346baSBosko Milekic */ 6239ed346baSBosko Milekic 6249ed346baSBosko Milekic /* 62515ec816aSJohn Baldwin * The backing function for the INVARIANTS-enabled mtx_assert() 6269ed346baSBosko Milekic */ 6271103f3b0SJohn Baldwin #ifdef INVARIANT_SUPPORT 6280cde2e34SJason Evans void 62956771ca7SJason Evans _mtx_assert(struct mtx *m, int what, const char *file, int line) 6300cde2e34SJason Evans { 6315cb0fbe4SJohn Baldwin 6321126349aSPaul Saab if (panicstr != NULL || dumping) 6335cb0fbe4SJohn Baldwin return; 634a10f4966SJake Burkholder switch (what) { 6350cde2e34SJason Evans case MA_OWNED: 6360cde2e34SJason Evans case MA_OWNED | MA_RECURSED: 6370cde2e34SJason Evans case MA_OWNED | MA_NOTRECURSED: 638a10f4966SJake Burkholder if (!mtx_owned(m)) 6390cde2e34SJason Evans panic("mutex %s not owned at %s:%d", 640aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 641a10f4966SJake Burkholder if (mtx_recursed(m)) { 642a10f4966SJake Burkholder if ((what & MA_NOTRECURSED) != 0) 6430cde2e34SJason Evans panic("mutex %s recursed at %s:%d", 644aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 645a10f4966SJake Burkholder } else if ((what & MA_RECURSED) != 0) { 6460cde2e34SJason Evans panic("mutex %s unrecursed at %s:%d", 647aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 6480cde2e34SJason Evans } 6490cde2e34SJason Evans break; 6500cde2e34SJason Evans case MA_NOTOWNED: 651a10f4966SJake Burkholder if (mtx_owned(m)) 6520cde2e34SJason Evans panic("mutex %s owned at %s:%d", 653aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 6540cde2e34SJason Evans break; 6550cde2e34SJason Evans default: 65656771ca7SJason Evans panic("unknown mtx_assert at %s:%d", file, line); 6570cde2e34SJason Evans } 6580cde2e34SJason Evans } 6590cde2e34SJason Evans #endif 6600cde2e34SJason Evans 6619ed346baSBosko Milekic /* 6629ed346baSBosko Milekic * The MUTEX_DEBUG-enabled mtx_validate() 66319284646SJohn Baldwin * 66419284646SJohn Baldwin * Most of these checks have been moved off into the LO_INITIALIZED flag 66519284646SJohn Baldwin * maintained by the witness code. 6669ed346baSBosko Milekic */ 66736412d79SJohn Baldwin #ifdef MUTEX_DEBUG 66836412d79SJohn Baldwin 6694d77a549SAlfred Perlstein void mtx_validate(struct mtx *); 67036412d79SJohn Baldwin 67119284646SJohn Baldwin void 67219284646SJohn Baldwin mtx_validate(struct mtx *m) 67336412d79SJohn Baldwin { 67436412d79SJohn Baldwin 67536412d79SJohn Baldwin /* 676fa669ab7SPoul-Henning Kamp * XXX: When kernacc() does not require Giant we can reenable this check 677fa669ab7SPoul-Henning Kamp */ 678fa669ab7SPoul-Henning Kamp #ifdef notyet 679fa669ab7SPoul-Henning Kamp /* 68076dcbd6fSBosko Milekic * Can't call kernacc() from early init386(), especially when 68176dcbd6fSBosko Milekic * initializing Giant mutex, because some stuff in kernacc() 68276dcbd6fSBosko Milekic * requires Giant itself. 68376dcbd6fSBosko Milekic */ 684ab07087eSBosko Milekic if (!cold) 685ab07087eSBosko Milekic if (!kernacc((caddr_t)m, sizeof(m), 686ab07087eSBosko Milekic VM_PROT_READ | VM_PROT_WRITE)) 68719284646SJohn Baldwin panic("Can't read and write to mutex %p", m); 68836412d79SJohn Baldwin #endif 68936412d79SJohn Baldwin } 69036412d79SJohn Baldwin #endif 69136412d79SJohn Baldwin 6929ed346baSBosko Milekic /* 693c27b5699SAndrew R. Reiter * General init routine used by the MTX_SYSINIT() macro. 694c27b5699SAndrew R. Reiter */ 695c27b5699SAndrew R. Reiter void 696c27b5699SAndrew R. Reiter mtx_sysinit(void *arg) 697c27b5699SAndrew R. Reiter { 698c27b5699SAndrew R. Reiter struct mtx_args *margs = arg; 699c27b5699SAndrew R. Reiter 7000c88508aSJohn Baldwin mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 701c27b5699SAndrew R. Reiter } 702c27b5699SAndrew R. Reiter 703c27b5699SAndrew R. Reiter /* 7049ed346baSBosko Milekic * Mutex initialization routine; initialize lock `m' of type contained in 7050c88508aSJohn Baldwin * `opts' with options contained in `opts' and name `name.' The optional 7060c88508aSJohn Baldwin * lock type `type' is used as a general lock category name for use with 7070c88508aSJohn Baldwin * witness. 7089ed346baSBosko Milekic */ 70936412d79SJohn Baldwin void 7100c88508aSJohn Baldwin mtx_init(struct mtx *m, const char *name, const char *type, int opts) 71136412d79SJohn Baldwin { 71283a81bcbSJohn Baldwin struct lock_class *class; 71383a81bcbSJohn Baldwin int flags; 7149ed346baSBosko Milekic 71519284646SJohn Baldwin MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 7167c0435b9SKip Macy MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0); 7179ed346baSBosko Milekic 71836412d79SJohn Baldwin #ifdef MUTEX_DEBUG 7199ed346baSBosko Milekic /* Diagnostic and error correction */ 72019284646SJohn Baldwin mtx_validate(m); 7216936206eSJohn Baldwin #endif 72236412d79SJohn Baldwin 72383a81bcbSJohn Baldwin /* Determine lock class and lock flags. */ 72419284646SJohn Baldwin if (opts & MTX_SPIN) 72583a81bcbSJohn Baldwin class = &lock_class_mtx_spin; 72619284646SJohn Baldwin else 72783a81bcbSJohn Baldwin class = &lock_class_mtx_sleep; 72883a81bcbSJohn Baldwin flags = 0; 72919284646SJohn Baldwin if (opts & MTX_QUIET) 73083a81bcbSJohn Baldwin flags |= LO_QUIET; 73119284646SJohn Baldwin if (opts & MTX_RECURSE) 73283a81bcbSJohn Baldwin flags |= LO_RECURSABLE; 73319284646SJohn Baldwin if ((opts & MTX_NOWITNESS) == 0) 73483a81bcbSJohn Baldwin flags |= LO_WITNESS; 735f22a4b62SJeff Roberson if (opts & MTX_DUPOK) 73683a81bcbSJohn Baldwin flags |= LO_DUPOK; 7377c0435b9SKip Macy if (opts & MTX_NOPROFILE) 7387c0435b9SKip Macy flags |= LO_NOPROFILE; 73919284646SJohn Baldwin 74083a81bcbSJohn Baldwin /* Initialize mutex. */ 74119284646SJohn Baldwin m->mtx_lock = MTX_UNOWNED; 74283a81bcbSJohn Baldwin m->mtx_recurse = 0; 7439ed346baSBosko Milekic 744aa89d8cdSJohn Baldwin lock_init(&m->lock_object, class, name, type, flags); 74536412d79SJohn Baldwin } 74636412d79SJohn Baldwin 7479ed346baSBosko Milekic /* 74819284646SJohn Baldwin * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 74919284646SJohn Baldwin * passed in as a flag here because if the corresponding mtx_init() was 75019284646SJohn Baldwin * called with MTX_QUIET set, then it will already be set in the mutex's 75119284646SJohn Baldwin * flags. 7529ed346baSBosko Milekic */ 75336412d79SJohn Baldwin void 75436412d79SJohn Baldwin mtx_destroy(struct mtx *m) 75536412d79SJohn Baldwin { 75636412d79SJohn Baldwin 75719284646SJohn Baldwin if (!mtx_owned(m)) 75819284646SJohn Baldwin MPASS(mtx_unowned(m)); 75919284646SJohn Baldwin else { 76008812b39SBosko Milekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 7619ed346baSBosko Milekic 762861a2308SScott Long /* Perform the non-mtx related part of mtx_unlock_spin(). */ 763aa89d8cdSJohn Baldwin if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin) 764861a2308SScott Long spinlock_exit(); 765764e4d54SJohn Baldwin else 766764e4d54SJohn Baldwin curthread->td_locks--; 767861a2308SScott Long 76819284646SJohn Baldwin /* Tell witness this isn't locked to make it happy. */ 769aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__, 770c86b6ff5SJohn Baldwin __LINE__); 77136412d79SJohn Baldwin } 7720384fff8SJason Evans 773186abbd7SJohn Baldwin m->mtx_lock = MTX_DESTROYED; 774aa89d8cdSJohn Baldwin lock_destroy(&m->lock_object); 7750384fff8SJason Evans } 776d23f5958SMatthew Dillon 777d23f5958SMatthew Dillon /* 778c53c013bSJohn Baldwin * Intialize the mutex code and system mutexes. This is called from the MD 779c53c013bSJohn Baldwin * startup code prior to mi_startup(). The per-CPU data space needs to be 780c53c013bSJohn Baldwin * setup before this is called. 781c53c013bSJohn Baldwin */ 782c53c013bSJohn Baldwin void 783c53c013bSJohn Baldwin mutex_init(void) 784c53c013bSJohn Baldwin { 785c53c013bSJohn Baldwin 786961a7b24SJohn Baldwin /* Setup turnstiles so that sleep mutexes work. */ 787961a7b24SJohn Baldwin init_turnstiles(); 788961a7b24SJohn Baldwin 789c53c013bSJohn Baldwin /* 790c53c013bSJohn Baldwin * Initialize mutexes. 791c53c013bSJohn Baldwin */ 7920c88508aSJohn Baldwin mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 7932502c107SJeff Roberson mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN); 7942502c107SJeff Roberson blocked_lock.mtx_lock = 0xdeadc0de; /* Always blocked. */ 7950c88508aSJohn Baldwin mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 7962502c107SJeff Roberson mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE); 7978c4b6380SJohn Baldwin mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 798c53c013bSJohn Baldwin mtx_lock(&Giant); 799c53c013bSJohn Baldwin } 800d272fe53SJohn Baldwin 801d272fe53SJohn Baldwin #ifdef DDB 802d272fe53SJohn Baldwin void 803d272fe53SJohn Baldwin db_show_mtx(struct lock_object *lock) 804d272fe53SJohn Baldwin { 805d272fe53SJohn Baldwin struct thread *td; 806d272fe53SJohn Baldwin struct mtx *m; 807d272fe53SJohn Baldwin 808d272fe53SJohn Baldwin m = (struct mtx *)lock; 809d272fe53SJohn Baldwin 810d272fe53SJohn Baldwin db_printf(" flags: {"); 81183a81bcbSJohn Baldwin if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 812d272fe53SJohn Baldwin db_printf("SPIN"); 813d272fe53SJohn Baldwin else 814d272fe53SJohn Baldwin db_printf("DEF"); 815aa89d8cdSJohn Baldwin if (m->lock_object.lo_flags & LO_RECURSABLE) 816d272fe53SJohn Baldwin db_printf(", RECURSE"); 817aa89d8cdSJohn Baldwin if (m->lock_object.lo_flags & LO_DUPOK) 818d272fe53SJohn Baldwin db_printf(", DUPOK"); 819d272fe53SJohn Baldwin db_printf("}\n"); 820d272fe53SJohn Baldwin db_printf(" state: {"); 821d272fe53SJohn Baldwin if (mtx_unowned(m)) 822d272fe53SJohn Baldwin db_printf("UNOWNED"); 823c0bfd703SJohn Baldwin else if (mtx_destroyed(m)) 824c0bfd703SJohn Baldwin db_printf("DESTROYED"); 825d272fe53SJohn Baldwin else { 826d272fe53SJohn Baldwin db_printf("OWNED"); 827d272fe53SJohn Baldwin if (m->mtx_lock & MTX_CONTESTED) 828d272fe53SJohn Baldwin db_printf(", CONTESTED"); 829d272fe53SJohn Baldwin if (m->mtx_lock & MTX_RECURSED) 830d272fe53SJohn Baldwin db_printf(", RECURSED"); 831d272fe53SJohn Baldwin } 832d272fe53SJohn Baldwin db_printf("}\n"); 833c0bfd703SJohn Baldwin if (!mtx_unowned(m) && !mtx_destroyed(m)) { 834d272fe53SJohn Baldwin td = mtx_owner(m); 835d272fe53SJohn Baldwin db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 836431f8906SJulian Elischer td->td_tid, td->td_proc->p_pid, td->td_name); 837d272fe53SJohn Baldwin if (mtx_recursed(m)) 838d272fe53SJohn Baldwin db_printf(" recursed: %d\n", m->mtx_recurse); 839d272fe53SJohn Baldwin } 840d272fe53SJohn Baldwin } 841d272fe53SJohn Baldwin #endif 842