10384fff8SJason Evans /*- 20384fff8SJason Evans * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 30384fff8SJason Evans * 40384fff8SJason Evans * Redistribution and use in source and binary forms, with or without 50384fff8SJason Evans * modification, are permitted provided that the following conditions 60384fff8SJason Evans * are met: 70384fff8SJason Evans * 1. Redistributions of source code must retain the above copyright 80384fff8SJason Evans * notice, this list of conditions and the following disclaimer. 90384fff8SJason Evans * 2. Redistributions in binary form must reproduce the above copyright 100384fff8SJason Evans * notice, this list of conditions and the following disclaimer in the 110384fff8SJason Evans * documentation and/or other materials provided with the distribution. 120384fff8SJason Evans * 3. Berkeley Software Design Inc's name may not be used to endorse or 130384fff8SJason Evans * promote products derived from this software without specific prior 140384fff8SJason Evans * written permission. 150384fff8SJason Evans * 160384fff8SJason Evans * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 170384fff8SJason Evans * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 180384fff8SJason Evans * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 190384fff8SJason Evans * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 200384fff8SJason Evans * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 210384fff8SJason Evans * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 220384fff8SJason Evans * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 230384fff8SJason Evans * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 240384fff8SJason Evans * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 250384fff8SJason Evans * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 260384fff8SJason Evans * SUCH DAMAGE. 270384fff8SJason Evans * 280384fff8SJason Evans * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2936412d79SJohn Baldwin * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 300384fff8SJason Evans */ 310384fff8SJason Evans 320384fff8SJason Evans /* 33ba48b69aSJohn Baldwin * Machine independent bits of mutex implementation. 340384fff8SJason Evans */ 350384fff8SJason Evans 36677b542eSDavid E. O'Brien #include <sys/cdefs.h> 37677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 38677b542eSDavid E. O'Brien 392498cf8cSJohn Baldwin #include "opt_adaptive_mutexes.h" 409c36c934SJohn Baldwin #include "opt_ddb.h" 41f5f9340bSFabien Thomas #include "opt_hwpmc_hooks.h" 429923b511SScott Long #include "opt_sched.h" 43a5a96a19SJohn Baldwin 440384fff8SJason Evans #include <sys/param.h> 456c35e809SDag-Erling Smørgrav #include <sys/systm.h> 4636412d79SJohn Baldwin #include <sys/bus.h> 471126349aSPaul Saab #include <sys/conf.h> 482d50560aSMarcel Moolenaar #include <sys/kdb.h> 4936412d79SJohn Baldwin #include <sys/kernel.h> 506c35e809SDag-Erling Smørgrav #include <sys/ktr.h> 5119284646SJohn Baldwin #include <sys/lock.h> 52fb919e4dSMark Murray #include <sys/malloc.h> 5319284646SJohn Baldwin #include <sys/mutex.h> 540384fff8SJason Evans #include <sys/proc.h> 55c4f7a187SJohn Baldwin #include <sys/resourcevar.h> 56b43179fbSJeff Roberson #include <sys/sched.h> 576c35e809SDag-Erling Smørgrav #include <sys/sbuf.h> 58a5a96a19SJohn Baldwin #include <sys/sysctl.h> 59961a7b24SJohn Baldwin #include <sys/turnstile.h> 6036412d79SJohn Baldwin #include <sys/vmmeter.h> 617c0435b9SKip Macy #include <sys/lock_profile.h> 620384fff8SJason Evans 6336412d79SJohn Baldwin #include <machine/atomic.h> 6436412d79SJohn Baldwin #include <machine/bus.h> 650384fff8SJason Evans #include <machine/cpu.h> 6636412d79SJohn Baldwin 679c36c934SJohn Baldwin #include <ddb/ddb.h> 689c36c934SJohn Baldwin 698c4b6380SJohn Baldwin #include <fs/devfs/devfs_int.h> 708c4b6380SJohn Baldwin 7136412d79SJohn Baldwin #include <vm/vm.h> 7236412d79SJohn Baldwin #include <vm/vm_extern.h> 7336412d79SJohn Baldwin 74cd6e6e4eSJohn Baldwin #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 75cd6e6e4eSJohn Baldwin #define ADAPTIVE_MUTEXES 76cd6e6e4eSJohn Baldwin #endif 77cd6e6e4eSJohn Baldwin 78f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 79f5f9340bSFabien Thomas #include <sys/pmckern.h> 80f5f9340bSFabien Thomas PMC_SOFT_DEFINE( , , lock, failed); 81f5f9340bSFabien Thomas #endif 82f5f9340bSFabien Thomas 83b9a80acaSStephan Uphoff /* 847f44c618SAttilio Rao * Return the mutex address when the lock cookie address is provided. 857f44c618SAttilio Rao * This functionality assumes that struct mtx* have a member named mtx_lock. 867f44c618SAttilio Rao */ 877f44c618SAttilio Rao #define mtxlock2mtx(c) (__containerof(c, struct mtx, mtx_lock)) 887f44c618SAttilio Rao 897f44c618SAttilio Rao /* 909ed346baSBosko Milekic * Internal utility macros. 910cde2e34SJason Evans */ 929ed346baSBosko Milekic #define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 930cde2e34SJason Evans 94c0bfd703SJohn Baldwin #define mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED) 95c0bfd703SJohn Baldwin 9649b94bfcSJohn Baldwin #define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) 979ed346baSBosko Milekic 98d576deedSPawel Jakub Dawidek static void assert_mtx(const struct lock_object *lock, int what); 99d272fe53SJohn Baldwin #ifdef DDB 100d576deedSPawel Jakub Dawidek static void db_show_mtx(const struct lock_object *lock); 101d272fe53SJohn Baldwin #endif 1027faf4d90SDavide Italiano static void lock_mtx(struct lock_object *lock, uintptr_t how); 1037faf4d90SDavide Italiano static void lock_spin(struct lock_object *lock, uintptr_t how); 104a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 105d576deedSPawel Jakub Dawidek static int owner_mtx(const struct lock_object *lock, 106d576deedSPawel Jakub Dawidek struct thread **owner); 107a5aedd68SStacey Son #endif 1087faf4d90SDavide Italiano static uintptr_t unlock_mtx(struct lock_object *lock); 1097faf4d90SDavide Italiano static uintptr_t unlock_spin(struct lock_object *lock); 110d272fe53SJohn Baldwin 1110cde2e34SJason Evans /* 11219284646SJohn Baldwin * Lock classes for sleep and spin mutexes. 1130cde2e34SJason Evans */ 11419284646SJohn Baldwin struct lock_class lock_class_mtx_sleep = { 115ae8dde30SJohn Baldwin .lc_name = "sleep mutex", 116ae8dde30SJohn Baldwin .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 117f9721b43SAttilio Rao .lc_assert = assert_mtx, 118d272fe53SJohn Baldwin #ifdef DDB 119ae8dde30SJohn Baldwin .lc_ddb_show = db_show_mtx, 120d272fe53SJohn Baldwin #endif 1216e21afd4SJohn Baldwin .lc_lock = lock_mtx, 1226e21afd4SJohn Baldwin .lc_unlock = unlock_mtx, 123a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 124a5aedd68SStacey Son .lc_owner = owner_mtx, 125a5aedd68SStacey Son #endif 12619284646SJohn Baldwin }; 12719284646SJohn Baldwin struct lock_class lock_class_mtx_spin = { 128ae8dde30SJohn Baldwin .lc_name = "spin mutex", 129ae8dde30SJohn Baldwin .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 130f9721b43SAttilio Rao .lc_assert = assert_mtx, 131d272fe53SJohn Baldwin #ifdef DDB 132ae8dde30SJohn Baldwin .lc_ddb_show = db_show_mtx, 133d272fe53SJohn Baldwin #endif 1346e21afd4SJohn Baldwin .lc_lock = lock_spin, 1356e21afd4SJohn Baldwin .lc_unlock = unlock_spin, 136a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 137a5aedd68SStacey Son .lc_owner = owner_mtx, 138a5aedd68SStacey Son #endif 1398484de75SJohn Baldwin }; 1408484de75SJohn Baldwin 1419ed346baSBosko Milekic /* 142c53c013bSJohn Baldwin * System-wide mutexes 143c53c013bSJohn Baldwin */ 1442502c107SJeff Roberson struct mtx blocked_lock; 145c53c013bSJohn Baldwin struct mtx Giant; 146c53c013bSJohn Baldwin 14767784314SPoul-Henning Kamp void 148d576deedSPawel Jakub Dawidek assert_mtx(const struct lock_object *lock, int what) 149f9721b43SAttilio Rao { 150f9721b43SAttilio Rao 151d576deedSPawel Jakub Dawidek mtx_assert((const struct mtx *)lock, what); 152f9721b43SAttilio Rao } 153f9721b43SAttilio Rao 15467784314SPoul-Henning Kamp void 1557faf4d90SDavide Italiano lock_mtx(struct lock_object *lock, uintptr_t how) 1566e21afd4SJohn Baldwin { 1576e21afd4SJohn Baldwin 1586e21afd4SJohn Baldwin mtx_lock((struct mtx *)lock); 1596e21afd4SJohn Baldwin } 1606e21afd4SJohn Baldwin 16167784314SPoul-Henning Kamp void 1627faf4d90SDavide Italiano lock_spin(struct lock_object *lock, uintptr_t how) 1636e21afd4SJohn Baldwin { 1646e21afd4SJohn Baldwin 1656e21afd4SJohn Baldwin panic("spin locks can only use msleep_spin"); 1666e21afd4SJohn Baldwin } 1676e21afd4SJohn Baldwin 1687faf4d90SDavide Italiano uintptr_t 1696e21afd4SJohn Baldwin unlock_mtx(struct lock_object *lock) 1706e21afd4SJohn Baldwin { 1716e21afd4SJohn Baldwin struct mtx *m; 1726e21afd4SJohn Baldwin 1736e21afd4SJohn Baldwin m = (struct mtx *)lock; 1746e21afd4SJohn Baldwin mtx_assert(m, MA_OWNED | MA_NOTRECURSED); 1756e21afd4SJohn Baldwin mtx_unlock(m); 1766e21afd4SJohn Baldwin return (0); 1776e21afd4SJohn Baldwin } 1786e21afd4SJohn Baldwin 1797faf4d90SDavide Italiano uintptr_t 1806e21afd4SJohn Baldwin unlock_spin(struct lock_object *lock) 1816e21afd4SJohn Baldwin { 1826e21afd4SJohn Baldwin 1836e21afd4SJohn Baldwin panic("spin locks can only use msleep_spin"); 1846e21afd4SJohn Baldwin } 1856e21afd4SJohn Baldwin 186a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 187a5aedd68SStacey Son int 188d576deedSPawel Jakub Dawidek owner_mtx(const struct lock_object *lock, struct thread **owner) 189a5aedd68SStacey Son { 190d576deedSPawel Jakub Dawidek const struct mtx *m = (const struct mtx *)lock; 191a5aedd68SStacey Son 192a5aedd68SStacey Son *owner = mtx_owner(m); 193a5aedd68SStacey Son return (mtx_unowned(m) == 0); 194a5aedd68SStacey Son } 195a5aedd68SStacey Son #endif 196a5aedd68SStacey Son 1970cde2e34SJason Evans /* 1986283b7d0SJohn Baldwin * Function versions of the inlined __mtx_* macros. These are used by 1996283b7d0SJohn Baldwin * modules and can also be called from assembly language if needed. 2006283b7d0SJohn Baldwin */ 2016283b7d0SJohn Baldwin void 2027f44c618SAttilio Rao __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line) 2036283b7d0SJohn Baldwin { 2047f44c618SAttilio Rao struct mtx *m; 2056283b7d0SJohn Baldwin 20635370593SAndriy Gapon if (SCHEDULER_STOPPED()) 20735370593SAndriy Gapon return; 2087f44c618SAttilio Rao 2097f44c618SAttilio Rao m = mtxlock2mtx(c); 2107f44c618SAttilio Rao 211cd2fe4e6SAttilio Rao KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 212e3ae0dfeSAttilio Rao ("mtx_lock() by idle thread %p on sleep mutex %s @ %s:%d", 213e3ae0dfeSAttilio Rao curthread, m->lock_object.lo_name, file, line)); 214186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 215186abbd7SJohn Baldwin ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 216aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 217aa89d8cdSJohn Baldwin ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 2180d975d63SJohn Baldwin file, line)); 219ac6b769bSAttilio Rao WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) | 220ac6b769bSAttilio Rao LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 2217c0435b9SKip Macy 222961135eaSJohn Baldwin __mtx_lock(m, curthread, opts, file, line); 223aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 224dde96c99SJohn Baldwin line); 225ac6b769bSAttilio Rao WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE, 226ac6b769bSAttilio Rao file, line); 227ce1c953eSMark Johnston TD_LOCKS_INC(curthread); 2286283b7d0SJohn Baldwin } 2296283b7d0SJohn Baldwin 2306283b7d0SJohn Baldwin void 2317f44c618SAttilio Rao __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line) 2326283b7d0SJohn Baldwin { 2337f44c618SAttilio Rao struct mtx *m; 23435370593SAndriy Gapon 23535370593SAndriy Gapon if (SCHEDULER_STOPPED()) 23635370593SAndriy Gapon return; 2377f44c618SAttilio Rao 2387f44c618SAttilio Rao m = mtxlock2mtx(c); 2397f44c618SAttilio Rao 240186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 241186abbd7SJohn Baldwin ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 242aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 243aa89d8cdSJohn Baldwin ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 2440d975d63SJohn Baldwin file, line)); 245aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 246aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 2470d975d63SJohn Baldwin line); 24821377ce0SJohn Baldwin mtx_assert(m, MA_OWNED); 249c66d7606SKip Macy 250961135eaSJohn Baldwin __mtx_unlock(m, curthread, opts, file, line); 251ce1c953eSMark Johnston TD_LOCKS_DEC(curthread); 2526283b7d0SJohn Baldwin } 2536283b7d0SJohn Baldwin 2546283b7d0SJohn Baldwin void 2557f44c618SAttilio Rao __mtx_lock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 2567f44c618SAttilio Rao int line) 2576283b7d0SJohn Baldwin { 2587f44c618SAttilio Rao struct mtx *m; 2596283b7d0SJohn Baldwin 26035370593SAndriy Gapon if (SCHEDULER_STOPPED()) 26135370593SAndriy Gapon return; 2627f44c618SAttilio Rao 2637f44c618SAttilio Rao m = mtxlock2mtx(c); 2647f44c618SAttilio Rao 265186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 266186abbd7SJohn Baldwin ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 267aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 2680d975d63SJohn Baldwin ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 269aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 270ad69e26bSJohn Baldwin if (mtx_owned(m)) 271ac6b769bSAttilio Rao KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 272ac6b769bSAttilio Rao (opts & MTX_RECURSE) != 0, 273ad69e26bSJohn Baldwin ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n", 274ad69e26bSJohn Baldwin m->lock_object.lo_name, file, line)); 275ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 276aa89d8cdSJohn Baldwin WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 27741313430SJohn Baldwin file, line, NULL); 278961135eaSJohn Baldwin __mtx_lock_spin(m, curthread, opts, file, line); 279aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 280dde96c99SJohn Baldwin line); 281aa89d8cdSJohn Baldwin WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 2826283b7d0SJohn Baldwin } 2836283b7d0SJohn Baldwin 2846283b7d0SJohn Baldwin void 2857f44c618SAttilio Rao __mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 2867f44c618SAttilio Rao int line) 2876283b7d0SJohn Baldwin { 2887f44c618SAttilio Rao struct mtx *m; 289c66d7606SKip Macy 29035370593SAndriy Gapon if (SCHEDULER_STOPPED()) 29135370593SAndriy Gapon return; 2927f44c618SAttilio Rao 2937f44c618SAttilio Rao m = mtxlock2mtx(c); 2947f44c618SAttilio Rao 295186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 296186abbd7SJohn Baldwin ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 297aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 2980d975d63SJohn Baldwin ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 299aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 300aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 301aa89d8cdSJohn Baldwin LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 302dde96c99SJohn Baldwin line); 3030d975d63SJohn Baldwin mtx_assert(m, MA_OWNED); 304c66d7606SKip Macy 305961135eaSJohn Baldwin __mtx_unlock_spin(m); 3066283b7d0SJohn Baldwin } 3076283b7d0SJohn Baldwin 3086283b7d0SJohn Baldwin /* 3099ed346baSBosko Milekic * The important part of mtx_trylock{,_flags}() 310eac09796SJohn Baldwin * Tries to acquire lock `m.' If this function is called on a mutex that 311eac09796SJohn Baldwin * is already owned, it will recursively acquire the lock. 3120cde2e34SJason Evans */ 3130cde2e34SJason Evans int 3147f44c618SAttilio Rao _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line) 3150cde2e34SJason Evans { 3167f44c618SAttilio Rao struct mtx *m; 3171723a064SJeff Roberson #ifdef LOCK_PROFILING 3187c0435b9SKip Macy uint64_t waittime = 0; 3191723a064SJeff Roberson int contested = 0; 3201723a064SJeff Roberson #endif 3211723a064SJeff Roberson int rval; 3220cde2e34SJason Evans 32335370593SAndriy Gapon if (SCHEDULER_STOPPED()) 32435370593SAndriy Gapon return (1); 32535370593SAndriy Gapon 3267f44c618SAttilio Rao m = mtxlock2mtx(c); 3277f44c618SAttilio Rao 328cd2fe4e6SAttilio Rao KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 329e3ae0dfeSAttilio Rao ("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d", 330e3ae0dfeSAttilio Rao curthread, m->lock_object.lo_name, file, line)); 331186abbd7SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 332186abbd7SJohn Baldwin ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 333aa89d8cdSJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 334aa89d8cdSJohn Baldwin ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 33583cece6fSJohn Baldwin file, line)); 3369ed346baSBosko Milekic 337ac6b769bSAttilio Rao if (mtx_owned(m) && ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 338ac6b769bSAttilio Rao (opts & MTX_RECURSE) != 0)) { 339eac09796SJohn Baldwin m->mtx_recurse++; 340eac09796SJohn Baldwin atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 341eac09796SJohn Baldwin rval = 1; 342eac09796SJohn Baldwin } else 343961135eaSJohn Baldwin rval = _mtx_obtain_lock(m, (uintptr_t)curthread); 344ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 3459ed346baSBosko Milekic 346aa89d8cdSJohn Baldwin LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line); 347764e4d54SJohn Baldwin if (rval) { 348aa89d8cdSJohn Baldwin WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 3492d96f0b1SJohn Baldwin file, line); 350ce1c953eSMark Johnston TD_LOCKS_INC(curthread); 351fe68a916SKip Macy if (m->mtx_recurse == 0) 35232cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, 353a5aedd68SStacey Son m, contested, waittime, file, line); 3547c0435b9SKip Macy 355764e4d54SJohn Baldwin } 3569ed346baSBosko Milekic 35719284646SJohn Baldwin return (rval); 3580cde2e34SJason Evans } 3590cde2e34SJason Evans 3600cde2e34SJason Evans /* 3617f44c618SAttilio Rao * __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 3629ed346baSBosko Milekic * 3639ed346baSBosko Milekic * We call this if the lock is either contested (i.e. we need to go to 3649ed346baSBosko Milekic * sleep waiting for it), or if we need to recurse on it. 3650cde2e34SJason Evans */ 3660cde2e34SJason Evans void 3677f44c618SAttilio Rao __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts, 3687f44c618SAttilio Rao const char *file, int line) 36936412d79SJohn Baldwin { 3707f44c618SAttilio Rao struct mtx *m; 3712502c107SJeff Roberson struct turnstile *ts; 3721723a064SJeff Roberson uintptr_t v; 373cd6e6e4eSJohn Baldwin #ifdef ADAPTIVE_MUTEXES 37476447e56SJohn Baldwin volatile struct thread *owner; 3752498cf8cSJohn Baldwin #endif 37602bd1bcdSIan Dowse #ifdef KTR 37702bd1bcdSIan Dowse int cont_logged = 0; 37802bd1bcdSIan Dowse #endif 3791723a064SJeff Roberson #ifdef LOCK_PROFILING 38070fe8436SKip Macy int contested = 0; 38170fe8436SKip Macy uint64_t waittime = 0; 3821723a064SJeff Roberson #endif 383a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 384a5aedd68SStacey Son uint64_t spin_cnt = 0; 385a5aedd68SStacey Son uint64_t sleep_cnt = 0; 386a5aedd68SStacey Son int64_t sleep_time = 0; 387076dd8ebSAndriy Gapon int64_t all_time = 0; 388a5aedd68SStacey Son #endif 38936412d79SJohn Baldwin 39035370593SAndriy Gapon if (SCHEDULER_STOPPED()) 39135370593SAndriy Gapon return; 39235370593SAndriy Gapon 3937f44c618SAttilio Rao m = mtxlock2mtx(c); 3947f44c618SAttilio Rao 3955fa8dd90SJohn Baldwin if (mtx_owned(m)) { 396ac6b769bSAttilio Rao KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 397ac6b769bSAttilio Rao (opts & MTX_RECURSE) != 0, 398eac09796SJohn Baldwin ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 399aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line)); 400ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 40136412d79SJohn Baldwin m->mtx_recurse++; 40208812b39SBosko Milekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 403aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 4045746a1d8SBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 40536412d79SJohn Baldwin return; 40636412d79SJohn Baldwin } 407ac6b769bSAttilio Rao opts &= ~MTX_RECURSE; 4089ed346baSBosko Milekic 409f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 410f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 411f5f9340bSFabien Thomas #endif 41270fe8436SKip Macy lock_profile_obtain_lock_failed(&m->lock_object, 41370fe8436SKip Macy &contested, &waittime); 414aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 41515ec816aSJohn Baldwin CTR4(KTR_LOCK, 41615ec816aSJohn Baldwin "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 417aa89d8cdSJohn Baldwin m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 418076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 419e2b25737SMark Johnston all_time -= lockstat_nsecs(&m->lock_object); 420076dd8ebSAndriy Gapon #endif 4211bd0eefbSJohn Baldwin 422fc4f686dSMateusz Guzik for (;;) { 423fc4f686dSMateusz Guzik if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) 424fc4f686dSMateusz Guzik break; 425a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 426a5aedd68SStacey Son spin_cnt++; 427a5aedd68SStacey Son #endif 42849aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES 42949aead8aSAttilio Rao /* 43049aead8aSAttilio Rao * If the owner is running on another CPU, spin until the 43149aead8aSAttilio Rao * owner stops running or the state of the lock changes. 43249aead8aSAttilio Rao */ 43349aead8aSAttilio Rao v = m->mtx_lock; 43449aead8aSAttilio Rao if (v != MTX_UNOWNED) { 43549aead8aSAttilio Rao owner = (struct thread *)(v & ~MTX_FLAGMASK); 43649aead8aSAttilio Rao if (TD_IS_RUNNING(owner)) { 43749aead8aSAttilio Rao if (LOCK_LOG_TEST(&m->lock_object, 0)) 43849aead8aSAttilio Rao CTR3(KTR_LOCK, 43949aead8aSAttilio Rao "%s: spinning on %p held by %p", 44049aead8aSAttilio Rao __func__, m, owner); 4412cba8dd3SJohn Baldwin KTR_STATE1(KTR_SCHED, "thread", 4422cba8dd3SJohn Baldwin sched_tdname((struct thread *)tid), 4432cba8dd3SJohn Baldwin "spinning", "lockname:\"%s\"", 4442cba8dd3SJohn Baldwin m->lock_object.lo_name); 44549aead8aSAttilio Rao while (mtx_owner(m) == owner && 446a5aedd68SStacey Son TD_IS_RUNNING(owner)) { 44749aead8aSAttilio Rao cpu_spinwait(); 448a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 449a5aedd68SStacey Son spin_cnt++; 450a5aedd68SStacey Son #endif 451a5aedd68SStacey Son } 4522cba8dd3SJohn Baldwin KTR_STATE0(KTR_SCHED, "thread", 4532cba8dd3SJohn Baldwin sched_tdname((struct thread *)tid), 4542cba8dd3SJohn Baldwin "running"); 45549aead8aSAttilio Rao continue; 45649aead8aSAttilio Rao } 45749aead8aSAttilio Rao } 45849aead8aSAttilio Rao #endif 45949aead8aSAttilio Rao 4602502c107SJeff Roberson ts = turnstile_trywait(&m->lock_object); 4615fa8dd90SJohn Baldwin v = m->mtx_lock; 4625fa8dd90SJohn Baldwin 46336412d79SJohn Baldwin /* 4649ed346baSBosko Milekic * Check if the lock has been released while spinning for 465961a7b24SJohn Baldwin * the turnstile chain lock. 46636412d79SJohn Baldwin */ 4675fa8dd90SJohn Baldwin if (v == MTX_UNOWNED) { 4682502c107SJeff Roberson turnstile_cancel(ts); 46936412d79SJohn Baldwin continue; 47036412d79SJohn Baldwin } 4719ed346baSBosko Milekic 47249aead8aSAttilio Rao #ifdef ADAPTIVE_MUTEXES 47349aead8aSAttilio Rao /* 474fa29f023SJohn Baldwin * The current lock owner might have started executing 475fa29f023SJohn Baldwin * on another CPU (or the lock could have changed 476fa29f023SJohn Baldwin * owners) while we were waiting on the turnstile 477fa29f023SJohn Baldwin * chain lock. If so, drop the turnstile lock and try 478fa29f023SJohn Baldwin * again. 47949aead8aSAttilio Rao */ 48049aead8aSAttilio Rao owner = (struct thread *)(v & ~MTX_FLAGMASK); 48149aead8aSAttilio Rao if (TD_IS_RUNNING(owner)) { 48249aead8aSAttilio Rao turnstile_cancel(ts); 48349aead8aSAttilio Rao continue; 48449aead8aSAttilio Rao } 48549aead8aSAttilio Rao #endif 48649aead8aSAttilio Rao 48736412d79SJohn Baldwin /* 4889ed346baSBosko Milekic * If the mutex isn't already contested and a failure occurs 4899ed346baSBosko Milekic * setting the contested bit, the mutex was either released 4909ed346baSBosko Milekic * or the state of the MTX_RECURSED bit changed. 49136412d79SJohn Baldwin */ 49236412d79SJohn Baldwin if ((v & MTX_CONTESTED) == 0 && 493122eceefSJohn Baldwin !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 4942502c107SJeff Roberson turnstile_cancel(ts); 49536412d79SJohn Baldwin continue; 49636412d79SJohn Baldwin } 49736412d79SJohn Baldwin 4989ed346baSBosko Milekic /* 4997feefcd6SJohn Baldwin * We definitely must sleep for this lock. 5009ed346baSBosko Milekic */ 50136412d79SJohn Baldwin mtx_assert(m, MA_NOTOWNED); 50236412d79SJohn Baldwin 50302bd1bcdSIan Dowse #ifdef KTR 50402bd1bcdSIan Dowse if (!cont_logged) { 50502bd1bcdSIan Dowse CTR6(KTR_CONTENTION, 50602bd1bcdSIan Dowse "contention: %p at %s:%d wants %s, taken by %s:%d", 507aa89d8cdSJohn Baldwin (void *)tid, file, line, m->lock_object.lo_name, 508aa89d8cdSJohn Baldwin WITNESS_FILE(&m->lock_object), 509aa89d8cdSJohn Baldwin WITNESS_LINE(&m->lock_object)); 51002bd1bcdSIan Dowse cont_logged = 1; 51102bd1bcdSIan Dowse } 51202bd1bcdSIan Dowse #endif 51336412d79SJohn Baldwin 5149ed346baSBosko Milekic /* 515961a7b24SJohn Baldwin * Block on the turnstile. 5169ed346baSBosko Milekic */ 517a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 518e2b25737SMark Johnston sleep_time -= lockstat_nsecs(&m->lock_object); 519a5aedd68SStacey Son #endif 5202502c107SJeff Roberson turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 521a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 522e2b25737SMark Johnston sleep_time += lockstat_nsecs(&m->lock_object); 523a5aedd68SStacey Son sleep_cnt++; 524a5aedd68SStacey Son #endif 52536412d79SJohn Baldwin } 526076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 527e2b25737SMark Johnston all_time += lockstat_nsecs(&m->lock_object); 528076dd8ebSAndriy Gapon #endif 52902bd1bcdSIan Dowse #ifdef KTR 53002bd1bcdSIan Dowse if (cont_logged) { 53102bd1bcdSIan Dowse CTR4(KTR_CONTENTION, 53202bd1bcdSIan Dowse "contention end: %s acquired by %p at %s:%d", 533aa89d8cdSJohn Baldwin m->lock_object.lo_name, (void *)tid, file, line); 53402bd1bcdSIan Dowse } 53502bd1bcdSIan Dowse #endif 53632cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested, 537eea4f254SJeff Roberson waittime, file, line); 538a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 539a5aedd68SStacey Son if (sleep_time) 54032cd0147SMark Johnston LOCKSTAT_RECORD1(adaptive__block, m, sleep_time); 541a5aedd68SStacey Son 542a5aedd68SStacey Son /* 543a5aedd68SStacey Son * Only record the loops spinning and not sleeping. 544a5aedd68SStacey Son */ 545a5aedd68SStacey Son if (spin_cnt > sleep_cnt) 54632cd0147SMark Johnston LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time); 547a5aedd68SStacey Son #endif 5489ed346baSBosko Milekic } 5499ed346baSBosko Milekic 5502502c107SJeff Roberson static void 5512502c107SJeff Roberson _mtx_lock_spin_failed(struct mtx *m) 5522502c107SJeff Roberson { 5532502c107SJeff Roberson struct thread *td; 5542502c107SJeff Roberson 5552502c107SJeff Roberson td = mtx_owner(m); 5562502c107SJeff Roberson 5572502c107SJeff Roberson /* If the mutex is unlocked, try again. */ 5582502c107SJeff Roberson if (td == NULL) 5592502c107SJeff Roberson return; 560b95b98b0SKonstantin Belousov 5612502c107SJeff Roberson printf( "spin lock %p (%s) held by %p (tid %d) too long\n", 5622502c107SJeff Roberson m, m->lock_object.lo_name, td, td->td_tid); 5632502c107SJeff Roberson #ifdef WITNESS 56498332c8cSAttilio Rao witness_display_spinlock(&m->lock_object, td, printf); 5652502c107SJeff Roberson #endif 5662502c107SJeff Roberson panic("spin lock held too long"); 5672502c107SJeff Roberson } 5682502c107SJeff Roberson 569b95b98b0SKonstantin Belousov #ifdef SMP 5709ed346baSBosko Milekic /* 5717f44c618SAttilio Rao * _mtx_lock_spin_cookie: the tougher part of acquiring an MTX_SPIN lock. 5729ed346baSBosko Milekic * 5739ed346baSBosko Milekic * This is only called if we need to actually spin for the lock. Recursion 5749ed346baSBosko Milekic * is handled inline. 5759ed346baSBosko Milekic */ 5769ed346baSBosko Milekic void 5777f44c618SAttilio Rao _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts, 5787f44c618SAttilio Rao const char *file, int line) 57936412d79SJohn Baldwin { 5807f44c618SAttilio Rao struct mtx *m; 5811723a064SJeff Roberson int i = 0; 5821723a064SJeff Roberson #ifdef LOCK_PROFILING 5831723a064SJeff Roberson int contested = 0; 58470fe8436SKip Macy uint64_t waittime = 0; 5851723a064SJeff Roberson #endif 586076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 587076dd8ebSAndriy Gapon int64_t spin_time = 0; 588076dd8ebSAndriy Gapon #endif 58936412d79SJohn Baldwin 59035370593SAndriy Gapon if (SCHEDULER_STOPPED()) 59135370593SAndriy Gapon return; 59235370593SAndriy Gapon 5937f44c618SAttilio Rao m = mtxlock2mtx(c); 5947f44c618SAttilio Rao 595aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 5965746a1d8SBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 5972cba8dd3SJohn Baldwin KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 5982cba8dd3SJohn Baldwin "spinning", "lockname:\"%s\"", m->lock_object.lo_name); 5999ed346baSBosko Milekic 600f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 601f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 602f5f9340bSFabien Thomas #endif 60370fe8436SKip Macy lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 604076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 605e2b25737SMark Johnston spin_time -= lockstat_nsecs(&m->lock_object); 606076dd8ebSAndriy Gapon #endif 607fc4f686dSMateusz Guzik for (;;) { 608fc4f686dSMateusz Guzik if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) 609fc4f686dSMateusz Guzik break; 6107141f2adSJohn Baldwin /* Give interrupts a chance while we spin. */ 611c6a37e84SJohn Baldwin spinlock_exit(); 61236412d79SJohn Baldwin while (m->mtx_lock != MTX_UNOWNED) { 613703fc290SJohn Baldwin if (i++ < 10000000) { 6149f1b87f1SMaxime Henrion cpu_spinwait(); 61536412d79SJohn Baldwin continue; 616703fc290SJohn Baldwin } 6170fa2168bSJohn Baldwin if (i < 60000000 || kdb_active || panicstr != NULL) 61836412d79SJohn Baldwin DELAY(1); 6192502c107SJeff Roberson else 6202502c107SJeff Roberson _mtx_lock_spin_failed(m); 6219f1b87f1SMaxime Henrion cpu_spinwait(); 62236412d79SJohn Baldwin } 623c6a37e84SJohn Baldwin spinlock_enter(); 62436412d79SJohn Baldwin } 625076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 626e2b25737SMark Johnston spin_time += lockstat_nsecs(&m->lock_object); 627076dd8ebSAndriy Gapon #endif 62836412d79SJohn Baldwin 629aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 6309ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 6312cba8dd3SJohn Baldwin KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 6322cba8dd3SJohn Baldwin "running"); 6339ed346baSBosko Milekic 634c6d48c87SMark Johnston #ifdef KDTRACE_HOOKS 63532cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m, 63632cd0147SMark Johnston contested, waittime, file, line); 637e2b25737SMark Johnston if (spin_time != 0) 63832cd0147SMark Johnston LOCKSTAT_RECORD1(spin__spin, m, spin_time); 639c6d48c87SMark Johnston #endif 64036412d79SJohn Baldwin } 64133fb8a38SJohn Baldwin #endif /* SMP */ 64236412d79SJohn Baldwin 6432502c107SJeff Roberson void 644ccdf2333SAttilio Rao thread_lock_flags_(struct thread *td, int opts, const char *file, int line) 6452502c107SJeff Roberson { 6462502c107SJeff Roberson struct mtx *m; 6472502c107SJeff Roberson uintptr_t tid; 6481723a064SJeff Roberson int i; 6491723a064SJeff Roberson #ifdef LOCK_PROFILING 6501723a064SJeff Roberson int contested = 0; 6511723a064SJeff Roberson uint64_t waittime = 0; 6521723a064SJeff Roberson #endif 653a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 654076dd8ebSAndriy Gapon int64_t spin_time = 0; 655a5aedd68SStacey Son #endif 6562502c107SJeff Roberson 6571723a064SJeff Roberson i = 0; 6582502c107SJeff Roberson tid = (uintptr_t)curthread; 65935370593SAndriy Gapon 660*f61d6c5aSMark Johnston if (SCHEDULER_STOPPED()) { 661*f61d6c5aSMark Johnston /* 662*f61d6c5aSMark Johnston * Ensure that spinlock sections are balanced even when the 663*f61d6c5aSMark Johnston * scheduler is stopped, since we may otherwise inadvertently 664*f61d6c5aSMark Johnston * re-enable interrupts while dumping core. 665*f61d6c5aSMark Johnston */ 666*f61d6c5aSMark Johnston spinlock_enter(); 66735370593SAndriy Gapon return; 668*f61d6c5aSMark Johnston } 66935370593SAndriy Gapon 670076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 671e2b25737SMark Johnston spin_time -= lockstat_nsecs(&td->td_lock->lock_object); 672076dd8ebSAndriy Gapon #endif 6732502c107SJeff Roberson for (;;) { 6742502c107SJeff Roberson retry: 6752502c107SJeff Roberson spinlock_enter(); 676710eacdcSJeff Roberson m = td->td_lock; 67713c85a48SJohn Baldwin KASSERT(m->mtx_lock != MTX_DESTROYED, 67813c85a48SJohn Baldwin ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 67913c85a48SJohn Baldwin KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 68013c85a48SJohn Baldwin ("thread_lock() of sleep mutex %s @ %s:%d", 68113c85a48SJohn Baldwin m->lock_object.lo_name, file, line)); 682ad69e26bSJohn Baldwin if (mtx_owned(m)) 683ad69e26bSJohn Baldwin KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 684ad69e26bSJohn Baldwin ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", 685ad69e26bSJohn Baldwin m->lock_object.lo_name, file, line)); 6862502c107SJeff Roberson WITNESS_CHECKORDER(&m->lock_object, 68741313430SJohn Baldwin opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 688fc4f686dSMateusz Guzik for (;;) { 689fc4f686dSMateusz Guzik if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) 690fc4f686dSMateusz Guzik break; 6912502c107SJeff Roberson if (m->mtx_lock == tid) { 6922502c107SJeff Roberson m->mtx_recurse++; 6932502c107SJeff Roberson break; 6942502c107SJeff Roberson } 695f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 696f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 697f5f9340bSFabien Thomas #endif 698eea4f254SJeff Roberson lock_profile_obtain_lock_failed(&m->lock_object, 699eea4f254SJeff Roberson &contested, &waittime); 7002502c107SJeff Roberson /* Give interrupts a chance while we spin. */ 7012502c107SJeff Roberson spinlock_exit(); 7022502c107SJeff Roberson while (m->mtx_lock != MTX_UNOWNED) { 7032502c107SJeff Roberson if (i++ < 10000000) 7042502c107SJeff Roberson cpu_spinwait(); 7052502c107SJeff Roberson else if (i < 60000000 || 7062502c107SJeff Roberson kdb_active || panicstr != NULL) 7072502c107SJeff Roberson DELAY(1); 7082502c107SJeff Roberson else 7092502c107SJeff Roberson _mtx_lock_spin_failed(m); 7102502c107SJeff Roberson cpu_spinwait(); 7112502c107SJeff Roberson if (m != td->td_lock) 7122502c107SJeff Roberson goto retry; 7132502c107SJeff Roberson } 7142502c107SJeff Roberson spinlock_enter(); 7152502c107SJeff Roberson } 7162502c107SJeff Roberson if (m == td->td_lock) 7172502c107SJeff Roberson break; 718961135eaSJohn Baldwin __mtx_unlock_spin(m); /* does spinlock_exit() */ 7192502c107SJeff Roberson } 720076dd8ebSAndriy Gapon #ifdef KDTRACE_HOOKS 721e2b25737SMark Johnston spin_time += lockstat_nsecs(&m->lock_object); 722076dd8ebSAndriy Gapon #endif 723eea4f254SJeff Roberson if (m->mtx_recurse == 0) 72432cd0147SMark Johnston LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m, 72532cd0147SMark Johnston contested, waittime, file, line); 72613c85a48SJohn Baldwin LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 72713c85a48SJohn Baldwin line); 7282502c107SJeff Roberson WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 7295002e195SMark Johnston #ifdef KDTRACE_HOOKS 730156fbc14SMark Johnston if (spin_time != 0) 73132cd0147SMark Johnston LOCKSTAT_RECORD1(thread__spin, m, spin_time); 7325002e195SMark Johnston #endif 7332502c107SJeff Roberson } 7342502c107SJeff Roberson 7352502c107SJeff Roberson struct mtx * 7362502c107SJeff Roberson thread_lock_block(struct thread *td) 7372502c107SJeff Roberson { 7382502c107SJeff Roberson struct mtx *lock; 7392502c107SJeff Roberson 7402502c107SJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 741710eacdcSJeff Roberson lock = td->td_lock; 7422502c107SJeff Roberson td->td_lock = &blocked_lock; 7432502c107SJeff Roberson mtx_unlock_spin(lock); 7442502c107SJeff Roberson 7452502c107SJeff Roberson return (lock); 7462502c107SJeff Roberson } 7472502c107SJeff Roberson 7482502c107SJeff Roberson void 7492502c107SJeff Roberson thread_lock_unblock(struct thread *td, struct mtx *new) 7502502c107SJeff Roberson { 7512502c107SJeff Roberson mtx_assert(new, MA_OWNED); 7522502c107SJeff Roberson MPASS(td->td_lock == &blocked_lock); 75365d32cd8SMatt Jacob atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new); 7542502c107SJeff Roberson } 7552502c107SJeff Roberson 7562502c107SJeff Roberson void 7572502c107SJeff Roberson thread_lock_set(struct thread *td, struct mtx *new) 7582502c107SJeff Roberson { 7592502c107SJeff Roberson struct mtx *lock; 7602502c107SJeff Roberson 7612502c107SJeff Roberson mtx_assert(new, MA_OWNED); 7622502c107SJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 763710eacdcSJeff Roberson lock = td->td_lock; 7642502c107SJeff Roberson td->td_lock = new; 7652502c107SJeff Roberson mtx_unlock_spin(lock); 7662502c107SJeff Roberson } 7672502c107SJeff Roberson 7689ed346baSBosko Milekic /* 7697f44c618SAttilio Rao * __mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 7709ed346baSBosko Milekic * 7719ed346baSBosko Milekic * We are only called here if the lock is recursed or contested (i.e. we 7729ed346baSBosko Milekic * need to wake up a blocked thread). 7739ed346baSBosko Milekic */ 77436412d79SJohn Baldwin void 7757f44c618SAttilio Rao __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line) 77636412d79SJohn Baldwin { 7777f44c618SAttilio Rao struct mtx *m; 778961a7b24SJohn Baldwin struct turnstile *ts; 7799ed346baSBosko Milekic 78035370593SAndriy Gapon if (SCHEDULER_STOPPED()) 78135370593SAndriy Gapon return; 78235370593SAndriy Gapon 7837f44c618SAttilio Rao m = mtxlock2mtx(c); 7847f44c618SAttilio Rao 78508812b39SBosko Milekic if (mtx_recursed(m)) { 78636412d79SJohn Baldwin if (--(m->mtx_recurse) == 0) 78708812b39SBosko Milekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 788aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 7899ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 79036412d79SJohn Baldwin return; 79136412d79SJohn Baldwin } 7929ed346baSBosko Milekic 7932502c107SJeff Roberson /* 7942502c107SJeff Roberson * We have to lock the chain before the turnstile so this turnstile 7952502c107SJeff Roberson * can be removed from the hash list if it is empty. 7962502c107SJeff Roberson */ 7972502c107SJeff Roberson turnstile_chain_lock(&m->lock_object); 798aa89d8cdSJohn Baldwin ts = turnstile_lookup(&m->lock_object); 799aa89d8cdSJohn Baldwin if (LOCK_LOG_TEST(&m->lock_object, opts)) 8009ed346baSBosko Milekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 801961a7b24SJohn Baldwin MPASS(ts != NULL); 8027aa4f685SJohn Baldwin turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 803961135eaSJohn Baldwin _mtx_release_lock_quick(m); 804bf9c6c31SJohn Baldwin 8052502c107SJeff Roberson /* 8062502c107SJeff Roberson * This turnstile is now no longer associated with the mutex. We can 8072502c107SJeff Roberson * unlock the chain lock so a new turnstile may take it's place. 8082502c107SJeff Roberson */ 8097aa4f685SJohn Baldwin turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 8102502c107SJeff Roberson turnstile_chain_unlock(&m->lock_object); 8119ed346baSBosko Milekic } 8129ed346baSBosko Milekic 8139ed346baSBosko Milekic /* 8149ed346baSBosko Milekic * All the unlocking of MTX_SPIN locks is done inline. 815961135eaSJohn Baldwin * See the __mtx_unlock_spin() macro for the details. 8169ed346baSBosko Milekic */ 8179ed346baSBosko Milekic 8189ed346baSBosko Milekic /* 81915ec816aSJohn Baldwin * The backing function for the INVARIANTS-enabled mtx_assert() 8209ed346baSBosko Milekic */ 8211103f3b0SJohn Baldwin #ifdef INVARIANT_SUPPORT 8220cde2e34SJason Evans void 8237f44c618SAttilio Rao __mtx_assert(const volatile uintptr_t *c, int what, const char *file, int line) 8240cde2e34SJason Evans { 8257f44c618SAttilio Rao const struct mtx *m; 8265cb0fbe4SJohn Baldwin 8271126349aSPaul Saab if (panicstr != NULL || dumping) 8285cb0fbe4SJohn Baldwin return; 8297f44c618SAttilio Rao 8307f44c618SAttilio Rao m = mtxlock2mtx(c); 8317f44c618SAttilio Rao 832a10f4966SJake Burkholder switch (what) { 8330cde2e34SJason Evans case MA_OWNED: 8340cde2e34SJason Evans case MA_OWNED | MA_RECURSED: 8350cde2e34SJason Evans case MA_OWNED | MA_NOTRECURSED: 836a10f4966SJake Burkholder if (!mtx_owned(m)) 8370cde2e34SJason Evans panic("mutex %s not owned at %s:%d", 838aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 839a10f4966SJake Burkholder if (mtx_recursed(m)) { 840a10f4966SJake Burkholder if ((what & MA_NOTRECURSED) != 0) 8410cde2e34SJason Evans panic("mutex %s recursed at %s:%d", 842aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 843a10f4966SJake Burkholder } else if ((what & MA_RECURSED) != 0) { 8440cde2e34SJason Evans panic("mutex %s unrecursed at %s:%d", 845aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 8460cde2e34SJason Evans } 8470cde2e34SJason Evans break; 8480cde2e34SJason Evans case MA_NOTOWNED: 849a10f4966SJake Burkholder if (mtx_owned(m)) 8500cde2e34SJason Evans panic("mutex %s owned at %s:%d", 851aa89d8cdSJohn Baldwin m->lock_object.lo_name, file, line); 8520cde2e34SJason Evans break; 8530cde2e34SJason Evans default: 85456771ca7SJason Evans panic("unknown mtx_assert at %s:%d", file, line); 8550cde2e34SJason Evans } 8560cde2e34SJason Evans } 8570cde2e34SJason Evans #endif 8580cde2e34SJason Evans 8599ed346baSBosko Milekic /* 860c27b5699SAndrew R. Reiter * General init routine used by the MTX_SYSINIT() macro. 861c27b5699SAndrew R. Reiter */ 862c27b5699SAndrew R. Reiter void 863c27b5699SAndrew R. Reiter mtx_sysinit(void *arg) 864c27b5699SAndrew R. Reiter { 865c27b5699SAndrew R. Reiter struct mtx_args *margs = arg; 866c27b5699SAndrew R. Reiter 8677f44c618SAttilio Rao mtx_init((struct mtx *)margs->ma_mtx, margs->ma_desc, NULL, 8687f44c618SAttilio Rao margs->ma_opts); 869c27b5699SAndrew R. Reiter } 870c27b5699SAndrew R. Reiter 871c27b5699SAndrew R. Reiter /* 8729ed346baSBosko Milekic * Mutex initialization routine; initialize lock `m' of type contained in 8730c88508aSJohn Baldwin * `opts' with options contained in `opts' and name `name.' The optional 8740c88508aSJohn Baldwin * lock type `type' is used as a general lock category name for use with 8750c88508aSJohn Baldwin * witness. 8769ed346baSBosko Milekic */ 87736412d79SJohn Baldwin void 8787f44c618SAttilio Rao _mtx_init(volatile uintptr_t *c, const char *name, const char *type, int opts) 87936412d79SJohn Baldwin { 8807f44c618SAttilio Rao struct mtx *m; 88183a81bcbSJohn Baldwin struct lock_class *class; 88283a81bcbSJohn Baldwin int flags; 8839ed346baSBosko Milekic 8847f44c618SAttilio Rao m = mtxlock2mtx(c); 8857f44c618SAttilio Rao 88619284646SJohn Baldwin MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 887fd07ddcfSDmitry Chagin MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE | MTX_NEW)) == 0); 888353998acSAttilio Rao ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock, 889353998acSAttilio Rao ("%s: mtx_lock not aligned for %s: %p", __func__, name, 890353998acSAttilio Rao &m->mtx_lock)); 8919ed346baSBosko Milekic 89283a81bcbSJohn Baldwin /* Determine lock class and lock flags. */ 89319284646SJohn Baldwin if (opts & MTX_SPIN) 89483a81bcbSJohn Baldwin class = &lock_class_mtx_spin; 89519284646SJohn Baldwin else 89683a81bcbSJohn Baldwin class = &lock_class_mtx_sleep; 89783a81bcbSJohn Baldwin flags = 0; 89819284646SJohn Baldwin if (opts & MTX_QUIET) 89983a81bcbSJohn Baldwin flags |= LO_QUIET; 90019284646SJohn Baldwin if (opts & MTX_RECURSE) 90183a81bcbSJohn Baldwin flags |= LO_RECURSABLE; 90219284646SJohn Baldwin if ((opts & MTX_NOWITNESS) == 0) 90383a81bcbSJohn Baldwin flags |= LO_WITNESS; 904f22a4b62SJeff Roberson if (opts & MTX_DUPOK) 90583a81bcbSJohn Baldwin flags |= LO_DUPOK; 9067c0435b9SKip Macy if (opts & MTX_NOPROFILE) 9077c0435b9SKip Macy flags |= LO_NOPROFILE; 908fd07ddcfSDmitry Chagin if (opts & MTX_NEW) 909fd07ddcfSDmitry Chagin flags |= LO_NEW; 91019284646SJohn Baldwin 91183a81bcbSJohn Baldwin /* Initialize mutex. */ 912b5fb43e5SJohn Baldwin lock_init(&m->lock_object, class, name, type, flags); 913b5fb43e5SJohn Baldwin 91419284646SJohn Baldwin m->mtx_lock = MTX_UNOWNED; 91583a81bcbSJohn Baldwin m->mtx_recurse = 0; 91636412d79SJohn Baldwin } 91736412d79SJohn Baldwin 9189ed346baSBosko Milekic /* 91919284646SJohn Baldwin * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 92019284646SJohn Baldwin * passed in as a flag here because if the corresponding mtx_init() was 92119284646SJohn Baldwin * called with MTX_QUIET set, then it will already be set in the mutex's 92219284646SJohn Baldwin * flags. 9239ed346baSBosko Milekic */ 92436412d79SJohn Baldwin void 9257f44c618SAttilio Rao _mtx_destroy(volatile uintptr_t *c) 92636412d79SJohn Baldwin { 9277f44c618SAttilio Rao struct mtx *m; 9287f44c618SAttilio Rao 9297f44c618SAttilio Rao m = mtxlock2mtx(c); 93036412d79SJohn Baldwin 93119284646SJohn Baldwin if (!mtx_owned(m)) 93219284646SJohn Baldwin MPASS(mtx_unowned(m)); 93319284646SJohn Baldwin else { 93408812b39SBosko Milekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 9359ed346baSBosko Milekic 936861a2308SScott Long /* Perform the non-mtx related part of mtx_unlock_spin(). */ 937aa89d8cdSJohn Baldwin if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin) 938861a2308SScott Long spinlock_exit(); 939764e4d54SJohn Baldwin else 940ce1c953eSMark Johnston TD_LOCKS_DEC(curthread); 941861a2308SScott Long 942d3df4af3SJeff Roberson lock_profile_release_lock(&m->lock_object); 94319284646SJohn Baldwin /* Tell witness this isn't locked to make it happy. */ 944aa89d8cdSJohn Baldwin WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__, 945c86b6ff5SJohn Baldwin __LINE__); 94636412d79SJohn Baldwin } 9470384fff8SJason Evans 948186abbd7SJohn Baldwin m->mtx_lock = MTX_DESTROYED; 949aa89d8cdSJohn Baldwin lock_destroy(&m->lock_object); 9500384fff8SJason Evans } 951d23f5958SMatthew Dillon 952d23f5958SMatthew Dillon /* 953c53c013bSJohn Baldwin * Intialize the mutex code and system mutexes. This is called from the MD 954c53c013bSJohn Baldwin * startup code prior to mi_startup(). The per-CPU data space needs to be 955c53c013bSJohn Baldwin * setup before this is called. 956c53c013bSJohn Baldwin */ 957c53c013bSJohn Baldwin void 958c53c013bSJohn Baldwin mutex_init(void) 959c53c013bSJohn Baldwin { 960c53c013bSJohn Baldwin 961961a7b24SJohn Baldwin /* Setup turnstiles so that sleep mutexes work. */ 962961a7b24SJohn Baldwin init_turnstiles(); 963961a7b24SJohn Baldwin 964c53c013bSJohn Baldwin /* 965c53c013bSJohn Baldwin * Initialize mutexes. 966c53c013bSJohn Baldwin */ 9670c88508aSJohn Baldwin mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 9682502c107SJeff Roberson mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN); 9692502c107SJeff Roberson blocked_lock.mtx_lock = 0xdeadc0de; /* Always blocked. */ 9700c88508aSJohn Baldwin mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 9716afb32fcSKonstantin Belousov mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN); 9725c7bebf9SKonstantin Belousov mtx_init(&proc0.p_statmtx, "pstatl", NULL, MTX_SPIN); 9735c7bebf9SKonstantin Belousov mtx_init(&proc0.p_itimmtx, "pitiml", NULL, MTX_SPIN); 9745c7bebf9SKonstantin Belousov mtx_init(&proc0.p_profmtx, "pprofl", NULL, MTX_SPIN); 9758c4b6380SJohn Baldwin mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 976c53c013bSJohn Baldwin mtx_lock(&Giant); 977c53c013bSJohn Baldwin } 978d272fe53SJohn Baldwin 979d272fe53SJohn Baldwin #ifdef DDB 980d272fe53SJohn Baldwin void 981d576deedSPawel Jakub Dawidek db_show_mtx(const struct lock_object *lock) 982d272fe53SJohn Baldwin { 983d272fe53SJohn Baldwin struct thread *td; 984d576deedSPawel Jakub Dawidek const struct mtx *m; 985d272fe53SJohn Baldwin 986d576deedSPawel Jakub Dawidek m = (const struct mtx *)lock; 987d272fe53SJohn Baldwin 988d272fe53SJohn Baldwin db_printf(" flags: {"); 98983a81bcbSJohn Baldwin if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 990d272fe53SJohn Baldwin db_printf("SPIN"); 991d272fe53SJohn Baldwin else 992d272fe53SJohn Baldwin db_printf("DEF"); 993aa89d8cdSJohn Baldwin if (m->lock_object.lo_flags & LO_RECURSABLE) 994d272fe53SJohn Baldwin db_printf(", RECURSE"); 995aa89d8cdSJohn Baldwin if (m->lock_object.lo_flags & LO_DUPOK) 996d272fe53SJohn Baldwin db_printf(", DUPOK"); 997d272fe53SJohn Baldwin db_printf("}\n"); 998d272fe53SJohn Baldwin db_printf(" state: {"); 999d272fe53SJohn Baldwin if (mtx_unowned(m)) 1000d272fe53SJohn Baldwin db_printf("UNOWNED"); 1001c0bfd703SJohn Baldwin else if (mtx_destroyed(m)) 1002c0bfd703SJohn Baldwin db_printf("DESTROYED"); 1003d272fe53SJohn Baldwin else { 1004d272fe53SJohn Baldwin db_printf("OWNED"); 1005d272fe53SJohn Baldwin if (m->mtx_lock & MTX_CONTESTED) 1006d272fe53SJohn Baldwin db_printf(", CONTESTED"); 1007d272fe53SJohn Baldwin if (m->mtx_lock & MTX_RECURSED) 1008d272fe53SJohn Baldwin db_printf(", RECURSED"); 1009d272fe53SJohn Baldwin } 1010d272fe53SJohn Baldwin db_printf("}\n"); 1011c0bfd703SJohn Baldwin if (!mtx_unowned(m) && !mtx_destroyed(m)) { 1012d272fe53SJohn Baldwin td = mtx_owner(m); 1013d272fe53SJohn Baldwin db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 1014431f8906SJulian Elischer td->td_tid, td->td_proc->p_pid, td->td_name); 1015d272fe53SJohn Baldwin if (mtx_recursed(m)) 1016d272fe53SJohn Baldwin db_printf(" recursed: %d\n", m->mtx_recurse); 1017d272fe53SJohn Baldwin } 1018d272fe53SJohn Baldwin } 1019d272fe53SJohn Baldwin #endif 1020