19454b2d8SWarner Losh /*- 28a36da99SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 38a36da99SPedro F. Giffuni * 4047dd67eSAttilio Rao * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org> 5047dd67eSAttilio Rao * All rights reserved. 653bf4bb2SPeter Wemm * 753bf4bb2SPeter Wemm * Redistribution and use in source and binary forms, with or without 853bf4bb2SPeter Wemm * modification, are permitted provided that the following conditions 953bf4bb2SPeter Wemm * are met: 1053bf4bb2SPeter Wemm * 1. Redistributions of source code must retain the above copyright 11047dd67eSAttilio Rao * notice(s), this list of conditions and the following disclaimer as 12047dd67eSAttilio Rao * the first lines of this file unmodified other than the possible 13047dd67eSAttilio Rao * addition of one or more copyright notices. 1453bf4bb2SPeter Wemm * 2. Redistributions in binary form must reproduce the above copyright 15047dd67eSAttilio Rao * notice(s), this list of conditions and the following disclaimer in the 1653bf4bb2SPeter Wemm * documentation and/or other materials provided with the distribution. 1753bf4bb2SPeter Wemm * 18047dd67eSAttilio Rao * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 19047dd67eSAttilio Rao * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20047dd67eSAttilio Rao * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21047dd67eSAttilio Rao * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 22047dd67eSAttilio Rao * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23047dd67eSAttilio Rao * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24047dd67eSAttilio Rao * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25047dd67eSAttilio Rao * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2653bf4bb2SPeter Wemm * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27047dd67eSAttilio Rao * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 28047dd67eSAttilio Rao * DAMAGE. 2953bf4bb2SPeter Wemm */ 3053bf4bb2SPeter Wemm 31047dd67eSAttilio Rao #include "opt_ddb.h" 32f5f9340bSFabien Thomas #include "opt_hwpmc_hooks.h" 33047dd67eSAttilio Rao 34677b542eSDavid E. O'Brien #include <sys/cdefs.h> 35677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 36677b542eSDavid E. O'Brien 3753bf4bb2SPeter Wemm #include <sys/param.h> 38cd2fe4e6SAttilio Rao #include <sys/kdb.h> 3961d80e90SJohn Baldwin #include <sys/ktr.h> 4053bf4bb2SPeter Wemm #include <sys/lock.h> 41047dd67eSAttilio Rao #include <sys/lock_profile.h> 428302d183SBruce Evans #include <sys/lockmgr.h> 43d8881ca3SJohn Baldwin #include <sys/mutex.h> 448302d183SBruce Evans #include <sys/proc.h> 45047dd67eSAttilio Rao #include <sys/sleepqueue.h> 46e8ddb61dSJeff Roberson #ifdef DEBUG_LOCKS 47e8ddb61dSJeff Roberson #include <sys/stack.h> 48e8ddb61dSJeff Roberson #endif 49651175c9SAttilio Rao #include <sys/sysctl.h> 50047dd67eSAttilio Rao #include <sys/systm.h> 5153bf4bb2SPeter Wemm 52047dd67eSAttilio Rao #include <machine/cpu.h> 536efc8a16SAttilio Rao 54be6847d7SJohn Baldwin #ifdef DDB 55be6847d7SJohn Baldwin #include <ddb/ddb.h> 56047dd67eSAttilio Rao #endif 57047dd67eSAttilio Rao 58f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 59f5f9340bSFabien Thomas #include <sys/pmckern.h> 60f5f9340bSFabien Thomas PMC_SOFT_DECLARE( , , lock, failed); 61f5f9340bSFabien Thomas #endif 62f5f9340bSFabien Thomas 63651175c9SAttilio Rao CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) == 64651175c9SAttilio Rao (LK_ADAPTIVE | LK_NOSHARE)); 65651175c9SAttilio Rao CTASSERT(LK_UNLOCKED == (LK_UNLOCKED & 66651175c9SAttilio Rao ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS))); 67047dd67eSAttilio Rao 68047dd67eSAttilio Rao #define SQ_EXCLUSIVE_QUEUE 0 69047dd67eSAttilio Rao #define SQ_SHARED_QUEUE 1 70047dd67eSAttilio Rao 71047dd67eSAttilio Rao #ifndef INVARIANTS 72047dd67eSAttilio Rao #define _lockmgr_assert(lk, what, file, line) 73047dd67eSAttilio Rao #endif 74ce1c953eSMark Johnston 75047dd67eSAttilio Rao #define TD_SLOCKS_INC(td) ((td)->td_lk_slocks++) 76047dd67eSAttilio Rao #define TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--) 77047dd67eSAttilio Rao 78047dd67eSAttilio Rao #ifndef DEBUG_LOCKS 79047dd67eSAttilio Rao #define STACK_PRINT(lk) 80047dd67eSAttilio Rao #define STACK_SAVE(lk) 81047dd67eSAttilio Rao #define STACK_ZERO(lk) 82047dd67eSAttilio Rao #else 83047dd67eSAttilio Rao #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack) 84047dd67eSAttilio Rao #define STACK_SAVE(lk) stack_save(&(lk)->lk_stack) 85047dd67eSAttilio Rao #define STACK_ZERO(lk) stack_zero(&(lk)->lk_stack) 86047dd67eSAttilio Rao #endif 87047dd67eSAttilio Rao 88047dd67eSAttilio Rao #define LOCK_LOG2(lk, string, arg1, arg2) \ 89047dd67eSAttilio Rao if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \ 90047dd67eSAttilio Rao CTR2(KTR_LOCK, (string), (arg1), (arg2)) 91047dd67eSAttilio Rao #define LOCK_LOG3(lk, string, arg1, arg2, arg3) \ 92047dd67eSAttilio Rao if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \ 93047dd67eSAttilio Rao CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3)) 94047dd67eSAttilio Rao 95e5f94314SAttilio Rao #define GIANT_DECLARE \ 96e5f94314SAttilio Rao int _i = 0; \ 97e5f94314SAttilio Rao WITNESS_SAVE_DECL(Giant) 98e5f94314SAttilio Rao #define GIANT_RESTORE() do { \ 99e5f94314SAttilio Rao if (_i > 0) { \ 100e5f94314SAttilio Rao while (_i--) \ 101e5f94314SAttilio Rao mtx_lock(&Giant); \ 102e5f94314SAttilio Rao WITNESS_RESTORE(&Giant.lock_object, Giant); \ 103e5f94314SAttilio Rao } \ 104e5f94314SAttilio Rao } while (0) 105e5f94314SAttilio Rao #define GIANT_SAVE() do { \ 106e5f94314SAttilio Rao if (mtx_owned(&Giant)) { \ 107e5f94314SAttilio Rao WITNESS_SAVE(&Giant.lock_object, Giant); \ 108e5f94314SAttilio Rao while (mtx_owned(&Giant)) { \ 109e5f94314SAttilio Rao _i++; \ 110e5f94314SAttilio Rao mtx_unlock(&Giant); \ 111e5f94314SAttilio Rao } \ 112e5f94314SAttilio Rao } \ 113e5f94314SAttilio Rao } while (0) 114e5f94314SAttilio Rao 11572ba3c08SKonstantin Belousov #define LK_CAN_SHARE(x, flags) \ 11672ba3c08SKonstantin Belousov (((x) & LK_SHARE) && \ 11772ba3c08SKonstantin Belousov (((x) & (LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) == 0 || \ 11872ba3c08SKonstantin Belousov (curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) || \ 11972ba3c08SKonstantin Belousov (curthread->td_pflags & TDP_DEADLKTREAT))) 120e5f94314SAttilio Rao #define LK_TRYOP(x) \ 121e5f94314SAttilio Rao ((x) & LK_NOWAIT) 122e5f94314SAttilio Rao 123e5f94314SAttilio Rao #define LK_CAN_WITNESS(x) \ 124e5f94314SAttilio Rao (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x)) 125e5f94314SAttilio Rao #define LK_TRYWIT(x) \ 126e5f94314SAttilio Rao (LK_TRYOP(x) ? LOP_TRYLOCK : 0) 127047dd67eSAttilio Rao 128651175c9SAttilio Rao #define LK_CAN_ADAPT(lk, f) \ 129651175c9SAttilio Rao (((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 && \ 130651175c9SAttilio Rao ((f) & LK_SLEEPFAIL) == 0) 131651175c9SAttilio Rao 132047dd67eSAttilio Rao #define lockmgr_disowned(lk) \ 133047dd67eSAttilio Rao (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC) 134047dd67eSAttilio Rao 135*10391db5SMateusz Guzik #define lockmgr_xlocked_v(v) \ 136*10391db5SMateusz Guzik (((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread) 137*10391db5SMateusz Guzik 138*10391db5SMateusz Guzik #define lockmgr_xlocked(lk) lockmgr_xlocked_v((lk)->lk_lock) 139047dd67eSAttilio Rao 140d576deedSPawel Jakub Dawidek static void assert_lockmgr(const struct lock_object *lock, int how); 141047dd67eSAttilio Rao #ifdef DDB 142d576deedSPawel Jakub Dawidek static void db_show_lockmgr(const struct lock_object *lock); 143be6847d7SJohn Baldwin #endif 1447faf4d90SDavide Italiano static void lock_lockmgr(struct lock_object *lock, uintptr_t how); 145a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 146d576deedSPawel Jakub Dawidek static int owner_lockmgr(const struct lock_object *lock, 147d576deedSPawel Jakub Dawidek struct thread **owner); 148a5aedd68SStacey Son #endif 1497faf4d90SDavide Italiano static uintptr_t unlock_lockmgr(struct lock_object *lock); 15061bd5e21SKip Macy 15161bd5e21SKip Macy struct lock_class lock_class_lockmgr = { 1523ff6d229SJohn Baldwin .lc_name = "lockmgr", 153047dd67eSAttilio Rao .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE, 154f9721b43SAttilio Rao .lc_assert = assert_lockmgr, 15561bd5e21SKip Macy #ifdef DDB 1566e21afd4SJohn Baldwin .lc_ddb_show = db_show_lockmgr, 15761bd5e21SKip Macy #endif 1586e21afd4SJohn Baldwin .lc_lock = lock_lockmgr, 159a5aedd68SStacey Son .lc_unlock = unlock_lockmgr, 160a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 161a5aedd68SStacey Son .lc_owner = owner_lockmgr, 162a5aedd68SStacey Son #endif 16361bd5e21SKip Macy }; 16461bd5e21SKip Macy 1651c6987ebSMateusz Guzik struct lockmgr_wait { 1661c6987ebSMateusz Guzik const char *iwmesg; 1671c6987ebSMateusz Guzik int ipri; 1681c6987ebSMateusz Guzik int itimo; 1691c6987ebSMateusz Guzik }; 1701c6987ebSMateusz Guzik 171c4a48867SMateusz Guzik static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp, 172c4a48867SMateusz Guzik int flags); 1731c6987ebSMateusz Guzik static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp); 1741c6987ebSMateusz Guzik 1751c6987ebSMateusz Guzik static void 1761c6987ebSMateusz Guzik lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper) 1771c6987ebSMateusz Guzik { 1781c6987ebSMateusz Guzik struct lock_class *class; 1791c6987ebSMateusz Guzik 1801c6987ebSMateusz Guzik if (flags & LK_INTERLOCK) { 1811c6987ebSMateusz Guzik class = LOCK_CLASS(ilk); 1821c6987ebSMateusz Guzik class->lc_unlock(ilk); 1831c6987ebSMateusz Guzik } 1841c6987ebSMateusz Guzik 1851c6987ebSMateusz Guzik if (__predict_false(wakeup_swapper)) 1861c6987ebSMateusz Guzik kick_proc0(); 1871c6987ebSMateusz Guzik } 188c4a48867SMateusz Guzik 189c4a48867SMateusz Guzik static void 190c4a48867SMateusz Guzik lockmgr_note_shared_acquire(struct lock *lk, int contested, 191c4a48867SMateusz Guzik uint64_t waittime, const char *file, int line, int flags) 192c4a48867SMateusz Guzik { 193c4a48867SMateusz Guzik 194c4a48867SMateusz Guzik lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime, 195c4a48867SMateusz Guzik file, line); 196c4a48867SMateusz Guzik LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line); 197c4a48867SMateusz Guzik WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line); 198c4a48867SMateusz Guzik TD_LOCKS_INC(curthread); 199c4a48867SMateusz Guzik TD_SLOCKS_INC(curthread); 200c4a48867SMateusz Guzik STACK_SAVE(lk); 201c4a48867SMateusz Guzik } 202c4a48867SMateusz Guzik 203c4a48867SMateusz Guzik static void 204c4a48867SMateusz Guzik lockmgr_note_shared_release(struct lock *lk, const char *file, int line) 205c4a48867SMateusz Guzik { 206c4a48867SMateusz Guzik 207c4a48867SMateusz Guzik lock_profile_release_lock(&lk->lock_object); 208c4a48867SMateusz Guzik WITNESS_UNLOCK(&lk->lock_object, 0, file, line); 209c4a48867SMateusz Guzik LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line); 210c4a48867SMateusz Guzik TD_LOCKS_DEC(curthread); 211c4a48867SMateusz Guzik TD_SLOCKS_DEC(curthread); 212c4a48867SMateusz Guzik } 213c4a48867SMateusz Guzik 214c4a48867SMateusz Guzik static void 215c4a48867SMateusz Guzik lockmgr_note_exclusive_acquire(struct lock *lk, int contested, 216c4a48867SMateusz Guzik uint64_t waittime, const char *file, int line, int flags) 217c4a48867SMateusz Guzik { 218c4a48867SMateusz Guzik 219c4a48867SMateusz Guzik lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime, 220c4a48867SMateusz Guzik file, line); 221c4a48867SMateusz Guzik LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line); 222c4a48867SMateusz Guzik WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file, 223c4a48867SMateusz Guzik line); 224c4a48867SMateusz Guzik TD_LOCKS_INC(curthread); 225c4a48867SMateusz Guzik STACK_SAVE(lk); 226c4a48867SMateusz Guzik } 227c4a48867SMateusz Guzik 228c4a48867SMateusz Guzik static void 229c4a48867SMateusz Guzik lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line) 230c4a48867SMateusz Guzik { 231c4a48867SMateusz Guzik 232c4a48867SMateusz Guzik lock_profile_release_lock(&lk->lock_object); 233c4a48867SMateusz Guzik LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, 234c4a48867SMateusz Guzik line); 235c4a48867SMateusz Guzik WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); 236c4a48867SMateusz Guzik TD_LOCKS_DEC(curthread); 237c4a48867SMateusz Guzik } 238c4a48867SMateusz Guzik 239047dd67eSAttilio Rao static __inline struct thread * 240d576deedSPawel Jakub Dawidek lockmgr_xholder(const struct lock *lk) 241047dd67eSAttilio Rao { 242047dd67eSAttilio Rao uintptr_t x; 243047dd67eSAttilio Rao 244047dd67eSAttilio Rao x = lk->lk_lock; 245047dd67eSAttilio Rao return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x)); 246047dd67eSAttilio Rao } 24784887fa3SAttilio Rao 24853bf4bb2SPeter Wemm /* 249047dd67eSAttilio Rao * It assumes sleepq_lock held and returns with this one unheld. 250047dd67eSAttilio Rao * It also assumes the generic interlock is sane and previously checked. 251047dd67eSAttilio Rao * If LK_INTERLOCK is specified the interlock is not reacquired after the 252047dd67eSAttilio Rao * sleep. 25353bf4bb2SPeter Wemm */ 254047dd67eSAttilio Rao static __inline int 255047dd67eSAttilio Rao sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk, 256047dd67eSAttilio Rao const char *wmesg, int pri, int timo, int queue) 257047dd67eSAttilio Rao { 258e5f94314SAttilio Rao GIANT_DECLARE; 259047dd67eSAttilio Rao struct lock_class *class; 260047dd67eSAttilio Rao int catch, error; 26153bf4bb2SPeter Wemm 262047dd67eSAttilio Rao class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL; 2635047a8fdSAttilio Rao catch = pri & PCATCH; 264047dd67eSAttilio Rao pri &= PRIMASK; 265047dd67eSAttilio Rao error = 0; 266047dd67eSAttilio Rao 267047dd67eSAttilio Rao LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk, 268047dd67eSAttilio Rao (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared"); 269047dd67eSAttilio Rao 270047dd67eSAttilio Rao if (flags & LK_INTERLOCK) 271047dd67eSAttilio Rao class->lc_unlock(ilk); 2722028867dSAttilio Rao if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0) 2732028867dSAttilio Rao lk->lk_exslpfail++; 274e5f94314SAttilio Rao GIANT_SAVE(); 275047dd67eSAttilio Rao sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ? 276047dd67eSAttilio Rao SLEEPQ_INTERRUPTIBLE : 0), queue); 277a115fb62SHans Petter Selasky if ((flags & LK_TIMELOCK) && timo) 278047dd67eSAttilio Rao sleepq_set_timeout(&lk->lock_object, timo); 279a115fb62SHans Petter Selasky 280047dd67eSAttilio Rao /* 281047dd67eSAttilio Rao * Decisional switch for real sleeping. 282047dd67eSAttilio Rao */ 283047dd67eSAttilio Rao if ((flags & LK_TIMELOCK) && timo && catch) 284047dd67eSAttilio Rao error = sleepq_timedwait_sig(&lk->lock_object, pri); 285047dd67eSAttilio Rao else if ((flags & LK_TIMELOCK) && timo) 286047dd67eSAttilio Rao error = sleepq_timedwait(&lk->lock_object, pri); 287047dd67eSAttilio Rao else if (catch) 288047dd67eSAttilio Rao error = sleepq_wait_sig(&lk->lock_object, pri); 289047dd67eSAttilio Rao else 290047dd67eSAttilio Rao sleepq_wait(&lk->lock_object, pri); 291e5f94314SAttilio Rao GIANT_RESTORE(); 292047dd67eSAttilio Rao if ((flags & LK_SLEEPFAIL) && error == 0) 293047dd67eSAttilio Rao error = ENOLCK; 294047dd67eSAttilio Rao 295047dd67eSAttilio Rao return (error); 296047dd67eSAttilio Rao } 297047dd67eSAttilio Rao 298da7bbd2cSJohn Baldwin static __inline int 299047dd67eSAttilio Rao wakeupshlk(struct lock *lk, const char *file, int line) 300047dd67eSAttilio Rao { 3010ad122a9SMateusz Guzik uintptr_t v, x, orig_x; 3022028867dSAttilio Rao u_int realexslp; 303da7bbd2cSJohn Baldwin int queue, wakeup_swapper; 304047dd67eSAttilio Rao 305da7bbd2cSJohn Baldwin wakeup_swapper = 0; 306047dd67eSAttilio Rao for (;;) { 307047dd67eSAttilio Rao x = lk->lk_lock; 3081c6987ebSMateusz Guzik if (lockmgr_sunlock_try(lk, &x)) 309047dd67eSAttilio Rao break; 310047dd67eSAttilio Rao 311047dd67eSAttilio Rao /* 312047dd67eSAttilio Rao * We should have a sharer with waiters, so enter the hard 313047dd67eSAttilio Rao * path in order to handle wakeups correctly. 314047dd67eSAttilio Rao */ 315047dd67eSAttilio Rao sleepq_lock(&lk->lock_object); 3160ad122a9SMateusz Guzik orig_x = lk->lk_lock; 3170ad122a9SMateusz Guzik retry_sleepq: 3180ad122a9SMateusz Guzik x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); 319047dd67eSAttilio Rao v = LK_UNLOCKED; 320047dd67eSAttilio Rao 321047dd67eSAttilio Rao /* 322047dd67eSAttilio Rao * If the lock has exclusive waiters, give them preference in 323047dd67eSAttilio Rao * order to avoid deadlock with shared runners up. 3242028867dSAttilio Rao * If interruptible sleeps left the exclusive queue empty 3252028867dSAttilio Rao * avoid a starvation for the threads sleeping on the shared 3262028867dSAttilio Rao * queue by giving them precedence and cleaning up the 3272028867dSAttilio Rao * exclusive waiters bit anyway. 328c636ba83SAttilio Rao * Please note that lk_exslpfail count may be lying about 329c636ba83SAttilio Rao * the real number of waiters with the LK_SLEEPFAIL flag on 330e3043798SPedro F. Giffuni * because they may be used in conjunction with interruptible 331aab9c8c2SAttilio Rao * sleeps so lk_exslpfail might be considered an 'upper limit' 332aab9c8c2SAttilio Rao * bound, including the edge cases. 333047dd67eSAttilio Rao */ 3342028867dSAttilio Rao realexslp = sleepq_sleepcnt(&lk->lock_object, 3352028867dSAttilio Rao SQ_EXCLUSIVE_QUEUE); 3362028867dSAttilio Rao if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) { 3372028867dSAttilio Rao if (lk->lk_exslpfail < realexslp) { 3382028867dSAttilio Rao lk->lk_exslpfail = 0; 339047dd67eSAttilio Rao queue = SQ_EXCLUSIVE_QUEUE; 340047dd67eSAttilio Rao v |= (x & LK_SHARED_WAITERS); 341047dd67eSAttilio Rao } else { 3422028867dSAttilio Rao lk->lk_exslpfail = 0; 3432028867dSAttilio Rao LOCK_LOG2(lk, 3442028867dSAttilio Rao "%s: %p has only LK_SLEEPFAIL sleepers", 3452028867dSAttilio Rao __func__, lk); 3462028867dSAttilio Rao LOCK_LOG2(lk, 3472028867dSAttilio Rao "%s: %p waking up threads on the exclusive queue", 3482028867dSAttilio Rao __func__, lk); 3492028867dSAttilio Rao wakeup_swapper = 3502028867dSAttilio Rao sleepq_broadcast(&lk->lock_object, 3512028867dSAttilio Rao SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE); 3522028867dSAttilio Rao queue = SQ_SHARED_QUEUE; 3532028867dSAttilio Rao } 3542028867dSAttilio Rao 3552028867dSAttilio Rao } else { 3569dbf7a62SAttilio Rao 3579dbf7a62SAttilio Rao /* 3589dbf7a62SAttilio Rao * Exclusive waiters sleeping with LK_SLEEPFAIL on 3599dbf7a62SAttilio Rao * and using interruptible sleeps/timeout may have 3609dbf7a62SAttilio Rao * left spourious lk_exslpfail counts on, so clean 3619dbf7a62SAttilio Rao * it up anyway. 3629dbf7a62SAttilio Rao */ 3639dbf7a62SAttilio Rao lk->lk_exslpfail = 0; 364047dd67eSAttilio Rao queue = SQ_SHARED_QUEUE; 365047dd67eSAttilio Rao } 366047dd67eSAttilio Rao 3670ad122a9SMateusz Guzik if (lockmgr_sunlock_try(lk, &orig_x)) { 368047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 3690ad122a9SMateusz Guzik break; 3700ad122a9SMateusz Guzik } 3710ad122a9SMateusz Guzik 3720ad122a9SMateusz Guzik x |= LK_SHARERS_LOCK(1); 3730ad122a9SMateusz Guzik if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) { 3740ad122a9SMateusz Guzik orig_x = x; 3750ad122a9SMateusz Guzik goto retry_sleepq; 376047dd67eSAttilio Rao } 377047dd67eSAttilio Rao LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", 378047dd67eSAttilio Rao __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : 379047dd67eSAttilio Rao "exclusive"); 3802028867dSAttilio Rao wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 381da7bbd2cSJohn Baldwin 0, queue); 382047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 383047dd67eSAttilio Rao break; 384047dd67eSAttilio Rao } 385047dd67eSAttilio Rao 386c4a48867SMateusz Guzik lockmgr_note_shared_release(lk, file, line); 387da7bbd2cSJohn Baldwin return (wakeup_swapper); 388047dd67eSAttilio Rao } 389047dd67eSAttilio Rao 390047dd67eSAttilio Rao static void 391d576deedSPawel Jakub Dawidek assert_lockmgr(const struct lock_object *lock, int what) 392f9721b43SAttilio Rao { 393f9721b43SAttilio Rao 394f9721b43SAttilio Rao panic("lockmgr locks do not support assertions"); 395f9721b43SAttilio Rao } 396f9721b43SAttilio Rao 397047dd67eSAttilio Rao static void 3987faf4d90SDavide Italiano lock_lockmgr(struct lock_object *lock, uintptr_t how) 3996e21afd4SJohn Baldwin { 4006e21afd4SJohn Baldwin 4016e21afd4SJohn Baldwin panic("lockmgr locks do not support sleep interlocking"); 4026e21afd4SJohn Baldwin } 4036e21afd4SJohn Baldwin 4047faf4d90SDavide Italiano static uintptr_t 4056e21afd4SJohn Baldwin unlock_lockmgr(struct lock_object *lock) 4066e21afd4SJohn Baldwin { 4076e21afd4SJohn Baldwin 4086e21afd4SJohn Baldwin panic("lockmgr locks do not support sleep interlocking"); 4096e21afd4SJohn Baldwin } 4106e21afd4SJohn Baldwin 411a5aedd68SStacey Son #ifdef KDTRACE_HOOKS 412a5aedd68SStacey Son static int 413d576deedSPawel Jakub Dawidek owner_lockmgr(const struct lock_object *lock, struct thread **owner) 414a5aedd68SStacey Son { 415a5aedd68SStacey Son 416a5aedd68SStacey Son panic("lockmgr locks do not support owner inquiring"); 417a5aedd68SStacey Son } 418a5aedd68SStacey Son #endif 419a5aedd68SStacey Son 42099448ed1SJohn Dyson void 421047dd67eSAttilio Rao lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags) 42299448ed1SJohn Dyson { 4236efc8a16SAttilio Rao int iflags; 4246efc8a16SAttilio Rao 425047dd67eSAttilio Rao MPASS((flags & ~LK_INIT_MASK) == 0); 426353998acSAttilio Rao ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock, 427353998acSAttilio Rao ("%s: lockmgr not aligned for %s: %p", __func__, wmesg, 428353998acSAttilio Rao &lk->lk_lock)); 42999448ed1SJohn Dyson 430f0830182SAttilio Rao iflags = LO_SLEEPABLE | LO_UPGRADABLE; 431f0830182SAttilio Rao if (flags & LK_CANRECURSE) 432f0830182SAttilio Rao iflags |= LO_RECURSABLE; 433047dd67eSAttilio Rao if ((flags & LK_NODUP) == 0) 4346efc8a16SAttilio Rao iflags |= LO_DUPOK; 4357fbfba7bSAttilio Rao if (flags & LK_NOPROFILE) 4367fbfba7bSAttilio Rao iflags |= LO_NOPROFILE; 437047dd67eSAttilio Rao if ((flags & LK_NOWITNESS) == 0) 4386efc8a16SAttilio Rao iflags |= LO_WITNESS; 4397fbfba7bSAttilio Rao if (flags & LK_QUIET) 4407fbfba7bSAttilio Rao iflags |= LO_QUIET; 441e63091eaSMarcel Moolenaar if (flags & LK_IS_VNODE) 442e63091eaSMarcel Moolenaar iflags |= LO_IS_VNODE; 443651175c9SAttilio Rao iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE); 444047dd67eSAttilio Rao 445b5fb43e5SJohn Baldwin lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags); 446047dd67eSAttilio Rao lk->lk_lock = LK_UNLOCKED; 447047dd67eSAttilio Rao lk->lk_recurse = 0; 4482028867dSAttilio Rao lk->lk_exslpfail = 0; 449047dd67eSAttilio Rao lk->lk_timo = timo; 450047dd67eSAttilio Rao lk->lk_pri = pri; 451047dd67eSAttilio Rao STACK_ZERO(lk); 45299448ed1SJohn Dyson } 45399448ed1SJohn Dyson 4543634d5b2SJohn Baldwin /* 4553634d5b2SJohn Baldwin * XXX: Gross hacks to manipulate external lock flags after 4563634d5b2SJohn Baldwin * initialization. Used for certain vnode and buf locks. 4573634d5b2SJohn Baldwin */ 4583634d5b2SJohn Baldwin void 4593634d5b2SJohn Baldwin lockallowshare(struct lock *lk) 4603634d5b2SJohn Baldwin { 4613634d5b2SJohn Baldwin 4623634d5b2SJohn Baldwin lockmgr_assert(lk, KA_XLOCKED); 4633634d5b2SJohn Baldwin lk->lock_object.lo_flags &= ~LK_NOSHARE; 4643634d5b2SJohn Baldwin } 4653634d5b2SJohn Baldwin 4663634d5b2SJohn Baldwin void 467575e02d9SKonstantin Belousov lockdisableshare(struct lock *lk) 468575e02d9SKonstantin Belousov { 469575e02d9SKonstantin Belousov 470575e02d9SKonstantin Belousov lockmgr_assert(lk, KA_XLOCKED); 471575e02d9SKonstantin Belousov lk->lock_object.lo_flags |= LK_NOSHARE; 472575e02d9SKonstantin Belousov } 473575e02d9SKonstantin Belousov 474575e02d9SKonstantin Belousov void 4753634d5b2SJohn Baldwin lockallowrecurse(struct lock *lk) 4763634d5b2SJohn Baldwin { 4773634d5b2SJohn Baldwin 4783634d5b2SJohn Baldwin lockmgr_assert(lk, KA_XLOCKED); 4793634d5b2SJohn Baldwin lk->lock_object.lo_flags |= LO_RECURSABLE; 4803634d5b2SJohn Baldwin } 4813634d5b2SJohn Baldwin 4823634d5b2SJohn Baldwin void 4833634d5b2SJohn Baldwin lockdisablerecurse(struct lock *lk) 4843634d5b2SJohn Baldwin { 4853634d5b2SJohn Baldwin 4863634d5b2SJohn Baldwin lockmgr_assert(lk, KA_XLOCKED); 4873634d5b2SJohn Baldwin lk->lock_object.lo_flags &= ~LO_RECURSABLE; 4883634d5b2SJohn Baldwin } 4893634d5b2SJohn Baldwin 490a18b1f1dSJason Evans void 491047dd67eSAttilio Rao lockdestroy(struct lock *lk) 492a18b1f1dSJason Evans { 493c91fcee7SJohn Baldwin 494047dd67eSAttilio Rao KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held")); 495047dd67eSAttilio Rao KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed")); 4962028867dSAttilio Rao KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters")); 497047dd67eSAttilio Rao lock_destroy(&lk->lock_object); 498047dd67eSAttilio Rao } 499047dd67eSAttilio Rao 500c4a48867SMateusz Guzik static bool __always_inline 501c4a48867SMateusz Guzik lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags) 502c4a48867SMateusz Guzik { 503c4a48867SMateusz Guzik 504c4a48867SMateusz Guzik /* 505c4a48867SMateusz Guzik * If no other thread has an exclusive lock, or 506c4a48867SMateusz Guzik * no exclusive waiter is present, bump the count of 507c4a48867SMateusz Guzik * sharers. Since we have to preserve the state of 508c4a48867SMateusz Guzik * waiters, if we fail to acquire the shared lock 509c4a48867SMateusz Guzik * loop back and retry. 510c4a48867SMateusz Guzik */ 511c4a48867SMateusz Guzik *xp = lk->lk_lock; 512c4a48867SMateusz Guzik while (LK_CAN_SHARE(*xp, flags)) { 513c4a48867SMateusz Guzik if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp, 514c4a48867SMateusz Guzik *xp + LK_ONE_SHARER)) { 515c4a48867SMateusz Guzik return (true); 516c4a48867SMateusz Guzik } 517c4a48867SMateusz Guzik } 518c4a48867SMateusz Guzik return (false); 519c4a48867SMateusz Guzik } 520c4a48867SMateusz Guzik 521c4a48867SMateusz Guzik static bool __always_inline 5221c6987ebSMateusz Guzik lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp) 523c4a48867SMateusz Guzik { 524c4a48867SMateusz Guzik 525c4a48867SMateusz Guzik for (;;) { 526c4a48867SMateusz Guzik /* 527c4a48867SMateusz Guzik * If there is more than one shared lock held, just drop one 528c4a48867SMateusz Guzik * and return. 529c4a48867SMateusz Guzik */ 5301c6987ebSMateusz Guzik if (LK_SHARERS(*xp) > 1) { 5311c6987ebSMateusz Guzik if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp, 5321c6987ebSMateusz Guzik *xp - LK_ONE_SHARER)) 533c4a48867SMateusz Guzik return (true); 534c4a48867SMateusz Guzik continue; 535c4a48867SMateusz Guzik } 536c4a48867SMateusz Guzik 537c4a48867SMateusz Guzik /* 538c4a48867SMateusz Guzik * If there are not waiters on the exclusive queue, drop the 539c4a48867SMateusz Guzik * lock quickly. 540c4a48867SMateusz Guzik */ 5411c6987ebSMateusz Guzik if ((*xp & LK_ALL_WAITERS) == 0) { 5421c6987ebSMateusz Guzik MPASS((*xp & ~LK_EXCLUSIVE_SPINNERS) == 543c4a48867SMateusz Guzik LK_SHARERS_LOCK(1)); 5441c6987ebSMateusz Guzik if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp, 545c4a48867SMateusz Guzik LK_UNLOCKED)) 546c4a48867SMateusz Guzik return (true); 547c4a48867SMateusz Guzik continue; 548c4a48867SMateusz Guzik } 549c4a48867SMateusz Guzik break; 550c4a48867SMateusz Guzik } 551c4a48867SMateusz Guzik return (false); 552c4a48867SMateusz Guzik } 553c4a48867SMateusz Guzik 5541c6987ebSMateusz Guzik static __noinline int 5551c6987ebSMateusz Guzik lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, 5561c6987ebSMateusz Guzik const char *file, int line, struct lockmgr_wait *lwa) 557c4a48867SMateusz Guzik { 5581c6987ebSMateusz Guzik uintptr_t tid, x; 5591c6987ebSMateusz Guzik int error = 0; 560047dd67eSAttilio Rao const char *iwmesg; 5611c6987ebSMateusz Guzik int ipri, itimo; 5621c6987ebSMateusz Guzik 5631723a064SJeff Roberson #ifdef LOCK_PROFILING 5641723a064SJeff Roberson uint64_t waittime = 0; 5651723a064SJeff Roberson int contested = 0; 5661723a064SJeff Roberson #endif 567047dd67eSAttilio Rao 5681c6987ebSMateusz Guzik if (__predict_false(panicstr != NULL)) 5691c6987ebSMateusz Guzik goto out; 5701c6987ebSMateusz Guzik 571047dd67eSAttilio Rao tid = (uintptr_t)curthread; 572047dd67eSAttilio Rao 573e5f94314SAttilio Rao if (LK_CAN_WITNESS(flags)) 574e5f94314SAttilio Rao WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, 57524150d37SJohn Baldwin file, line, flags & LK_INTERLOCK ? ilk : NULL); 576047dd67eSAttilio Rao for (;;) { 577c4a48867SMateusz Guzik if (lockmgr_slock_try(lk, &x, flags)) 578047dd67eSAttilio Rao break; 579f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 580f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 581f5f9340bSFabien Thomas #endif 582047dd67eSAttilio Rao lock_profile_obtain_lock_failed(&lk->lock_object, 583047dd67eSAttilio Rao &contested, &waittime); 584047dd67eSAttilio Rao 585047dd67eSAttilio Rao /* 58696f1567fSKonstantin Belousov * If the lock is already held by curthread in 587047dd67eSAttilio Rao * exclusive way avoid a deadlock. 588047dd67eSAttilio Rao */ 589047dd67eSAttilio Rao if (LK_HOLDER(x) == tid) { 590047dd67eSAttilio Rao LOCK_LOG2(lk, 59196f1567fSKonstantin Belousov "%s: %p already held in exclusive mode", 592047dd67eSAttilio Rao __func__, lk); 593047dd67eSAttilio Rao error = EDEADLK; 594047dd67eSAttilio Rao break; 595a18b1f1dSJason Evans } 596a18b1f1dSJason Evans 597a18b1f1dSJason Evans /* 598047dd67eSAttilio Rao * If the lock is expected to not sleep just give up 599047dd67eSAttilio Rao * and return. 600d7a7e179SAttilio Rao */ 601047dd67eSAttilio Rao if (LK_TRYOP(flags)) { 602047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p fails the try operation", 603047dd67eSAttilio Rao __func__, lk); 604047dd67eSAttilio Rao error = EBUSY; 605047dd67eSAttilio Rao break; 606047dd67eSAttilio Rao } 607047dd67eSAttilio Rao 608047dd67eSAttilio Rao /* 609047dd67eSAttilio Rao * Acquire the sleepqueue chain lock because we 610047dd67eSAttilio Rao * probabilly will need to manipulate waiters flags. 611047dd67eSAttilio Rao */ 612047dd67eSAttilio Rao sleepq_lock(&lk->lock_object); 613047dd67eSAttilio Rao x = lk->lk_lock; 6140ad122a9SMateusz Guzik retry_sleepq: 615047dd67eSAttilio Rao 616047dd67eSAttilio Rao /* 617047dd67eSAttilio Rao * if the lock can be acquired in shared mode, try 618047dd67eSAttilio Rao * again. 619047dd67eSAttilio Rao */ 62072ba3c08SKonstantin Belousov if (LK_CAN_SHARE(x, flags)) { 621047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 622047dd67eSAttilio Rao continue; 623047dd67eSAttilio Rao } 624047dd67eSAttilio Rao 625047dd67eSAttilio Rao /* 626047dd67eSAttilio Rao * Try to set the LK_SHARED_WAITERS flag. If we fail, 627047dd67eSAttilio Rao * loop back and retry. 628047dd67eSAttilio Rao */ 629047dd67eSAttilio Rao if ((x & LK_SHARED_WAITERS) == 0) { 6300ad122a9SMateusz Guzik if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x, 631047dd67eSAttilio Rao x | LK_SHARED_WAITERS)) { 6320ad122a9SMateusz Guzik goto retry_sleepq; 633047dd67eSAttilio Rao } 634047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p set shared waiters flag", 635047dd67eSAttilio Rao __func__, lk); 636047dd67eSAttilio Rao } 637047dd67eSAttilio Rao 6381c6987ebSMateusz Guzik if (lwa == NULL) { 6391c6987ebSMateusz Guzik iwmesg = lk->lock_object.lo_name; 6401c6987ebSMateusz Guzik ipri = lk->lk_pri; 6411c6987ebSMateusz Guzik itimo = lk->lk_timo; 6421c6987ebSMateusz Guzik } else { 6431c6987ebSMateusz Guzik iwmesg = lwa->iwmesg; 6441c6987ebSMateusz Guzik ipri = lwa->ipri; 6451c6987ebSMateusz Guzik itimo = lwa->itimo; 6461c6987ebSMateusz Guzik } 6471c6987ebSMateusz Guzik 648047dd67eSAttilio Rao /* 649047dd67eSAttilio Rao * As far as we have been unable to acquire the 650047dd67eSAttilio Rao * shared lock and the shared waiters flag is set, 651047dd67eSAttilio Rao * we will sleep. 652047dd67eSAttilio Rao */ 653047dd67eSAttilio Rao error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, 654047dd67eSAttilio Rao SQ_SHARED_QUEUE); 655047dd67eSAttilio Rao flags &= ~LK_INTERLOCK; 656047dd67eSAttilio Rao if (error) { 657047dd67eSAttilio Rao LOCK_LOG3(lk, 658047dd67eSAttilio Rao "%s: interrupted sleep for %p with %d", 659047dd67eSAttilio Rao __func__, lk, error); 660047dd67eSAttilio Rao break; 661047dd67eSAttilio Rao } 662047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 663047dd67eSAttilio Rao __func__, lk); 664047dd67eSAttilio Rao } 665047dd67eSAttilio Rao if (error == 0) { 666c4a48867SMateusz Guzik #ifdef LOCK_PROFILING 667c4a48867SMateusz Guzik lockmgr_note_shared_acquire(lk, contested, waittime, 668c4a48867SMateusz Guzik file, line, flags); 669c4a48867SMateusz Guzik #else 670c4a48867SMateusz Guzik lockmgr_note_shared_acquire(lk, 0, 0, file, line, 671c4a48867SMateusz Guzik flags); 672c4a48867SMateusz Guzik #endif 673047dd67eSAttilio Rao } 674047dd67eSAttilio Rao 6751c6987ebSMateusz Guzik out: 6761c6987ebSMateusz Guzik lockmgr_exit(flags, ilk, 0); 6771c6987ebSMateusz Guzik return (error); 678047dd67eSAttilio Rao } 679047dd67eSAttilio Rao 6801c6987ebSMateusz Guzik static __noinline int 6811c6987ebSMateusz Guzik lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, 6821c6987ebSMateusz Guzik const char *file, int line, struct lockmgr_wait *lwa) 6831c6987ebSMateusz Guzik { 6841c6987ebSMateusz Guzik struct lock_class *class; 6851c6987ebSMateusz Guzik uintptr_t tid, x, v; 6861c6987ebSMateusz Guzik int error = 0; 6871c6987ebSMateusz Guzik const char *iwmesg; 6881c6987ebSMateusz Guzik int ipri, itimo; 6897c6fe803SKonstantin Belousov 6901c6987ebSMateusz Guzik #ifdef LOCK_PROFILING 6911c6987ebSMateusz Guzik uint64_t waittime = 0; 6921c6987ebSMateusz Guzik int contested = 0; 6931c6987ebSMateusz Guzik #endif 694047dd67eSAttilio Rao 6951c6987ebSMateusz Guzik if (__predict_false(panicstr != NULL)) 6961c6987ebSMateusz Guzik goto out; 6971c6987ebSMateusz Guzik 6981c6987ebSMateusz Guzik tid = (uintptr_t)curthread; 6991c6987ebSMateusz Guzik 700e5f94314SAttilio Rao if (LK_CAN_WITNESS(flags)) 701e5f94314SAttilio Rao WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER | 70224150d37SJohn Baldwin LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ? 70324150d37SJohn Baldwin ilk : NULL); 704047dd67eSAttilio Rao 705047dd67eSAttilio Rao /* 70696f1567fSKonstantin Belousov * If curthread already holds the lock and this one is 707047dd67eSAttilio Rao * allowed to recurse, simply recurse on it. 708047dd67eSAttilio Rao */ 709047dd67eSAttilio Rao if (lockmgr_xlocked(lk)) { 710047dd67eSAttilio Rao if ((flags & LK_CANRECURSE) == 0 && 711f0830182SAttilio Rao (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) { 712047dd67eSAttilio Rao /* 713047dd67eSAttilio Rao * If the lock is expected to not panic just 714047dd67eSAttilio Rao * give up and return. 715047dd67eSAttilio Rao */ 716047dd67eSAttilio Rao if (LK_TRYOP(flags)) { 717047dd67eSAttilio Rao LOCK_LOG2(lk, 718047dd67eSAttilio Rao "%s: %p fails the try operation", 719047dd67eSAttilio Rao __func__, lk); 720047dd67eSAttilio Rao error = EBUSY; 7211c6987ebSMateusz Guzik goto out; 722047dd67eSAttilio Rao } 7231c6987ebSMateusz Guzik if (flags & LK_INTERLOCK) { 7241c6987ebSMateusz Guzik class = LOCK_CLASS(ilk); 725047dd67eSAttilio Rao class->lc_unlock(ilk); 7261c6987ebSMateusz Guzik } 72783fc34eaSGleb Smirnoff panic("%s: recursing on non recursive lockmgr %p " 72883fc34eaSGleb Smirnoff "@ %s:%d\n", __func__, lk, file, line); 729047dd67eSAttilio Rao } 730047dd67eSAttilio Rao lk->lk_recurse++; 731047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p recursing", __func__, lk); 732047dd67eSAttilio Rao LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, 733047dd67eSAttilio Rao lk->lk_recurse, file, line); 734e5f94314SAttilio Rao WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | 735e5f94314SAttilio Rao LK_TRYWIT(flags), file, line); 736047dd67eSAttilio Rao TD_LOCKS_INC(curthread); 7371c6987ebSMateusz Guzik goto out; 738047dd67eSAttilio Rao } 739047dd67eSAttilio Rao 740fc4f686dSMateusz Guzik for (;;) { 741fc4f686dSMateusz Guzik if (lk->lk_lock == LK_UNLOCKED && 742fc4f686dSMateusz Guzik atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) 743fc4f686dSMateusz Guzik break; 744f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 745f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 746f5f9340bSFabien Thomas #endif 747047dd67eSAttilio Rao lock_profile_obtain_lock_failed(&lk->lock_object, 748047dd67eSAttilio Rao &contested, &waittime); 749047dd67eSAttilio Rao 750047dd67eSAttilio Rao /* 751047dd67eSAttilio Rao * If the lock is expected to not sleep just give up 752047dd67eSAttilio Rao * and return. 753047dd67eSAttilio Rao */ 754047dd67eSAttilio Rao if (LK_TRYOP(flags)) { 755047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p fails the try operation", 756047dd67eSAttilio Rao __func__, lk); 757047dd67eSAttilio Rao error = EBUSY; 758047dd67eSAttilio Rao break; 759047dd67eSAttilio Rao } 760047dd67eSAttilio Rao 761047dd67eSAttilio Rao /* 762047dd67eSAttilio Rao * Acquire the sleepqueue chain lock because we 763047dd67eSAttilio Rao * probabilly will need to manipulate waiters flags. 764047dd67eSAttilio Rao */ 765047dd67eSAttilio Rao sleepq_lock(&lk->lock_object); 766047dd67eSAttilio Rao x = lk->lk_lock; 7670ad122a9SMateusz Guzik retry_sleepq: 768047dd67eSAttilio Rao 769047dd67eSAttilio Rao /* 770047dd67eSAttilio Rao * if the lock has been released while we spun on 771047dd67eSAttilio Rao * the sleepqueue chain lock just try again. 772047dd67eSAttilio Rao */ 773047dd67eSAttilio Rao if (x == LK_UNLOCKED) { 774047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 775047dd67eSAttilio Rao continue; 776047dd67eSAttilio Rao } 777047dd67eSAttilio Rao 778047dd67eSAttilio Rao /* 779047dd67eSAttilio Rao * The lock can be in the state where there is a 780047dd67eSAttilio Rao * pending queue of waiters, but still no owner. 781047dd67eSAttilio Rao * This happens when the lock is contested and an 782047dd67eSAttilio Rao * owner is going to claim the lock. 783047dd67eSAttilio Rao * If curthread is the one successfully acquiring it 784047dd67eSAttilio Rao * claim lock ownership and return, preserving waiters 785047dd67eSAttilio Rao * flags. 786047dd67eSAttilio Rao */ 787651175c9SAttilio Rao v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); 788651175c9SAttilio Rao if ((x & ~v) == LK_UNLOCKED) { 789651175c9SAttilio Rao v &= ~LK_EXCLUSIVE_SPINNERS; 7900ad122a9SMateusz Guzik if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x, 791047dd67eSAttilio Rao tid | v)) { 792047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 793047dd67eSAttilio Rao LOCK_LOG2(lk, 794047dd67eSAttilio Rao "%s: %p claimed by a new writer", 795047dd67eSAttilio Rao __func__, lk); 796047dd67eSAttilio Rao break; 797047dd67eSAttilio Rao } 7980ad122a9SMateusz Guzik goto retry_sleepq; 799047dd67eSAttilio Rao } 800047dd67eSAttilio Rao 801047dd67eSAttilio Rao /* 802047dd67eSAttilio Rao * Try to set the LK_EXCLUSIVE_WAITERS flag. If we 803047dd67eSAttilio Rao * fail, loop back and retry. 804047dd67eSAttilio Rao */ 805047dd67eSAttilio Rao if ((x & LK_EXCLUSIVE_WAITERS) == 0) { 8060ad122a9SMateusz Guzik if (!atomic_fcmpset_ptr(&lk->lk_lock, &x, 807047dd67eSAttilio Rao x | LK_EXCLUSIVE_WAITERS)) { 8080ad122a9SMateusz Guzik goto retry_sleepq; 809047dd67eSAttilio Rao } 810047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p set excl waiters flag", 811047dd67eSAttilio Rao __func__, lk); 812047dd67eSAttilio Rao } 813047dd67eSAttilio Rao 8141c6987ebSMateusz Guzik if (lwa == NULL) { 8151c6987ebSMateusz Guzik iwmesg = lk->lock_object.lo_name; 8161c6987ebSMateusz Guzik ipri = lk->lk_pri; 8171c6987ebSMateusz Guzik itimo = lk->lk_timo; 8181c6987ebSMateusz Guzik } else { 8191c6987ebSMateusz Guzik iwmesg = lwa->iwmesg; 8201c6987ebSMateusz Guzik ipri = lwa->ipri; 8211c6987ebSMateusz Guzik itimo = lwa->itimo; 8221c6987ebSMateusz Guzik } 8231c6987ebSMateusz Guzik 824047dd67eSAttilio Rao /* 825047dd67eSAttilio Rao * As far as we have been unable to acquire the 826047dd67eSAttilio Rao * exclusive lock and the exclusive waiters flag 827047dd67eSAttilio Rao * is set, we will sleep. 828047dd67eSAttilio Rao */ 829047dd67eSAttilio Rao error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, 830047dd67eSAttilio Rao SQ_EXCLUSIVE_QUEUE); 831047dd67eSAttilio Rao flags &= ~LK_INTERLOCK; 832047dd67eSAttilio Rao if (error) { 833047dd67eSAttilio Rao LOCK_LOG3(lk, 834047dd67eSAttilio Rao "%s: interrupted sleep for %p with %d", 835047dd67eSAttilio Rao __func__, lk, error); 836047dd67eSAttilio Rao break; 837047dd67eSAttilio Rao } 838047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 839047dd67eSAttilio Rao __func__, lk); 840047dd67eSAttilio Rao } 841047dd67eSAttilio Rao if (error == 0) { 842c4a48867SMateusz Guzik #ifdef LOCK_PROFILING 843c4a48867SMateusz Guzik lockmgr_note_exclusive_acquire(lk, contested, waittime, 844c4a48867SMateusz Guzik file, line, flags); 845c4a48867SMateusz Guzik #else 846c4a48867SMateusz Guzik lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, 847c4a48867SMateusz Guzik flags); 848c4a48867SMateusz Guzik #endif 849047dd67eSAttilio Rao } 8501c6987ebSMateusz Guzik 8511c6987ebSMateusz Guzik out: 8521c6987ebSMateusz Guzik lockmgr_exit(flags, ilk, 0); 8531c6987ebSMateusz Guzik return (error); 8541c6987ebSMateusz Guzik } 8551c6987ebSMateusz Guzik 8561c6987ebSMateusz Guzik static __noinline int 8571c6987ebSMateusz Guzik lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk, 8581c6987ebSMateusz Guzik const char *file, int line, struct lockmgr_wait *lwa) 8591c6987ebSMateusz Guzik { 8601c6987ebSMateusz Guzik uintptr_t tid, x, v; 8611c6987ebSMateusz Guzik int error = 0; 8621c6987ebSMateusz Guzik int wakeup_swapper = 0; 8631c6987ebSMateusz Guzik int op; 8641c6987ebSMateusz Guzik 8651c6987ebSMateusz Guzik if (__predict_false(panicstr != NULL)) 8661c6987ebSMateusz Guzik goto out; 8671c6987ebSMateusz Guzik 8681c6987ebSMateusz Guzik tid = (uintptr_t)curthread; 8691c6987ebSMateusz Guzik 8701c6987ebSMateusz Guzik _lockmgr_assert(lk, KA_SLOCKED, file, line); 8711c6987ebSMateusz Guzik v = lk->lk_lock; 8721c6987ebSMateusz Guzik x = v & LK_ALL_WAITERS; 8731c6987ebSMateusz Guzik v &= LK_EXCLUSIVE_SPINNERS; 8741c6987ebSMateusz Guzik 8751c6987ebSMateusz Guzik /* 8761c6987ebSMateusz Guzik * Try to switch from one shared lock to an exclusive one. 8771c6987ebSMateusz Guzik * We need to preserve waiters flags during the operation. 8781c6987ebSMateusz Guzik */ 8791c6987ebSMateusz Guzik if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, 8801c6987ebSMateusz Guzik tid | x)) { 8811c6987ebSMateusz Guzik LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, 8821c6987ebSMateusz Guzik line); 8831c6987ebSMateusz Guzik WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | 8841c6987ebSMateusz Guzik LK_TRYWIT(flags), file, line); 8851c6987ebSMateusz Guzik TD_SLOCKS_DEC(curthread); 8861c6987ebSMateusz Guzik goto out; 8871c6987ebSMateusz Guzik } 8881c6987ebSMateusz Guzik 8891c6987ebSMateusz Guzik op = flags & LK_TYPE_MASK; 8901c6987ebSMateusz Guzik 8911c6987ebSMateusz Guzik /* 8921c6987ebSMateusz Guzik * In LK_TRYUPGRADE mode, do not drop the lock, 8931c6987ebSMateusz Guzik * returning EBUSY instead. 8941c6987ebSMateusz Guzik */ 8951c6987ebSMateusz Guzik if (op == LK_TRYUPGRADE) { 8961c6987ebSMateusz Guzik LOCK_LOG2(lk, "%s: %p failed the nowait upgrade", 8971c6987ebSMateusz Guzik __func__, lk); 8981c6987ebSMateusz Guzik error = EBUSY; 8991c6987ebSMateusz Guzik goto out; 9001c6987ebSMateusz Guzik } 9011c6987ebSMateusz Guzik 9021c6987ebSMateusz Guzik /* 9031c6987ebSMateusz Guzik * We have been unable to succeed in upgrading, so just 9041c6987ebSMateusz Guzik * give up the shared lock. 9051c6987ebSMateusz Guzik */ 9061c6987ebSMateusz Guzik wakeup_swapper |= wakeupshlk(lk, file, line); 9071c6987ebSMateusz Guzik error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa); 9081c6987ebSMateusz Guzik flags &= ~LK_INTERLOCK; 9091c6987ebSMateusz Guzik out: 9101c6987ebSMateusz Guzik lockmgr_exit(flags, ilk, wakeup_swapper); 9111c6987ebSMateusz Guzik return (error); 9121c6987ebSMateusz Guzik } 9131c6987ebSMateusz Guzik 9141c6987ebSMateusz Guzik int 9151c6987ebSMateusz Guzik lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk, 9161c6987ebSMateusz Guzik const char *file, int line) 9171c6987ebSMateusz Guzik { 9181c6987ebSMateusz Guzik struct lock_class *class; 9191c6987ebSMateusz Guzik uintptr_t x, tid; 9201c6987ebSMateusz Guzik u_int op; 9211c6987ebSMateusz Guzik bool locked; 9221c6987ebSMateusz Guzik 923b543c98cSConrad Meyer if (__predict_false(panicstr != NULL)) 924b543c98cSConrad Meyer return (0); 925b543c98cSConrad Meyer 9261c6987ebSMateusz Guzik op = flags & LK_TYPE_MASK; 9271c6987ebSMateusz Guzik locked = false; 9281c6987ebSMateusz Guzik switch (op) { 9291c6987ebSMateusz Guzik case LK_SHARED: 9301c6987ebSMateusz Guzik if (LK_CAN_WITNESS(flags)) 9311c6987ebSMateusz Guzik WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, 9321c6987ebSMateusz Guzik file, line, flags & LK_INTERLOCK ? ilk : NULL); 9331c6987ebSMateusz Guzik if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE)) 9341c6987ebSMateusz Guzik break; 9351c6987ebSMateusz Guzik if (lockmgr_slock_try(lk, &x, flags)) { 9361c6987ebSMateusz Guzik lockmgr_note_shared_acquire(lk, 0, 0, 9371c6987ebSMateusz Guzik file, line, flags); 9381c6987ebSMateusz Guzik locked = true; 9391c6987ebSMateusz Guzik } else { 9401c6987ebSMateusz Guzik return (lockmgr_slock_hard(lk, flags, ilk, file, line, 9411c6987ebSMateusz Guzik NULL)); 9421c6987ebSMateusz Guzik } 9431c6987ebSMateusz Guzik break; 9441c6987ebSMateusz Guzik case LK_EXCLUSIVE: 9451c6987ebSMateusz Guzik if (LK_CAN_WITNESS(flags)) 9461c6987ebSMateusz Guzik WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER | 9471c6987ebSMateusz Guzik LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ? 9481c6987ebSMateusz Guzik ilk : NULL); 9491c6987ebSMateusz Guzik tid = (uintptr_t)curthread; 9501c6987ebSMateusz Guzik if (lk->lk_lock == LK_UNLOCKED && 9511c6987ebSMateusz Guzik atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { 9521c6987ebSMateusz Guzik lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, 9531c6987ebSMateusz Guzik flags); 9541c6987ebSMateusz Guzik locked = true; 9551c6987ebSMateusz Guzik } else { 9561c6987ebSMateusz Guzik return (lockmgr_xlock_hard(lk, flags, ilk, file, line, 9571c6987ebSMateusz Guzik NULL)); 9581c6987ebSMateusz Guzik } 9591c6987ebSMateusz Guzik break; 9601c6987ebSMateusz Guzik case LK_UPGRADE: 9611c6987ebSMateusz Guzik case LK_TRYUPGRADE: 9621c6987ebSMateusz Guzik return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL)); 9631c6987ebSMateusz Guzik default: 9641c6987ebSMateusz Guzik break; 9651c6987ebSMateusz Guzik } 9661c6987ebSMateusz Guzik if (__predict_true(locked)) { 9671c6987ebSMateusz Guzik if (__predict_false(flags & LK_INTERLOCK)) { 9681c6987ebSMateusz Guzik class = LOCK_CLASS(ilk); 9691c6987ebSMateusz Guzik class->lc_unlock(ilk); 9701c6987ebSMateusz Guzik } 9711c6987ebSMateusz Guzik return (0); 9721c6987ebSMateusz Guzik } else { 9731c6987ebSMateusz Guzik return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT, 9741c6987ebSMateusz Guzik LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line)); 9751c6987ebSMateusz Guzik } 9761c6987ebSMateusz Guzik } 9771c6987ebSMateusz Guzik 9781c6987ebSMateusz Guzik static __noinline int 9791c6987ebSMateusz Guzik lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, 9801c6987ebSMateusz Guzik const char *file, int line) 9811c6987ebSMateusz Guzik 9821c6987ebSMateusz Guzik { 9831c6987ebSMateusz Guzik int wakeup_swapper = 0; 9841c6987ebSMateusz Guzik 9851c6987ebSMateusz Guzik if (__predict_false(panicstr != NULL)) 9861c6987ebSMateusz Guzik goto out; 9871c6987ebSMateusz Guzik 9881c6987ebSMateusz Guzik wakeup_swapper = wakeupshlk(lk, file, line); 9891c6987ebSMateusz Guzik 9901c6987ebSMateusz Guzik out: 9911c6987ebSMateusz Guzik lockmgr_exit(flags, ilk, wakeup_swapper); 9921c6987ebSMateusz Guzik return (0); 9931c6987ebSMateusz Guzik } 9941c6987ebSMateusz Guzik 9951c6987ebSMateusz Guzik static __noinline int 9961c6987ebSMateusz Guzik lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, 9971c6987ebSMateusz Guzik const char *file, int line) 9981c6987ebSMateusz Guzik { 9991c6987ebSMateusz Guzik uintptr_t tid, v; 10001c6987ebSMateusz Guzik int wakeup_swapper = 0; 10011c6987ebSMateusz Guzik u_int realexslp; 10021c6987ebSMateusz Guzik int queue; 10031c6987ebSMateusz Guzik 10041c6987ebSMateusz Guzik if (__predict_false(panicstr != NULL)) 10051c6987ebSMateusz Guzik goto out; 10061c6987ebSMateusz Guzik 10071c6987ebSMateusz Guzik tid = (uintptr_t)curthread; 10081c6987ebSMateusz Guzik 10091c6987ebSMateusz Guzik /* 10101c6987ebSMateusz Guzik * As first option, treact the lock as if it has not 10111c6987ebSMateusz Guzik * any waiter. 10121c6987ebSMateusz Guzik * Fix-up the tid var if the lock has been disowned. 10131c6987ebSMateusz Guzik */ 10141c6987ebSMateusz Guzik if (LK_HOLDER(x) == LK_KERNPROC) 10151c6987ebSMateusz Guzik tid = LK_KERNPROC; 10161c6987ebSMateusz Guzik else { 10171c6987ebSMateusz Guzik WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); 10181c6987ebSMateusz Guzik TD_LOCKS_DEC(curthread); 10191c6987ebSMateusz Guzik } 10201c6987ebSMateusz Guzik LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line); 10211c6987ebSMateusz Guzik 10221c6987ebSMateusz Guzik /* 10231c6987ebSMateusz Guzik * The lock is held in exclusive mode. 10241c6987ebSMateusz Guzik * If the lock is recursed also, then unrecurse it. 10251c6987ebSMateusz Guzik */ 1026*10391db5SMateusz Guzik if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) { 10271c6987ebSMateusz Guzik LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk); 10281c6987ebSMateusz Guzik lk->lk_recurse--; 10291c6987ebSMateusz Guzik goto out; 10301c6987ebSMateusz Guzik } 10311c6987ebSMateusz Guzik if (tid != LK_KERNPROC) 10321c6987ebSMateusz Guzik lock_profile_release_lock(&lk->lock_object); 10331c6987ebSMateusz Guzik 1034*10391db5SMateusz Guzik if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) 10351c6987ebSMateusz Guzik goto out; 10361c6987ebSMateusz Guzik 10371c6987ebSMateusz Guzik sleepq_lock(&lk->lock_object); 10381c6987ebSMateusz Guzik x = lk->lk_lock; 10391c6987ebSMateusz Guzik v = LK_UNLOCKED; 10401c6987ebSMateusz Guzik 10411c6987ebSMateusz Guzik /* 10421c6987ebSMateusz Guzik * If the lock has exclusive waiters, give them 10431c6987ebSMateusz Guzik * preference in order to avoid deadlock with 10441c6987ebSMateusz Guzik * shared runners up. 10451c6987ebSMateusz Guzik * If interruptible sleeps left the exclusive queue 10461c6987ebSMateusz Guzik * empty avoid a starvation for the threads sleeping 10471c6987ebSMateusz Guzik * on the shared queue by giving them precedence 10481c6987ebSMateusz Guzik * and cleaning up the exclusive waiters bit anyway. 10491c6987ebSMateusz Guzik * Please note that lk_exslpfail count may be lying 10501c6987ebSMateusz Guzik * about the real number of waiters with the 10511c6987ebSMateusz Guzik * LK_SLEEPFAIL flag on because they may be used in 10521c6987ebSMateusz Guzik * conjunction with interruptible sleeps so 10531c6987ebSMateusz Guzik * lk_exslpfail might be considered an 'upper limit' 10541c6987ebSMateusz Guzik * bound, including the edge cases. 10551c6987ebSMateusz Guzik */ 10561c6987ebSMateusz Guzik MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); 10571c6987ebSMateusz Guzik realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE); 10581c6987ebSMateusz Guzik if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) { 10591c6987ebSMateusz Guzik if (lk->lk_exslpfail < realexslp) { 10601c6987ebSMateusz Guzik lk->lk_exslpfail = 0; 10611c6987ebSMateusz Guzik queue = SQ_EXCLUSIVE_QUEUE; 10621c6987ebSMateusz Guzik v |= (x & LK_SHARED_WAITERS); 10631c6987ebSMateusz Guzik } else { 10641c6987ebSMateusz Guzik lk->lk_exslpfail = 0; 10651c6987ebSMateusz Guzik LOCK_LOG2(lk, 10661c6987ebSMateusz Guzik "%s: %p has only LK_SLEEPFAIL sleepers", 10671c6987ebSMateusz Guzik __func__, lk); 10681c6987ebSMateusz Guzik LOCK_LOG2(lk, 10691c6987ebSMateusz Guzik "%s: %p waking up threads on the exclusive queue", 10701c6987ebSMateusz Guzik __func__, lk); 10711c6987ebSMateusz Guzik wakeup_swapper = sleepq_broadcast(&lk->lock_object, 10721c6987ebSMateusz Guzik SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE); 10731c6987ebSMateusz Guzik queue = SQ_SHARED_QUEUE; 10741c6987ebSMateusz Guzik } 10751c6987ebSMateusz Guzik } else { 10761c6987ebSMateusz Guzik 10771c6987ebSMateusz Guzik /* 10781c6987ebSMateusz Guzik * Exclusive waiters sleeping with LK_SLEEPFAIL 10791c6987ebSMateusz Guzik * on and using interruptible sleeps/timeout 10801c6987ebSMateusz Guzik * may have left spourious lk_exslpfail counts 10811c6987ebSMateusz Guzik * on, so clean it up anyway. 10821c6987ebSMateusz Guzik */ 10831c6987ebSMateusz Guzik lk->lk_exslpfail = 0; 10841c6987ebSMateusz Guzik queue = SQ_SHARED_QUEUE; 10851c6987ebSMateusz Guzik } 10861c6987ebSMateusz Guzik 10871c6987ebSMateusz Guzik LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", 10881c6987ebSMateusz Guzik __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : 10891c6987ebSMateusz Guzik "exclusive"); 10901c6987ebSMateusz Guzik atomic_store_rel_ptr(&lk->lk_lock, v); 10911c6987ebSMateusz Guzik wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); 10921c6987ebSMateusz Guzik sleepq_release(&lk->lock_object); 10931c6987ebSMateusz Guzik 10941c6987ebSMateusz Guzik out: 10951c6987ebSMateusz Guzik lockmgr_exit(flags, ilk, wakeup_swapper); 10961c6987ebSMateusz Guzik return (0); 10971c6987ebSMateusz Guzik } 10981c6987ebSMateusz Guzik 10991c6987ebSMateusz Guzik int 11001c6987ebSMateusz Guzik lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk) 11011c6987ebSMateusz Guzik { 11021c6987ebSMateusz Guzik struct lock_class *class; 11031c6987ebSMateusz Guzik uintptr_t x, tid; 11041c6987ebSMateusz Guzik bool unlocked; 11051c6987ebSMateusz Guzik const char *file; 11061c6987ebSMateusz Guzik int line; 11071c6987ebSMateusz Guzik 1108b543c98cSConrad Meyer if (__predict_false(panicstr != NULL)) 1109b543c98cSConrad Meyer return (0); 1110b543c98cSConrad Meyer 11111c6987ebSMateusz Guzik file = __FILE__; 11121c6987ebSMateusz Guzik line = __LINE__; 11131c6987ebSMateusz Guzik 11141c6987ebSMateusz Guzik _lockmgr_assert(lk, KA_LOCKED, file, line); 11151c6987ebSMateusz Guzik unlocked = false; 11161c6987ebSMateusz Guzik x = lk->lk_lock; 11171c6987ebSMateusz Guzik if (__predict_true(x & LK_SHARE) != 0) { 11181c6987ebSMateusz Guzik if (lockmgr_sunlock_try(lk, &x)) { 11191c6987ebSMateusz Guzik lockmgr_note_shared_release(lk, file, line); 11201c6987ebSMateusz Guzik unlocked = true; 11211c6987ebSMateusz Guzik } else { 11221c6987ebSMateusz Guzik return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line)); 11231c6987ebSMateusz Guzik } 11241c6987ebSMateusz Guzik } else { 11251c6987ebSMateusz Guzik tid = (uintptr_t)curthread; 11261c6987ebSMateusz Guzik if (!lockmgr_recursed(lk) && 11271c6987ebSMateusz Guzik atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) { 11281c6987ebSMateusz Guzik lockmgr_note_exclusive_release(lk, file, line); 11291c6987ebSMateusz Guzik unlocked = true; 11301c6987ebSMateusz Guzik } else { 11311c6987ebSMateusz Guzik return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line)); 11321c6987ebSMateusz Guzik } 11331c6987ebSMateusz Guzik } 11341c6987ebSMateusz Guzik if (__predict_false(flags & LK_INTERLOCK)) { 11351c6987ebSMateusz Guzik class = LOCK_CLASS(ilk); 11361c6987ebSMateusz Guzik class->lc_unlock(ilk); 11371c6987ebSMateusz Guzik } 11381c6987ebSMateusz Guzik return (0); 11391c6987ebSMateusz Guzik } 11401c6987ebSMateusz Guzik 11411c6987ebSMateusz Guzik int 11421c6987ebSMateusz Guzik __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, 11431c6987ebSMateusz Guzik const char *wmesg, int pri, int timo, const char *file, int line) 11441c6987ebSMateusz Guzik { 11451c6987ebSMateusz Guzik GIANT_DECLARE; 11461c6987ebSMateusz Guzik struct lockmgr_wait lwa; 11471c6987ebSMateusz Guzik struct lock_class *class; 11481c6987ebSMateusz Guzik const char *iwmesg; 11491c6987ebSMateusz Guzik uintptr_t tid, v, x; 11501c6987ebSMateusz Guzik u_int op, realexslp; 11511c6987ebSMateusz Guzik int error, ipri, itimo, queue, wakeup_swapper; 11521c6987ebSMateusz Guzik #ifdef LOCK_PROFILING 11531c6987ebSMateusz Guzik uint64_t waittime = 0; 11541c6987ebSMateusz Guzik int contested = 0; 11551c6987ebSMateusz Guzik #endif 11561c6987ebSMateusz Guzik 1157b543c98cSConrad Meyer if (panicstr != NULL) 1158b543c98cSConrad Meyer return (0); 1159b543c98cSConrad Meyer 11601c6987ebSMateusz Guzik error = 0; 11611c6987ebSMateusz Guzik tid = (uintptr_t)curthread; 11621c6987ebSMateusz Guzik op = (flags & LK_TYPE_MASK); 11631c6987ebSMateusz Guzik iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg; 11641c6987ebSMateusz Guzik ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri; 11651c6987ebSMateusz Guzik itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo; 11661c6987ebSMateusz Guzik 11671c6987ebSMateusz Guzik lwa.iwmesg = iwmesg; 11681c6987ebSMateusz Guzik lwa.ipri = ipri; 11691c6987ebSMateusz Guzik lwa.itimo = itimo; 11701c6987ebSMateusz Guzik 11711c6987ebSMateusz Guzik MPASS((flags & ~LK_TOTAL_MASK) == 0); 11721c6987ebSMateusz Guzik KASSERT((op & (op - 1)) == 0, 11731c6987ebSMateusz Guzik ("%s: Invalid requested operation @ %s:%d", __func__, file, line)); 11741c6987ebSMateusz Guzik KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 || 11751c6987ebSMateusz Guzik (op != LK_DOWNGRADE && op != LK_RELEASE), 11761c6987ebSMateusz Guzik ("%s: Invalid flags in regard of the operation desired @ %s:%d", 11771c6987ebSMateusz Guzik __func__, file, line)); 11781c6987ebSMateusz Guzik KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL, 11791c6987ebSMateusz Guzik ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d", 11801c6987ebSMateusz Guzik __func__, file, line)); 11811c6987ebSMateusz Guzik KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 11821c6987ebSMateusz Guzik ("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread, 11831c6987ebSMateusz Guzik lk->lock_object.lo_name, file, line)); 11841c6987ebSMateusz Guzik 11851c6987ebSMateusz Guzik class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL; 11861c6987ebSMateusz Guzik 11871c6987ebSMateusz Guzik if (lk->lock_object.lo_flags & LK_NOSHARE) { 11881c6987ebSMateusz Guzik switch (op) { 11891c6987ebSMateusz Guzik case LK_SHARED: 11901c6987ebSMateusz Guzik op = LK_EXCLUSIVE; 11911c6987ebSMateusz Guzik break; 11921c6987ebSMateusz Guzik case LK_UPGRADE: 11931c6987ebSMateusz Guzik case LK_TRYUPGRADE: 11941c6987ebSMateusz Guzik case LK_DOWNGRADE: 11951c6987ebSMateusz Guzik _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, 11961c6987ebSMateusz Guzik file, line); 11971c6987ebSMateusz Guzik if (flags & LK_INTERLOCK) 11981c6987ebSMateusz Guzik class->lc_unlock(ilk); 11991c6987ebSMateusz Guzik return (0); 12001c6987ebSMateusz Guzik } 12011c6987ebSMateusz Guzik } 12021c6987ebSMateusz Guzik 12031c6987ebSMateusz Guzik wakeup_swapper = 0; 12041c6987ebSMateusz Guzik switch (op) { 12051c6987ebSMateusz Guzik case LK_SHARED: 12061c6987ebSMateusz Guzik return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa)); 12071c6987ebSMateusz Guzik break; 12081c6987ebSMateusz Guzik case LK_UPGRADE: 12091c6987ebSMateusz Guzik case LK_TRYUPGRADE: 12101c6987ebSMateusz Guzik return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa)); 12111c6987ebSMateusz Guzik break; 12121c6987ebSMateusz Guzik case LK_EXCLUSIVE: 12131c6987ebSMateusz Guzik return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa)); 1214047dd67eSAttilio Rao break; 1215047dd67eSAttilio Rao case LK_DOWNGRADE: 12161c7d98d0SAttilio Rao _lockmgr_assert(lk, KA_XLOCKED, file, line); 1217e5f94314SAttilio Rao LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line); 1218e5f94314SAttilio Rao WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line); 12191c7d98d0SAttilio Rao 12201c7d98d0SAttilio Rao /* 12211c7d98d0SAttilio Rao * Panic if the lock is recursed. 12221c7d98d0SAttilio Rao */ 12231c7d98d0SAttilio Rao if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) { 12241c7d98d0SAttilio Rao if (flags & LK_INTERLOCK) 12251c7d98d0SAttilio Rao class->lc_unlock(ilk); 12261c7d98d0SAttilio Rao panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n", 12271c7d98d0SAttilio Rao __func__, iwmesg, file, line); 12281c7d98d0SAttilio Rao } 1229e5f94314SAttilio Rao TD_SLOCKS_INC(curthread); 1230047dd67eSAttilio Rao 1231047dd67eSAttilio Rao /* 1232047dd67eSAttilio Rao * In order to preserve waiters flags, just spin. 1233047dd67eSAttilio Rao */ 1234047dd67eSAttilio Rao for (;;) { 1235651175c9SAttilio Rao x = lk->lk_lock; 1236651175c9SAttilio Rao MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); 1237651175c9SAttilio Rao x &= LK_ALL_WAITERS; 1238047dd67eSAttilio Rao if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, 1239e5f94314SAttilio Rao LK_SHARERS_LOCK(1) | x)) 1240047dd67eSAttilio Rao break; 1241047dd67eSAttilio Rao cpu_spinwait(); 1242047dd67eSAttilio Rao } 1243047dd67eSAttilio Rao break; 1244047dd67eSAttilio Rao case LK_RELEASE: 1245047dd67eSAttilio Rao _lockmgr_assert(lk, KA_LOCKED, file, line); 1246047dd67eSAttilio Rao x = lk->lk_lock; 1247047dd67eSAttilio Rao 12481c6987ebSMateusz Guzik if (__predict_true(x & LK_SHARE) != 0) { 12491c6987ebSMateusz Guzik return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line)); 1250047dd67eSAttilio Rao } else { 12511c6987ebSMateusz Guzik return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line)); 12522028867dSAttilio Rao } 1253047dd67eSAttilio Rao break; 1254047dd67eSAttilio Rao case LK_DRAIN: 1255e5f94314SAttilio Rao if (LK_CAN_WITNESS(flags)) 1256e5f94314SAttilio Rao WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER | 125724150d37SJohn Baldwin LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ? 125824150d37SJohn Baldwin ilk : NULL); 1259047dd67eSAttilio Rao 1260047dd67eSAttilio Rao /* 126196f1567fSKonstantin Belousov * Trying to drain a lock we already own will result in a 1262047dd67eSAttilio Rao * deadlock. 1263047dd67eSAttilio Rao */ 1264047dd67eSAttilio Rao if (lockmgr_xlocked(lk)) { 1265047dd67eSAttilio Rao if (flags & LK_INTERLOCK) 1266047dd67eSAttilio Rao class->lc_unlock(ilk); 1267047dd67eSAttilio Rao panic("%s: draining %s with the lock held @ %s:%d\n", 1268047dd67eSAttilio Rao __func__, iwmesg, file, line); 1269047dd67eSAttilio Rao } 1270047dd67eSAttilio Rao 1271fc4f686dSMateusz Guzik for (;;) { 1272fc4f686dSMateusz Guzik if (lk->lk_lock == LK_UNLOCKED && 1273fc4f686dSMateusz Guzik atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) 1274fc4f686dSMateusz Guzik break; 1275fc4f686dSMateusz Guzik 1276f5f9340bSFabien Thomas #ifdef HWPMC_HOOKS 1277f5f9340bSFabien Thomas PMC_SOFT_CALL( , , lock, failed); 1278f5f9340bSFabien Thomas #endif 1279047dd67eSAttilio Rao lock_profile_obtain_lock_failed(&lk->lock_object, 1280047dd67eSAttilio Rao &contested, &waittime); 1281047dd67eSAttilio Rao 1282047dd67eSAttilio Rao /* 1283047dd67eSAttilio Rao * If the lock is expected to not sleep just give up 1284047dd67eSAttilio Rao * and return. 1285047dd67eSAttilio Rao */ 1286047dd67eSAttilio Rao if (LK_TRYOP(flags)) { 1287047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p fails the try operation", 1288047dd67eSAttilio Rao __func__, lk); 1289047dd67eSAttilio Rao error = EBUSY; 1290047dd67eSAttilio Rao break; 1291047dd67eSAttilio Rao } 1292047dd67eSAttilio Rao 1293047dd67eSAttilio Rao /* 1294047dd67eSAttilio Rao * Acquire the sleepqueue chain lock because we 1295047dd67eSAttilio Rao * probabilly will need to manipulate waiters flags. 1296047dd67eSAttilio Rao */ 1297047dd67eSAttilio Rao sleepq_lock(&lk->lock_object); 1298047dd67eSAttilio Rao x = lk->lk_lock; 1299047dd67eSAttilio Rao 1300047dd67eSAttilio Rao /* 1301047dd67eSAttilio Rao * if the lock has been released while we spun on 1302047dd67eSAttilio Rao * the sleepqueue chain lock just try again. 1303047dd67eSAttilio Rao */ 1304047dd67eSAttilio Rao if (x == LK_UNLOCKED) { 1305047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 1306047dd67eSAttilio Rao continue; 1307047dd67eSAttilio Rao } 1308047dd67eSAttilio Rao 1309651175c9SAttilio Rao v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); 1310651175c9SAttilio Rao if ((x & ~v) == LK_UNLOCKED) { 1311651175c9SAttilio Rao v = (x & ~LK_EXCLUSIVE_SPINNERS); 13122028867dSAttilio Rao 13132028867dSAttilio Rao /* 13142028867dSAttilio Rao * If interruptible sleeps left the exclusive 13152028867dSAttilio Rao * queue empty avoid a starvation for the 13162028867dSAttilio Rao * threads sleeping on the shared queue by 13172028867dSAttilio Rao * giving them precedence and cleaning up the 13182028867dSAttilio Rao * exclusive waiters bit anyway. 1319c636ba83SAttilio Rao * Please note that lk_exslpfail count may be 1320c636ba83SAttilio Rao * lying about the real number of waiters with 1321c636ba83SAttilio Rao * the LK_SLEEPFAIL flag on because they may 1322e3043798SPedro F. Giffuni * be used in conjunction with interruptible 1323aab9c8c2SAttilio Rao * sleeps so lk_exslpfail might be considered 1324aab9c8c2SAttilio Rao * an 'upper limit' bound, including the edge 1325c636ba83SAttilio Rao * cases. 13262028867dSAttilio Rao */ 1327047dd67eSAttilio Rao if (v & LK_EXCLUSIVE_WAITERS) { 1328047dd67eSAttilio Rao queue = SQ_EXCLUSIVE_QUEUE; 1329047dd67eSAttilio Rao v &= ~LK_EXCLUSIVE_WAITERS; 1330047dd67eSAttilio Rao } else { 13319dbf7a62SAttilio Rao 13329dbf7a62SAttilio Rao /* 13339dbf7a62SAttilio Rao * Exclusive waiters sleeping with 13349dbf7a62SAttilio Rao * LK_SLEEPFAIL on and using 13359dbf7a62SAttilio Rao * interruptible sleeps/timeout may 13369dbf7a62SAttilio Rao * have left spourious lk_exslpfail 13379dbf7a62SAttilio Rao * counts on, so clean it up anyway. 13389dbf7a62SAttilio Rao */ 1339047dd67eSAttilio Rao MPASS(v & LK_SHARED_WAITERS); 13409dbf7a62SAttilio Rao lk->lk_exslpfail = 0; 1341047dd67eSAttilio Rao queue = SQ_SHARED_QUEUE; 1342047dd67eSAttilio Rao v &= ~LK_SHARED_WAITERS; 1343047dd67eSAttilio Rao } 13442028867dSAttilio Rao if (queue == SQ_EXCLUSIVE_QUEUE) { 13452028867dSAttilio Rao realexslp = 13462028867dSAttilio Rao sleepq_sleepcnt(&lk->lock_object, 13472028867dSAttilio Rao SQ_EXCLUSIVE_QUEUE); 13482028867dSAttilio Rao if (lk->lk_exslpfail >= realexslp) { 13492028867dSAttilio Rao lk->lk_exslpfail = 0; 13502028867dSAttilio Rao queue = SQ_SHARED_QUEUE; 13512028867dSAttilio Rao v &= ~LK_SHARED_WAITERS; 13522028867dSAttilio Rao if (realexslp != 0) { 13532028867dSAttilio Rao LOCK_LOG2(lk, 13542028867dSAttilio Rao "%s: %p has only LK_SLEEPFAIL sleepers", 13552028867dSAttilio Rao __func__, lk); 13562028867dSAttilio Rao LOCK_LOG2(lk, 13572028867dSAttilio Rao "%s: %p waking up threads on the exclusive queue", 13582028867dSAttilio Rao __func__, lk); 13592028867dSAttilio Rao wakeup_swapper = 13602028867dSAttilio Rao sleepq_broadcast( 13612028867dSAttilio Rao &lk->lock_object, 13622028867dSAttilio Rao SLEEPQ_LK, 0, 13632028867dSAttilio Rao SQ_EXCLUSIVE_QUEUE); 13642028867dSAttilio Rao } 13652028867dSAttilio Rao } else 13662028867dSAttilio Rao lk->lk_exslpfail = 0; 13672028867dSAttilio Rao } 1368047dd67eSAttilio Rao if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) { 1369047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 1370047dd67eSAttilio Rao continue; 1371047dd67eSAttilio Rao } 1372047dd67eSAttilio Rao LOCK_LOG3(lk, 1373047dd67eSAttilio Rao "%s: %p waking up all threads on the %s queue", 1374047dd67eSAttilio Rao __func__, lk, queue == SQ_SHARED_QUEUE ? 1375047dd67eSAttilio Rao "shared" : "exclusive"); 1376814f26daSJohn Baldwin wakeup_swapper |= sleepq_broadcast( 1377da7bbd2cSJohn Baldwin &lk->lock_object, SLEEPQ_LK, 0, queue); 1378047dd67eSAttilio Rao 1379047dd67eSAttilio Rao /* 1380047dd67eSAttilio Rao * If shared waiters have been woken up we need 1381047dd67eSAttilio Rao * to wait for one of them to acquire the lock 1382047dd67eSAttilio Rao * before to set the exclusive waiters in 1383047dd67eSAttilio Rao * order to avoid a deadlock. 1384047dd67eSAttilio Rao */ 1385047dd67eSAttilio Rao if (queue == SQ_SHARED_QUEUE) { 1386047dd67eSAttilio Rao for (v = lk->lk_lock; 1387047dd67eSAttilio Rao (v & LK_SHARE) && !LK_SHARERS(v); 1388047dd67eSAttilio Rao v = lk->lk_lock) 1389047dd67eSAttilio Rao cpu_spinwait(); 1390047dd67eSAttilio Rao } 1391047dd67eSAttilio Rao } 1392047dd67eSAttilio Rao 1393047dd67eSAttilio Rao /* 1394047dd67eSAttilio Rao * Try to set the LK_EXCLUSIVE_WAITERS flag. If we 1395047dd67eSAttilio Rao * fail, loop back and retry. 1396047dd67eSAttilio Rao */ 1397047dd67eSAttilio Rao if ((x & LK_EXCLUSIVE_WAITERS) == 0) { 1398047dd67eSAttilio Rao if (!atomic_cmpset_ptr(&lk->lk_lock, x, 1399047dd67eSAttilio Rao x | LK_EXCLUSIVE_WAITERS)) { 1400047dd67eSAttilio Rao sleepq_release(&lk->lock_object); 1401047dd67eSAttilio Rao continue; 1402047dd67eSAttilio Rao } 1403047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p set drain waiters flag", 1404047dd67eSAttilio Rao __func__, lk); 1405047dd67eSAttilio Rao } 1406047dd67eSAttilio Rao 1407047dd67eSAttilio Rao /* 1408047dd67eSAttilio Rao * As far as we have been unable to acquire the 1409047dd67eSAttilio Rao * exclusive lock and the exclusive waiters flag 1410047dd67eSAttilio Rao * is set, we will sleep. 1411047dd67eSAttilio Rao */ 1412047dd67eSAttilio Rao if (flags & LK_INTERLOCK) { 1413047dd67eSAttilio Rao class->lc_unlock(ilk); 1414047dd67eSAttilio Rao flags &= ~LK_INTERLOCK; 1415047dd67eSAttilio Rao } 1416e5f94314SAttilio Rao GIANT_SAVE(); 1417047dd67eSAttilio Rao sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK, 1418047dd67eSAttilio Rao SQ_EXCLUSIVE_QUEUE); 1419047dd67eSAttilio Rao sleepq_wait(&lk->lock_object, ipri & PRIMASK); 1420e5f94314SAttilio Rao GIANT_RESTORE(); 1421047dd67eSAttilio Rao LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 1422047dd67eSAttilio Rao __func__, lk); 1423047dd67eSAttilio Rao } 1424047dd67eSAttilio Rao 1425047dd67eSAttilio Rao if (error == 0) { 1426047dd67eSAttilio Rao lock_profile_obtain_lock_success(&lk->lock_object, 1427047dd67eSAttilio Rao contested, waittime, file, line); 1428047dd67eSAttilio Rao LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0, 1429047dd67eSAttilio Rao lk->lk_recurse, file, line); 1430e5f94314SAttilio Rao WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | 1431e5f94314SAttilio Rao LK_TRYWIT(flags), file, line); 1432047dd67eSAttilio Rao TD_LOCKS_INC(curthread); 1433047dd67eSAttilio Rao STACK_SAVE(lk); 1434047dd67eSAttilio Rao } 1435047dd67eSAttilio Rao break; 1436047dd67eSAttilio Rao default: 1437047dd67eSAttilio Rao if (flags & LK_INTERLOCK) 1438047dd67eSAttilio Rao class->lc_unlock(ilk); 1439047dd67eSAttilio Rao panic("%s: unknown lockmgr request 0x%x\n", __func__, op); 1440047dd67eSAttilio Rao } 1441047dd67eSAttilio Rao 1442047dd67eSAttilio Rao if (flags & LK_INTERLOCK) 1443047dd67eSAttilio Rao class->lc_unlock(ilk); 1444da7bbd2cSJohn Baldwin if (wakeup_swapper) 1445da7bbd2cSJohn Baldwin kick_proc0(); 1446047dd67eSAttilio Rao 1447047dd67eSAttilio Rao return (error); 1448047dd67eSAttilio Rao } 1449047dd67eSAttilio Rao 1450d7a7e179SAttilio Rao void 1451047dd67eSAttilio Rao _lockmgr_disown(struct lock *lk, const char *file, int line) 1452047dd67eSAttilio Rao { 1453047dd67eSAttilio Rao uintptr_t tid, x; 1454047dd67eSAttilio Rao 145535370593SAndriy Gapon if (SCHEDULER_STOPPED()) 145635370593SAndriy Gapon return; 145735370593SAndriy Gapon 1458047dd67eSAttilio Rao tid = (uintptr_t)curthread; 14591c7d98d0SAttilio Rao _lockmgr_assert(lk, KA_XLOCKED, file, line); 14601c7d98d0SAttilio Rao 14611c7d98d0SAttilio Rao /* 14621c7d98d0SAttilio Rao * Panic if the lock is recursed. 14631c7d98d0SAttilio Rao */ 14641c7d98d0SAttilio Rao if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) 14651c7d98d0SAttilio Rao panic("%s: disown a recursed lockmgr @ %s:%d\n", 14661c7d98d0SAttilio Rao __func__, file, line); 1467047dd67eSAttilio Rao 1468047dd67eSAttilio Rao /* 146996f1567fSKonstantin Belousov * If the owner is already LK_KERNPROC just skip the whole operation. 1470047dd67eSAttilio Rao */ 1471047dd67eSAttilio Rao if (LK_HOLDER(lk->lk_lock) != tid) 1472047dd67eSAttilio Rao return; 147304a28689SJeff Roberson lock_profile_release_lock(&lk->lock_object); 1474e5f94314SAttilio Rao LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line); 1475e5f94314SAttilio Rao WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); 1476e5f94314SAttilio Rao TD_LOCKS_DEC(curthread); 1477337c5ff4SAttilio Rao STACK_SAVE(lk); 1478047dd67eSAttilio Rao 1479047dd67eSAttilio Rao /* 1480047dd67eSAttilio Rao * In order to preserve waiters flags, just spin. 1481047dd67eSAttilio Rao */ 1482047dd67eSAttilio Rao for (;;) { 1483651175c9SAttilio Rao x = lk->lk_lock; 1484651175c9SAttilio Rao MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); 1485651175c9SAttilio Rao x &= LK_ALL_WAITERS; 148622dd228dSAttilio Rao if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, 1487e5f94314SAttilio Rao LK_KERNPROC | x)) 1488047dd67eSAttilio Rao return; 1489047dd67eSAttilio Rao cpu_spinwait(); 1490047dd67eSAttilio Rao } 1491047dd67eSAttilio Rao } 1492047dd67eSAttilio Rao 1493047dd67eSAttilio Rao void 1494d576deedSPawel Jakub Dawidek lockmgr_printinfo(const struct lock *lk) 1495d7a7e179SAttilio Rao { 1496d7a7e179SAttilio Rao struct thread *td; 1497047dd67eSAttilio Rao uintptr_t x; 1498d7a7e179SAttilio Rao 1499047dd67eSAttilio Rao if (lk->lk_lock == LK_UNLOCKED) 1500047dd67eSAttilio Rao printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name); 1501047dd67eSAttilio Rao else if (lk->lk_lock & LK_SHARE) 1502047dd67eSAttilio Rao printf("lock type %s: SHARED (count %ju)\n", 1503047dd67eSAttilio Rao lk->lock_object.lo_name, 1504047dd67eSAttilio Rao (uintmax_t)LK_SHARERS(lk->lk_lock)); 1505047dd67eSAttilio Rao else { 1506047dd67eSAttilio Rao td = lockmgr_xholder(lk); 1507e64b4fa8SKonstantin Belousov if (td == (struct thread *)LK_KERNPROC) 1508e64b4fa8SKonstantin Belousov printf("lock type %s: EXCL by KERNPROC\n", 1509e64b4fa8SKonstantin Belousov lk->lock_object.lo_name); 1510e64b4fa8SKonstantin Belousov else 15112573ea5fSIvan Voras printf("lock type %s: EXCL by thread %p " 1512e64b4fa8SKonstantin Belousov "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name, 1513e64b4fa8SKonstantin Belousov td, td->td_proc->p_pid, td->td_proc->p_comm, 1514e64b4fa8SKonstantin Belousov td->td_tid); 1515d7a7e179SAttilio Rao } 1516d7a7e179SAttilio Rao 1517047dd67eSAttilio Rao x = lk->lk_lock; 1518047dd67eSAttilio Rao if (x & LK_EXCLUSIVE_WAITERS) 1519047dd67eSAttilio Rao printf(" with exclusive waiters pending\n"); 1520047dd67eSAttilio Rao if (x & LK_SHARED_WAITERS) 1521047dd67eSAttilio Rao printf(" with shared waiters pending\n"); 1522651175c9SAttilio Rao if (x & LK_EXCLUSIVE_SPINNERS) 1523651175c9SAttilio Rao printf(" with exclusive spinners pending\n"); 1524047dd67eSAttilio Rao 1525047dd67eSAttilio Rao STACK_PRINT(lk); 1526047dd67eSAttilio Rao } 1527047dd67eSAttilio Rao 152899448ed1SJohn Dyson int 1529d576deedSPawel Jakub Dawidek lockstatus(const struct lock *lk) 153099448ed1SJohn Dyson { 1531047dd67eSAttilio Rao uintptr_t v, x; 1532047dd67eSAttilio Rao int ret; 153399448ed1SJohn Dyson 1534047dd67eSAttilio Rao ret = LK_SHARED; 1535047dd67eSAttilio Rao x = lk->lk_lock; 1536047dd67eSAttilio Rao v = LK_HOLDER(x); 15370e9eb108SAttilio Rao 1538047dd67eSAttilio Rao if ((x & LK_SHARE) == 0) { 1539047dd67eSAttilio Rao if (v == (uintptr_t)curthread || v == LK_KERNPROC) 1540047dd67eSAttilio Rao ret = LK_EXCLUSIVE; 15416bdfe06aSEivind Eklund else 1542047dd67eSAttilio Rao ret = LK_EXCLOTHER; 1543047dd67eSAttilio Rao } else if (x == LK_UNLOCKED) 1544047dd67eSAttilio Rao ret = 0; 154599448ed1SJohn Dyson 1546047dd67eSAttilio Rao return (ret); 154753bf4bb2SPeter Wemm } 1548be6847d7SJohn Baldwin 154984887fa3SAttilio Rao #ifdef INVARIANT_SUPPORT 1550de5b1952SAlexander Leidinger 1551de5b1952SAlexander Leidinger FEATURE(invariant_support, 1552de5b1952SAlexander Leidinger "Support for modules compiled with INVARIANTS option"); 1553de5b1952SAlexander Leidinger 155484887fa3SAttilio Rao #ifndef INVARIANTS 155584887fa3SAttilio Rao #undef _lockmgr_assert 155684887fa3SAttilio Rao #endif 155784887fa3SAttilio Rao 155884887fa3SAttilio Rao void 1559d576deedSPawel Jakub Dawidek _lockmgr_assert(const struct lock *lk, int what, const char *file, int line) 156084887fa3SAttilio Rao { 156184887fa3SAttilio Rao int slocked = 0; 156284887fa3SAttilio Rao 156384887fa3SAttilio Rao if (panicstr != NULL) 156484887fa3SAttilio Rao return; 156584887fa3SAttilio Rao switch (what) { 156684887fa3SAttilio Rao case KA_SLOCKED: 156784887fa3SAttilio Rao case KA_SLOCKED | KA_NOTRECURSED: 156884887fa3SAttilio Rao case KA_SLOCKED | KA_RECURSED: 156984887fa3SAttilio Rao slocked = 1; 157084887fa3SAttilio Rao case KA_LOCKED: 157184887fa3SAttilio Rao case KA_LOCKED | KA_NOTRECURSED: 157284887fa3SAttilio Rao case KA_LOCKED | KA_RECURSED: 1573e5f94314SAttilio Rao #ifdef WITNESS 1574e5f94314SAttilio Rao 1575e5f94314SAttilio Rao /* 1576e5f94314SAttilio Rao * We cannot trust WITNESS if the lock is held in exclusive 1577e5f94314SAttilio Rao * mode and a call to lockmgr_disown() happened. 1578e5f94314SAttilio Rao * Workaround this skipping the check if the lock is held in 1579e5f94314SAttilio Rao * exclusive mode even for the KA_LOCKED case. 1580e5f94314SAttilio Rao */ 1581e5f94314SAttilio Rao if (slocked || (lk->lk_lock & LK_SHARE)) { 1582e5f94314SAttilio Rao witness_assert(&lk->lock_object, what, file, line); 1583e5f94314SAttilio Rao break; 1584e5f94314SAttilio Rao } 1585e5f94314SAttilio Rao #endif 1586047dd67eSAttilio Rao if (lk->lk_lock == LK_UNLOCKED || 1587047dd67eSAttilio Rao ((lk->lk_lock & LK_SHARE) == 0 && (slocked || 1588047dd67eSAttilio Rao (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))))) 158984887fa3SAttilio Rao panic("Lock %s not %slocked @ %s:%d\n", 1590047dd67eSAttilio Rao lk->lock_object.lo_name, slocked ? "share" : "", 159184887fa3SAttilio Rao file, line); 1592047dd67eSAttilio Rao 1593047dd67eSAttilio Rao if ((lk->lk_lock & LK_SHARE) == 0) { 1594047dd67eSAttilio Rao if (lockmgr_recursed(lk)) { 159584887fa3SAttilio Rao if (what & KA_NOTRECURSED) 159684887fa3SAttilio Rao panic("Lock %s recursed @ %s:%d\n", 1597047dd67eSAttilio Rao lk->lock_object.lo_name, file, 1598047dd67eSAttilio Rao line); 159984887fa3SAttilio Rao } else if (what & KA_RECURSED) 160084887fa3SAttilio Rao panic("Lock %s not recursed @ %s:%d\n", 1601047dd67eSAttilio Rao lk->lock_object.lo_name, file, line); 160284887fa3SAttilio Rao } 160384887fa3SAttilio Rao break; 160484887fa3SAttilio Rao case KA_XLOCKED: 160584887fa3SAttilio Rao case KA_XLOCKED | KA_NOTRECURSED: 160684887fa3SAttilio Rao case KA_XLOCKED | KA_RECURSED: 1607047dd67eSAttilio Rao if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)) 160884887fa3SAttilio Rao panic("Lock %s not exclusively locked @ %s:%d\n", 1609047dd67eSAttilio Rao lk->lock_object.lo_name, file, line); 1610047dd67eSAttilio Rao if (lockmgr_recursed(lk)) { 161184887fa3SAttilio Rao if (what & KA_NOTRECURSED) 161284887fa3SAttilio Rao panic("Lock %s recursed @ %s:%d\n", 1613047dd67eSAttilio Rao lk->lock_object.lo_name, file, line); 161484887fa3SAttilio Rao } else if (what & KA_RECURSED) 161584887fa3SAttilio Rao panic("Lock %s not recursed @ %s:%d\n", 1616047dd67eSAttilio Rao lk->lock_object.lo_name, file, line); 161784887fa3SAttilio Rao break; 161884887fa3SAttilio Rao case KA_UNLOCKED: 1619047dd67eSAttilio Rao if (lockmgr_xlocked(lk) || lockmgr_disowned(lk)) 162084887fa3SAttilio Rao panic("Lock %s exclusively locked @ %s:%d\n", 1621047dd67eSAttilio Rao lk->lock_object.lo_name, file, line); 162284887fa3SAttilio Rao break; 162384887fa3SAttilio Rao default: 1624047dd67eSAttilio Rao panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file, 1625047dd67eSAttilio Rao line); 162684887fa3SAttilio Rao } 162784887fa3SAttilio Rao } 1628047dd67eSAttilio Rao #endif 162984887fa3SAttilio Rao 1630be6847d7SJohn Baldwin #ifdef DDB 1631462a7addSJohn Baldwin int 1632462a7addSJohn Baldwin lockmgr_chain(struct thread *td, struct thread **ownerp) 1633462a7addSJohn Baldwin { 1634047dd67eSAttilio Rao struct lock *lk; 1635462a7addSJohn Baldwin 1636047dd67eSAttilio Rao lk = td->td_wchan; 1637462a7addSJohn Baldwin 1638047dd67eSAttilio Rao if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr) 1639462a7addSJohn Baldwin return (0); 1640047dd67eSAttilio Rao db_printf("blocked on lockmgr %s", lk->lock_object.lo_name); 1641047dd67eSAttilio Rao if (lk->lk_lock & LK_SHARE) 1642047dd67eSAttilio Rao db_printf("SHARED (count %ju)\n", 1643047dd67eSAttilio Rao (uintmax_t)LK_SHARERS(lk->lk_lock)); 1644047dd67eSAttilio Rao else 1645047dd67eSAttilio Rao db_printf("EXCL\n"); 1646047dd67eSAttilio Rao *ownerp = lockmgr_xholder(lk); 1647462a7addSJohn Baldwin 1648462a7addSJohn Baldwin return (1); 1649462a7addSJohn Baldwin } 1650462a7addSJohn Baldwin 1651047dd67eSAttilio Rao static void 1652d576deedSPawel Jakub Dawidek db_show_lockmgr(const struct lock_object *lock) 1653be6847d7SJohn Baldwin { 1654be6847d7SJohn Baldwin struct thread *td; 1655d576deedSPawel Jakub Dawidek const struct lock *lk; 1656be6847d7SJohn Baldwin 1657d576deedSPawel Jakub Dawidek lk = (const struct lock *)lock; 1658be6847d7SJohn Baldwin 1659be6847d7SJohn Baldwin db_printf(" state: "); 1660047dd67eSAttilio Rao if (lk->lk_lock == LK_UNLOCKED) 1661be6847d7SJohn Baldwin db_printf("UNLOCKED\n"); 1662047dd67eSAttilio Rao else if (lk->lk_lock & LK_SHARE) 1663047dd67eSAttilio Rao db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock)); 1664047dd67eSAttilio Rao else { 1665047dd67eSAttilio Rao td = lockmgr_xholder(lk); 1666047dd67eSAttilio Rao if (td == (struct thread *)LK_KERNPROC) 1667047dd67eSAttilio Rao db_printf("XLOCK: LK_KERNPROC\n"); 1668047dd67eSAttilio Rao else 1669047dd67eSAttilio Rao db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1670047dd67eSAttilio Rao td->td_tid, td->td_proc->p_pid, 1671047dd67eSAttilio Rao td->td_proc->p_comm); 1672047dd67eSAttilio Rao if (lockmgr_recursed(lk)) 1673047dd67eSAttilio Rao db_printf(" recursed: %d\n", lk->lk_recurse); 1674047dd67eSAttilio Rao } 1675047dd67eSAttilio Rao db_printf(" waiters: "); 1676047dd67eSAttilio Rao switch (lk->lk_lock & LK_ALL_WAITERS) { 1677047dd67eSAttilio Rao case LK_SHARED_WAITERS: 1678047dd67eSAttilio Rao db_printf("shared\n"); 1679e5023dd9SEdward Tomasz Napierala break; 1680047dd67eSAttilio Rao case LK_EXCLUSIVE_WAITERS: 1681047dd67eSAttilio Rao db_printf("exclusive\n"); 1682047dd67eSAttilio Rao break; 1683047dd67eSAttilio Rao case LK_ALL_WAITERS: 1684047dd67eSAttilio Rao db_printf("shared and exclusive\n"); 1685047dd67eSAttilio Rao break; 1686047dd67eSAttilio Rao default: 1687047dd67eSAttilio Rao db_printf("none\n"); 1688047dd67eSAttilio Rao } 1689651175c9SAttilio Rao db_printf(" spinners: "); 1690651175c9SAttilio Rao if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS) 1691651175c9SAttilio Rao db_printf("exclusive\n"); 1692651175c9SAttilio Rao else 1693651175c9SAttilio Rao db_printf("none\n"); 1694be6847d7SJohn Baldwin } 1695be6847d7SJohn Baldwin #endif 1696