19454b2d8SWarner Losh /*- 253bf4bb2SPeter Wemm * Copyright (c) 1995 353bf4bb2SPeter Wemm * The Regents of the University of California. All rights reserved. 453bf4bb2SPeter Wemm * 503e9c6c1SJohn Dyson * Copyright (C) 1997 603e9c6c1SJohn Dyson * John S. Dyson. All rights reserved. 703e9c6c1SJohn Dyson * 853bf4bb2SPeter Wemm * This code contains ideas from software contributed to Berkeley by 953bf4bb2SPeter Wemm * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 1053bf4bb2SPeter Wemm * System project at Carnegie-Mellon University. 1153bf4bb2SPeter Wemm * 1253bf4bb2SPeter Wemm * Redistribution and use in source and binary forms, with or without 1353bf4bb2SPeter Wemm * modification, are permitted provided that the following conditions 1453bf4bb2SPeter Wemm * are met: 1553bf4bb2SPeter Wemm * 1. Redistributions of source code must retain the above copyright 1653bf4bb2SPeter Wemm * notice, this list of conditions and the following disclaimer. 1753bf4bb2SPeter Wemm * 2. Redistributions in binary form must reproduce the above copyright 1853bf4bb2SPeter Wemm * notice, this list of conditions and the following disclaimer in the 1953bf4bb2SPeter Wemm * documentation and/or other materials provided with the distribution. 2053bf4bb2SPeter Wemm * 3. All advertising materials mentioning features or use of this software 2153bf4bb2SPeter Wemm * must display the following acknowledgement: 2253bf4bb2SPeter Wemm * This product includes software developed by the University of 2353bf4bb2SPeter Wemm * California, Berkeley and its contributors. 2453bf4bb2SPeter Wemm * 4. Neither the name of the University nor the names of its contributors 2553bf4bb2SPeter Wemm * may be used to endorse or promote products derived from this software 2653bf4bb2SPeter Wemm * without specific prior written permission. 2753bf4bb2SPeter Wemm * 2853bf4bb2SPeter Wemm * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2953bf4bb2SPeter Wemm * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 3053bf4bb2SPeter Wemm * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 3153bf4bb2SPeter Wemm * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 3253bf4bb2SPeter Wemm * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 3353bf4bb2SPeter Wemm * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 3453bf4bb2SPeter Wemm * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 3553bf4bb2SPeter Wemm * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3653bf4bb2SPeter Wemm * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3753bf4bb2SPeter Wemm * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3853bf4bb2SPeter Wemm * SUCH DAMAGE. 3953bf4bb2SPeter Wemm * 4053bf4bb2SPeter Wemm * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 4153bf4bb2SPeter Wemm */ 4253bf4bb2SPeter Wemm 43677b542eSDavid E. O'Brien #include <sys/cdefs.h> 44677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 45677b542eSDavid E. O'Brien 46be6847d7SJohn Baldwin #include "opt_ddb.h" 477c0435b9SKip Macy #include "opt_global.h" 48be6847d7SJohn Baldwin 4953bf4bb2SPeter Wemm #include <sys/param.h> 50c30bf5c3SRobert Watson #include <sys/kdb.h> 519722d88fSJason Evans #include <sys/kernel.h> 5261d80e90SJohn Baldwin #include <sys/ktr.h> 5353bf4bb2SPeter Wemm #include <sys/lock.h> 548302d183SBruce Evans #include <sys/lockmgr.h> 55d8881ca3SJohn Baldwin #include <sys/mutex.h> 568302d183SBruce Evans #include <sys/proc.h> 574bdb9b11SPeter Wemm #include <sys/systm.h> 587c0435b9SKip Macy #include <sys/lock_profile.h> 59e8ddb61dSJeff Roberson #ifdef DEBUG_LOCKS 60e8ddb61dSJeff Roberson #include <sys/stack.h> 61e8ddb61dSJeff Roberson #endif 6253bf4bb2SPeter Wemm 636efc8a16SAttilio Rao #define LOCKMGR_TRYOP(x) ((x) & LK_NOWAIT) 646efc8a16SAttilio Rao #define LOCKMGR_TRYW(x) (LOCKMGR_TRYOP((x)) ? LOP_TRYLOCK : 0) 6584887fa3SAttilio Rao #define LOCKMGR_UNHELD(x) (((x) & (LK_HAVE_EXCL | LK_SHARE_NONZERO)) == 0) 6684887fa3SAttilio Rao #define LOCKMGR_NOTOWNER(td) ((td) != curthread && (td) != LK_KERNPROC) 676efc8a16SAttilio Rao 68f9721b43SAttilio Rao static void assert_lockmgr(struct lock_object *lock, int what); 69be6847d7SJohn Baldwin #ifdef DDB 70be6847d7SJohn Baldwin #include <ddb/ddb.h> 7161bd5e21SKip Macy static void db_show_lockmgr(struct lock_object *lock); 72be6847d7SJohn Baldwin #endif 736e21afd4SJohn Baldwin static void lock_lockmgr(struct lock_object *lock, int how); 746e21afd4SJohn Baldwin static int unlock_lockmgr(struct lock_object *lock); 7561bd5e21SKip Macy 7661bd5e21SKip Macy struct lock_class lock_class_lockmgr = { 773ff6d229SJohn Baldwin .lc_name = "lockmgr", 783ff6d229SJohn Baldwin .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, 79f9721b43SAttilio Rao .lc_assert = assert_lockmgr, 8061bd5e21SKip Macy #ifdef DDB 816e21afd4SJohn Baldwin .lc_ddb_show = db_show_lockmgr, 8261bd5e21SKip Macy #endif 836e21afd4SJohn Baldwin .lc_lock = lock_lockmgr, 846e21afd4SJohn Baldwin .lc_unlock = unlock_lockmgr, 8561bd5e21SKip Macy }; 8661bd5e21SKip Macy 8784887fa3SAttilio Rao #ifndef INVARIANTS 8884887fa3SAttilio Rao #define _lockmgr_assert(lkp, what, file, line) 8984887fa3SAttilio Rao #endif 9084887fa3SAttilio Rao 9153bf4bb2SPeter Wemm /* 9253bf4bb2SPeter Wemm * Locking primitives implementation. 9353bf4bb2SPeter Wemm * Locks provide shared/exclusive sychronization. 9453bf4bb2SPeter Wemm */ 9553bf4bb2SPeter Wemm 966e21afd4SJohn Baldwin void 97f9721b43SAttilio Rao assert_lockmgr(struct lock_object *lock, int what) 98f9721b43SAttilio Rao { 99f9721b43SAttilio Rao 100f9721b43SAttilio Rao panic("lockmgr locks do not support assertions"); 101f9721b43SAttilio Rao } 102f9721b43SAttilio Rao 103f9721b43SAttilio Rao void 1046e21afd4SJohn Baldwin lock_lockmgr(struct lock_object *lock, int how) 1056e21afd4SJohn Baldwin { 1066e21afd4SJohn Baldwin 1076e21afd4SJohn Baldwin panic("lockmgr locks do not support sleep interlocking"); 1086e21afd4SJohn Baldwin } 1096e21afd4SJohn Baldwin 1106e21afd4SJohn Baldwin int 1116e21afd4SJohn Baldwin unlock_lockmgr(struct lock_object *lock) 1126e21afd4SJohn Baldwin { 1136e21afd4SJohn Baldwin 1146e21afd4SJohn Baldwin panic("lockmgr locks do not support sleep interlocking"); 1156e21afd4SJohn Baldwin } 1166e21afd4SJohn Baldwin 117d7a7e179SAttilio Rao #define COUNT(td, x) ((td)->td_locks += (x)) 11899448ed1SJohn Dyson #define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \ 11999448ed1SJohn Dyson LK_SHARE_NONZERO | LK_WAIT_NONZERO) 12099448ed1SJohn Dyson 121fe68a916SKip Macy static int acquire(struct lock **lkpp, int extflags, int wanted, int *contested, uint64_t *waittime); 12299448ed1SJohn Dyson static int acquiredrain(struct lock *lkp, int extflags) ; 12303e9c6c1SJohn Dyson 124a96ab770SJeff Roberson static __inline void 125f158df07SJeff Roberson sharelock(struct thread *td, struct lock *lkp, int incr) { 12603e9c6c1SJohn Dyson lkp->lk_flags |= LK_SHARE_NONZERO; 12703e9c6c1SJohn Dyson lkp->lk_sharecount += incr; 128f158df07SJeff Roberson COUNT(td, incr); 12903e9c6c1SJohn Dyson } 13003e9c6c1SJohn Dyson 131a96ab770SJeff Roberson static __inline void 132f158df07SJeff Roberson shareunlock(struct thread *td, struct lock *lkp, int decr) { 133219cbf59SEivind Eklund 1345526d2d9SEivind Eklund KASSERT(lkp->lk_sharecount >= decr, ("shareunlock: count < decr")); 13503e9c6c1SJohn Dyson 136f158df07SJeff Roberson COUNT(td, -decr); 1379b2e5badSJohn Dyson if (lkp->lk_sharecount == decr) { 13803e9c6c1SJohn Dyson lkp->lk_flags &= ~LK_SHARE_NONZERO; 1399b2e5badSJohn Dyson if (lkp->lk_flags & (LK_WANT_UPGRADE | LK_WANT_EXCL)) { 1409b2e5badSJohn Dyson wakeup(lkp); 1419b2e5badSJohn Dyson } 1429b2e5badSJohn Dyson lkp->lk_sharecount = 0; 1439b2e5badSJohn Dyson } else { 1449b2e5badSJohn Dyson lkp->lk_sharecount -= decr; 1459b2e5badSJohn Dyson } 14603e9c6c1SJohn Dyson } 14703e9c6c1SJohn Dyson 14803e9c6c1SJohn Dyson static int 149fe68a916SKip Macy acquire(struct lock **lkpp, int extflags, int wanted, int *contested, uint64_t *waittime) 15041bd6c15SJeff Roberson { 151c6964d3bSKirk McKusick struct lock *lkp = *lkpp; 15220728d8fSJeff Roberson int error; 153c06394f5SJohn Baldwin CTR3(KTR_LOCK, 154ff381670SRobert Watson "acquire(): lkp == %p, extflags == 0x%x, wanted == 0x%x", 155a18b1f1dSJason Evans lkp, extflags, wanted); 156a18b1f1dSJason Evans 15720728d8fSJeff Roberson if ((extflags & LK_NOWAIT) && (lkp->lk_flags & wanted)) 15803e9c6c1SJohn Dyson return EBUSY; 15920728d8fSJeff Roberson error = 0; 160fe68a916SKip Macy if ((lkp->lk_flags & wanted) != 0) 161fe68a916SKip Macy lock_profile_obtain_lock_failed(&lkp->lk_object, contested, waittime); 162fe68a916SKip Macy 16303e9c6c1SJohn Dyson while ((lkp->lk_flags & wanted) != 0) { 16420728d8fSJeff Roberson CTR2(KTR_LOCK, 16520728d8fSJeff Roberson "acquire(): lkp == %p, lk_flags == 0x%x sleeping", 16620728d8fSJeff Roberson lkp, lkp->lk_flags); 16703e9c6c1SJohn Dyson lkp->lk_flags |= LK_WAIT_NONZERO; 16803e9c6c1SJohn Dyson lkp->lk_waitcount++; 16996fde7daSJake Burkholder error = msleep(lkp, lkp->lk_interlock, lkp->lk_prio, 17023b59018SMatthew Dillon lkp->lk_wmesg, 17123b59018SMatthew Dillon ((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0)); 1729b2e5badSJohn Dyson lkp->lk_waitcount--; 17320728d8fSJeff Roberson if (lkp->lk_waitcount == 0) 17420728d8fSJeff Roberson lkp->lk_flags &= ~LK_WAIT_NONZERO; 17520728d8fSJeff Roberson if (error) 17620728d8fSJeff Roberson break; 17703e9c6c1SJohn Dyson if (extflags & LK_SLEEPFAIL) { 17820728d8fSJeff Roberson error = ENOLCK; 17920728d8fSJeff Roberson break; 18003e9c6c1SJohn Dyson } 181c6964d3bSKirk McKusick if (lkp->lk_newlock != NULL) { 182c6964d3bSKirk McKusick mtx_lock(lkp->lk_newlock->lk_interlock); 183c6964d3bSKirk McKusick mtx_unlock(lkp->lk_interlock); 184c6964d3bSKirk McKusick if (lkp->lk_waitcount == 0) 185c6964d3bSKirk McKusick wakeup((void *)(&lkp->lk_newlock)); 186c6964d3bSKirk McKusick *lkpp = lkp = lkp->lk_newlock; 187c6964d3bSKirk McKusick } 18803e9c6c1SJohn Dyson } 18920728d8fSJeff Roberson mtx_assert(lkp->lk_interlock, MA_OWNED); 19020728d8fSJeff Roberson return (error); 19103e9c6c1SJohn Dyson } 19203e9c6c1SJohn Dyson 19353bf4bb2SPeter Wemm /* 19453bf4bb2SPeter Wemm * Set, change, or release a lock. 19553bf4bb2SPeter Wemm * 19653bf4bb2SPeter Wemm * Shared requests increment the shared count. Exclusive requests set the 19753bf4bb2SPeter Wemm * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 19853bf4bb2SPeter Wemm * accepted shared locks and shared-to-exclusive upgrades to go away. 19953bf4bb2SPeter Wemm */ 20053bf4bb2SPeter Wemm int 2010e9eb108SAttilio Rao _lockmgr(struct lock *lkp, u_int flags, struct mtx *interlkp, char *file, 2020e9eb108SAttilio Rao int line) 2037c0435b9SKip Macy 20453bf4bb2SPeter Wemm { 2050e9eb108SAttilio Rao struct thread *td; 20653bf4bb2SPeter Wemm int error; 207635962afSJohn Baldwin int extflags, lockflags; 208fe68a916SKip Macy int contested = 0; 209fe68a916SKip Macy uint64_t waitstart = 0; 21053bf4bb2SPeter Wemm 21153bf4bb2SPeter Wemm error = 0; 2120e9eb108SAttilio Rao td = curthread; 21303e9c6c1SJohn Dyson 21484887fa3SAttilio Rao #ifdef INVARIANTS 21584887fa3SAttilio Rao if (lkp->lk_flags & LK_DESTROYED) { 21684887fa3SAttilio Rao if (flags & LK_INTERLOCK) 21784887fa3SAttilio Rao mtx_unlock(interlkp); 21884887fa3SAttilio Rao if (panicstr != NULL) 21984887fa3SAttilio Rao return (0); 22084887fa3SAttilio Rao panic("%s: %p lockmgr is destroyed", __func__, lkp); 22184887fa3SAttilio Rao } 22284887fa3SAttilio Rao #endif 22317661e5aSJeff Roberson if ((flags & LK_INTERNAL) == 0) 2249ed346baSBosko Milekic mtx_lock(lkp->lk_interlock); 22541bd6c15SJeff Roberson CTR6(KTR_LOCK, 22641bd6c15SJeff Roberson "lockmgr(): lkp == %p (lk_wmesg == \"%s\"), owner == %p, exclusivecount == %d, flags == 0x%x, " 22741bd6c15SJeff Roberson "td == %p", lkp, lkp->lk_wmesg, lkp->lk_lockholder, 22841bd6c15SJeff Roberson lkp->lk_exclusivecount, flags, td); 229e8ddb61dSJeff Roberson #ifdef DEBUG_LOCKS 230e8ddb61dSJeff Roberson { 231e8ddb61dSJeff Roberson struct stack stack; /* XXX */ 232e8ddb61dSJeff Roberson stack_save(&stack); 233e37a4994SPawel Jakub Dawidek CTRSTACK(KTR_LOCK, &stack, 0, 1); 234e8ddb61dSJeff Roberson } 23541bd6c15SJeff Roberson #endif 23641bd6c15SJeff Roberson 23798689e1eSAlfred Perlstein if (flags & LK_INTERLOCK) { 2386157b69fSAlfred Perlstein mtx_assert(interlkp, MA_OWNED | MA_NOTRECURSED); 2399ed346baSBosko Milekic mtx_unlock(interlkp); 24098689e1eSAlfred Perlstein } 24103e9c6c1SJohn Dyson 24217661e5aSJeff Roberson if ((flags & (LK_NOWAIT|LK_RELEASE)) == 0) 24326306795SJohn Baldwin WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, 244aa89d8cdSJohn Baldwin &lkp->lk_interlock->lock_object, 24526306795SJohn Baldwin "Acquiring lockmgr lock \"%s\"", lkp->lk_wmesg); 24617661e5aSJeff Roberson 2473f085c22SJohn Baldwin if (panicstr != NULL) { 2483f085c22SJohn Baldwin mtx_unlock(lkp->lk_interlock); 2493f085c22SJohn Baldwin return (0); 2503f085c22SJohn Baldwin } 251c4c0ec5bSJeff Roberson if ((lkp->lk_flags & LK_NOSHARE) && 252c4c0ec5bSJeff Roberson (flags & LK_TYPE_MASK) == LK_SHARED) { 253c4c0ec5bSJeff Roberson flags &= ~LK_TYPE_MASK; 254c4c0ec5bSJeff Roberson flags |= LK_EXCLUSIVE; 255c4c0ec5bSJeff Roberson } 25653bf4bb2SPeter Wemm extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 25753bf4bb2SPeter Wemm 25853bf4bb2SPeter Wemm switch (flags & LK_TYPE_MASK) { 25953bf4bb2SPeter Wemm 26053bf4bb2SPeter Wemm case LK_SHARED: 2616efc8a16SAttilio Rao if (!LOCKMGR_TRYOP(extflags)) 2626efc8a16SAttilio Rao WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER, file, 2636efc8a16SAttilio Rao line); 264beef8a36SJulian Elischer /* 265beef8a36SJulian Elischer * If we are not the exclusive lock holder, we have to block 266beef8a36SJulian Elischer * while there is an exclusive lock holder or while an 267beef8a36SJulian Elischer * exclusive lock request or upgrade request is in progress. 268beef8a36SJulian Elischer * 269fa2a4d05STim J. Robbins * However, if TDP_DEADLKTREAT is set, we override exclusive 270beef8a36SJulian Elischer * lock requests or upgrade requests ( but not the exclusive 271beef8a36SJulian Elischer * lock itself ). 272beef8a36SJulian Elischer */ 273d7a7e179SAttilio Rao if (lkp->lk_lockholder != td) { 274635962afSJohn Baldwin lockflags = LK_HAVE_EXCL; 2759032b51eSAttilio Rao if (!(td->td_pflags & TDP_DEADLKTREAT)) 276bce98419SJohn Baldwin lockflags |= LK_WANT_EXCL | LK_WANT_UPGRADE; 277fe68a916SKip Macy error = acquire(&lkp, extflags, lockflags, &contested, &waitstart); 27853bf4bb2SPeter Wemm if (error) 27953bf4bb2SPeter Wemm break; 280f158df07SJeff Roberson sharelock(td, lkp, 1); 2817c0435b9SKip Macy if (lkp->lk_sharecount == 1) 282fe68a916SKip Macy lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line); 2836efc8a16SAttilio Rao WITNESS_LOCK(&lkp->lk_object, LOCKMGR_TRYW(extflags), 2846efc8a16SAttilio Rao file, line); 2857c0435b9SKip Macy 2867181624aSJeff Roberson #if defined(DEBUG_LOCKS) 2877499fd8dSJeff Roberson stack_save(&lkp->lk_stack); 2887181624aSJeff Roberson #endif 28953bf4bb2SPeter Wemm break; 29053bf4bb2SPeter Wemm } 29153bf4bb2SPeter Wemm /* 29253bf4bb2SPeter Wemm * We hold an exclusive lock, so downgrade it to shared. 29353bf4bb2SPeter Wemm * An alternative would be to fail with EDEADLK. 29453bf4bb2SPeter Wemm */ 29593b0017fSPhilippe Charnier /* FALLTHROUGH downgrade */ 29653bf4bb2SPeter Wemm 29753bf4bb2SPeter Wemm case LK_DOWNGRADE: 29884887fa3SAttilio Rao _lockmgr_assert(lkp, KA_XLOCKED, file, line); 299f158df07SJeff Roberson sharelock(td, lkp, lkp->lk_exclusivecount); 3006efc8a16SAttilio Rao WITNESS_DOWNGRADE(&lkp->lk_object, 0, file, line); 301f158df07SJeff Roberson COUNT(td, -lkp->lk_exclusivecount); 30253bf4bb2SPeter Wemm lkp->lk_exclusivecount = 0; 30353bf4bb2SPeter Wemm lkp->lk_flags &= ~LK_HAVE_EXCL; 3046f8132a8SJulian Elischer lkp->lk_lockholder = LK_NOPROC; 30553bf4bb2SPeter Wemm if (lkp->lk_waitcount) 30653bf4bb2SPeter Wemm wakeup((void *)lkp); 30753bf4bb2SPeter Wemm break; 30853bf4bb2SPeter Wemm 30953bf4bb2SPeter Wemm case LK_UPGRADE: 31053bf4bb2SPeter Wemm /* 31153bf4bb2SPeter Wemm * Upgrade a shared lock to an exclusive one. If another 31253bf4bb2SPeter Wemm * shared lock has already requested an upgrade to an 31353bf4bb2SPeter Wemm * exclusive lock, our shared lock is released and an 31453bf4bb2SPeter Wemm * exclusive lock is requested (which will be granted 31553bf4bb2SPeter Wemm * after the upgrade). If we return an error, the file 31653bf4bb2SPeter Wemm * will always be unlocked. 31753bf4bb2SPeter Wemm */ 31884887fa3SAttilio Rao _lockmgr_assert(lkp, KA_SLOCKED, file, line); 319f158df07SJeff Roberson shareunlock(td, lkp, 1); 3207c0435b9SKip Macy if (lkp->lk_sharecount == 0) 3217c0435b9SKip Macy lock_profile_release_lock(&lkp->lk_object); 32253bf4bb2SPeter Wemm /* 32353bf4bb2SPeter Wemm * If we are just polling, check to see if we will block. 32453bf4bb2SPeter Wemm */ 32553bf4bb2SPeter Wemm if ((extflags & LK_NOWAIT) && 32653bf4bb2SPeter Wemm ((lkp->lk_flags & LK_WANT_UPGRADE) || 32753bf4bb2SPeter Wemm lkp->lk_sharecount > 1)) { 32853bf4bb2SPeter Wemm error = EBUSY; 3296efc8a16SAttilio Rao WITNESS_UNLOCK(&lkp->lk_object, 0, file, line); 33053bf4bb2SPeter Wemm break; 33153bf4bb2SPeter Wemm } 33253bf4bb2SPeter Wemm if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 33353bf4bb2SPeter Wemm /* 33453bf4bb2SPeter Wemm * We are first shared lock to request an upgrade, so 33553bf4bb2SPeter Wemm * request upgrade and wait for the shared count to 33653bf4bb2SPeter Wemm * drop to zero, then take exclusive lock. 33753bf4bb2SPeter Wemm */ 33853bf4bb2SPeter Wemm lkp->lk_flags |= LK_WANT_UPGRADE; 339fe68a916SKip Macy error = acquire(&lkp, extflags, LK_SHARE_NONZERO, &contested, &waitstart); 34053bf4bb2SPeter Wemm lkp->lk_flags &= ~LK_WANT_UPGRADE; 3419b2e5badSJohn Dyson 3424cef6d5aSAlexander Kabaev if (error) { 3434cef6d5aSAlexander Kabaev if ((lkp->lk_flags & ( LK_WANT_EXCL | LK_WAIT_NONZERO)) == (LK_WANT_EXCL | LK_WAIT_NONZERO)) 3444cef6d5aSAlexander Kabaev wakeup((void *)lkp); 3456efc8a16SAttilio Rao WITNESS_UNLOCK(&lkp->lk_object, 0, file, line); 34653bf4bb2SPeter Wemm break; 3474cef6d5aSAlexander Kabaev } 34853bf4bb2SPeter Wemm if (lkp->lk_exclusivecount != 0) 34953bf4bb2SPeter Wemm panic("lockmgr: non-zero exclusive count"); 350d8b8e875SPaul Saab lkp->lk_flags |= LK_HAVE_EXCL; 351d7a7e179SAttilio Rao lkp->lk_lockholder = td; 35253bf4bb2SPeter Wemm lkp->lk_exclusivecount = 1; 3536efc8a16SAttilio Rao WITNESS_UPGRADE(&lkp->lk_object, LOP_EXCLUSIVE | 3546efc8a16SAttilio Rao LOP_TRYLOCK, file, line); 355f158df07SJeff Roberson COUNT(td, 1); 356fe68a916SKip Macy lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line); 35715a1057cSEivind Eklund #if defined(DEBUG_LOCKS) 3587499fd8dSJeff Roberson stack_save(&lkp->lk_stack); 35915a1057cSEivind Eklund #endif 36053bf4bb2SPeter Wemm break; 36153bf4bb2SPeter Wemm } 36253bf4bb2SPeter Wemm /* 36353bf4bb2SPeter Wemm * Someone else has requested upgrade. Release our shared 36453bf4bb2SPeter Wemm * lock, awaken upgrade requestor if we are the last shared 36553bf4bb2SPeter Wemm * lock, then request an exclusive lock. 36653bf4bb2SPeter Wemm */ 3676efc8a16SAttilio Rao WITNESS_UNLOCK(&lkp->lk_object, 0, file, line); 36803e9c6c1SJohn Dyson if ( (lkp->lk_flags & (LK_SHARE_NONZERO|LK_WAIT_NONZERO)) == 36903e9c6c1SJohn Dyson LK_WAIT_NONZERO) 37053bf4bb2SPeter Wemm wakeup((void *)lkp); 37193b0017fSPhilippe Charnier /* FALLTHROUGH exclusive request */ 37253bf4bb2SPeter Wemm 37353bf4bb2SPeter Wemm case LK_EXCLUSIVE: 3746efc8a16SAttilio Rao if (!LOCKMGR_TRYOP(extflags)) 3756efc8a16SAttilio Rao WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER | 3766efc8a16SAttilio Rao LOP_EXCLUSIVE, file, line); 377d7a7e179SAttilio Rao if (lkp->lk_lockholder == td) { 37853bf4bb2SPeter Wemm /* 37953bf4bb2SPeter Wemm * Recursive lock. 38053bf4bb2SPeter Wemm */ 38133638e93SKirk McKusick if ((extflags & (LK_NOWAIT | LK_CANRECURSE)) == 0) 38253bf4bb2SPeter Wemm panic("lockmgr: locking against myself"); 38333638e93SKirk McKusick if ((extflags & LK_CANRECURSE) != 0) { 38453bf4bb2SPeter Wemm lkp->lk_exclusivecount++; 3856efc8a16SAttilio Rao WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE | 3866efc8a16SAttilio Rao LOCKMGR_TRYW(extflags), file, line); 387f158df07SJeff Roberson COUNT(td, 1); 38853bf4bb2SPeter Wemm break; 38953bf4bb2SPeter Wemm } 39033638e93SKirk McKusick } 39153bf4bb2SPeter Wemm /* 39253bf4bb2SPeter Wemm * If we are just polling, check to see if we will sleep. 39353bf4bb2SPeter Wemm */ 39403e9c6c1SJohn Dyson if ((extflags & LK_NOWAIT) && 39503e9c6c1SJohn Dyson (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO))) { 39653bf4bb2SPeter Wemm error = EBUSY; 39753bf4bb2SPeter Wemm break; 39853bf4bb2SPeter Wemm } 39953bf4bb2SPeter Wemm /* 40053bf4bb2SPeter Wemm * Try to acquire the want_exclusive flag. 40153bf4bb2SPeter Wemm */ 402fe68a916SKip Macy error = acquire(&lkp, extflags, (LK_HAVE_EXCL | LK_WANT_EXCL), &contested, &waitstart); 40353bf4bb2SPeter Wemm if (error) 40453bf4bb2SPeter Wemm break; 40553bf4bb2SPeter Wemm lkp->lk_flags |= LK_WANT_EXCL; 40653bf4bb2SPeter Wemm /* 40753bf4bb2SPeter Wemm * Wait for shared locks and upgrades to finish. 40853bf4bb2SPeter Wemm */ 409fe68a916SKip Macy error = acquire(&lkp, extflags, LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, &contested, &waitstart); 41053bf4bb2SPeter Wemm lkp->lk_flags &= ~LK_WANT_EXCL; 4114cef6d5aSAlexander Kabaev if (error) { 4124cef6d5aSAlexander Kabaev if (lkp->lk_flags & LK_WAIT_NONZERO) 4134cef6d5aSAlexander Kabaev wakeup((void *)lkp); 41453bf4bb2SPeter Wemm break; 4154cef6d5aSAlexander Kabaev } 41653bf4bb2SPeter Wemm lkp->lk_flags |= LK_HAVE_EXCL; 417d7a7e179SAttilio Rao lkp->lk_lockholder = td; 41853bf4bb2SPeter Wemm if (lkp->lk_exclusivecount != 0) 41953bf4bb2SPeter Wemm panic("lockmgr: non-zero exclusive count"); 42053bf4bb2SPeter Wemm lkp->lk_exclusivecount = 1; 4216efc8a16SAttilio Rao WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE | 4226efc8a16SAttilio Rao LOCKMGR_TRYW(extflags), file, line); 423f158df07SJeff Roberson COUNT(td, 1); 424fe68a916SKip Macy lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line); 42515a1057cSEivind Eklund #if defined(DEBUG_LOCKS) 4267499fd8dSJeff Roberson stack_save(&lkp->lk_stack); 42715a1057cSEivind Eklund #endif 42853bf4bb2SPeter Wemm break; 42953bf4bb2SPeter Wemm 43053bf4bb2SPeter Wemm case LK_RELEASE: 43184887fa3SAttilio Rao _lockmgr_assert(lkp, KA_LOCKED, file, line); 43253bf4bb2SPeter Wemm if (lkp->lk_exclusivecount != 0) { 4336efc8a16SAttilio Rao if (lkp->lk_lockholder != LK_KERNPROC) { 4346efc8a16SAttilio Rao WITNESS_UNLOCK(&lkp->lk_object, LOP_EXCLUSIVE, 4356efc8a16SAttilio Rao file, line); 436f158df07SJeff Roberson COUNT(td, -1); 4376efc8a16SAttilio Rao } 43884887fa3SAttilio Rao if (lkp->lk_exclusivecount-- == 1) { 43953bf4bb2SPeter Wemm lkp->lk_flags &= ~LK_HAVE_EXCL; 44053bf4bb2SPeter Wemm lkp->lk_lockholder = LK_NOPROC; 4417c0435b9SKip Macy lock_profile_release_lock(&lkp->lk_object); 44253bf4bb2SPeter Wemm } 4436efc8a16SAttilio Rao } else if (lkp->lk_flags & LK_SHARE_NONZERO) { 4446efc8a16SAttilio Rao WITNESS_UNLOCK(&lkp->lk_object, 0, file, line); 445f158df07SJeff Roberson shareunlock(td, lkp, 1); 4461f71de49SSuleiman Souhlal } 4471f71de49SSuleiman Souhlal 44803e9c6c1SJohn Dyson if (lkp->lk_flags & LK_WAIT_NONZERO) 44953bf4bb2SPeter Wemm wakeup((void *)lkp); 45053bf4bb2SPeter Wemm break; 45153bf4bb2SPeter Wemm 45253bf4bb2SPeter Wemm case LK_DRAIN: 45353bf4bb2SPeter Wemm /* 45453bf4bb2SPeter Wemm * Check that we do not already hold the lock, as it can 45553bf4bb2SPeter Wemm * never drain if we do. Unfortunately, we have no way to 45653bf4bb2SPeter Wemm * check for holding a shared lock, but at least we can 45753bf4bb2SPeter Wemm * check for an exclusive one. 45853bf4bb2SPeter Wemm */ 4596efc8a16SAttilio Rao if (!LOCKMGR_TRYOP(extflags)) 4606efc8a16SAttilio Rao WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER | 4616efc8a16SAttilio Rao LOP_EXCLUSIVE, file, line); 462d7a7e179SAttilio Rao if (lkp->lk_lockholder == td) 46353bf4bb2SPeter Wemm panic("lockmgr: draining against myself"); 46403e9c6c1SJohn Dyson 46503e9c6c1SJohn Dyson error = acquiredrain(lkp, extflags); 46603e9c6c1SJohn Dyson if (error) 46753bf4bb2SPeter Wemm break; 46853bf4bb2SPeter Wemm lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 469d7a7e179SAttilio Rao lkp->lk_lockholder = td; 47053bf4bb2SPeter Wemm lkp->lk_exclusivecount = 1; 4716efc8a16SAttilio Rao WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE | 4726efc8a16SAttilio Rao LOCKMGR_TRYW(extflags), file, line); 473f158df07SJeff Roberson COUNT(td, 1); 47415a1057cSEivind Eklund #if defined(DEBUG_LOCKS) 4757499fd8dSJeff Roberson stack_save(&lkp->lk_stack); 47615a1057cSEivind Eklund #endif 47753bf4bb2SPeter Wemm break; 47853bf4bb2SPeter Wemm 47953bf4bb2SPeter Wemm default: 4809ed346baSBosko Milekic mtx_unlock(lkp->lk_interlock); 48153bf4bb2SPeter Wemm panic("lockmgr: unknown locktype request %d", 48253bf4bb2SPeter Wemm flags & LK_TYPE_MASK); 48353bf4bb2SPeter Wemm /* NOTREACHED */ 48453bf4bb2SPeter Wemm } 48503e9c6c1SJohn Dyson if ((lkp->lk_flags & LK_WAITDRAIN) && 48603e9c6c1SJohn Dyson (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 48703e9c6c1SJohn Dyson LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0) { 48853bf4bb2SPeter Wemm lkp->lk_flags &= ~LK_WAITDRAIN; 48953bf4bb2SPeter Wemm wakeup((void *)&lkp->lk_flags); 49053bf4bb2SPeter Wemm } 4919ed346baSBosko Milekic mtx_unlock(lkp->lk_interlock); 49253bf4bb2SPeter Wemm return (error); 49353bf4bb2SPeter Wemm } 49453bf4bb2SPeter Wemm 49599448ed1SJohn Dyson static int 49699448ed1SJohn Dyson acquiredrain(struct lock *lkp, int extflags) { 49799448ed1SJohn Dyson int error; 49899448ed1SJohn Dyson 49999448ed1SJohn Dyson if ((extflags & LK_NOWAIT) && (lkp->lk_flags & LK_ALL)) { 50099448ed1SJohn Dyson return EBUSY; 50199448ed1SJohn Dyson } 50299448ed1SJohn Dyson while (lkp->lk_flags & LK_ALL) { 50399448ed1SJohn Dyson lkp->lk_flags |= LK_WAITDRAIN; 50496fde7daSJake Burkholder error = msleep(&lkp->lk_flags, lkp->lk_interlock, lkp->lk_prio, 50523b59018SMatthew Dillon lkp->lk_wmesg, 50623b59018SMatthew Dillon ((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0)); 50799448ed1SJohn Dyson if (error) 50899448ed1SJohn Dyson return error; 50999448ed1SJohn Dyson if (extflags & LK_SLEEPFAIL) { 51099448ed1SJohn Dyson return ENOLCK; 51199448ed1SJohn Dyson } 51299448ed1SJohn Dyson } 51399448ed1SJohn Dyson return 0; 51499448ed1SJohn Dyson } 51599448ed1SJohn Dyson 51699448ed1SJohn Dyson /* 51799448ed1SJohn Dyson * Initialize a lock; required before use. 51899448ed1SJohn Dyson */ 51999448ed1SJohn Dyson void 52099448ed1SJohn Dyson lockinit(lkp, prio, wmesg, timo, flags) 52199448ed1SJohn Dyson struct lock *lkp; 52299448ed1SJohn Dyson int prio; 52304858e7eSEivind Eklund const char *wmesg; 52499448ed1SJohn Dyson int timo; 52599448ed1SJohn Dyson int flags; 52699448ed1SJohn Dyson { 5276efc8a16SAttilio Rao int iflags; 5286efc8a16SAttilio Rao 529c06394f5SJohn Baldwin CTR5(KTR_LOCK, "lockinit(): lkp == %p, prio == %d, wmesg == \"%s\", " 530a18b1f1dSJason Evans "timo == %d, flags = 0x%x\n", lkp, prio, wmesg, timo, flags); 53199448ed1SJohn Dyson 532857d9c60SDon Lewis lkp->lk_interlock = mtx_pool_alloc(mtxpool_lockbuilder); 5336efc8a16SAttilio Rao lkp->lk_flags = (flags & LK_EXTFLG_MASK) & ~(LK_NOWITNESS | LK_NODUP); 53499448ed1SJohn Dyson lkp->lk_sharecount = 0; 53599448ed1SJohn Dyson lkp->lk_waitcount = 0; 53699448ed1SJohn Dyson lkp->lk_exclusivecount = 0; 53799448ed1SJohn Dyson lkp->lk_prio = prio; 53899448ed1SJohn Dyson lkp->lk_timo = timo; 53999448ed1SJohn Dyson lkp->lk_lockholder = LK_NOPROC; 540c6964d3bSKirk McKusick lkp->lk_newlock = NULL; 5416efc8a16SAttilio Rao iflags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE; 5426efc8a16SAttilio Rao if (!(flags & LK_NODUP)) 5436efc8a16SAttilio Rao iflags |= LO_DUPOK; 5446efc8a16SAttilio Rao if (!(flags & LK_NOWITNESS)) 5456efc8a16SAttilio Rao iflags |= LO_WITNESS; 5463a096f6cSKirk McKusick #ifdef DEBUG_LOCKS 5477499fd8dSJeff Roberson stack_zero(&lkp->lk_stack); 5483a096f6cSKirk McKusick #endif 5496efc8a16SAttilio Rao lock_init(&lkp->lk_object, &lock_class_lockmgr, wmesg, NULL, iflags); 55099448ed1SJohn Dyson } 55199448ed1SJohn Dyson 55299448ed1SJohn Dyson /* 553a18b1f1dSJason Evans * Destroy a lock. 554a18b1f1dSJason Evans */ 555a18b1f1dSJason Evans void 556a18b1f1dSJason Evans lockdestroy(lkp) 557a18b1f1dSJason Evans struct lock *lkp; 558a18b1f1dSJason Evans { 559c91fcee7SJohn Baldwin 560c06394f5SJohn Baldwin CTR2(KTR_LOCK, "lockdestroy(): lkp == %p (lk_wmesg == \"%s\")", 561a18b1f1dSJason Evans lkp, lkp->lk_wmesg); 56284887fa3SAttilio Rao KASSERT((lkp->lk_flags & (LK_HAVE_EXCL | LK_SHARE_NONZERO)) == 0, 56384887fa3SAttilio Rao ("lockmgr still held")); 56484887fa3SAttilio Rao KASSERT(lkp->lk_exclusivecount == 0, ("lockmgr still recursed")); 56584887fa3SAttilio Rao lkp->lk_flags = LK_DESTROYED; 566ab2dab16SJohn Baldwin lock_destroy(&lkp->lk_object); 567a18b1f1dSJason Evans } 568a18b1f1dSJason Evans 569a18b1f1dSJason Evans /* 570d7a7e179SAttilio Rao * Disown the lockmgr. 571d7a7e179SAttilio Rao */ 572d7a7e179SAttilio Rao void 5736efc8a16SAttilio Rao _lockmgr_disown(struct lock *lkp, const char *file, int line) 574d7a7e179SAttilio Rao { 575d7a7e179SAttilio Rao struct thread *td; 576d7a7e179SAttilio Rao 577d7a7e179SAttilio Rao td = curthread; 57884887fa3SAttilio Rao KASSERT(panicstr != NULL || (lkp->lk_flags & LK_DESTROYED) == 0, 57984887fa3SAttilio Rao ("%s: %p lockmgr is destroyed", __func__, lkp)); 58084887fa3SAttilio Rao _lockmgr_assert(lkp, KA_XLOCKED | KA_NOTRECURSED, file, line); 581d7a7e179SAttilio Rao 582d7a7e179SAttilio Rao /* 583d7a7e179SAttilio Rao * Drop the lock reference and switch the owner. This will result 584d7a7e179SAttilio Rao * in an atomic operation like td_lock is only accessed by curthread 585d1127e66SAttilio Rao * and lk_lockholder only needs one write. Note also that the lock 586d1127e66SAttilio Rao * owner can be alredy KERNPROC, so in that case just skip the 587d1127e66SAttilio Rao * decrement. 588d7a7e179SAttilio Rao */ 5896efc8a16SAttilio Rao if (lkp->lk_lockholder == td) { 5906efc8a16SAttilio Rao WITNESS_UNLOCK(&lkp->lk_object, LOP_EXCLUSIVE, file, line); 591d7a7e179SAttilio Rao td->td_locks--; 5926efc8a16SAttilio Rao } 593d7a7e179SAttilio Rao lkp->lk_lockholder = LK_KERNPROC; 594d7a7e179SAttilio Rao } 595d7a7e179SAttilio Rao 596d7a7e179SAttilio Rao /* 59799448ed1SJohn Dyson * Determine the status of a lock. 59899448ed1SJohn Dyson */ 59999448ed1SJohn Dyson int 600b40ce416SJulian Elischer lockstatus(lkp, td) 60199448ed1SJohn Dyson struct lock *lkp; 602b40ce416SJulian Elischer struct thread *td; 60399448ed1SJohn Dyson { 60499448ed1SJohn Dyson int lock_type = 0; 6052b59d50cSRobert Watson int interlocked; 60699448ed1SJohn Dyson 6072433c488SAttilio Rao KASSERT(td == curthread, 6080e9eb108SAttilio Rao ("%s: thread passed argument (%p) is not valid", __func__, td)); 60984887fa3SAttilio Rao KASSERT((lkp->lk_flags & LK_DESTROYED) == 0, 61084887fa3SAttilio Rao ("%s: %p lockmgr is destroyed", __func__, lkp)); 6110e9eb108SAttilio Rao 6122b59d50cSRobert Watson if (!kdb_active) { 6132b59d50cSRobert Watson interlocked = 1; 6149ed346baSBosko Milekic mtx_lock(lkp->lk_interlock); 6152b59d50cSRobert Watson } else 6162b59d50cSRobert Watson interlocked = 0; 6176bdfe06aSEivind Eklund if (lkp->lk_exclusivecount != 0) { 6182433c488SAttilio Rao if (lkp->lk_lockholder == td) 61999448ed1SJohn Dyson lock_type = LK_EXCLUSIVE; 6206bdfe06aSEivind Eklund else 6216bdfe06aSEivind Eklund lock_type = LK_EXCLOTHER; 6226bdfe06aSEivind Eklund } else if (lkp->lk_sharecount != 0) 62399448ed1SJohn Dyson lock_type = LK_SHARED; 6242b59d50cSRobert Watson if (interlocked) 6259ed346baSBosko Milekic mtx_unlock(lkp->lk_interlock); 62699448ed1SJohn Dyson return (lock_type); 62799448ed1SJohn Dyson } 62899448ed1SJohn Dyson 62953bf4bb2SPeter Wemm /* 63004aa807cSTor Egge * Determine the number of waiters on a lock. 63104aa807cSTor Egge */ 63204aa807cSTor Egge int 63304aa807cSTor Egge lockwaiters(lkp) 63404aa807cSTor Egge struct lock *lkp; 63504aa807cSTor Egge { 63604aa807cSTor Egge int count; 63704aa807cSTor Egge 63884887fa3SAttilio Rao KASSERT((lkp->lk_flags & LK_DESTROYED) == 0, 63984887fa3SAttilio Rao ("%s: %p lockmgr is destroyed", __func__, lkp)); 64004aa807cSTor Egge mtx_lock(lkp->lk_interlock); 64104aa807cSTor Egge count = lkp->lk_waitcount; 64204aa807cSTor Egge mtx_unlock(lkp->lk_interlock); 64304aa807cSTor Egge return (count); 64404aa807cSTor Egge } 64504aa807cSTor Egge 64604aa807cSTor Egge /* 64753bf4bb2SPeter Wemm * Print out information about state of a lock. Used by VOP_PRINT 6480e61ac7bSPoul-Henning Kamp * routines to display status about contained locks. 64953bf4bb2SPeter Wemm */ 650a1ce9d5cSPeter Wemm void 65153bf4bb2SPeter Wemm lockmgr_printinfo(lkp) 65253bf4bb2SPeter Wemm struct lock *lkp; 65353bf4bb2SPeter Wemm { 65453bf4bb2SPeter Wemm 65553bf4bb2SPeter Wemm if (lkp->lk_sharecount) 65653bf4bb2SPeter Wemm printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 65753bf4bb2SPeter Wemm lkp->lk_sharecount); 65853bf4bb2SPeter Wemm else if (lkp->lk_flags & LK_HAVE_EXCL) 659c969c60cSAlexander Kabaev printf(" lock type %s: EXCL (count %d) by thread %p (pid %d)", 660c969c60cSAlexander Kabaev lkp->lk_wmesg, lkp->lk_exclusivecount, 661c969c60cSAlexander Kabaev lkp->lk_lockholder, lkp->lk_lockholder->td_proc->p_pid); 66253bf4bb2SPeter Wemm if (lkp->lk_waitcount > 0) 66353bf4bb2SPeter Wemm printf(" with %d pending", lkp->lk_waitcount); 6647499fd8dSJeff Roberson #ifdef DEBUG_LOCKS 6659ccca7d1SRobert Watson stack_print_ddb(&lkp->lk_stack); 6667499fd8dSJeff Roberson #endif 66753bf4bb2SPeter Wemm } 668be6847d7SJohn Baldwin 66984887fa3SAttilio Rao #ifdef INVARIANT_SUPPORT 67084887fa3SAttilio Rao #ifndef INVARIANTS 67184887fa3SAttilio Rao #undef _lockmgr_assert 67284887fa3SAttilio Rao #endif 67384887fa3SAttilio Rao 67484887fa3SAttilio Rao void 67584887fa3SAttilio Rao _lockmgr_assert(struct lock *lkp, int what, const char *file, int line) 67684887fa3SAttilio Rao { 67784887fa3SAttilio Rao struct thread *td; 67884887fa3SAttilio Rao u_int x; 67984887fa3SAttilio Rao int slocked = 0; 68084887fa3SAttilio Rao 68184887fa3SAttilio Rao x = lkp->lk_flags; 68284887fa3SAttilio Rao td = lkp->lk_lockholder; 68384887fa3SAttilio Rao if (panicstr != NULL) 68484887fa3SAttilio Rao return; 68584887fa3SAttilio Rao switch (what) { 68684887fa3SAttilio Rao case KA_SLOCKED: 68784887fa3SAttilio Rao case KA_SLOCKED | KA_NOTRECURSED: 68884887fa3SAttilio Rao case KA_SLOCKED | KA_RECURSED: 68984887fa3SAttilio Rao slocked = 1; 69084887fa3SAttilio Rao case KA_LOCKED: 69184887fa3SAttilio Rao case KA_LOCKED | KA_NOTRECURSED: 69284887fa3SAttilio Rao case KA_LOCKED | KA_RECURSED: 69384887fa3SAttilio Rao #ifdef WITNESS 69484887fa3SAttilio Rao /* 69584887fa3SAttilio Rao * We cannot trust WITNESS if the lock is held in 69684887fa3SAttilio Rao * exclusive mode and a call to lockmgr_disown() happened. 69784887fa3SAttilio Rao * Workaround this skipping the check if the lock is 69884887fa3SAttilio Rao * held in exclusive mode even for the KA_LOCKED case. 69984887fa3SAttilio Rao */ 70084887fa3SAttilio Rao if (slocked || (x & LK_HAVE_EXCL) == 0) { 70184887fa3SAttilio Rao witness_assert(&lkp->lk_object, what, file, line); 70284887fa3SAttilio Rao break; 70384887fa3SAttilio Rao } 70484887fa3SAttilio Rao #endif 70584887fa3SAttilio Rao if (LOCKMGR_UNHELD(x) || ((x & LK_SHARE_NONZERO) == 0 && 70684887fa3SAttilio Rao (slocked || LOCKMGR_NOTOWNER(td)))) 70784887fa3SAttilio Rao panic("Lock %s not %slocked @ %s:%d\n", 70884887fa3SAttilio Rao lkp->lk_object.lo_name, slocked ? "share " : "", 70984887fa3SAttilio Rao file, line); 71084887fa3SAttilio Rao if ((x & LK_SHARE_NONZERO) == 0) { 71184887fa3SAttilio Rao if (lockmgr_recursed(lkp)) { 71284887fa3SAttilio Rao if (what & KA_NOTRECURSED) 71384887fa3SAttilio Rao panic("Lock %s recursed @ %s:%d\n", 71484887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 71584887fa3SAttilio Rao } else if (what & KA_RECURSED) 71684887fa3SAttilio Rao panic("Lock %s not recursed @ %s:%d\n", 71784887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 71884887fa3SAttilio Rao } 71984887fa3SAttilio Rao break; 72084887fa3SAttilio Rao case KA_XLOCKED: 72184887fa3SAttilio Rao case KA_XLOCKED | KA_NOTRECURSED: 72284887fa3SAttilio Rao case KA_XLOCKED | KA_RECURSED: 72384887fa3SAttilio Rao if ((x & LK_HAVE_EXCL) == 0 || LOCKMGR_NOTOWNER(td)) 72484887fa3SAttilio Rao panic("Lock %s not exclusively locked @ %s:%d\n", 72584887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 72684887fa3SAttilio Rao if (lockmgr_recursed(lkp)) { 72784887fa3SAttilio Rao if (what & KA_NOTRECURSED) 72884887fa3SAttilio Rao panic("Lock %s recursed @ %s:%d\n", 72984887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 73084887fa3SAttilio Rao } else if (what & KA_RECURSED) 73184887fa3SAttilio Rao panic("Lock %s not recursed @ %s:%d\n", 73284887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 73384887fa3SAttilio Rao break; 73484887fa3SAttilio Rao case KA_UNLOCKED: 73584887fa3SAttilio Rao if (td == curthread || td == LK_KERNPROC) 73684887fa3SAttilio Rao panic("Lock %s exclusively locked @ %s:%d\n", 73784887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 73884887fa3SAttilio Rao break; 73984887fa3SAttilio Rao case KA_HELD: 74084887fa3SAttilio Rao case KA_UNHELD: 74184887fa3SAttilio Rao if (LOCKMGR_UNHELD(x)) { 74284887fa3SAttilio Rao if (what & KA_HELD) 74384887fa3SAttilio Rao panic("Lock %s not locked by anyone @ %s:%d\n", 74484887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 74584887fa3SAttilio Rao } else if (what & KA_UNHELD) 74684887fa3SAttilio Rao panic("Lock %s locked by someone @ %s:%d\n", 74784887fa3SAttilio Rao lkp->lk_object.lo_name, file, line); 74884887fa3SAttilio Rao break; 74984887fa3SAttilio Rao default: 75084887fa3SAttilio Rao panic("Unknown lockmgr lock assertion: 0x%x @ %s:%d", what, 75184887fa3SAttilio Rao file, line); 75284887fa3SAttilio Rao } 75384887fa3SAttilio Rao } 75484887fa3SAttilio Rao #endif /* INVARIANT_SUPPORT */ 75584887fa3SAttilio Rao 756be6847d7SJohn Baldwin #ifdef DDB 757462a7addSJohn Baldwin /* 758462a7addSJohn Baldwin * Check to see if a thread that is blocked on a sleep queue is actually 759462a7addSJohn Baldwin * blocked on a 'struct lock'. If so, output some details and return true. 760462a7addSJohn Baldwin * If the lock has an exclusive owner, return that in *ownerp. 761462a7addSJohn Baldwin */ 762462a7addSJohn Baldwin int 763462a7addSJohn Baldwin lockmgr_chain(struct thread *td, struct thread **ownerp) 764462a7addSJohn Baldwin { 765462a7addSJohn Baldwin struct lock *lkp; 766462a7addSJohn Baldwin 767462a7addSJohn Baldwin lkp = td->td_wchan; 768462a7addSJohn Baldwin 769462a7addSJohn Baldwin /* Simple test to see if wchan points to a lockmgr lock. */ 770ab2dab16SJohn Baldwin if (LOCK_CLASS(&lkp->lk_object) == &lock_class_lockmgr && 771ab2dab16SJohn Baldwin lkp->lk_wmesg == td->td_wmesg) 7726d257b6eSJohn Baldwin goto ok; 7736d257b6eSJohn Baldwin 7746d257b6eSJohn Baldwin /* 7756d257b6eSJohn Baldwin * If this thread is doing a DRAIN, then it would be asleep on 7766d257b6eSJohn Baldwin * &lkp->lk_flags rather than lkp. 7776d257b6eSJohn Baldwin */ 7786d257b6eSJohn Baldwin lkp = (struct lock *)((char *)td->td_wchan - 7796d257b6eSJohn Baldwin offsetof(struct lock, lk_flags)); 780ab2dab16SJohn Baldwin if (LOCK_CLASS(&lkp->lk_object) == &lock_class_lockmgr && 781ab2dab16SJohn Baldwin lkp->lk_wmesg == td->td_wmesg && (lkp->lk_flags & LK_WAITDRAIN)) 7826d257b6eSJohn Baldwin goto ok; 7836d257b6eSJohn Baldwin 7846d257b6eSJohn Baldwin /* Doen't seem to be a lockmgr lock. */ 785462a7addSJohn Baldwin return (0); 786462a7addSJohn Baldwin 7876d257b6eSJohn Baldwin ok: 788462a7addSJohn Baldwin /* Ok, we think we have a lockmgr lock, so output some details. */ 789462a7addSJohn Baldwin db_printf("blocked on lk \"%s\" ", lkp->lk_wmesg); 790462a7addSJohn Baldwin if (lkp->lk_sharecount) { 791462a7addSJohn Baldwin db_printf("SHARED (count %d)\n", lkp->lk_sharecount); 792462a7addSJohn Baldwin *ownerp = NULL; 793462a7addSJohn Baldwin } else { 794462a7addSJohn Baldwin db_printf("EXCL (count %d)\n", lkp->lk_exclusivecount); 795462a7addSJohn Baldwin *ownerp = lkp->lk_lockholder; 796462a7addSJohn Baldwin } 797462a7addSJohn Baldwin return (1); 798462a7addSJohn Baldwin } 799462a7addSJohn Baldwin 80061bd5e21SKip Macy void 80161bd5e21SKip Macy db_show_lockmgr(struct lock_object *lock) 802be6847d7SJohn Baldwin { 803be6847d7SJohn Baldwin struct thread *td; 804be6847d7SJohn Baldwin struct lock *lkp; 805be6847d7SJohn Baldwin 80661bd5e21SKip Macy lkp = (struct lock *)lock; 807be6847d7SJohn Baldwin 808be6847d7SJohn Baldwin db_printf(" lock type: %s\n", lkp->lk_wmesg); 809be6847d7SJohn Baldwin db_printf(" state: "); 810be6847d7SJohn Baldwin if (lkp->lk_sharecount) 811be6847d7SJohn Baldwin db_printf("SHARED (count %d)\n", lkp->lk_sharecount); 812be6847d7SJohn Baldwin else if (lkp->lk_flags & LK_HAVE_EXCL) { 813be6847d7SJohn Baldwin td = lkp->lk_lockholder; 814be6847d7SJohn Baldwin db_printf("EXCL (count %d) %p ", lkp->lk_exclusivecount, td); 815be6847d7SJohn Baldwin db_printf("(tid %d, pid %d, \"%s\")\n", td->td_tid, 816431f8906SJulian Elischer td->td_proc->p_pid, td->td_name); 817be6847d7SJohn Baldwin } else 818be6847d7SJohn Baldwin db_printf("UNLOCKED\n"); 819be6847d7SJohn Baldwin if (lkp->lk_waitcount > 0) 820be6847d7SJohn Baldwin db_printf(" waiters: %d\n", lkp->lk_waitcount); 821be6847d7SJohn Baldwin } 822be6847d7SJohn Baldwin #endif 823