xref: /freebsd/sys/kern/kern_lock.c (revision a52a51a2d5900e475c3dc6203e09a376316ce90f)

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice(s), this list of conditions and the following disclaimer as
 *    the first lines of this file unmodified other than the possible
 *    addition of one or more copyright notices.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice(s), this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 */

#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"

#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/lock_profile.h>
#include <sys/lockmgr.h>
#include <sys/lockstat.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sleepqueue.h>
#ifdef DEBUG_LOCKS
#include <sys/stack.h>
#endif
#include <sys/sysctl.h>
#include <sys/systm.h>

#include <machine/cpu.h>

#ifdef DDB
#include <ddb/ddb.h>
#endif

#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DECLARE( , , lock, failed);
#endif

/*
 * Hack. There should be prio_t or similar so that this is not necessary.
 */
_Static_assert((PRILASTFLAG * 2) - 1 <= USHRT_MAX,
    "prio flags wont fit in u_short pri in struct lock");

CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
    ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));

#define	SQ_EXCLUSIVE_QUEUE	0
#define	SQ_SHARED_QUEUE		1

#ifndef INVARIANTS
#define	_lockmgr_assert(lk, what, file, line)
#endif

#define	TD_SLOCKS_INC(td)	((td)->td_lk_slocks++)
#define	TD_SLOCKS_DEC(td)	((td)->td_lk_slocks--)

#ifndef DEBUG_LOCKS
#define	STACK_PRINT(lk)
#define	STACK_SAVE(lk)
#define	STACK_ZERO(lk)
#else
#define	STACK_PRINT(lk)	stack_print_ddb(&(lk)->lk_stack)
#define	STACK_SAVE(lk)	stack_save(&(lk)->lk_stack)
#define	STACK_ZERO(lk)	stack_zero(&(lk)->lk_stack)
#endif

#define	LOCK_LOG2(lk, string, arg1, arg2)				\
	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
		CTR2(KTR_LOCK, (string), (arg1), (arg2))
#define	LOCK_LOG3(lk, string, arg1, arg2, arg3)				\
	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
		CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))

#define	GIANT_DECLARE							\
	int _i = 0;							\
	WITNESS_SAVE_DECL(Giant)
#define	GIANT_RESTORE() do {						\
	if (__predict_false(_i > 0)) {					\
		while (_i--)						\
			mtx_lock(&Giant);				\
		WITNESS_RESTORE(&Giant.lock_object, Giant);		\
	}								\
} while (0)
#define	GIANT_SAVE() do {						\
	if (__predict_false(mtx_owned(&Giant))) {			\
		WITNESS_SAVE(&Giant.lock_object, Giant);		\
		while (mtx_owned(&Giant)) {				\
			_i++;						\
			mtx_unlock(&Giant);				\
		}							\
	}								\
} while (0)

static __always_inline bool
LK_CAN_SHARE(uintptr_t x, int flags, bool fp)
{

	if ((x & (LK_SHARE | LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) ==
	    LK_SHARE)
		return (true);
	if (fp || (!(x & LK_SHARE)))
		return (false);
	if ((curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||
	    (curthread->td_pflags & TDP_DEADLKTREAT))
		return (true);
	return (false);
}

#define	LK_TRYOP(x)							\
	((x) & LK_NOWAIT)

#define	LK_CAN_WITNESS(x)						\
	(((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
#define	LK_TRYWIT(x)							\
	(LK_TRYOP(x) ? LOP_TRYLOCK : 0)

#define	lockmgr_xlocked_v(v)						\
	(((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)

#define	lockmgr_xlocked(lk) lockmgr_xlocked_v(lockmgr_read_value(lk))

static void	assert_lockmgr(const struct lock_object *lock, int how);
#ifdef DDB
static void	db_show_lockmgr(const struct lock_object *lock);
#endif
static void	lock_lockmgr(struct lock_object *lock, uintptr_t how);
#ifdef KDTRACE_HOOKS
static int	owner_lockmgr(const struct lock_object *lock,
		    struct thread **owner);
#endif
static uintptr_t unlock_lockmgr(struct lock_object *lock);

struct lock_class lock_class_lockmgr = {
	.lc_name = "lockmgr",
	.lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
	.lc_assert = assert_lockmgr,
#ifdef DDB
	.lc_ddb_show = db_show_lockmgr,
#endif
	.lc_lock = lock_lockmgr,
	.lc_unlock = unlock_lockmgr,
#ifdef KDTRACE_HOOKS
	.lc_owner = owner_lockmgr,
#endif
};

static __read_mostly bool lk_adaptive = true;
static SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging");
SYSCTL_BOOL(_debug_lockmgr, OID_AUTO, adaptive_spinning, CTLFLAG_RW, &lk_adaptive,
    0, "");
#define lockmgr_delay  locks_delay

struct lockmgr_wait {
	const char *iwmesg;
	int ipri;
	int itimo;
};

static __always_inline bool lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
    int flags, bool fp);
static __always_inline bool lockmgr_sunlock_try(struct lock *lk,
    uintptr_t *xp);

static void
lockmgr_exit(u_int flags, struct lock_object *ilk)
{
	struct lock_class *class;

	if (flags & LK_INTERLOCK) {
		class = LOCK_CLASS(ilk);
		class->lc_unlock(ilk);
	}
}

static void
lockmgr_note_shared_acquire(struct lock *lk, int contested,
    uint64_t waittime, const char *file, int line, int flags)
{

	LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
	    waittime, file, line, LOCKSTAT_READER);
	LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
	WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
	TD_LOCKS_INC(curthread);
	TD_SLOCKS_INC(curthread);
	STACK_SAVE(lk);
}

static void
lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
{

	WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
	TD_LOCKS_DEC(curthread);
	TD_SLOCKS_DEC(curthread);
}

static void
lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
    uint64_t waittime, const char *file, int line, int flags)
{

	LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
	    waittime, file, line, LOCKSTAT_WRITER);
	LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
	WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
	    line);
	TD_LOCKS_INC(curthread);
	STACK_SAVE(lk);
}

static void
lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
{

	if (!lockmgr_disowned(lk)) {
		WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
		TD_LOCKS_DEC(curthread);
	}
	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
	    line);
}

static __inline struct thread *
lockmgr_xholder(const struct lock *lk)
{
	uintptr_t x;

	x = lockmgr_read_value(lk);
	return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
}

/*
 * It assumes sleepq_lock held and returns with this one unheld.
 * It also assumes the generic interlock is sane and previously checked.
 * If LK_INTERLOCK is specified the interlock is not reacquired after the
 * sleep.
 */
static __inline int
sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
    const char *wmesg, int pri, int timo, int queue)
{
	GIANT_DECLARE;
	struct lock_class *class;
	int catch, error;

	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
	catch = pri & PCATCH;
	pri &= PRIMASK;
	error = 0;

	LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
	    (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");

	if (flags & LK_INTERLOCK)
		class->lc_unlock(ilk);
	if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0) {
		if (lk->lk_exslpfail < USHRT_MAX)
			lk->lk_exslpfail++;
	}
	GIANT_SAVE();
	sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
	    SLEEPQ_INTERRUPTIBLE : 0), queue);
	if ((flags & LK_TIMELOCK) && timo)
		sleepq_set_timeout(&lk->lock_object, timo);

	/*
	 * Decisional switch for real sleeping.
	 */
	if ((flags & LK_TIMELOCK) && timo && catch)
		error = sleepq_timedwait_sig(&lk->lock_object, pri);
	else if ((flags & LK_TIMELOCK) && timo)
		error = sleepq_timedwait(&lk->lock_object, pri);
	else if (catch)
		error = sleepq_wait_sig(&lk->lock_object, pri);
	else
		sleepq_wait(&lk->lock_object, pri);
	GIANT_RESTORE();
	if ((flags & LK_SLEEPFAIL) && error == 0)
		error = ENOLCK;

	return (error);
}

static __inline void
wakeupshlk(struct lock *lk, const char *file, int line)
{
	uintptr_t v, x, orig_x;
	u_int realexslp;
	int queue;

	for (;;) {
		x = lockmgr_read_value(lk);
		if (lockmgr_sunlock_try(lk, &x))
			break;

		/*
		 * We should have a sharer with waiters, so enter the hard
		 * path in order to handle wakeups correctly.
		 */
		sleepq_lock(&lk->lock_object);
		orig_x = lockmgr_read_value(lk);
retry_sleepq:
		x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
		v = LK_UNLOCKED;

		/*
		 * If the lock has exclusive waiters, give them preference in
		 * order to avoid deadlock with shared runners up.
		 * If interruptible sleeps left the exclusive queue empty
		 * avoid a starvation for the threads sleeping on the shared
		 * queue by giving them precedence and cleaning up the
		 * exclusive waiters bit anyway.
		 * Please note that lk_exslpfail count may be lying about
		 * the real number of waiters with the LK_SLEEPFAIL flag on
		 * because they may be used in conjunction with interruptible
		 * sleeps so lk_exslpfail might be considered an 'upper limit'
		 * bound, including the edge cases.
		 */
		realexslp = sleepq_sleepcnt(&lk->lock_object,
		    SQ_EXCLUSIVE_QUEUE);
		if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
			if (lk->lk_exslpfail != USHRT_MAX && lk->lk_exslpfail < realexslp) {
				lk->lk_exslpfail = 0;
				queue = SQ_EXCLUSIVE_QUEUE;
				v |= (x & LK_SHARED_WAITERS);
			} else {
				lk->lk_exslpfail = 0;
				LOCK_LOG2(lk,
				    "%s: %p has only LK_SLEEPFAIL sleepers",
				    __func__, lk);
				LOCK_LOG2(lk,
			    "%s: %p waking up threads on the exclusive queue",
				    __func__, lk);
				sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
				    SQ_EXCLUSIVE_QUEUE);
				queue = SQ_SHARED_QUEUE;
			}
		} else {
			/*
			 * Exclusive waiters sleeping with LK_SLEEPFAIL on
			 * and using interruptible sleeps/timeout may have
			 * left spourious lk_exslpfail counts on, so clean
			 * it up anyway.
			 */
			lk->lk_exslpfail = 0;
			queue = SQ_SHARED_QUEUE;
		}

		if (lockmgr_sunlock_try(lk, &orig_x)) {
			sleepq_release(&lk->lock_object);
			break;
		}

		x |= LK_SHARERS_LOCK(1);
		if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
			orig_x = x;
			goto retry_sleepq;
		}
		LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
		    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
		    "exclusive");
		sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
		sleepq_release(&lk->lock_object);
		break;
	}

	LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
}

static void
assert_lockmgr(const struct lock_object *lock, int what)
{

	panic("lockmgr locks do not support assertions");
}

static void
lock_lockmgr(struct lock_object *lock, uintptr_t how)
{

	panic("lockmgr locks do not support sleep interlocking");
}

static uintptr_t
unlock_lockmgr(struct lock_object *lock)
{

	panic("lockmgr locks do not support sleep interlocking");
}

#ifdef KDTRACE_HOOKS
static int
owner_lockmgr(const struct lock_object *lock, struct thread **owner)
{

	panic("lockmgr locks do not support owner inquiring");
}
#endif

void
lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
{
	int iflags;

	MPASS((flags & ~LK_INIT_MASK) == 0);
	ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
            ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
            &lk->lk_lock));

	iflags = LO_SLEEPABLE | LO_UPGRADABLE;
	if (flags & LK_CANRECURSE)
		iflags |= LO_RECURSABLE;
	if ((flags & LK_NODUP) == 0)
		iflags |= LO_DUPOK;
	if (flags & LK_NOPROFILE)
		iflags |= LO_NOPROFILE;
	if ((flags & LK_NOWITNESS) == 0)
		iflags |= LO_WITNESS;
	if (flags & LK_QUIET)
		iflags |= LO_QUIET;
	if (flags & LK_IS_VNODE)
		iflags |= LO_IS_VNODE;
	if (flags & LK_NEW)
		iflags |= LO_NEW;
	iflags |= flags & LK_NOSHARE;

	lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
	lk->lk_lock = LK_UNLOCKED;
	lk->lk_recurse = 0;
	lk->lk_exslpfail = 0;
	lk->lk_timo = timo;
	lk->lk_pri = pri;
	STACK_ZERO(lk);
}

/*
 * XXX: Gross hacks to manipulate external lock flags after
 * initialization.  Used for certain vnode and buf locks.
 */
void
lockallowshare(struct lock *lk)
{

	lockmgr_assert(lk, KA_XLOCKED);
	lk->lock_object.lo_flags &= ~LK_NOSHARE;
}

void
lockdisableshare(struct lock *lk)
{

	lockmgr_assert(lk, KA_XLOCKED);
	lk->lock_object.lo_flags |= LK_NOSHARE;
}

void
lockallowrecurse(struct lock *lk)
{

	lockmgr_assert(lk, KA_XLOCKED);
	lk->lock_object.lo_flags |= LO_RECURSABLE;
}

void
lockdisablerecurse(struct lock *lk)
{

	lockmgr_assert(lk, KA_XLOCKED);
	lk->lock_object.lo_flags &= ~LO_RECURSABLE;
}

void
lockdestroy(struct lock *lk)
{

	KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
	KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
	KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
	lock_destroy(&lk->lock_object);
}

static __always_inline bool
lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags, bool fp)
{

	/*
	 * If no other thread has an exclusive lock, or
	 * no exclusive waiter is present, bump the count of
	 * sharers.  Since we have to preserve the state of
	 * waiters, if we fail to acquire the shared lock
	 * loop back and retry.
	 */
	while (LK_CAN_SHARE(*xp, flags, fp)) {
		if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
		    *xp + LK_ONE_SHARER)) {
			return (true);
		}
	}
	return (false);
}

static __always_inline bool
lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp)
{

	for (;;) {
		if (LK_SHARERS(*xp) > 1 || !(*xp & LK_ALL_WAITERS)) {
			if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
			    *xp - LK_ONE_SHARER))
				return (true);
			continue;
		}
		break;
	}
	return (false);
}

static bool
lockmgr_slock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp,
    int flags)
{
	struct thread *owner;
	uintptr_t x;

	x = *xp;
	MPASS(x != LK_UNLOCKED);
	owner = (struct thread *)LK_HOLDER(x);
	for (;;) {
		MPASS(owner != curthread);
		if (owner == (struct thread *)LK_KERNPROC)
			return (false);
		if ((x & LK_SHARE) && LK_SHARERS(x) > 0)
			return (false);
		if (owner == NULL)
			return (false);
		if (!TD_IS_RUNNING(owner))
			return (false);
		if ((x & LK_ALL_WAITERS) != 0)
			return (false);
		lock_delay(lda);
		x = lockmgr_read_value(lk);
		if (LK_CAN_SHARE(x, flags, false)) {
			*xp = x;
			return (true);
		}
		owner = (struct thread *)LK_HOLDER(x);
	}
}

static __noinline int
lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
    const char *file, int line, struct lockmgr_wait *lwa)
{
	uintptr_t tid, x;
	int error = 0;
	const char *iwmesg;
	int ipri, itimo;

#ifdef KDTRACE_HOOKS
	uint64_t sleep_time = 0;
#endif
#ifdef LOCK_PROFILING
	uint64_t waittime = 0;
	int contested = 0;
#endif
	struct lock_delay_arg lda;

	if (SCHEDULER_STOPPED())
		goto out;

	tid = (uintptr_t)curthread;

	if (LK_CAN_WITNESS(flags))
		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
		    file, line, flags & LK_INTERLOCK ? ilk : NULL);
	x = lockmgr_read_value(lk);
	lock_delay_arg_init(&lda, &lockmgr_delay);
	if (!lk_adaptive)
		flags &= ~LK_ADAPTIVE;
	/*
	 * The lock may already be locked exclusive by curthread,
	 * avoid deadlock.
	 */
	if (LK_HOLDER(x) == tid) {
		LOCK_LOG2(lk,
		    "%s: %p already held in exclusive mode",
		    __func__, lk);
		error = EDEADLK;
		goto out;
	}

	for (;;) {
		if (lockmgr_slock_try(lk, &x, flags, false))
			break;

		lock_profile_obtain_lock_failed(&lk->lock_object, false,
		    &contested, &waittime);

		if ((flags & (LK_ADAPTIVE | LK_INTERLOCK)) == LK_ADAPTIVE) {
			if (lockmgr_slock_adaptive(&lda, lk, &x, flags))
				continue;
		}

#ifdef HWPMC_HOOKS
		PMC_SOFT_CALL( , , lock, failed);
#endif

		/*
		 * If the lock is expected to not sleep just give up
		 * and return.
		 */
		if (LK_TRYOP(flags)) {
			LOCK_LOG2(lk, "%s: %p fails the try operation",
			    __func__, lk);
			error = EBUSY;
			break;
		}

		/*
		 * Acquire the sleepqueue chain lock because we
		 * probabilly will need to manipulate waiters flags.
		 */
		sleepq_lock(&lk->lock_object);
		x = lockmgr_read_value(lk);
retry_sleepq:

		/*
		 * if the lock can be acquired in shared mode, try
		 * again.
		 */
		if (LK_CAN_SHARE(x, flags, false)) {
			sleepq_release(&lk->lock_object);
			continue;
		}

		/*
		 * Try to set the LK_SHARED_WAITERS flag.  If we fail,
		 * loop back and retry.
		 */
		if ((x & LK_SHARED_WAITERS) == 0) {
			if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
			    x | LK_SHARED_WAITERS)) {
				goto retry_sleepq;
			}
			LOCK_LOG2(lk, "%s: %p set shared waiters flag",
			    __func__, lk);
		}

		if (lwa == NULL) {
			iwmesg = lk->lock_object.lo_name;
			ipri = lk->lk_pri;
			itimo = lk->lk_timo;
		} else {
			iwmesg = lwa->iwmesg;
			ipri = lwa->ipri;
			itimo = lwa->itimo;
		}

		/*
		 * As far as we have been unable to acquire the
		 * shared lock and the shared waiters flag is set,
		 * we will sleep.
		 */
#ifdef KDTRACE_HOOKS
		sleep_time -= lockstat_nsecs(&lk->lock_object);
#endif
		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
		    SQ_SHARED_QUEUE);
#ifdef KDTRACE_HOOKS
		sleep_time += lockstat_nsecs(&lk->lock_object);
#endif
		flags &= ~LK_INTERLOCK;
		if (error) {
			LOCK_LOG3(lk,
			    "%s: interrupted sleep for %p with %d",
			    __func__, lk, error);
			break;
		}
		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
		    __func__, lk);
		x = lockmgr_read_value(lk);
	}
	if (error == 0) {
#ifdef KDTRACE_HOOKS
		if (sleep_time != 0)
			LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
			    LOCKSTAT_READER, (x & LK_SHARE) == 0,
			    (x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
#endif
#ifdef LOCK_PROFILING
		lockmgr_note_shared_acquire(lk, contested, waittime,
		    file, line, flags);
#else
		lockmgr_note_shared_acquire(lk, 0, 0, file, line,
		    flags);
#endif
	}

out:
	lockmgr_exit(flags, ilk);
	return (error);
}

static bool
lockmgr_xlock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp)
{
	struct thread *owner;
	uintptr_t x;

	x = *xp;
	MPASS(x != LK_UNLOCKED);
	owner = (struct thread *)LK_HOLDER(x);
	for (;;) {
		MPASS(owner != curthread);
		if (owner == NULL)
			return (false);
		if ((x & LK_SHARE) && LK_SHARERS(x) > 0)
			return (false);
		if (owner == (struct thread *)LK_KERNPROC)
			return (false);
		if (!TD_IS_RUNNING(owner))
			return (false);
		if ((x & LK_ALL_WAITERS) != 0)
			return (false);
		lock_delay(lda);
		x = lockmgr_read_value(lk);
		if (x == LK_UNLOCKED) {
			*xp = x;
			return (true);
		}
		owner = (struct thread *)LK_HOLDER(x);
	}
}

static __noinline int
lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
    const char *file, int line, struct lockmgr_wait *lwa)
{
	struct lock_class *class;
	uintptr_t tid, x, v;
	int error = 0;
	const char *iwmesg;
	int ipri, itimo;

#ifdef KDTRACE_HOOKS
	uint64_t sleep_time = 0;
#endif
#ifdef LOCK_PROFILING
	uint64_t waittime = 0;
	int contested = 0;
#endif
	struct lock_delay_arg lda;

	if (SCHEDULER_STOPPED())
		goto out;

	tid = (uintptr_t)curthread;

	if (LK_CAN_WITNESS(flags))
		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
		    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
		    ilk : NULL);

	/*
	 * If curthread already holds the lock and this one is
	 * allowed to recurse, simply recurse on it.
	 */
	if (lockmgr_xlocked(lk)) {
		if ((flags & LK_CANRECURSE) == 0 &&
		    (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
			/*
			 * If the lock is expected to not panic just
			 * give up and return.
			 */
			if (LK_TRYOP(flags)) {
				LOCK_LOG2(lk,
				    "%s: %p fails the try operation",
				    __func__, lk);
				error = EBUSY;
				goto out;
			}
			if (flags & LK_INTERLOCK) {
				class = LOCK_CLASS(ilk);
				class->lc_unlock(ilk);
			}
			STACK_PRINT(lk);
			panic("%s: recursing on non recursive lockmgr %p "
			    "@ %s:%d\n", __func__, lk, file, line);
		}
		atomic_set_ptr(&lk->lk_lock, LK_WRITER_RECURSED);
		lk->lk_recurse++;
		LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
		LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
		    lk->lk_recurse, file, line);
		WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
		    LK_TRYWIT(flags), file, line);
		TD_LOCKS_INC(curthread);
		goto out;
	}

	x = LK_UNLOCKED;
	lock_delay_arg_init(&lda, &lockmgr_delay);
	if (!lk_adaptive)
		flags &= ~LK_ADAPTIVE;
	for (;;) {
		if (x == LK_UNLOCKED) {
			if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x, tid))
				break;
			continue;
		}

		lock_profile_obtain_lock_failed(&lk->lock_object, false,
		    &contested, &waittime);

		if ((flags & (LK_ADAPTIVE | LK_INTERLOCK)) == LK_ADAPTIVE) {
			if (lockmgr_xlock_adaptive(&lda, lk, &x))
				continue;
		}
#ifdef HWPMC_HOOKS
		PMC_SOFT_CALL( , , lock, failed);
#endif

		/*
		 * If the lock is expected to not sleep just give up
		 * and return.
		 */
		if (LK_TRYOP(flags)) {
			LOCK_LOG2(lk, "%s: %p fails the try operation",
			    __func__, lk);
			error = EBUSY;
			break;
		}

		/*
		 * Acquire the sleepqueue chain lock because we
		 * probabilly will need to manipulate waiters flags.
		 */
		sleepq_lock(&lk->lock_object);
		x = lockmgr_read_value(lk);
retry_sleepq:

		/*
		 * if the lock has been released while we spun on
		 * the sleepqueue chain lock just try again.
		 */
		if (x == LK_UNLOCKED) {
			sleepq_release(&lk->lock_object);
			continue;
		}

		/*
		 * The lock can be in the state where there is a
		 * pending queue of waiters, but still no owner.
		 * This happens when the lock is contested and an
		 * owner is going to claim the lock.
		 * If curthread is the one successfully acquiring it
		 * claim lock ownership and return, preserving waiters
		 * flags.
		 */
		v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
		if ((x & ~v) == LK_UNLOCKED) {
			v &= ~LK_EXCLUSIVE_SPINNERS;
			if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
			    tid | v)) {
				sleepq_release(&lk->lock_object);
				LOCK_LOG2(lk,
				    "%s: %p claimed by a new writer",
				    __func__, lk);
				break;
			}
			goto retry_sleepq;
		}

		/*
		 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
		 * fail, loop back and retry.
		 */
		if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
			if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
			    x | LK_EXCLUSIVE_WAITERS)) {
				goto retry_sleepq;
			}
			LOCK_LOG2(lk, "%s: %p set excl waiters flag",
			    __func__, lk);
		}

		if (lwa == NULL) {
			iwmesg = lk->lock_object.lo_name;
			ipri = lk->lk_pri;
			itimo = lk->lk_timo;
		} else {
			iwmesg = lwa->iwmesg;
			ipri = lwa->ipri;
			itimo = lwa->itimo;
		}

		/*
		 * As far as we have been unable to acquire the
		 * exclusive lock and the exclusive waiters flag
		 * is set, we will sleep.
		 */
#ifdef KDTRACE_HOOKS
		sleep_time -= lockstat_nsecs(&lk->lock_object);
#endif
		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
		    SQ_EXCLUSIVE_QUEUE);
#ifdef KDTRACE_HOOKS
		sleep_time += lockstat_nsecs(&lk->lock_object);
#endif
		flags &= ~LK_INTERLOCK;
		if (error) {
			LOCK_LOG3(lk,
			    "%s: interrupted sleep for %p with %d",
			    __func__, lk, error);
			break;
		}
		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
		    __func__, lk);
		x = lockmgr_read_value(lk);
	}
	if (error == 0) {
#ifdef KDTRACE_HOOKS
		if (sleep_time != 0)
			LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
			    LOCKSTAT_WRITER, (x & LK_SHARE) == 0,
			    (x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
#endif
#ifdef LOCK_PROFILING
		lockmgr_note_exclusive_acquire(lk, contested, waittime,
		    file, line, flags);
#else
		lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
		    flags);
#endif
	}

out:
	lockmgr_exit(flags, ilk);
	return (error);
}

static __noinline int
lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
    const char *file, int line, struct lockmgr_wait *lwa)
{
	uintptr_t tid, v, setv;
	int error = 0;
	int op;

	if (SCHEDULER_STOPPED())
		goto out;

	tid = (uintptr_t)curthread;

	_lockmgr_assert(lk, KA_SLOCKED, file, line);

	op = flags & LK_TYPE_MASK;
	v = lockmgr_read_value(lk);
	for (;;) {
		if (LK_SHARERS(v) > 1) {
			if (op == LK_TRYUPGRADE) {
				LOCK_LOG2(lk, "%s: %p failed the nowait upgrade",
				    __func__, lk);
				error = EBUSY;
				goto out;
			}
			if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &v,
			    v - LK_ONE_SHARER)) {
				lockmgr_note_shared_release(lk, file, line);
				goto out_xlock;
			}
			continue;
		}
		MPASS((v & ~LK_ALL_WAITERS) == LK_SHARERS_LOCK(1));

		setv = tid;
		setv |= (v & LK_ALL_WAITERS);

		/*
		 * Try to switch from one shared lock to an exclusive one.
		 * We need to preserve waiters flags during the operation.
		 */
		if (atomic_fcmpset_ptr(&lk->lk_lock, &v, setv)) {
			LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
			    line);
			WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
			    LK_TRYWIT(flags), file, line);
			LOCKSTAT_RECORD0(lockmgr__upgrade, lk);
			TD_SLOCKS_DEC(curthread);
			goto out;
		}
	}

out_xlock:
	error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
	flags &= ~LK_INTERLOCK;
out:
	lockmgr_exit(flags, ilk);
	return (error);
}

int
lockmgr_lock_flags(struct lock *lk, u_int flags, struct lock_object *ilk,
    const char *file, int line)
{
	struct lock_class *class;
	uintptr_t x, tid;
	u_int op;
	bool locked;

	if (SCHEDULER_STOPPED())
		return (0);

	op = flags & LK_TYPE_MASK;
	locked = false;
	switch (op) {
	case LK_SHARED:
		if (LK_CAN_WITNESS(flags))
			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
			    file, line, flags & LK_INTERLOCK ? ilk : NULL);
		if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
			break;
		x = lockmgr_read_value(lk);
		if (lockmgr_slock_try(lk, &x, flags, true)) {
			lockmgr_note_shared_acquire(lk, 0, 0,
			    file, line, flags);
			locked = true;
		} else {
			return (lockmgr_slock_hard(lk, flags, ilk, file, line,
			    NULL));
		}
		break;
	case LK_EXCLUSIVE:
		if (LK_CAN_WITNESS(flags))
			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
			    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
			    ilk : NULL);
		tid = (uintptr_t)curthread;
		if (lockmgr_read_value(lk) == LK_UNLOCKED &&
		    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
			lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
			    flags);
			locked = true;
		} else {
			return (lockmgr_xlock_hard(lk, flags, ilk, file, line,
			    NULL));
		}
		break;
	case LK_UPGRADE:
	case LK_TRYUPGRADE:
		return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL));
	default:
		break;
	}
	if (__predict_true(locked)) {
		if (__predict_false(flags & LK_INTERLOCK)) {
			class = LOCK_CLASS(ilk);
			class->lc_unlock(ilk);
		}
		return (0);
	} else {
		return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
		    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
	}
}

static __noinline int
lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
    const char *file, int line)
{
	if (!SCHEDULER_STOPPED())
		wakeupshlk(lk, file, line);
	lockmgr_exit(flags, ilk);
	return (0);
}

static __noinline int
lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
    const char *file, int line)
{
	uintptr_t tid, v;
	u_int realexslp;
	int queue;

	if (SCHEDULER_STOPPED())
		goto out;

	tid = (uintptr_t)curthread;

	/*
	 * As first option, treact the lock as if it has not
	 * any waiter.
	 * Fix-up the tid var if the lock has been disowned.
	 */
	if (lockmgr_disowned_v(x))
		tid = LK_KERNPROC;

	/*
	 * The lock is held in exclusive mode.
	 * If the lock is recursed also, then unrecurse it.
	 */
	if (lockmgr_recursed_v(x)) {
		LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);
		lk->lk_recurse--;
		if (lk->lk_recurse == 0)
			atomic_clear_ptr(&lk->lk_lock, LK_WRITER_RECURSED);
		goto out;
	}
	if (tid != LK_KERNPROC)
		LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk,
		    LOCKSTAT_WRITER);

	if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
		goto out;

	sleepq_lock(&lk->lock_object);
	x = lockmgr_read_value(lk);
	v = LK_UNLOCKED;

	/*
	 * If the lock has exclusive waiters, give them
	 * preference in order to avoid deadlock with
	 * shared runners up.
	 * If interruptible sleeps left the exclusive queue
	 * empty avoid a starvation for the threads sleeping
	 * on the shared queue by giving them precedence
	 * and cleaning up the exclusive waiters bit anyway.
	 * Please note that lk_exslpfail count may be lying
	 * about the real number of waiters with the
	 * LK_SLEEPFAIL flag on because they may be used in
	 * conjunction with interruptible sleeps so
	 * lk_exslpfail might be considered an 'upper limit'
	 * bound, including the edge cases.
	 */
	MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
	realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
	if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
		if (lk->lk_exslpfail != USHRT_MAX && lk->lk_exslpfail < realexslp) {
			lk->lk_exslpfail = 0;
			queue = SQ_EXCLUSIVE_QUEUE;
			v |= (x & LK_SHARED_WAITERS);
		} else {
			lk->lk_exslpfail = 0;
			LOCK_LOG2(lk,
			    "%s: %p has only LK_SLEEPFAIL sleepers",
			    __func__, lk);
			LOCK_LOG2(lk,
			    "%s: %p waking up threads on the exclusive queue",
			    __func__, lk);
			sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
			    SQ_EXCLUSIVE_QUEUE);
			queue = SQ_SHARED_QUEUE;
		}
	} else {
		/*
		 * Exclusive waiters sleeping with LK_SLEEPFAIL
		 * on and using interruptible sleeps/timeout
		 * may have left spourious lk_exslpfail counts
		 * on, so clean it up anyway.
		 */
		lk->lk_exslpfail = 0;
		queue = SQ_SHARED_QUEUE;
	}

	LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
	    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
	    "exclusive");
	atomic_store_rel_ptr(&lk->lk_lock, v);
	sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
	sleepq_release(&lk->lock_object);

out:
	lockmgr_exit(flags, ilk);
	return (0);
}

/*
 * Lightweight entry points for common operations.
 *
 * Functionality is similar to sx locks, in that none of the additional lockmgr
 * features are supported. To be clear, these are NOT supported:
 * 1. shared locking disablement
 * 2. returning with an error after sleep
 * 3. unlocking the interlock
 *
 * If in doubt, use lockmgr_lock_flags.
 */
int
lockmgr_slock(struct lock *lk, u_int flags, const char *file, int line)
{
	uintptr_t x;

	MPASS((flags & LK_TYPE_MASK) == LK_SHARED);
	MPASS((flags & LK_INTERLOCK) == 0);
	MPASS((lk->lock_object.lo_flags & LK_NOSHARE) == 0);

	if (LK_CAN_WITNESS(flags))
		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
		    file, line, NULL);
	x = lockmgr_read_value(lk);
	if (__predict_true(lockmgr_slock_try(lk, &x, flags, true))) {
		lockmgr_note_shared_acquire(lk, 0, 0, file, line, flags);
		return (0);
	}

	return (lockmgr_slock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL));
}

int
lockmgr_xlock(struct lock *lk, u_int flags, const char *file, int line)
{
	uintptr_t tid;

	MPASS((flags & LK_TYPE_MASK) == LK_EXCLUSIVE);
	MPASS((flags & LK_INTERLOCK) == 0);

	if (LK_CAN_WITNESS(flags))
		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
		    LOP_EXCLUSIVE, file, line, NULL);
	tid = (uintptr_t)curthread;
	if (atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
		lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
		    flags);
		return (0);
	}

	return (lockmgr_xlock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL));
}

int
lockmgr_unlock(struct lock *lk)
{
	uintptr_t x, tid;
	const char *file;
	int line;

	file = __FILE__;
	line = __LINE__;

	_lockmgr_assert(lk, KA_LOCKED, file, line);
	x = lockmgr_read_value(lk);
	if (__predict_true(x & LK_SHARE) != 0) {
		lockmgr_note_shared_release(lk, file, line);
		if (lockmgr_sunlock_try(lk, &x)) {
			LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
		} else {
			return (lockmgr_sunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
		}
	} else {
		tid = (uintptr_t)curthread;
		lockmgr_note_exclusive_release(lk, file, line);
		if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
			LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk,LOCKSTAT_WRITER);
		} else {
			return (lockmgr_xunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
		}
	}
	return (0);
}

int
__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
    const char *wmesg, int pri, int timo, const char *file, int line)
{
	GIANT_DECLARE;
	struct lockmgr_wait lwa;
	struct lock_class *class;
	const char *iwmesg;
	uintptr_t tid, v, x;
	u_int op, realexslp;
	int error, ipri, itimo, queue;
#ifdef LOCK_PROFILING
	uint64_t waittime = 0;
	int contested = 0;
#endif

	if (SCHEDULER_STOPPED())
		return (0);

	error = 0;
	tid = (uintptr_t)curthread;
	op = (flags & LK_TYPE_MASK);
	iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
	ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
	itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;

	lwa.iwmesg = iwmesg;
	lwa.ipri = ipri;
	lwa.itimo = itimo;

	MPASS((flags & ~LK_TOTAL_MASK) == 0);
	KASSERT((op & (op - 1)) == 0,
	    ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
	KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
	    (op != LK_DOWNGRADE && op != LK_RELEASE),
	    ("%s: Invalid flags in regard of the operation desired @ %s:%d",
	    __func__, file, line));
	KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
	    ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
	    __func__, file, line));
	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
	    ("%s: idle thread %p on lockmgr %p @ %s:%d", __func__, curthread,
	    lk, file, line));

	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;

	if (lk->lock_object.lo_flags & LK_NOSHARE) {
		switch (op) {
		case LK_SHARED:
			op = LK_EXCLUSIVE;
			break;
		case LK_UPGRADE:
		case LK_TRYUPGRADE:
		case LK_DOWNGRADE:
			_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
			    file, line);
			if (flags & LK_INTERLOCK)
				class->lc_unlock(ilk);
			return (0);
		}
	}

	switch (op) {
	case LK_SHARED:
		return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
		break;
	case LK_UPGRADE:
	case LK_TRYUPGRADE:
		return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
		break;
	case LK_EXCLUSIVE:
		return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
		break;
	case LK_DOWNGRADE:
		_lockmgr_assert(lk, KA_XLOCKED, file, line);
		WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);

		/*
		 * Panic if the lock is recursed.
		 */
		if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
			if (flags & LK_INTERLOCK)
				class->lc_unlock(ilk);
			panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n",
			    __func__, iwmesg, file, line);
		}
		TD_SLOCKS_INC(curthread);

		/*
		 * In order to preserve waiters flags, just spin.
		 */
		for (;;) {
			x = lockmgr_read_value(lk);
			MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
			x &= LK_ALL_WAITERS;
			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
			    LK_SHARERS_LOCK(1) | x))
				break;
			cpu_spinwait();
		}
		LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
		LOCKSTAT_RECORD0(lockmgr__downgrade, lk);
		break;
	case LK_RELEASE:
		_lockmgr_assert(lk, KA_LOCKED, file, line);
		x = lockmgr_read_value(lk);

		if (__predict_true(x & LK_SHARE) != 0) {
			lockmgr_note_shared_release(lk, file, line);
			return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
		} else {
			lockmgr_note_exclusive_release(lk, file, line);
			return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
		}
		break;
	case LK_DRAIN:
		if (LK_CAN_WITNESS(flags))
			WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
			    LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
			    ilk : NULL);

		/*
		 * Trying to drain a lock we already own will result in a
		 * deadlock.
		 */
		if (lockmgr_xlocked(lk)) {
			if (flags & LK_INTERLOCK)
				class->lc_unlock(ilk);
			panic("%s: draining %s with the lock held @ %s:%d\n",
			    __func__, iwmesg, file, line);
		}

		for (;;) {
			if (lk->lk_lock == LK_UNLOCKED &&
			    atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
				break;

#ifdef HWPMC_HOOKS
			PMC_SOFT_CALL( , , lock, failed);
#endif
			lock_profile_obtain_lock_failed(&lk->lock_object, false,
			    &contested, &waittime);

			/*
			 * If the lock is expected to not sleep just give up
			 * and return.
			 */
			if (LK_TRYOP(flags)) {
				LOCK_LOG2(lk, "%s: %p fails the try operation",
				    __func__, lk);
				error = EBUSY;
				break;
			}

			/*
			 * Acquire the sleepqueue chain lock because we
			 * probabilly will need to manipulate waiters flags.
			 */
			sleepq_lock(&lk->lock_object);
			x = lockmgr_read_value(lk);

			/*
			 * if the lock has been released while we spun on
			 * the sleepqueue chain lock just try again.
			 */
			if (x == LK_UNLOCKED) {
				sleepq_release(&lk->lock_object);
				continue;
			}

			v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
			if ((x & ~v) == LK_UNLOCKED) {
				v = (x & ~LK_EXCLUSIVE_SPINNERS);

				/*
				 * If interruptible sleeps left the exclusive
				 * queue empty avoid a starvation for the
				 * threads sleeping on the shared queue by
				 * giving them precedence and cleaning up the
				 * exclusive waiters bit anyway.
				 * Please note that lk_exslpfail count may be
				 * lying about the real number of waiters with
				 * the LK_SLEEPFAIL flag on because they may
				 * be used in conjunction with interruptible
				 * sleeps so lk_exslpfail might be considered
				 * an 'upper limit' bound, including the edge
				 * cases.
				 */
				if (v & LK_EXCLUSIVE_WAITERS) {
					queue = SQ_EXCLUSIVE_QUEUE;
					v &= ~LK_EXCLUSIVE_WAITERS;
				} else {
					/*
					 * Exclusive waiters sleeping with
					 * LK_SLEEPFAIL on and using
					 * interruptible sleeps/timeout may
					 * have left spourious lk_exslpfail
					 * counts on, so clean it up anyway.
					 */
					MPASS(v & LK_SHARED_WAITERS);
					lk->lk_exslpfail = 0;
					queue = SQ_SHARED_QUEUE;
					v &= ~LK_SHARED_WAITERS;
				}
				if (queue == SQ_EXCLUSIVE_QUEUE) {
					realexslp =
					    sleepq_sleepcnt(&lk->lock_object,
					    SQ_EXCLUSIVE_QUEUE);
					if (lk->lk_exslpfail >= realexslp) {
						lk->lk_exslpfail = 0;
						queue = SQ_SHARED_QUEUE;
						v &= ~LK_SHARED_WAITERS;
						if (realexslp != 0) {
							LOCK_LOG2(lk,
					"%s: %p has only LK_SLEEPFAIL sleepers",
							    __func__, lk);
							LOCK_LOG2(lk,
			"%s: %p waking up threads on the exclusive queue",
							    __func__, lk);
							sleepq_broadcast(
							    &lk->lock_object,
							    SLEEPQ_LK, 0,
							    SQ_EXCLUSIVE_QUEUE);
						}
					} else
						lk->lk_exslpfail = 0;
				}
				if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
					sleepq_release(&lk->lock_object);
					continue;
				}
				LOCK_LOG3(lk,
				"%s: %p waking up all threads on the %s queue",
				    __func__, lk, queue == SQ_SHARED_QUEUE ?
				    "shared" : "exclusive");
				sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
				    queue);

				/*
				 * If shared waiters have been woken up we need
				 * to wait for one of them to acquire the lock
				 * before to set the exclusive waiters in
				 * order to avoid a deadlock.
				 */
				if (queue == SQ_SHARED_QUEUE) {
					for (v = lk->lk_lock;
					    (v & LK_SHARE) && !LK_SHARERS(v);
					    v = lk->lk_lock)
						cpu_spinwait();
				}
			}

			/*
			 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
			 * fail, loop back and retry.
			 */
			if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
				if (!atomic_cmpset_ptr(&lk->lk_lock, x,
				    x | LK_EXCLUSIVE_WAITERS)) {
					sleepq_release(&lk->lock_object);
					continue;
				}
				LOCK_LOG2(lk, "%s: %p set drain waiters flag",
				    __func__, lk);
			}

			/*
			 * As far as we have been unable to acquire the
			 * exclusive lock and the exclusive waiters flag
			 * is set, we will sleep.
			 */
			if (flags & LK_INTERLOCK) {
				class->lc_unlock(ilk);
				flags &= ~LK_INTERLOCK;
			}
			GIANT_SAVE();
			sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
			    SQ_EXCLUSIVE_QUEUE);
			sleepq_wait(&lk->lock_object, ipri & PRIMASK);
			GIANT_RESTORE();
			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
			    __func__, lk);
		}

		if (error == 0) {
			lock_profile_obtain_lock_success(&lk->lock_object,
			    false, contested, waittime, file, line);
			LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
			    lk->lk_recurse, file, line);
			WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
			    LK_TRYWIT(flags), file, line);
			TD_LOCKS_INC(curthread);
			STACK_SAVE(lk);
		}
		break;
	default:
		if (flags & LK_INTERLOCK)
			class->lc_unlock(ilk);
		panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
	}

	if (flags & LK_INTERLOCK)
		class->lc_unlock(ilk);

	return (error);
}

void
_lockmgr_disown(struct lock *lk, const char *file, int line)
{
	uintptr_t tid, x;

	if (SCHEDULER_STOPPED())
		return;

	tid = (uintptr_t)curthread;
	_lockmgr_assert(lk, KA_XLOCKED, file, line);

	/*
	 * Panic if the lock is recursed.
	 */
	if (lockmgr_xlocked(lk) && lockmgr_recursed(lk))
		panic("%s: disown a recursed lockmgr @ %s:%d\n",
		    __func__,  file, line);

	/*
	 * If the owner is already LK_KERNPROC just skip the whole operation.
	 */
	if (LK_HOLDER(lk->lk_lock) != tid)
		return;
	lock_profile_release_lock(&lk->lock_object, false);
	LOCKSTAT_RECORD1(lockmgr__disown, lk, LOCKSTAT_WRITER);
	LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
	TD_LOCKS_DEC(curthread);
	STACK_SAVE(lk);

	/*
	 * In order to preserve waiters flags, just spin.
	 */
	for (;;) {
		x = lockmgr_read_value(lk);
		MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
		x &= LK_ALL_WAITERS;
		if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
		    LK_KERNPROC | x))
			return;
		cpu_spinwait();
	}
}

void
lockmgr_printinfo(const struct lock *lk)
{
	struct thread *td;
	uintptr_t x;

	if (lk->lk_lock == LK_UNLOCKED)
		printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
	else if (lk->lk_lock & LK_SHARE)
		printf("lock type %s: SHARED (count %ju)\n",
		    lk->lock_object.lo_name,
		    (uintmax_t)LK_SHARERS(lk->lk_lock));
	else {
		td = lockmgr_xholder(lk);
		if (td == (struct thread *)LK_KERNPROC)
			printf("lock type %s: EXCL by KERNPROC\n",
			    lk->lock_object.lo_name);
		else
			printf("lock type %s: EXCL by thread %p "
			    "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
			    td, td->td_proc->p_pid, td->td_proc->p_comm,
			    td->td_tid);
	}

	x = lk->lk_lock;
	if (x & LK_EXCLUSIVE_WAITERS)
		printf(" with exclusive waiters pending\n");
	if (x & LK_SHARED_WAITERS)
		printf(" with shared waiters pending\n");
	if (x & LK_EXCLUSIVE_SPINNERS)
		printf(" with exclusive spinners pending\n");

	STACK_PRINT(lk);
}

int
lockstatus(const struct lock *lk)
{
	uintptr_t v, x;
	int ret;

	ret = LK_SHARED;
	x = lockmgr_read_value(lk);
	v = LK_HOLDER(x);

	if ((x & LK_SHARE) == 0) {
		if (v == (uintptr_t)curthread || v == LK_KERNPROC)
			ret = LK_EXCLUSIVE;
		else
			ret = LK_EXCLOTHER;
	} else if (x == LK_UNLOCKED)
		ret = 0;

	return (ret);
}

#ifdef INVARIANT_SUPPORT

FEATURE(invariant_support,
    "Support for modules compiled with INVARIANTS option");

#ifndef INVARIANTS
#undef	_lockmgr_assert
#endif

void
_lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
{
	int slocked = 0;

	if (SCHEDULER_STOPPED())
		return;
	switch (what) {
	case KA_SLOCKED:
	case KA_SLOCKED | KA_NOTRECURSED:
	case KA_SLOCKED | KA_RECURSED:
		slocked = 1;
	case KA_LOCKED:
	case KA_LOCKED | KA_NOTRECURSED:
	case KA_LOCKED | KA_RECURSED:
#ifdef WITNESS

		/*
		 * We cannot trust WITNESS if the lock is held in exclusive
		 * mode and a call to lockmgr_disown() happened.
		 * Workaround this skipping the check if the lock is held in
		 * exclusive mode even for the KA_LOCKED case.
		 */
		if (slocked || (lk->lk_lock & LK_SHARE)) {
			witness_assert(&lk->lock_object, what, file, line);
			break;
		}
#endif
		if (lk->lk_lock == LK_UNLOCKED ||
		    ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
		    (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
			panic("Lock %s not %slocked @ %s:%d\n",
			    lk->lock_object.lo_name, slocked ? "share" : "",
			    file, line);

		if ((lk->lk_lock & LK_SHARE) == 0) {
			if (lockmgr_recursed(lk)) {
				if (what & KA_NOTRECURSED)
					panic("Lock %s recursed @ %s:%d\n",
					    lk->lock_object.lo_name, file,
					    line);
			} else if (what & KA_RECURSED)
				panic("Lock %s not recursed @ %s:%d\n",
				    lk->lock_object.lo_name, file, line);
		}
		break;
	case KA_XLOCKED:
	case KA_XLOCKED | KA_NOTRECURSED:
	case KA_XLOCKED | KA_RECURSED:
		if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
			panic("Lock %s not exclusively locked @ %s:%d\n",
			    lk->lock_object.lo_name, file, line);
		if (lockmgr_recursed(lk)) {
			if (what & KA_NOTRECURSED)
				panic("Lock %s recursed @ %s:%d\n",
				    lk->lock_object.lo_name, file, line);
		} else if (what & KA_RECURSED)
			panic("Lock %s not recursed @ %s:%d\n",
			    lk->lock_object.lo_name, file, line);
		break;
	case KA_UNLOCKED:
		if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
			panic("Lock %s exclusively locked @ %s:%d\n",
			    lk->lock_object.lo_name, file, line);
		break;
	default:
		panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
		    line);
	}
}
#endif

#ifdef DDB
int
lockmgr_chain(struct thread *td, struct thread **ownerp)
{
	const struct lock *lk;

	lk = td->td_wchan;

	if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
		return (0);
	db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
	if (lk->lk_lock & LK_SHARE)
		db_printf("SHARED (count %ju)\n",
		    (uintmax_t)LK_SHARERS(lk->lk_lock));
	else
		db_printf("EXCL\n");
	*ownerp = lockmgr_xholder(lk);

	return (1);
}

static void
db_show_lockmgr(const struct lock_object *lock)
{
	struct thread *td;
	const struct lock *lk;

	lk = (const struct lock *)lock;

	db_printf(" state: ");
	if (lk->lk_lock == LK_UNLOCKED)
		db_printf("UNLOCKED\n");
	else if (lk->lk_lock & LK_SHARE)
		db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
	else {
		td = lockmgr_xholder(lk);
		if (td == (struct thread *)LK_KERNPROC)
			db_printf("XLOCK: LK_KERNPROC\n");
		else
			db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
			    td->td_tid, td->td_proc->p_pid,
			    td->td_proc->p_comm);
		if (lockmgr_recursed(lk))
			db_printf(" recursed: %d\n", lk->lk_recurse);
	}
	db_printf(" waiters: ");
	switch (lk->lk_lock & LK_ALL_WAITERS) {
	case LK_SHARED_WAITERS:
		db_printf("shared\n");
		break;
	case LK_EXCLUSIVE_WAITERS:
		db_printf("exclusive\n");
		break;
	case LK_ALL_WAITERS:
		db_printf("shared and exclusive\n");
		break;
	default:
		db_printf("none\n");
	}
	db_printf(" spinners: ");
	if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
		db_printf("exclusive\n");
	else
		db_printf("none\n");
}
#endif