port/threads/rwlock.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2016 by Delphix. All rights reserved.
 * Copyright 2024 Oxide Computer Company
 */

#include "lint.h"
#include "thr_uberdata.h"
#include <sys/sdt.h>

#define	TRY_FLAG		0x10
#define	READ_LOCK		0
#define	WRITE_LOCK		1
#define	READ_LOCK_TRY		(READ_LOCK | TRY_FLAG)
#define	WRITE_LOCK_TRY		(WRITE_LOCK | TRY_FLAG)

#define	NLOCKS	4	/* initial number of readlock_t structs allocated */

#define	ASSERT_CONSISTENT_STATE(readers)		\
	ASSERT(!((readers) & URW_WRITE_LOCKED) ||	\
		((readers) & ~URW_HAS_WAITERS) == URW_WRITE_LOCKED)

/*
 * Find/allocate an entry for rwlp in our array of rwlocks held for reading.
 * We must be deferring signals for this to be safe.
 * Else if we are returning an entry with ul_rdlockcnt == 0,
 * it could be reassigned behind our back in a signal handler.
 */
static readlock_t *
rwl_entry(rwlock_t *rwlp)
{
	ulwp_t *self = curthread;
	readlock_t *remembered = NULL;
	readlock_t *readlockp;
	uint_t nlocks;

	/* we must be deferring signals */
	ASSERT((self->ul_critical + self->ul_sigdefer) != 0);

	if ((nlocks = self->ul_rdlockcnt) != 0)
		readlockp = self->ul_readlock.array;
	else {
		nlocks = 1;
		readlockp = &self->ul_readlock.single;
	}

	for (; nlocks; nlocks--, readlockp++) {
		if (readlockp->rd_rwlock == rwlp)
			return (readlockp);
		if (readlockp->rd_count == 0 && remembered == NULL)
			remembered = readlockp;
	}
	if (remembered != NULL) {
		remembered->rd_rwlock = rwlp;
		return (remembered);
	}

	/*
	 * No entry available.  Allocate more space, converting the single
	 * readlock_t entry into an array of readlock_t entries if necessary.
	 */
	if ((nlocks = self->ul_rdlockcnt) == 0) {
		/*
		 * Initial allocation of the readlock_t array.
		 * Convert the single entry into an array.
		 */
		self->ul_rdlockcnt = nlocks = NLOCKS;
		readlockp = lmalloc(nlocks * sizeof (readlock_t));
		/*
		 * The single readlock_t becomes the first entry in the array.
		 */
		*readlockp = self->ul_readlock.single;
		self->ul_readlock.single.rd_count = 0;
		self->ul_readlock.array = readlockp;
		/*
		 * Return the next available entry in the array.
		 */
		(++readlockp)->rd_rwlock = rwlp;
		return (readlockp);
	}
	/*
	 * Reallocate the array, double the size each time.
	 */
	readlockp = lmalloc(nlocks * 2 * sizeof (readlock_t));
	(void) memcpy(readlockp, self->ul_readlock.array,
	    nlocks * sizeof (readlock_t));
	lfree(self->ul_readlock.array, nlocks * sizeof (readlock_t));
	self->ul_readlock.array = readlockp;
	self->ul_rdlockcnt *= 2;
	/*
	 * Return the next available entry in the newly allocated array.
	 */
	(readlockp += nlocks)->rd_rwlock = rwlp;
	return (readlockp);
}

/*
 * Free the array of rwlocks held for reading.
 */
void
rwl_free(ulwp_t *ulwp)
{
	uint_t nlocks;

	if ((nlocks = ulwp->ul_rdlockcnt) != 0)
		lfree(ulwp->ul_readlock.array, nlocks * sizeof (readlock_t));
	ulwp->ul_rdlockcnt = 0;
	ulwp->ul_readlock.single.rd_rwlock = NULL;
	ulwp->ul_readlock.single.rd_count = 0;
}

/*
 * Check if a reader version of the lock is held by the current thread.
 */
#pragma weak _rw_read_held = rw_read_held
int
rw_read_held(rwlock_t *rwlp)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t readers;
	ulwp_t *self = curthread;
	readlock_t *readlockp;
	uint_t nlocks;
	int rval = 0;

	no_preempt(self);

	readers = *rwstate;
	ASSERT_CONSISTENT_STATE(readers);
	if (!(readers & URW_WRITE_LOCKED) &&
	    (readers & URW_READERS_MASK) != 0) {
		/*
		 * The lock is held for reading by some thread.
		 * Search our array of rwlocks held for reading for a match.
		 */
		if ((nlocks = self->ul_rdlockcnt) != 0)
			readlockp = self->ul_readlock.array;
		else {
			nlocks = 1;
			readlockp = &self->ul_readlock.single;
		}
		for (; nlocks; nlocks--, readlockp++) {
			if (readlockp->rd_rwlock == rwlp) {
				if (readlockp->rd_count)
					rval = 1;
				break;
			}
		}
	}

	preempt(self);
	return (rval);
}

/*
 * Check if a writer version of the lock is held by the current thread.
 */
#pragma weak _rw_write_held = rw_write_held
int
rw_write_held(rwlock_t *rwlp)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t readers;
	ulwp_t *self = curthread;
	int rval;

	no_preempt(self);

	readers = *rwstate;
	ASSERT_CONSISTENT_STATE(readers);
	rval = ((readers & URW_WRITE_LOCKED) &&
	    rwlp->rwlock_owner == (uintptr_t)self &&
	    (rwlp->rwlock_type == USYNC_THREAD ||
	    rwlp->rwlock_ownerpid == self->ul_uberdata->pid));

	preempt(self);
	return (rval);
}

#pragma weak _rwlock_init = rwlock_init
int
rwlock_init(rwlock_t *rwlp, int type, void *arg __unused)
{
	ulwp_t *self = curthread;

	if (type != USYNC_THREAD && type != USYNC_PROCESS)
		return (EINVAL);
	/*
	 * Once reinitialized, we can no longer be holding a read or write lock.
	 * We can do nothing about other threads that are holding read locks.
	 */
	sigoff(self);
	rwl_entry(rwlp)->rd_count = 0;
	sigon(self);
	(void) memset(rwlp, 0, sizeof (*rwlp));
	rwlp->rwlock_type = (uint16_t)type;
	rwlp->rwlock_magic = RWL_MAGIC;
	rwlp->mutex.mutex_type = (uint8_t)type;
	rwlp->mutex.mutex_flag = LOCK_INITED;
	rwlp->mutex.mutex_magic = MUTEX_MAGIC;

	/*
	 * This should be at the beginning of the function,
	 * but for the sake of old broken applications that
	 * do not have proper alignment for their rwlocks
	 * (and don't check the return code from rwlock_init),
	 * we put it here, after initializing the rwlock regardless.
	 */
	if (((uintptr_t)rwlp & (_LONG_LONG_ALIGNMENT - 1)) &&
	    self->ul_misaligned == 0)
		return (EINVAL);

	return (0);
}

#pragma weak pthread_rwlock_destroy = rwlock_destroy
#pragma weak _rwlock_destroy = rwlock_destroy
int
rwlock_destroy(rwlock_t *rwlp)
{
	ulwp_t *self = curthread;

	/*
	 * Once destroyed, we can no longer be holding a read or write lock.
	 * We can do nothing about other threads that are holding read locks.
	 */
	sigoff(self);
	rwl_entry(rwlp)->rd_count = 0;
	sigon(self);
	rwlp->rwlock_magic = 0;
	tdb_sync_obj_deregister(rwlp);
	return (0);
}

/*
 * The following four functions:
 *	read_lock_try()
 *	read_unlock_try()
 *	write_lock_try()
 *	write_unlock_try()
 * lie at the heart of the fast-path code for rwlocks,
 * both process-private and process-shared.
 *
 * They are called once without recourse to any other locking primitives.
 * If they succeed, we are done and the fast-path code was successful.
 * If they fail, we have to deal with lock queues, either to enqueue
 * ourself and sleep or to dequeue and wake up someone else (slow paths).
 *
 * Unless 'ignore_waiters_flag' is true (a condition that applies only
 * when read_lock_try() or write_lock_try() is called from code that
 * is already in the slow path and has already acquired the queue lock),
 * these functions will always fail if the waiters flag, URW_HAS_WAITERS,
 * is set in the 'rwstate' word.  Thus, setting the waiters flag on the
 * rwlock and acquiring the queue lock guarantees exclusive access to
 * the rwlock (and is the only way to guarantee exclusive access).
 */

/*
 * Attempt to acquire a readers lock.  Return true on success.
 */
static int
read_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t mask = ignore_waiters_flag?
	    URW_WRITE_LOCKED : (URW_HAS_WAITERS | URW_WRITE_LOCKED);
	uint32_t readers;
	ulwp_t *self = curthread;

	no_preempt(self);
	while (((readers = *rwstate) & mask) == 0) {
		if (atomic_cas_32(rwstate, readers, readers + 1) == readers) {
			preempt(self);
			return (1);
		}
	}
	preempt(self);
	return (0);
}

/*
 * Attempt to release a reader lock.  Return true on success.
 */
static int
read_unlock_try(rwlock_t *rwlp)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t readers;
	ulwp_t *self = curthread;

	no_preempt(self);
	while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
		if (atomic_cas_32(rwstate, readers, readers - 1) == readers) {
			preempt(self);
			return (1);
		}
	}
	preempt(self);
	return (0);
}

/*
 * Attempt to acquire a writer lock.  Return true on success.
 */
static int
write_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t mask = ignore_waiters_flag?
	    (URW_WRITE_LOCKED | URW_READERS_MASK) :
	    (URW_HAS_WAITERS | URW_WRITE_LOCKED | URW_READERS_MASK);
	ulwp_t *self = curthread;
	uint32_t readers;

	no_preempt(self);
	while (((readers = *rwstate) & mask) == 0) {
		if (atomic_cas_32(rwstate, readers, readers | URW_WRITE_LOCKED)
		    == readers) {
			preempt(self);
			return (1);
		}
	}
	preempt(self);
	return (0);
}

/*
 * Attempt to release a writer lock.  Return true on success.
 */
static int
write_unlock_try(rwlock_t *rwlp)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t readers;
	ulwp_t *self = curthread;

	no_preempt(self);
	while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
		if (atomic_cas_32(rwstate, readers, 0) == readers) {
			preempt(self);
			return (1);
		}
	}
	preempt(self);
	return (0);
}

/*
 * Release a process-private rwlock and wake up any thread(s) sleeping on it.
 * This is called when a thread releases a lock that appears to have waiters.
 */
static void
rw_queue_release(rwlock_t *rwlp)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	queue_head_t *qp;
	uint32_t readers;
	uint32_t writer;
	ulwp_t **ulwpp;
	ulwp_t *ulwp;
	ulwp_t *prev;
	int nlwpid = 0;
	int more;
	int maxlwps = MAXLWPS;
	lwpid_t buffer[MAXLWPS];
	lwpid_t *lwpid = buffer;

	qp = queue_lock(rwlp, MX);

	/*
	 * Here is where we actually drop the lock,
	 * but we retain the URW_HAS_WAITERS flag, if it is already set.
	 */
	readers = *rwstate;
	ASSERT_CONSISTENT_STATE(readers);
	if (readers & URW_WRITE_LOCKED)	/* drop the writer lock */
		atomic_and_32(rwstate, ~URW_WRITE_LOCKED);
	else				/* drop the readers lock */
		atomic_dec_32(rwstate);
	if (!(readers & URW_HAS_WAITERS)) {	/* no waiters */
		queue_unlock(qp);
		return;
	}

	/*
	 * The presence of the URW_HAS_WAITERS flag causes all rwlock
	 * code to go through the slow path, acquiring queue_lock(qp).
	 * Therefore, the rest of this code is safe because we are
	 * holding the queue lock and the URW_HAS_WAITERS flag is set.
	 */

	readers = *rwstate;		/* must fetch the value again */
	ASSERT_CONSISTENT_STATE(readers);
	ASSERT(readers & URW_HAS_WAITERS);
	readers &= URW_READERS_MASK;	/* count of current readers */
	writer = 0;			/* no current writer */

	/*
	 * Examine the queue of waiters in priority order and prepare
	 * to wake up as many readers as we encounter before encountering
	 * a writer.  If the highest priority thread on the queue is a
	 * writer, stop there and wake it up.
	 *
	 * We keep track of lwpids that are to be unparked in lwpid[].
	 * __lwp_unpark_all() is called to unpark all of them after
	 * they have been removed from the sleep queue and the sleep
	 * queue lock has been dropped.  If we run out of space in our
	 * on-stack buffer, we need to allocate more but we can't call
	 * lmalloc() because we are holding a queue lock when the overflow
	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
	 * either because the application may have allocated a small
	 * stack and we don't want to overrun the stack.  So we call
	 * alloc_lwpids() to allocate a bigger buffer using the mmap()
	 * system call directly since that path acquires no locks.
	 */
	while ((ulwpp = queue_slot(qp, &prev, &more)) != NULL) {
		ulwp = *ulwpp;
		ASSERT(ulwp->ul_wchan == rwlp);
		if (ulwp->ul_writer) {
			if (writer != 0 || readers != 0)
				break;
			/* one writer to wake */
			writer++;
		} else {
			if (writer != 0)
				break;
			/* at least one reader to wake */
			readers++;
			if (nlwpid == maxlwps)
				lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
		}
		queue_unlink(qp, ulwpp, prev);
		ulwp->ul_sleepq = NULL;
		ulwp->ul_wchan = NULL;
		if (writer) {
			/*
			 * Hand off the lock to the writer we will be waking.
			 */
			ASSERT((*rwstate & ~URW_HAS_WAITERS) == 0);
			atomic_or_32(rwstate, URW_WRITE_LOCKED);
			rwlp->rwlock_owner = (uintptr_t)ulwp;
		}
		lwpid[nlwpid++] = ulwp->ul_lwpid;
	}

	/*
	 * This modification of rwstate must be done last.
	 * The presence of the URW_HAS_WAITERS flag causes all rwlock
	 * code to go through the slow path, acquiring queue_lock(qp).
	 * Otherwise the read_lock_try() and write_lock_try() fast paths
	 * are effective.
	 */
	if (ulwpp == NULL)
		atomic_and_32(rwstate, ~URW_HAS_WAITERS);

	if (nlwpid == 0) {
		queue_unlock(qp);
	} else {
		ulwp_t *self = curthread;
		no_preempt(self);
		queue_unlock(qp);
		if (nlwpid == 1)
			(void) __lwp_unpark(lwpid[0]);
		else
			(void) __lwp_unpark_all(lwpid, nlwpid);
		preempt(self);
	}
	if (lwpid != buffer)
		(void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
}

/*
 * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock,
 * and trywrlock for process-shared (USYNC_PROCESS) rwlocks.
 *
 * Note: if the lock appears to be contended we call __lwp_rwlock_rdlock()
 * or __lwp_rwlock_wrlock() holding the mutex. These return with the mutex
 * released, and if they need to sleep will release the mutex first. In the
 * event of a spurious wakeup, these will return EAGAIN (because it is much
 * easier for us to re-acquire the mutex here).
 */
int
shared_rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	mutex_t *mp = &rwlp->mutex;
	int try_flag;
	int error;

	try_flag = (rd_wr & TRY_FLAG);
	rd_wr &= ~TRY_FLAG;
	ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK);

	if (!try_flag) {
		DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr);
	}

	do {
		if (try_flag && (*rwstate & URW_WRITE_LOCKED)) {
			error = EBUSY;
			break;
		}
		if ((error = mutex_lock(mp)) != 0)
			break;
		if (rd_wr == READ_LOCK) {
			if (read_lock_try(rwlp, 0)) {
				(void) mutex_unlock(mp);
				break;
			}
		} else {
			if (write_lock_try(rwlp, 0)) {
				(void) mutex_unlock(mp);
				break;
			}
		}
		atomic_or_32(rwstate, URW_HAS_WAITERS);

#ifdef DEBUG
		uint32_t readers;
		readers = *rwstate;
		ASSERT_CONSISTENT_STATE(readers);
#endif
		/*
		 * The calls to __lwp_rwlock_*() below will release the mutex,
		 * so we need a dtrace probe here.  The owner field of the
		 * mutex is cleared in the kernel when the mutex is released,
		 * so we should not clear it here.
		 */
		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
		/*
		 * The waiters bit may be inaccurate.
		 * Only the kernel knows for sure.
		 */
		if (rd_wr == READ_LOCK) {
			if (try_flag)
				error = __lwp_rwlock_tryrdlock(rwlp);
			else
				error = __lwp_rwlock_rdlock(rwlp, tsp);
		} else {
			if (try_flag)
				error = __lwp_rwlock_trywrlock(rwlp);
			else
				error = __lwp_rwlock_wrlock(rwlp, tsp);
		}
	} while (error == EAGAIN || error == EINTR);

	if (!try_flag) {
		DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
	}

	return (error);
}

/*
 * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock,
 * and trywrlock for process-private (USYNC_THREAD) rwlocks.
 */
int
rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t readers;
	ulwp_t *self = curthread;
	queue_head_t *qp;
	ulwp_t *ulwp;
	int try_flag;
	int ignore_waiters_flag;
	int error = 0;

	try_flag = (rd_wr & TRY_FLAG);
	rd_wr &= ~TRY_FLAG;
	ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK);

	if (!try_flag) {
		DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr);
	}

	qp = queue_lock(rwlp, MX);
	/* initial attempt to acquire the lock fails if there are waiters */
	ignore_waiters_flag = 0;
	while (error == 0) {
		if (rd_wr == READ_LOCK) {
			if (read_lock_try(rwlp, ignore_waiters_flag))
				break;
		} else {
			if (write_lock_try(rwlp, ignore_waiters_flag))
				break;
		}
		/* subsequent attempts do not fail due to waiters */
		ignore_waiters_flag = 1;
		atomic_or_32(rwstate, URW_HAS_WAITERS);
		readers = *rwstate;
		ASSERT_CONSISTENT_STATE(readers);
		if ((readers & URW_WRITE_LOCKED) ||
		    (rd_wr == WRITE_LOCK &&
		    (readers & URW_READERS_MASK) != 0))
			/* EMPTY */;	/* somebody holds the lock */
		else if ((ulwp = queue_waiter(qp)) == NULL) {
			atomic_and_32(rwstate, ~URW_HAS_WAITERS);
			ignore_waiters_flag = 0;
			continue;	/* no queued waiters, start over */
		} else {
			/*
			 * Do a priority check on the queued waiter (the
			 * highest priority thread on the queue) to see
			 * if we should defer to it or just grab the lock.
			 */
			int our_pri = real_priority(self);
			int his_pri = real_priority(ulwp);

			if (rd_wr == WRITE_LOCK) {
				/*
				 * We defer to a queued thread that has
				 * a higher priority than ours.
				 */
				if (his_pri <= our_pri) {
					/*
					 * Don't defer, just grab the lock.
					 */
					continue;
				}
			} else {
				/*
				 * We defer to a queued thread that has
				 * a higher priority than ours or that
				 * is a writer whose priority equals ours.
				 */
				if (his_pri < our_pri ||
				    (his_pri == our_pri && !ulwp->ul_writer)) {
					/*
					 * Don't defer, just grab the lock.
					 */
					continue;
				}
			}
		}
		/*
		 * We are about to block.
		 * If we're doing a trylock, return EBUSY instead.
		 */
		if (try_flag) {
			error = EBUSY;
			break;
		}
		/*
		 * Enqueue writers ahead of readers.
		 */
		self->ul_writer = rd_wr;	/* *must* be 0 or 1 */
		enqueue(qp, self, 0);
		set_parking_flag(self, 1);
		queue_unlock(qp);
		if ((error = __lwp_park(tsp, 0)) == EINTR)
			error = 0;
		set_parking_flag(self, 0);
		qp = queue_lock(rwlp, MX);
		if (self->ul_sleepq && dequeue_self(qp) == 0) {
			atomic_and_32(rwstate, ~URW_HAS_WAITERS);
			ignore_waiters_flag = 0;
		}
		self->ul_writer = 0;
		if (rd_wr == WRITE_LOCK &&
		    (*rwstate & URW_WRITE_LOCKED) &&
		    rwlp->rwlock_owner == (uintptr_t)self) {
			/*
			 * We acquired the lock by hand-off
			 * from the previous owner,
			 */
			error = 0;	/* timedlock did not fail */
			break;
		}
	}

	/*
	 * Make one final check to see if there are any threads left
	 * on the rwlock queue.  Clear the URW_HAS_WAITERS flag if not.
	 */
	if (qp->qh_root == NULL || qp->qh_root->qr_head == NULL)
		atomic_and_32(rwstate, ~URW_HAS_WAITERS);

	queue_unlock(qp);

	if (!try_flag) {
		DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
	}

	return (error);
}

int
rw_rdlock_impl(rwlock_t *rwlp, timespec_t *tsp)
{
	ulwp_t *self = curthread;
	uberdata_t *udp = self->ul_uberdata;
	readlock_t *readlockp;
	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
	int error;

	/*
	 * If we already hold a readers lock on this rwlock,
	 * just increment our reference count and return.
	 */
	sigoff(self);
	readlockp = rwl_entry(rwlp);
	if (readlockp->rd_count != 0) {
		if (readlockp->rd_count == READ_LOCK_MAX) {
			sigon(self);
			error = EAGAIN;
			goto out;
		}
		sigon(self);
		error = 0;
		goto out;
	}
	sigon(self);

	/*
	 * If we hold the writer lock, bail out.
	 */
	if (rw_write_held(rwlp)) {
		if (self->ul_error_detection)
			rwlock_error(rwlp, "rwlock_rdlock",
			    "calling thread owns the writer lock");
		error = EDEADLK;
		goto out;
	}

	if (read_lock_try(rwlp, 0))
		error = 0;
	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
		error = shared_rwlock_lock(rwlp, tsp, READ_LOCK);
	else						/* user-level */
		error = rwlock_lock(rwlp, tsp, READ_LOCK);

out:
	if (error == 0) {
		sigoff(self);
		rwl_entry(rwlp)->rd_count++;
		sigon(self);
		if (rwsp)
			tdb_incr(rwsp->rw_rdlock);
		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
	} else {
		DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK, error);
	}

	return (error);
}

#pragma weak pthread_rwlock_rdlock = rw_rdlock
#pragma weak _rw_rdlock = rw_rdlock
int
rw_rdlock(rwlock_t *rwlp)
{
	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
	return (rw_rdlock_impl(rwlp, NULL));
}

void
lrw_rdlock(rwlock_t *rwlp)
{
	enter_critical(curthread);
	(void) rw_rdlock_impl(rwlp, NULL);
}

int
pthread_rwlock_relclockrdlock_np(pthread_rwlock_t *restrict rwlp,
    clockid_t clock, const struct timespec *restrict reltime)
{
	timespec_t tslocal = *reltime;
	int error;

	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);

	switch (clock) {
	case CLOCK_REALTIME:
	case CLOCK_HIGHRES:
		break;
	default:
		return (EINVAL);
	}

	error = rw_rdlock_impl((rwlock_t *)rwlp, &tslocal);
	if (error == ETIME)
		error = ETIMEDOUT;
	return (error);
}

int
pthread_rwlock_reltimedrdlock_np(pthread_rwlock_t *restrict rwlp,
    const struct timespec *restrict reltime)
{
	return (pthread_rwlock_relclockrdlock_np(rwlp, CLOCK_REALTIME,
	    reltime));
}

int
pthread_rwlock_clockrdlock(pthread_rwlock_t *restrict rwlp, clockid_t clock,
    const struct timespec *restrict abstime)
{
	timespec_t tslocal;
	int error;

	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);

	switch (clock) {
	case CLOCK_REALTIME:
	case CLOCK_HIGHRES:
		break;
	default:
		return (EINVAL);
	}

	abstime_to_reltime(clock, abstime, &tslocal);
	error = rw_rdlock_impl((rwlock_t *)rwlp, &tslocal);
	if (error == ETIME)
		error = ETIMEDOUT;
	return (error);
}

int
pthread_rwlock_timedrdlock(pthread_rwlock_t *restrict rwlp,
    const struct timespec *restrict abstime)
{
	return (pthread_rwlock_clockrdlock(rwlp, CLOCK_REALTIME, abstime));
}

int
rw_wrlock_impl(rwlock_t *rwlp, timespec_t *tsp)
{
	ulwp_t *self = curthread;
	uberdata_t *udp = self->ul_uberdata;
	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
	int error;

	/*
	 * If we hold a readers lock on this rwlock, bail out.
	 */
	if (rw_read_held(rwlp)) {
		if (self->ul_error_detection)
			rwlock_error(rwlp, "rwlock_wrlock",
			    "calling thread owns the readers lock");
		error = EDEADLK;
		goto out;
	}

	/*
	 * If we hold the writer lock, bail out.
	 */
	if (rw_write_held(rwlp)) {
		if (self->ul_error_detection)
			rwlock_error(rwlp, "rwlock_wrlock",
			    "calling thread owns the writer lock");
		error = EDEADLK;
		goto out;
	}

	if (write_lock_try(rwlp, 0))
		error = 0;
	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
		error = shared_rwlock_lock(rwlp, tsp, WRITE_LOCK);
	else						/* user-level */
		error = rwlock_lock(rwlp, tsp, WRITE_LOCK);

out:
	if (error == 0) {
		rwlp->rwlock_owner = (uintptr_t)self;
		if (rwlp->rwlock_type == USYNC_PROCESS)
			rwlp->rwlock_ownerpid = udp->pid;
		if (rwsp) {
			tdb_incr(rwsp->rw_wrlock);
			rwsp->rw_wrlock_begin_hold = gethrtime();
		}
		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
	} else {
		DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK, error);
	}
	return (error);
}

#pragma weak pthread_rwlock_wrlock = rw_wrlock
#pragma weak _rw_wrlock = rw_wrlock
int
rw_wrlock(rwlock_t *rwlp)
{
	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
	return (rw_wrlock_impl(rwlp, NULL));
}

void
lrw_wrlock(rwlock_t *rwlp)
{
	enter_critical(curthread);
	(void) rw_wrlock_impl(rwlp, NULL);
}

int
pthread_rwlock_relclockwrlock_np(pthread_rwlock_t *restrict rwlp,
    clockid_t clock, const struct timespec *restrict reltime)
{
	timespec_t tslocal = *reltime;
	int error;

	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);

	switch (clock) {
	case CLOCK_REALTIME:
	case CLOCK_HIGHRES:
		break;
	default:
		return (EINVAL);
	}

	error = rw_wrlock_impl((rwlock_t *)rwlp, &tslocal);
	if (error == ETIME)
		error = ETIMEDOUT;
	return (error);
}

int
pthread_rwlock_reltimedwrlock_np(pthread_rwlock_t *restrict rwlp,
    const struct timespec *restrict reltime)
{
	return (pthread_rwlock_relclockwrlock_np(rwlp, CLOCK_REALTIME,
	    reltime));
}

int
pthread_rwlock_clockwrlock(pthread_rwlock_t *rwlp, clockid_t clock,
    const timespec_t *abstime)
{
	timespec_t tslocal;
	int error;

	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);

	switch (clock) {
	case CLOCK_REALTIME:
	case CLOCK_HIGHRES:
		break;
	default:
		return (EINVAL);
	}

	abstime_to_reltime(clock, abstime, &tslocal);
	error = rw_wrlock_impl((rwlock_t *)rwlp, &tslocal);
	if (error == ETIME)
		error = ETIMEDOUT;
	return (error);
}

int
pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlp, const timespec_t *abstime)
{
	return (pthread_rwlock_clockwrlock(rwlp, CLOCK_REALTIME, abstime));
}

#pragma weak pthread_rwlock_tryrdlock = rw_tryrdlock
int
rw_tryrdlock(rwlock_t *rwlp)
{
	ulwp_t *self = curthread;
	uberdata_t *udp = self->ul_uberdata;
	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
	readlock_t *readlockp;
	int error;

	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);

	if (rwsp)
		tdb_incr(rwsp->rw_rdlock_try);

	/*
	 * If we already hold a readers lock on this rwlock,
	 * just increment our reference count and return.
	 */
	sigoff(self);
	readlockp = rwl_entry(rwlp);
	if (readlockp->rd_count != 0) {
		if (readlockp->rd_count == READ_LOCK_MAX) {
			sigon(self);
			error = EAGAIN;
			goto out;
		}
		sigon(self);
		error = 0;
		goto out;
	}
	sigon(self);

	if (read_lock_try(rwlp, 0))
		error = 0;
	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
		error = shared_rwlock_lock(rwlp, NULL, READ_LOCK_TRY);
	else						/* user-level */
		error = rwlock_lock(rwlp, NULL, READ_LOCK_TRY);

out:
	if (error == 0) {
		sigoff(self);
		rwl_entry(rwlp)->rd_count++;
		sigon(self);
		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
	} else {
		if (rwsp)
			tdb_incr(rwsp->rw_rdlock_try_fail);
		if (error != EBUSY) {
			DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK,
			    error);
		}
	}

	return (error);
}

#pragma weak pthread_rwlock_trywrlock = rw_trywrlock
int
rw_trywrlock(rwlock_t *rwlp)
{
	ulwp_t *self = curthread;
	uberdata_t *udp = self->ul_uberdata;
	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
	int error;

	ASSERT(!self->ul_critical || self->ul_bindflags);

	if (rwsp)
		tdb_incr(rwsp->rw_wrlock_try);

	if (write_lock_try(rwlp, 0))
		error = 0;
	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
		error = shared_rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);
	else						/* user-level */
		error = rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);

	if (error == 0) {
		rwlp->rwlock_owner = (uintptr_t)self;
		if (rwlp->rwlock_type == USYNC_PROCESS)
			rwlp->rwlock_ownerpid = udp->pid;
		if (rwsp)
			rwsp->rw_wrlock_begin_hold = gethrtime();
		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
	} else {
		if (rwsp)
			tdb_incr(rwsp->rw_wrlock_try_fail);
		if (error != EBUSY) {
			DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK,
			    error);
		}
	}
	return (error);
}

#pragma weak pthread_rwlock_unlock = rw_unlock
#pragma weak _rw_unlock = rw_unlock
int
rw_unlock(rwlock_t *rwlp)
{
	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
	uint32_t readers;
	ulwp_t *self = curthread;
	uberdata_t *udp = self->ul_uberdata;
	tdb_rwlock_stats_t *rwsp;
	int rd_wr;

	readers = *rwstate;
	ASSERT_CONSISTENT_STATE(readers);
	if (readers & URW_WRITE_LOCKED) {
		rd_wr = WRITE_LOCK;
		readers = 0;
	} else {
		rd_wr = READ_LOCK;
		readers &= URW_READERS_MASK;
	}

	if (rd_wr == WRITE_LOCK) {
		/*
		 * Since the writer lock is held, we'd better be
		 * holding it, else we cannot legitimately be here.
		 */
		if (!rw_write_held(rwlp)) {
			if (self->ul_error_detection)
				rwlock_error(rwlp, "rwlock_unlock",
				    "writer lock held, "
				    "but not by the calling thread");
			return (EPERM);
		}
		if ((rwsp = RWLOCK_STATS(rwlp, udp)) != NULL) {
			if (rwsp->rw_wrlock_begin_hold)
				rwsp->rw_wrlock_hold_time +=
				    gethrtime() - rwsp->rw_wrlock_begin_hold;
			rwsp->rw_wrlock_begin_hold = 0;
		}
		rwlp->rwlock_owner = 0;
		rwlp->rwlock_ownerpid = 0;
	} else if (readers > 0) {
		/*
		 * A readers lock is held; if we don't hold one, bail out.
		 */
		readlock_t *readlockp;

		sigoff(self);
		readlockp = rwl_entry(rwlp);
		if (readlockp->rd_count == 0) {
			sigon(self);
			if (self->ul_error_detection)
				rwlock_error(rwlp, "rwlock_unlock",
				    "readers lock held, "
				    "but not by the calling thread");
			return (EPERM);
		}
		/*
		 * If we hold more than one readers lock on this rwlock,
		 * just decrement our reference count and return.
		 */
		if (--readlockp->rd_count != 0) {
			sigon(self);
			goto out;
		}
		sigon(self);
	} else {
		/*
		 * This is a usage error.
		 * No thread should release an unowned lock.
		 */
		if (self->ul_error_detection)
			rwlock_error(rwlp, "rwlock_unlock", "lock not owned");
		return (EPERM);
	}

	if (rd_wr == WRITE_LOCK && write_unlock_try(rwlp)) {
		/* EMPTY */;
	} else if (rd_wr == READ_LOCK && read_unlock_try(rwlp)) {
		/* EMPTY */;
	} else if (rwlp->rwlock_type == USYNC_PROCESS) {
		(void) mutex_lock(&rwlp->mutex);
		(void) __lwp_rwlock_unlock(rwlp);
		(void) mutex_unlock(&rwlp->mutex);
	} else {
		rw_queue_release(rwlp);
	}

out:
	DTRACE_PROBE2(plockstat, rw__release, rwlp, rd_wr);
	return (0);
}

void
lrw_unlock(rwlock_t *rwlp)
{
	(void) rw_unlock(rwlp);
	exit_critical(curthread);
}