xref: /titanic_41/usr/src/lib/libc/port/threads/rwlock.c (revision bbbbacb4bedf3d3959b4c3ec8ba1e07c8c10251d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include "lint.h"
27 #include "thr_uberdata.h"
28 #include <sys/sdt.h>
29 
30 #define	TRY_FLAG		0x10
31 #define	READ_LOCK		0
32 #define	WRITE_LOCK		1
33 #define	READ_LOCK_TRY		(READ_LOCK | TRY_FLAG)
34 #define	WRITE_LOCK_TRY		(WRITE_LOCK | TRY_FLAG)
35 
36 #define	NLOCKS	4	/* initial number of readlock_t structs allocated */
37 
38 #define	ASSERT_CONSISTENT_STATE(readers)		\
39 	ASSERT(!((readers) & URW_WRITE_LOCKED) ||	\
40 		((readers) & ~URW_HAS_WAITERS) == URW_WRITE_LOCKED)
41 
42 /*
43  * Find/allocate an entry for rwlp in our array of rwlocks held for reading.
44  * We must be deferring signals for this to be safe.
45  * Else if we are returning an entry with ul_rdlockcnt == 0,
46  * it could be reassigned behind our back in a signal handler.
47  */
48 static readlock_t *
rwl_entry(rwlock_t * rwlp)49 rwl_entry(rwlock_t *rwlp)
50 {
51 	ulwp_t *self = curthread;
52 	readlock_t *remembered = NULL;
53 	readlock_t *readlockp;
54 	uint_t nlocks;
55 
56 	/* we must be deferring signals */
57 	ASSERT((self->ul_critical + self->ul_sigdefer) != 0);
58 
59 	if ((nlocks = self->ul_rdlockcnt) != 0)
60 		readlockp = self->ul_readlock.array;
61 	else {
62 		nlocks = 1;
63 		readlockp = &self->ul_readlock.single;
64 	}
65 
66 	for (; nlocks; nlocks--, readlockp++) {
67 		if (readlockp->rd_rwlock == rwlp)
68 			return (readlockp);
69 		if (readlockp->rd_count == 0 && remembered == NULL)
70 			remembered = readlockp;
71 	}
72 	if (remembered != NULL) {
73 		remembered->rd_rwlock = rwlp;
74 		return (remembered);
75 	}
76 
77 	/*
78 	 * No entry available.  Allocate more space, converting the single
79 	 * readlock_t entry into an array of readlock_t entries if necessary.
80 	 */
81 	if ((nlocks = self->ul_rdlockcnt) == 0) {
82 		/*
83 		 * Initial allocation of the readlock_t array.
84 		 * Convert the single entry into an array.
85 		 */
86 		self->ul_rdlockcnt = nlocks = NLOCKS;
87 		readlockp = lmalloc(nlocks * sizeof (readlock_t));
88 		/*
89 		 * The single readlock_t becomes the first entry in the array.
90 		 */
91 		*readlockp = self->ul_readlock.single;
92 		self->ul_readlock.single.rd_count = 0;
93 		self->ul_readlock.array = readlockp;
94 		/*
95 		 * Return the next available entry in the array.
96 		 */
97 		(++readlockp)->rd_rwlock = rwlp;
98 		return (readlockp);
99 	}
100 	/*
101 	 * Reallocate the array, double the size each time.
102 	 */
103 	readlockp = lmalloc(nlocks * 2 * sizeof (readlock_t));
104 	(void) memcpy(readlockp, self->ul_readlock.array,
105 	    nlocks * sizeof (readlock_t));
106 	lfree(self->ul_readlock.array, nlocks * sizeof (readlock_t));
107 	self->ul_readlock.array = readlockp;
108 	self->ul_rdlockcnt *= 2;
109 	/*
110 	 * Return the next available entry in the newly allocated array.
111 	 */
112 	(readlockp += nlocks)->rd_rwlock = rwlp;
113 	return (readlockp);
114 }
115 
116 /*
117  * Free the array of rwlocks held for reading.
118  */
119 void
rwl_free(ulwp_t * ulwp)120 rwl_free(ulwp_t *ulwp)
121 {
122 	uint_t nlocks;
123 
124 	if ((nlocks = ulwp->ul_rdlockcnt) != 0)
125 		lfree(ulwp->ul_readlock.array, nlocks * sizeof (readlock_t));
126 	ulwp->ul_rdlockcnt = 0;
127 	ulwp->ul_readlock.single.rd_rwlock = NULL;
128 	ulwp->ul_readlock.single.rd_count = 0;
129 }
130 
131 /*
132  * Check if a reader version of the lock is held by the current thread.
133  */
134 #pragma weak _rw_read_held = rw_read_held
135 int
rw_read_held(rwlock_t * rwlp)136 rw_read_held(rwlock_t *rwlp)
137 {
138 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
139 	uint32_t readers;
140 	ulwp_t *self = curthread;
141 	readlock_t *readlockp;
142 	uint_t nlocks;
143 	int rval = 0;
144 
145 	no_preempt(self);
146 
147 	readers = *rwstate;
148 	ASSERT_CONSISTENT_STATE(readers);
149 	if (!(readers & URW_WRITE_LOCKED) &&
150 	    (readers & URW_READERS_MASK) != 0) {
151 		/*
152 		 * The lock is held for reading by some thread.
153 		 * Search our array of rwlocks held for reading for a match.
154 		 */
155 		if ((nlocks = self->ul_rdlockcnt) != 0)
156 			readlockp = self->ul_readlock.array;
157 		else {
158 			nlocks = 1;
159 			readlockp = &self->ul_readlock.single;
160 		}
161 		for (; nlocks; nlocks--, readlockp++) {
162 			if (readlockp->rd_rwlock == rwlp) {
163 				if (readlockp->rd_count)
164 					rval = 1;
165 				break;
166 			}
167 		}
168 	}
169 
170 	preempt(self);
171 	return (rval);
172 }
173 
174 /*
175  * Check if a writer version of the lock is held by the current thread.
176  */
177 #pragma weak _rw_write_held = rw_write_held
178 int
rw_write_held(rwlock_t * rwlp)179 rw_write_held(rwlock_t *rwlp)
180 {
181 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
182 	uint32_t readers;
183 	ulwp_t *self = curthread;
184 	int rval;
185 
186 	no_preempt(self);
187 
188 	readers = *rwstate;
189 	ASSERT_CONSISTENT_STATE(readers);
190 	rval = ((readers & URW_WRITE_LOCKED) &&
191 	    rwlp->rwlock_owner == (uintptr_t)self &&
192 	    (rwlp->rwlock_type == USYNC_THREAD ||
193 	    rwlp->rwlock_ownerpid == self->ul_uberdata->pid));
194 
195 	preempt(self);
196 	return (rval);
197 }
198 
199 #pragma weak _rwlock_init = rwlock_init
200 /* ARGSUSED2 */
201 int
rwlock_init(rwlock_t * rwlp,int type,void * arg)202 rwlock_init(rwlock_t *rwlp, int type, void *arg)
203 {
204 	ulwp_t *self = curthread;
205 
206 	if (type != USYNC_THREAD && type != USYNC_PROCESS)
207 		return (EINVAL);
208 	/*
209 	 * Once reinitialized, we can no longer be holding a read or write lock.
210 	 * We can do nothing about other threads that are holding read locks.
211 	 */
212 	sigoff(self);
213 	rwl_entry(rwlp)->rd_count = 0;
214 	sigon(self);
215 	(void) memset(rwlp, 0, sizeof (*rwlp));
216 	rwlp->rwlock_type = (uint16_t)type;
217 	rwlp->rwlock_magic = RWL_MAGIC;
218 	rwlp->mutex.mutex_type = (uint8_t)type;
219 	rwlp->mutex.mutex_flag = LOCK_INITED;
220 	rwlp->mutex.mutex_magic = MUTEX_MAGIC;
221 
222 	/*
223 	 * This should be at the beginning of the function,
224 	 * but for the sake of old broken applications that
225 	 * do not have proper alignment for their rwlocks
226 	 * (and don't check the return code from rwlock_init),
227 	 * we put it here, after initializing the rwlock regardless.
228 	 */
229 	if (((uintptr_t)rwlp & (_LONG_LONG_ALIGNMENT - 1)) &&
230 	    self->ul_misaligned == 0)
231 		return (EINVAL);
232 
233 	return (0);
234 }
235 
236 #pragma weak pthread_rwlock_destroy = rwlock_destroy
237 #pragma weak _rwlock_destroy = rwlock_destroy
238 int
rwlock_destroy(rwlock_t * rwlp)239 rwlock_destroy(rwlock_t *rwlp)
240 {
241 	ulwp_t *self = curthread;
242 
243 	/*
244 	 * Once destroyed, we can no longer be holding a read or write lock.
245 	 * We can do nothing about other threads that are holding read locks.
246 	 */
247 	sigoff(self);
248 	rwl_entry(rwlp)->rd_count = 0;
249 	sigon(self);
250 	rwlp->rwlock_magic = 0;
251 	tdb_sync_obj_deregister(rwlp);
252 	return (0);
253 }
254 
255 /*
256  * The following four functions:
257  *	read_lock_try()
258  *	read_unlock_try()
259  *	write_lock_try()
260  *	write_unlock_try()
261  * lie at the heart of the fast-path code for rwlocks,
262  * both process-private and process-shared.
263  *
264  * They are called once without recourse to any other locking primitives.
265  * If they succeed, we are done and the fast-path code was successful.
266  * If they fail, we have to deal with lock queues, either to enqueue
267  * ourself and sleep or to dequeue and wake up someone else (slow paths).
268  *
269  * Unless 'ignore_waiters_flag' is true (a condition that applies only
270  * when read_lock_try() or write_lock_try() is called from code that
271  * is already in the slow path and has already acquired the queue lock),
272  * these functions will always fail if the waiters flag, URW_HAS_WAITERS,
273  * is set in the 'rwstate' word.  Thus, setting the waiters flag on the
274  * rwlock and acquiring the queue lock guarantees exclusive access to
275  * the rwlock (and is the only way to guarantee exclusive access).
276  */
277 
278 /*
279  * Attempt to acquire a readers lock.  Return true on success.
280  */
281 static int
read_lock_try(rwlock_t * rwlp,int ignore_waiters_flag)282 read_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
283 {
284 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
285 	uint32_t mask = ignore_waiters_flag?
286 	    URW_WRITE_LOCKED : (URW_HAS_WAITERS | URW_WRITE_LOCKED);
287 	uint32_t readers;
288 	ulwp_t *self = curthread;
289 
290 	no_preempt(self);
291 	while (((readers = *rwstate) & mask) == 0) {
292 		if (atomic_cas_32(rwstate, readers, readers + 1) == readers) {
293 			preempt(self);
294 			return (1);
295 		}
296 	}
297 	preempt(self);
298 	return (0);
299 }
300 
301 /*
302  * Attempt to release a reader lock.  Return true on success.
303  */
304 static int
read_unlock_try(rwlock_t * rwlp)305 read_unlock_try(rwlock_t *rwlp)
306 {
307 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
308 	uint32_t readers;
309 	ulwp_t *self = curthread;
310 
311 	no_preempt(self);
312 	while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
313 		if (atomic_cas_32(rwstate, readers, readers - 1) == readers) {
314 			preempt(self);
315 			return (1);
316 		}
317 	}
318 	preempt(self);
319 	return (0);
320 }
321 
322 /*
323  * Attempt to acquire a writer lock.  Return true on success.
324  */
325 static int
write_lock_try(rwlock_t * rwlp,int ignore_waiters_flag)326 write_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
327 {
328 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
329 	uint32_t mask = ignore_waiters_flag?
330 	    (URW_WRITE_LOCKED | URW_READERS_MASK) :
331 	    (URW_HAS_WAITERS | URW_WRITE_LOCKED | URW_READERS_MASK);
332 	ulwp_t *self = curthread;
333 	uint32_t readers;
334 
335 	no_preempt(self);
336 	while (((readers = *rwstate) & mask) == 0) {
337 		if (atomic_cas_32(rwstate, readers, readers | URW_WRITE_LOCKED)
338 		    == readers) {
339 			preempt(self);
340 			return (1);
341 		}
342 	}
343 	preempt(self);
344 	return (0);
345 }
346 
347 /*
348  * Attempt to release a writer lock.  Return true on success.
349  */
350 static int
write_unlock_try(rwlock_t * rwlp)351 write_unlock_try(rwlock_t *rwlp)
352 {
353 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
354 	uint32_t readers;
355 	ulwp_t *self = curthread;
356 
357 	no_preempt(self);
358 	while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
359 		if (atomic_cas_32(rwstate, readers, 0) == readers) {
360 			preempt(self);
361 			return (1);
362 		}
363 	}
364 	preempt(self);
365 	return (0);
366 }
367 
368 /*
369  * Release a process-private rwlock and wake up any thread(s) sleeping on it.
370  * This is called when a thread releases a lock that appears to have waiters.
371  */
372 static void
rw_queue_release(rwlock_t * rwlp)373 rw_queue_release(rwlock_t *rwlp)
374 {
375 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
376 	queue_head_t *qp;
377 	uint32_t readers;
378 	uint32_t writer;
379 	ulwp_t **ulwpp;
380 	ulwp_t *ulwp;
381 	ulwp_t *prev;
382 	int nlwpid = 0;
383 	int more;
384 	int maxlwps = MAXLWPS;
385 	lwpid_t buffer[MAXLWPS];
386 	lwpid_t *lwpid = buffer;
387 
388 	qp = queue_lock(rwlp, MX);
389 
390 	/*
391 	 * Here is where we actually drop the lock,
392 	 * but we retain the URW_HAS_WAITERS flag, if it is already set.
393 	 */
394 	readers = *rwstate;
395 	ASSERT_CONSISTENT_STATE(readers);
396 	if (readers & URW_WRITE_LOCKED)	/* drop the writer lock */
397 		atomic_and_32(rwstate, ~URW_WRITE_LOCKED);
398 	else				/* drop the readers lock */
399 		atomic_dec_32(rwstate);
400 	if (!(readers & URW_HAS_WAITERS)) {	/* no waiters */
401 		queue_unlock(qp);
402 		return;
403 	}
404 
405 	/*
406 	 * The presence of the URW_HAS_WAITERS flag causes all rwlock
407 	 * code to go through the slow path, acquiring queue_lock(qp).
408 	 * Therefore, the rest of this code is safe because we are
409 	 * holding the queue lock and the URW_HAS_WAITERS flag is set.
410 	 */
411 
412 	readers = *rwstate;		/* must fetch the value again */
413 	ASSERT_CONSISTENT_STATE(readers);
414 	ASSERT(readers & URW_HAS_WAITERS);
415 	readers &= URW_READERS_MASK;	/* count of current readers */
416 	writer = 0;			/* no current writer */
417 
418 	/*
419 	 * Examine the queue of waiters in priority order and prepare
420 	 * to wake up as many readers as we encounter before encountering
421 	 * a writer.  If the highest priority thread on the queue is a
422 	 * writer, stop there and wake it up.
423 	 *
424 	 * We keep track of lwpids that are to be unparked in lwpid[].
425 	 * __lwp_unpark_all() is called to unpark all of them after
426 	 * they have been removed from the sleep queue and the sleep
427 	 * queue lock has been dropped.  If we run out of space in our
428 	 * on-stack buffer, we need to allocate more but we can't call
429 	 * lmalloc() because we are holding a queue lock when the overflow
430 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
431 	 * either because the application may have allocated a small
432 	 * stack and we don't want to overrun the stack.  So we call
433 	 * alloc_lwpids() to allocate a bigger buffer using the mmap()
434 	 * system call directly since that path acquires no locks.
435 	 */
436 	while ((ulwpp = queue_slot(qp, &prev, &more)) != NULL) {
437 		ulwp = *ulwpp;
438 		ASSERT(ulwp->ul_wchan == rwlp);
439 		if (ulwp->ul_writer) {
440 			if (writer != 0 || readers != 0)
441 				break;
442 			/* one writer to wake */
443 			writer++;
444 		} else {
445 			if (writer != 0)
446 				break;
447 			/* at least one reader to wake */
448 			readers++;
449 			if (nlwpid == maxlwps)
450 				lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
451 		}
452 		queue_unlink(qp, ulwpp, prev);
453 		ulwp->ul_sleepq = NULL;
454 		ulwp->ul_wchan = NULL;
455 		if (writer) {
456 			/*
457 			 * Hand off the lock to the writer we will be waking.
458 			 */
459 			ASSERT((*rwstate & ~URW_HAS_WAITERS) == 0);
460 			atomic_or_32(rwstate, URW_WRITE_LOCKED);
461 			rwlp->rwlock_owner = (uintptr_t)ulwp;
462 		}
463 		lwpid[nlwpid++] = ulwp->ul_lwpid;
464 	}
465 
466 	/*
467 	 * This modification of rwstate must be done last.
468 	 * The presence of the URW_HAS_WAITERS flag causes all rwlock
469 	 * code to go through the slow path, acquiring queue_lock(qp).
470 	 * Otherwise the read_lock_try() and write_lock_try() fast paths
471 	 * are effective.
472 	 */
473 	if (ulwpp == NULL)
474 		atomic_and_32(rwstate, ~URW_HAS_WAITERS);
475 
476 	if (nlwpid == 0) {
477 		queue_unlock(qp);
478 	} else {
479 		ulwp_t *self = curthread;
480 		no_preempt(self);
481 		queue_unlock(qp);
482 		if (nlwpid == 1)
483 			(void) __lwp_unpark(lwpid[0]);
484 		else
485 			(void) __lwp_unpark_all(lwpid, nlwpid);
486 		preempt(self);
487 	}
488 	if (lwpid != buffer)
489 		(void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
490 }
491 
492 /*
493  * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock,
494  * and trywrlock for process-shared (USYNC_PROCESS) rwlocks.
495  *
496  * Note: if the lock appears to be contended we call __lwp_rwlock_rdlock()
497  * or __lwp_rwlock_wrlock() holding the mutex. These return with the mutex
498  * released, and if they need to sleep will release the mutex first. In the
499  * event of a spurious wakeup, these will return EAGAIN (because it is much
500  * easier for us to re-acquire the mutex here).
501  */
502 int
shared_rwlock_lock(rwlock_t * rwlp,timespec_t * tsp,int rd_wr)503 shared_rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
504 {
505 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
506 	mutex_t *mp = &rwlp->mutex;
507 	uint32_t readers;
508 	int try_flag;
509 	int error;
510 
511 	try_flag = (rd_wr & TRY_FLAG);
512 	rd_wr &= ~TRY_FLAG;
513 	ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK);
514 
515 	if (!try_flag) {
516 		DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr);
517 	}
518 
519 	do {
520 		if (try_flag && (*rwstate & URW_WRITE_LOCKED)) {
521 			error = EBUSY;
522 			break;
523 		}
524 		if ((error = mutex_lock(mp)) != 0)
525 			break;
526 		if (rd_wr == READ_LOCK) {
527 			if (read_lock_try(rwlp, 0)) {
528 				(void) mutex_unlock(mp);
529 				break;
530 			}
531 		} else {
532 			if (write_lock_try(rwlp, 0)) {
533 				(void) mutex_unlock(mp);
534 				break;
535 			}
536 		}
537 		atomic_or_32(rwstate, URW_HAS_WAITERS);
538 		readers = *rwstate;
539 		ASSERT_CONSISTENT_STATE(readers);
540 		/*
541 		 * The calls to __lwp_rwlock_*() below will release the mutex,
542 		 * so we need a dtrace probe here.  The owner field of the
543 		 * mutex is cleared in the kernel when the mutex is released,
544 		 * so we should not clear it here.
545 		 */
546 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
547 		/*
548 		 * The waiters bit may be inaccurate.
549 		 * Only the kernel knows for sure.
550 		 */
551 		if (rd_wr == READ_LOCK) {
552 			if (try_flag)
553 				error = __lwp_rwlock_tryrdlock(rwlp);
554 			else
555 				error = __lwp_rwlock_rdlock(rwlp, tsp);
556 		} else {
557 			if (try_flag)
558 				error = __lwp_rwlock_trywrlock(rwlp);
559 			else
560 				error = __lwp_rwlock_wrlock(rwlp, tsp);
561 		}
562 	} while (error == EAGAIN || error == EINTR);
563 
564 	if (!try_flag) {
565 		DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
566 	}
567 
568 	return (error);
569 }
570 
571 /*
572  * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock,
573  * and trywrlock for process-private (USYNC_THREAD) rwlocks.
574  */
575 int
rwlock_lock(rwlock_t * rwlp,timespec_t * tsp,int rd_wr)576 rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
577 {
578 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
579 	uint32_t readers;
580 	ulwp_t *self = curthread;
581 	queue_head_t *qp;
582 	ulwp_t *ulwp;
583 	int try_flag;
584 	int ignore_waiters_flag;
585 	int error = 0;
586 
587 	try_flag = (rd_wr & TRY_FLAG);
588 	rd_wr &= ~TRY_FLAG;
589 	ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK);
590 
591 	if (!try_flag) {
592 		DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr);
593 	}
594 
595 	qp = queue_lock(rwlp, MX);
596 	/* initial attempt to acquire the lock fails if there are waiters */
597 	ignore_waiters_flag = 0;
598 	while (error == 0) {
599 		if (rd_wr == READ_LOCK) {
600 			if (read_lock_try(rwlp, ignore_waiters_flag))
601 				break;
602 		} else {
603 			if (write_lock_try(rwlp, ignore_waiters_flag))
604 				break;
605 		}
606 		/* subsequent attempts do not fail due to waiters */
607 		ignore_waiters_flag = 1;
608 		atomic_or_32(rwstate, URW_HAS_WAITERS);
609 		readers = *rwstate;
610 		ASSERT_CONSISTENT_STATE(readers);
611 		if ((readers & URW_WRITE_LOCKED) ||
612 		    (rd_wr == WRITE_LOCK &&
613 		    (readers & URW_READERS_MASK) != 0))
614 			/* EMPTY */;	/* somebody holds the lock */
615 		else if ((ulwp = queue_waiter(qp)) == NULL) {
616 			atomic_and_32(rwstate, ~URW_HAS_WAITERS);
617 			ignore_waiters_flag = 0;
618 			continue;	/* no queued waiters, start over */
619 		} else {
620 			/*
621 			 * Do a priority check on the queued waiter (the
622 			 * highest priority thread on the queue) to see
623 			 * if we should defer to him or just grab the lock.
624 			 */
625 			int our_pri = real_priority(self);
626 			int his_pri = real_priority(ulwp);
627 
628 			if (rd_wr == WRITE_LOCK) {
629 				/*
630 				 * We defer to a queued thread that has
631 				 * a higher priority than ours.
632 				 */
633 				if (his_pri <= our_pri) {
634 					/*
635 					 * Don't defer, just grab the lock.
636 					 */
637 					continue;
638 				}
639 			} else {
640 				/*
641 				 * We defer to a queued thread that has
642 				 * a higher priority than ours or that
643 				 * is a writer whose priority equals ours.
644 				 */
645 				if (his_pri < our_pri ||
646 				    (his_pri == our_pri && !ulwp->ul_writer)) {
647 					/*
648 					 * Don't defer, just grab the lock.
649 					 */
650 					continue;
651 				}
652 			}
653 		}
654 		/*
655 		 * We are about to block.
656 		 * If we're doing a trylock, return EBUSY instead.
657 		 */
658 		if (try_flag) {
659 			error = EBUSY;
660 			break;
661 		}
662 		/*
663 		 * Enqueue writers ahead of readers.
664 		 */
665 		self->ul_writer = rd_wr;	/* *must* be 0 or 1 */
666 		enqueue(qp, self, 0);
667 		set_parking_flag(self, 1);
668 		queue_unlock(qp);
669 		if ((error = __lwp_park(tsp, 0)) == EINTR)
670 			error = 0;
671 		set_parking_flag(self, 0);
672 		qp = queue_lock(rwlp, MX);
673 		if (self->ul_sleepq && dequeue_self(qp) == 0) {
674 			atomic_and_32(rwstate, ~URW_HAS_WAITERS);
675 			ignore_waiters_flag = 0;
676 		}
677 		self->ul_writer = 0;
678 		if (rd_wr == WRITE_LOCK &&
679 		    (*rwstate & URW_WRITE_LOCKED) &&
680 		    rwlp->rwlock_owner == (uintptr_t)self) {
681 			/*
682 			 * We acquired the lock by hand-off
683 			 * from the previous owner,
684 			 */
685 			error = 0;	/* timedlock did not fail */
686 			break;
687 		}
688 	}
689 
690 	/*
691 	 * Make one final check to see if there are any threads left
692 	 * on the rwlock queue.  Clear the URW_HAS_WAITERS flag if not.
693 	 */
694 	if (qp->qh_root == NULL || qp->qh_root->qr_head == NULL)
695 		atomic_and_32(rwstate, ~URW_HAS_WAITERS);
696 
697 	queue_unlock(qp);
698 
699 	if (!try_flag) {
700 		DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
701 	}
702 
703 	return (error);
704 }
705 
706 int
rw_rdlock_impl(rwlock_t * rwlp,timespec_t * tsp)707 rw_rdlock_impl(rwlock_t *rwlp, timespec_t *tsp)
708 {
709 	ulwp_t *self = curthread;
710 	uberdata_t *udp = self->ul_uberdata;
711 	readlock_t *readlockp;
712 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
713 	int error;
714 
715 	/*
716 	 * If we already hold a readers lock on this rwlock,
717 	 * just increment our reference count and return.
718 	 */
719 	sigoff(self);
720 	readlockp = rwl_entry(rwlp);
721 	if (readlockp->rd_count != 0) {
722 		if (readlockp->rd_count == READ_LOCK_MAX) {
723 			sigon(self);
724 			error = EAGAIN;
725 			goto out;
726 		}
727 		sigon(self);
728 		error = 0;
729 		goto out;
730 	}
731 	sigon(self);
732 
733 	/*
734 	 * If we hold the writer lock, bail out.
735 	 */
736 	if (rw_write_held(rwlp)) {
737 		if (self->ul_error_detection)
738 			rwlock_error(rwlp, "rwlock_rdlock",
739 			    "calling thread owns the writer lock");
740 		error = EDEADLK;
741 		goto out;
742 	}
743 
744 	if (read_lock_try(rwlp, 0))
745 		error = 0;
746 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
747 		error = shared_rwlock_lock(rwlp, tsp, READ_LOCK);
748 	else						/* user-level */
749 		error = rwlock_lock(rwlp, tsp, READ_LOCK);
750 
751 out:
752 	if (error == 0) {
753 		sigoff(self);
754 		rwl_entry(rwlp)->rd_count++;
755 		sigon(self);
756 		if (rwsp)
757 			tdb_incr(rwsp->rw_rdlock);
758 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
759 	} else {
760 		DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK, error);
761 	}
762 
763 	return (error);
764 }
765 
766 #pragma weak pthread_rwlock_rdlock = rw_rdlock
767 #pragma weak _rw_rdlock = rw_rdlock
768 int
rw_rdlock(rwlock_t * rwlp)769 rw_rdlock(rwlock_t *rwlp)
770 {
771 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
772 	return (rw_rdlock_impl(rwlp, NULL));
773 }
774 
775 void
lrw_rdlock(rwlock_t * rwlp)776 lrw_rdlock(rwlock_t *rwlp)
777 {
778 	enter_critical(curthread);
779 	(void) rw_rdlock_impl(rwlp, NULL);
780 }
781 
782 int
pthread_rwlock_reltimedrdlock_np(pthread_rwlock_t * _RESTRICT_KYWD rwlp,const struct timespec * _RESTRICT_KYWD reltime)783 pthread_rwlock_reltimedrdlock_np(pthread_rwlock_t *_RESTRICT_KYWD rwlp,
784     const struct timespec *_RESTRICT_KYWD reltime)
785 {
786 	timespec_t tslocal = *reltime;
787 	int error;
788 
789 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
790 	error = rw_rdlock_impl((rwlock_t *)rwlp, &tslocal);
791 	if (error == ETIME)
792 		error = ETIMEDOUT;
793 	return (error);
794 }
795 
796 int
pthread_rwlock_timedrdlock(pthread_rwlock_t * _RESTRICT_KYWD rwlp,const struct timespec * _RESTRICT_KYWD abstime)797 pthread_rwlock_timedrdlock(pthread_rwlock_t *_RESTRICT_KYWD rwlp,
798     const struct timespec *_RESTRICT_KYWD abstime)
799 {
800 	timespec_t tslocal;
801 	int error;
802 
803 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
804 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
805 	error = rw_rdlock_impl((rwlock_t *)rwlp, &tslocal);
806 	if (error == ETIME)
807 		error = ETIMEDOUT;
808 	return (error);
809 }
810 
811 int
rw_wrlock_impl(rwlock_t * rwlp,timespec_t * tsp)812 rw_wrlock_impl(rwlock_t *rwlp, timespec_t *tsp)
813 {
814 	ulwp_t *self = curthread;
815 	uberdata_t *udp = self->ul_uberdata;
816 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
817 	int error;
818 
819 	/*
820 	 * If we hold a readers lock on this rwlock, bail out.
821 	 */
822 	if (rw_read_held(rwlp)) {
823 		if (self->ul_error_detection)
824 			rwlock_error(rwlp, "rwlock_wrlock",
825 			    "calling thread owns the readers lock");
826 		error = EDEADLK;
827 		goto out;
828 	}
829 
830 	/*
831 	 * If we hold the writer lock, bail out.
832 	 */
833 	if (rw_write_held(rwlp)) {
834 		if (self->ul_error_detection)
835 			rwlock_error(rwlp, "rwlock_wrlock",
836 			    "calling thread owns the writer lock");
837 		error = EDEADLK;
838 		goto out;
839 	}
840 
841 	if (write_lock_try(rwlp, 0))
842 		error = 0;
843 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
844 		error = shared_rwlock_lock(rwlp, tsp, WRITE_LOCK);
845 	else						/* user-level */
846 		error = rwlock_lock(rwlp, tsp, WRITE_LOCK);
847 
848 out:
849 	if (error == 0) {
850 		rwlp->rwlock_owner = (uintptr_t)self;
851 		if (rwlp->rwlock_type == USYNC_PROCESS)
852 			rwlp->rwlock_ownerpid = udp->pid;
853 		if (rwsp) {
854 			tdb_incr(rwsp->rw_wrlock);
855 			rwsp->rw_wrlock_begin_hold = gethrtime();
856 		}
857 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
858 	} else {
859 		DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK, error);
860 	}
861 	return (error);
862 }
863 
864 #pragma weak pthread_rwlock_wrlock = rw_wrlock
865 #pragma weak _rw_wrlock = rw_wrlock
866 int
rw_wrlock(rwlock_t * rwlp)867 rw_wrlock(rwlock_t *rwlp)
868 {
869 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
870 	return (rw_wrlock_impl(rwlp, NULL));
871 }
872 
873 void
lrw_wrlock(rwlock_t * rwlp)874 lrw_wrlock(rwlock_t *rwlp)
875 {
876 	enter_critical(curthread);
877 	(void) rw_wrlock_impl(rwlp, NULL);
878 }
879 
880 int
pthread_rwlock_reltimedwrlock_np(pthread_rwlock_t * _RESTRICT_KYWD rwlp,const struct timespec * _RESTRICT_KYWD reltime)881 pthread_rwlock_reltimedwrlock_np(pthread_rwlock_t *_RESTRICT_KYWD rwlp,
882     const struct timespec *_RESTRICT_KYWD reltime)
883 {
884 	timespec_t tslocal = *reltime;
885 	int error;
886 
887 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
888 	error = rw_wrlock_impl((rwlock_t *)rwlp, &tslocal);
889 	if (error == ETIME)
890 		error = ETIMEDOUT;
891 	return (error);
892 }
893 
894 int
pthread_rwlock_timedwrlock(pthread_rwlock_t * rwlp,const timespec_t * abstime)895 pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlp, const timespec_t *abstime)
896 {
897 	timespec_t tslocal;
898 	int error;
899 
900 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
901 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
902 	error = rw_wrlock_impl((rwlock_t *)rwlp, &tslocal);
903 	if (error == ETIME)
904 		error = ETIMEDOUT;
905 	return (error);
906 }
907 
908 #pragma weak pthread_rwlock_tryrdlock = rw_tryrdlock
909 int
rw_tryrdlock(rwlock_t * rwlp)910 rw_tryrdlock(rwlock_t *rwlp)
911 {
912 	ulwp_t *self = curthread;
913 	uberdata_t *udp = self->ul_uberdata;
914 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
915 	readlock_t *readlockp;
916 	int error;
917 
918 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
919 
920 	if (rwsp)
921 		tdb_incr(rwsp->rw_rdlock_try);
922 
923 	/*
924 	 * If we already hold a readers lock on this rwlock,
925 	 * just increment our reference count and return.
926 	 */
927 	sigoff(self);
928 	readlockp = rwl_entry(rwlp);
929 	if (readlockp->rd_count != 0) {
930 		if (readlockp->rd_count == READ_LOCK_MAX) {
931 			sigon(self);
932 			error = EAGAIN;
933 			goto out;
934 		}
935 		sigon(self);
936 		error = 0;
937 		goto out;
938 	}
939 	sigon(self);
940 
941 	if (read_lock_try(rwlp, 0))
942 		error = 0;
943 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
944 		error = shared_rwlock_lock(rwlp, NULL, READ_LOCK_TRY);
945 	else						/* user-level */
946 		error = rwlock_lock(rwlp, NULL, READ_LOCK_TRY);
947 
948 out:
949 	if (error == 0) {
950 		sigoff(self);
951 		rwl_entry(rwlp)->rd_count++;
952 		sigon(self);
953 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
954 	} else {
955 		if (rwsp)
956 			tdb_incr(rwsp->rw_rdlock_try_fail);
957 		if (error != EBUSY) {
958 			DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK,
959 			    error);
960 		}
961 	}
962 
963 	return (error);
964 }
965 
966 #pragma weak pthread_rwlock_trywrlock = rw_trywrlock
967 int
rw_trywrlock(rwlock_t * rwlp)968 rw_trywrlock(rwlock_t *rwlp)
969 {
970 	ulwp_t *self = curthread;
971 	uberdata_t *udp = self->ul_uberdata;
972 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
973 	int error;
974 
975 	ASSERT(!self->ul_critical || self->ul_bindflags);
976 
977 	if (rwsp)
978 		tdb_incr(rwsp->rw_wrlock_try);
979 
980 	if (write_lock_try(rwlp, 0))
981 		error = 0;
982 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
983 		error = shared_rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);
984 	else						/* user-level */
985 		error = rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);
986 
987 	if (error == 0) {
988 		rwlp->rwlock_owner = (uintptr_t)self;
989 		if (rwlp->rwlock_type == USYNC_PROCESS)
990 			rwlp->rwlock_ownerpid = udp->pid;
991 		if (rwsp)
992 			rwsp->rw_wrlock_begin_hold = gethrtime();
993 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
994 	} else {
995 		if (rwsp)
996 			tdb_incr(rwsp->rw_wrlock_try_fail);
997 		if (error != EBUSY) {
998 			DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK,
999 			    error);
1000 		}
1001 	}
1002 	return (error);
1003 }
1004 
1005 #pragma weak pthread_rwlock_unlock = rw_unlock
1006 #pragma weak _rw_unlock = rw_unlock
1007 int
rw_unlock(rwlock_t * rwlp)1008 rw_unlock(rwlock_t *rwlp)
1009 {
1010 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
1011 	uint32_t readers;
1012 	ulwp_t *self = curthread;
1013 	uberdata_t *udp = self->ul_uberdata;
1014 	tdb_rwlock_stats_t *rwsp;
1015 	int rd_wr;
1016 
1017 	readers = *rwstate;
1018 	ASSERT_CONSISTENT_STATE(readers);
1019 	if (readers & URW_WRITE_LOCKED) {
1020 		rd_wr = WRITE_LOCK;
1021 		readers = 0;
1022 	} else {
1023 		rd_wr = READ_LOCK;
1024 		readers &= URW_READERS_MASK;
1025 	}
1026 
1027 	if (rd_wr == WRITE_LOCK) {
1028 		/*
1029 		 * Since the writer lock is held, we'd better be
1030 		 * holding it, else we cannot legitimately be here.
1031 		 */
1032 		if (!rw_write_held(rwlp)) {
1033 			if (self->ul_error_detection)
1034 				rwlock_error(rwlp, "rwlock_unlock",
1035 				    "writer lock held, "
1036 				    "but not by the calling thread");
1037 			return (EPERM);
1038 		}
1039 		if ((rwsp = RWLOCK_STATS(rwlp, udp)) != NULL) {
1040 			if (rwsp->rw_wrlock_begin_hold)
1041 				rwsp->rw_wrlock_hold_time +=
1042 				    gethrtime() - rwsp->rw_wrlock_begin_hold;
1043 			rwsp->rw_wrlock_begin_hold = 0;
1044 		}
1045 		rwlp->rwlock_owner = 0;
1046 		rwlp->rwlock_ownerpid = 0;
1047 	} else if (readers > 0) {
1048 		/*
1049 		 * A readers lock is held; if we don't hold one, bail out.
1050 		 */
1051 		readlock_t *readlockp;
1052 
1053 		sigoff(self);
1054 		readlockp = rwl_entry(rwlp);
1055 		if (readlockp->rd_count == 0) {
1056 			sigon(self);
1057 			if (self->ul_error_detection)
1058 				rwlock_error(rwlp, "rwlock_unlock",
1059 				    "readers lock held, "
1060 				    "but not by the calling thread");
1061 			return (EPERM);
1062 		}
1063 		/*
1064 		 * If we hold more than one readers lock on this rwlock,
1065 		 * just decrement our reference count and return.
1066 		 */
1067 		if (--readlockp->rd_count != 0) {
1068 			sigon(self);
1069 			goto out;
1070 		}
1071 		sigon(self);
1072 	} else {
1073 		/*
1074 		 * This is a usage error.
1075 		 * No thread should release an unowned lock.
1076 		 */
1077 		if (self->ul_error_detection)
1078 			rwlock_error(rwlp, "rwlock_unlock", "lock not owned");
1079 		return (EPERM);
1080 	}
1081 
1082 	if (rd_wr == WRITE_LOCK && write_unlock_try(rwlp)) {
1083 		/* EMPTY */;
1084 	} else if (rd_wr == READ_LOCK && read_unlock_try(rwlp)) {
1085 		/* EMPTY */;
1086 	} else if (rwlp->rwlock_type == USYNC_PROCESS) {
1087 		(void) mutex_lock(&rwlp->mutex);
1088 		(void) __lwp_rwlock_unlock(rwlp);
1089 		(void) mutex_unlock(&rwlp->mutex);
1090 	} else {
1091 		rw_queue_release(rwlp);
1092 	}
1093 
1094 out:
1095 	DTRACE_PROBE2(plockstat, rw__release, rwlp, rd_wr);
1096 	return (0);
1097 }
1098 
1099 void
lrw_unlock(rwlock_t * rwlp)1100 lrw_unlock(rwlock_t *rwlp)
1101 {
1102 	(void) rw_unlock(rwlp);
1103 	exit_critical(curthread);
1104 }
1105