1*7c478bd9Sstevel@tonic-gate /* 2*7c478bd9Sstevel@tonic-gate * CDDL HEADER START 3*7c478bd9Sstevel@tonic-gate * 4*7c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*7c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*7c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*7c478bd9Sstevel@tonic-gate * with the License. 8*7c478bd9Sstevel@tonic-gate * 9*7c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*7c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*7c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 12*7c478bd9Sstevel@tonic-gate * and limitations under the License. 13*7c478bd9Sstevel@tonic-gate * 14*7c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*7c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*7c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*7c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*7c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*7c478bd9Sstevel@tonic-gate * 20*7c478bd9Sstevel@tonic-gate * CDDL HEADER END 21*7c478bd9Sstevel@tonic-gate */ 22*7c478bd9Sstevel@tonic-gate /* 23*7c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*7c478bd9Sstevel@tonic-gate * Use is subject to license terms. 25*7c478bd9Sstevel@tonic-gate */ 26*7c478bd9Sstevel@tonic-gate 27*7c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*7c478bd9Sstevel@tonic-gate 29*7c478bd9Sstevel@tonic-gate #include "lint.h" 30*7c478bd9Sstevel@tonic-gate #include "thr_uberdata.h" 31*7c478bd9Sstevel@tonic-gate 32*7c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 33*7c478bd9Sstevel@tonic-gate 34*7c478bd9Sstevel@tonic-gate #define TRY_FLAG 0x10 35*7c478bd9Sstevel@tonic-gate #define READ_LOCK 0 36*7c478bd9Sstevel@tonic-gate #define WRITE_LOCK 1 37*7c478bd9Sstevel@tonic-gate #define READ_LOCK_TRY (READ_LOCK | TRY_FLAG) 38*7c478bd9Sstevel@tonic-gate #define WRITE_LOCK_TRY (WRITE_LOCK | TRY_FLAG) 39*7c478bd9Sstevel@tonic-gate 40*7c478bd9Sstevel@tonic-gate #define NLOCKS 4 /* initial number of readlock_t structs allocated */ 41*7c478bd9Sstevel@tonic-gate 42*7c478bd9Sstevel@tonic-gate /* 43*7c478bd9Sstevel@tonic-gate * Find/allocate an entry for rwlp in our array of rwlocks held for reading. 44*7c478bd9Sstevel@tonic-gate */ 45*7c478bd9Sstevel@tonic-gate static readlock_t * 46*7c478bd9Sstevel@tonic-gate rwl_entry(rwlock_t *rwlp) 47*7c478bd9Sstevel@tonic-gate { 48*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 49*7c478bd9Sstevel@tonic-gate readlock_t *remembered = NULL; 50*7c478bd9Sstevel@tonic-gate readlock_t *readlockp; 51*7c478bd9Sstevel@tonic-gate uint_t nlocks; 52*7c478bd9Sstevel@tonic-gate 53*7c478bd9Sstevel@tonic-gate if ((nlocks = self->ul_rdlocks) != 0) 54*7c478bd9Sstevel@tonic-gate readlockp = self->ul_readlock.array; 55*7c478bd9Sstevel@tonic-gate else { 56*7c478bd9Sstevel@tonic-gate nlocks = 1; 57*7c478bd9Sstevel@tonic-gate readlockp = &self->ul_readlock.single; 58*7c478bd9Sstevel@tonic-gate } 59*7c478bd9Sstevel@tonic-gate 60*7c478bd9Sstevel@tonic-gate for (; nlocks; nlocks--, readlockp++) { 61*7c478bd9Sstevel@tonic-gate if (readlockp->rd_rwlock == rwlp) 62*7c478bd9Sstevel@tonic-gate return (readlockp); 63*7c478bd9Sstevel@tonic-gate if (readlockp->rd_count == 0 && remembered == NULL) 64*7c478bd9Sstevel@tonic-gate remembered = readlockp; 65*7c478bd9Sstevel@tonic-gate } 66*7c478bd9Sstevel@tonic-gate if (remembered != NULL) { 67*7c478bd9Sstevel@tonic-gate remembered->rd_rwlock = rwlp; 68*7c478bd9Sstevel@tonic-gate return (remembered); 69*7c478bd9Sstevel@tonic-gate } 70*7c478bd9Sstevel@tonic-gate 71*7c478bd9Sstevel@tonic-gate /* 72*7c478bd9Sstevel@tonic-gate * No entry available. Allocate more space, converting the single 73*7c478bd9Sstevel@tonic-gate * readlock_t entry into an array of readlock_t entries if necessary. 74*7c478bd9Sstevel@tonic-gate */ 75*7c478bd9Sstevel@tonic-gate if ((nlocks = self->ul_rdlocks) == 0) { 76*7c478bd9Sstevel@tonic-gate /* 77*7c478bd9Sstevel@tonic-gate * Initial allocation of the readlock_t array. 78*7c478bd9Sstevel@tonic-gate * Convert the single entry into an array. 79*7c478bd9Sstevel@tonic-gate */ 80*7c478bd9Sstevel@tonic-gate self->ul_rdlocks = nlocks = NLOCKS; 81*7c478bd9Sstevel@tonic-gate readlockp = lmalloc(nlocks * sizeof (readlock_t)); 82*7c478bd9Sstevel@tonic-gate /* 83*7c478bd9Sstevel@tonic-gate * The single readlock_t becomes the first entry in the array. 84*7c478bd9Sstevel@tonic-gate */ 85*7c478bd9Sstevel@tonic-gate *readlockp = self->ul_readlock.single; 86*7c478bd9Sstevel@tonic-gate self->ul_readlock.single.rd_count = 0; 87*7c478bd9Sstevel@tonic-gate self->ul_readlock.array = readlockp; 88*7c478bd9Sstevel@tonic-gate /* 89*7c478bd9Sstevel@tonic-gate * Return the next available entry in the array. 90*7c478bd9Sstevel@tonic-gate */ 91*7c478bd9Sstevel@tonic-gate (++readlockp)->rd_rwlock = rwlp; 92*7c478bd9Sstevel@tonic-gate return (readlockp); 93*7c478bd9Sstevel@tonic-gate } 94*7c478bd9Sstevel@tonic-gate /* 95*7c478bd9Sstevel@tonic-gate * Reallocate the array, double the size each time. 96*7c478bd9Sstevel@tonic-gate */ 97*7c478bd9Sstevel@tonic-gate readlockp = lmalloc(nlocks * 2 * sizeof (readlock_t)); 98*7c478bd9Sstevel@tonic-gate (void) _memcpy(readlockp, self->ul_readlock.array, 99*7c478bd9Sstevel@tonic-gate nlocks * sizeof (readlock_t)); 100*7c478bd9Sstevel@tonic-gate lfree(self->ul_readlock.array, nlocks * sizeof (readlock_t)); 101*7c478bd9Sstevel@tonic-gate self->ul_readlock.array = readlockp; 102*7c478bd9Sstevel@tonic-gate self->ul_rdlocks *= 2; 103*7c478bd9Sstevel@tonic-gate /* 104*7c478bd9Sstevel@tonic-gate * Return the next available entry in the newly allocated array. 105*7c478bd9Sstevel@tonic-gate */ 106*7c478bd9Sstevel@tonic-gate (readlockp += nlocks)->rd_rwlock = rwlp; 107*7c478bd9Sstevel@tonic-gate return (readlockp); 108*7c478bd9Sstevel@tonic-gate } 109*7c478bd9Sstevel@tonic-gate 110*7c478bd9Sstevel@tonic-gate /* 111*7c478bd9Sstevel@tonic-gate * Free the array of rwlocks held for reading. 112*7c478bd9Sstevel@tonic-gate */ 113*7c478bd9Sstevel@tonic-gate void 114*7c478bd9Sstevel@tonic-gate rwl_free(ulwp_t *ulwp) 115*7c478bd9Sstevel@tonic-gate { 116*7c478bd9Sstevel@tonic-gate uint_t nlocks; 117*7c478bd9Sstevel@tonic-gate 118*7c478bd9Sstevel@tonic-gate if ((nlocks = ulwp->ul_rdlocks) != 0) 119*7c478bd9Sstevel@tonic-gate lfree(ulwp->ul_readlock.array, nlocks * sizeof (readlock_t)); 120*7c478bd9Sstevel@tonic-gate ulwp->ul_rdlocks = 0; 121*7c478bd9Sstevel@tonic-gate ulwp->ul_readlock.single.rd_rwlock = NULL; 122*7c478bd9Sstevel@tonic-gate ulwp->ul_readlock.single.rd_count = 0; 123*7c478bd9Sstevel@tonic-gate } 124*7c478bd9Sstevel@tonic-gate 125*7c478bd9Sstevel@tonic-gate /* 126*7c478bd9Sstevel@tonic-gate * Check if a reader version of the lock is held by the current thread. 127*7c478bd9Sstevel@tonic-gate * rw_read_is_held() is private to libc. 128*7c478bd9Sstevel@tonic-gate */ 129*7c478bd9Sstevel@tonic-gate #pragma weak rw_read_is_held = _rw_read_held 130*7c478bd9Sstevel@tonic-gate #pragma weak rw_read_held = _rw_read_held 131*7c478bd9Sstevel@tonic-gate int 132*7c478bd9Sstevel@tonic-gate _rw_read_held(rwlock_t *rwlp) 133*7c478bd9Sstevel@tonic-gate { 134*7c478bd9Sstevel@tonic-gate ulwp_t *self; 135*7c478bd9Sstevel@tonic-gate readlock_t *readlockp; 136*7c478bd9Sstevel@tonic-gate uint_t nlocks; 137*7c478bd9Sstevel@tonic-gate 138*7c478bd9Sstevel@tonic-gate /* quick answer */ 139*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { 140*7c478bd9Sstevel@tonic-gate if (!((uint32_t)rwlp->rwlock_readers & URW_READERS_MASK)) 141*7c478bd9Sstevel@tonic-gate return (0); 142*7c478bd9Sstevel@tonic-gate } else if (rwlp->rwlock_readers <= 0) { 143*7c478bd9Sstevel@tonic-gate return (0); 144*7c478bd9Sstevel@tonic-gate } 145*7c478bd9Sstevel@tonic-gate 146*7c478bd9Sstevel@tonic-gate /* 147*7c478bd9Sstevel@tonic-gate * The lock is held for reading by some thread. 148*7c478bd9Sstevel@tonic-gate * Search our array of rwlocks held for reading for a match. 149*7c478bd9Sstevel@tonic-gate */ 150*7c478bd9Sstevel@tonic-gate self = curthread; 151*7c478bd9Sstevel@tonic-gate if ((nlocks = self->ul_rdlocks) != 0) 152*7c478bd9Sstevel@tonic-gate readlockp = self->ul_readlock.array; 153*7c478bd9Sstevel@tonic-gate else { 154*7c478bd9Sstevel@tonic-gate nlocks = 1; 155*7c478bd9Sstevel@tonic-gate readlockp = &self->ul_readlock.single; 156*7c478bd9Sstevel@tonic-gate } 157*7c478bd9Sstevel@tonic-gate 158*7c478bd9Sstevel@tonic-gate for (; nlocks; nlocks--, readlockp++) 159*7c478bd9Sstevel@tonic-gate if (readlockp->rd_rwlock == rwlp) 160*7c478bd9Sstevel@tonic-gate return (readlockp->rd_count? 1 : 0); 161*7c478bd9Sstevel@tonic-gate 162*7c478bd9Sstevel@tonic-gate return (0); 163*7c478bd9Sstevel@tonic-gate } 164*7c478bd9Sstevel@tonic-gate 165*7c478bd9Sstevel@tonic-gate /* 166*7c478bd9Sstevel@tonic-gate * Check if a writer version of the lock is held by the current thread. 167*7c478bd9Sstevel@tonic-gate * rw_write_is_held() is private to libc. 168*7c478bd9Sstevel@tonic-gate */ 169*7c478bd9Sstevel@tonic-gate #pragma weak rw_write_is_held = _rw_write_held 170*7c478bd9Sstevel@tonic-gate #pragma weak rw_write_held = _rw_write_held 171*7c478bd9Sstevel@tonic-gate int 172*7c478bd9Sstevel@tonic-gate _rw_write_held(rwlock_t *rwlp) 173*7c478bd9Sstevel@tonic-gate { 174*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 175*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 176*7c478bd9Sstevel@tonic-gate 177*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) 178*7c478bd9Sstevel@tonic-gate return (((uint32_t)rwlp->rwlock_readers & URW_WRITE_LOCKED) && 179*7c478bd9Sstevel@tonic-gate (rwlp->rwlock_ownerpid == udp->pid) && 180*7c478bd9Sstevel@tonic-gate (rwlp->rwlock_owner == (uintptr_t)self)); 181*7c478bd9Sstevel@tonic-gate 182*7c478bd9Sstevel@tonic-gate /* USYNC_THREAD */ 183*7c478bd9Sstevel@tonic-gate return (rwlp->rwlock_readers == -1 && mutex_is_held(&rwlp->mutex)); 184*7c478bd9Sstevel@tonic-gate } 185*7c478bd9Sstevel@tonic-gate 186*7c478bd9Sstevel@tonic-gate #pragma weak rwlock_init = __rwlock_init 187*7c478bd9Sstevel@tonic-gate #pragma weak _rwlock_init = __rwlock_init 188*7c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 189*7c478bd9Sstevel@tonic-gate int 190*7c478bd9Sstevel@tonic-gate __rwlock_init(rwlock_t *rwlp, int type, void *arg) 191*7c478bd9Sstevel@tonic-gate { 192*7c478bd9Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 193*7c478bd9Sstevel@tonic-gate return (EINVAL); 194*7c478bd9Sstevel@tonic-gate /* 195*7c478bd9Sstevel@tonic-gate * Once reinitialized, we can no longer be holding a read or write lock. 196*7c478bd9Sstevel@tonic-gate * We can do nothing about other threads that are holding read locks. 197*7c478bd9Sstevel@tonic-gate */ 198*7c478bd9Sstevel@tonic-gate if (rw_read_is_held(rwlp)) 199*7c478bd9Sstevel@tonic-gate rwl_entry(rwlp)->rd_count = 0; 200*7c478bd9Sstevel@tonic-gate (void) _memset(rwlp, 0, sizeof (*rwlp)); 201*7c478bd9Sstevel@tonic-gate rwlp->rwlock_type = (uint16_t)type; 202*7c478bd9Sstevel@tonic-gate rwlp->rwlock_magic = RWL_MAGIC; 203*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers = 0; 204*7c478bd9Sstevel@tonic-gate rwlp->mutex.mutex_type = (uint8_t)type; 205*7c478bd9Sstevel@tonic-gate rwlp->mutex.mutex_flag = LOCK_INITED; 206*7c478bd9Sstevel@tonic-gate rwlp->mutex.mutex_magic = MUTEX_MAGIC; 207*7c478bd9Sstevel@tonic-gate rwlp->readercv.cond_type = (uint16_t)type; 208*7c478bd9Sstevel@tonic-gate rwlp->readercv.cond_magic = COND_MAGIC; 209*7c478bd9Sstevel@tonic-gate rwlp->writercv.cond_type = (uint16_t)type; 210*7c478bd9Sstevel@tonic-gate rwlp->writercv.cond_magic = COND_MAGIC; 211*7c478bd9Sstevel@tonic-gate return (0); 212*7c478bd9Sstevel@tonic-gate } 213*7c478bd9Sstevel@tonic-gate 214*7c478bd9Sstevel@tonic-gate #pragma weak rwlock_destroy = __rwlock_destroy 215*7c478bd9Sstevel@tonic-gate #pragma weak _rwlock_destroy = __rwlock_destroy 216*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_destroy = __rwlock_destroy 217*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_rwlock_destroy = __rwlock_destroy 218*7c478bd9Sstevel@tonic-gate int 219*7c478bd9Sstevel@tonic-gate __rwlock_destroy(rwlock_t *rwlp) 220*7c478bd9Sstevel@tonic-gate { 221*7c478bd9Sstevel@tonic-gate /* 222*7c478bd9Sstevel@tonic-gate * Once destroyed, we can no longer be holding a read or write lock. 223*7c478bd9Sstevel@tonic-gate * We can do nothing about other threads that are holding read locks. 224*7c478bd9Sstevel@tonic-gate */ 225*7c478bd9Sstevel@tonic-gate if (rw_read_is_held(rwlp)) 226*7c478bd9Sstevel@tonic-gate rwl_entry(rwlp)->rd_count = 0; 227*7c478bd9Sstevel@tonic-gate rwlp->rwlock_magic = 0; 228*7c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(rwlp); 229*7c478bd9Sstevel@tonic-gate return (0); 230*7c478bd9Sstevel@tonic-gate } 231*7c478bd9Sstevel@tonic-gate 232*7c478bd9Sstevel@tonic-gate /* 233*7c478bd9Sstevel@tonic-gate * Wake up the next thread sleeping on the rwlock queue and then 234*7c478bd9Sstevel@tonic-gate * drop the queue lock. Return non-zero if we wake up someone. 235*7c478bd9Sstevel@tonic-gate * 236*7c478bd9Sstevel@tonic-gate * This is called whenever a thread releases the lock and whenever a 237*7c478bd9Sstevel@tonic-gate * thread successfully or unsuccessfully attempts to acquire the lock. 238*7c478bd9Sstevel@tonic-gate * (Basically, whenever the state of the queue might have changed.) 239*7c478bd9Sstevel@tonic-gate * 240*7c478bd9Sstevel@tonic-gate * We wake up at most one thread. If there are more threads to be 241*7c478bd9Sstevel@tonic-gate * awakened, the next one will be waked up by the thread we wake up. 242*7c478bd9Sstevel@tonic-gate * This ensures that queued threads will acquire the lock in priority 243*7c478bd9Sstevel@tonic-gate * order and that queued writers will take precedence over queued 244*7c478bd9Sstevel@tonic-gate * readers of the same priority. 245*7c478bd9Sstevel@tonic-gate */ 246*7c478bd9Sstevel@tonic-gate static int 247*7c478bd9Sstevel@tonic-gate rw_queue_release(queue_head_t *qp, rwlock_t *rwlp) 248*7c478bd9Sstevel@tonic-gate { 249*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 250*7c478bd9Sstevel@tonic-gate int more; 251*7c478bd9Sstevel@tonic-gate 252*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_readers >= 0 && rwlp->rwlock_mwaiters) { 253*7c478bd9Sstevel@tonic-gate /* 254*7c478bd9Sstevel@tonic-gate * The lock is free or at least is available to readers 255*7c478bd9Sstevel@tonic-gate * and there are (or might be) waiters on the queue. 256*7c478bd9Sstevel@tonic-gate */ 257*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_readers != 0 && 258*7c478bd9Sstevel@tonic-gate (ulwp = queue_waiter(qp, rwlp)) == NULL) 259*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mwaiters = 0; 260*7c478bd9Sstevel@tonic-gate else if (rwlp->rwlock_readers == 0 || !ulwp->ul_writer) { 261*7c478bd9Sstevel@tonic-gate if ((ulwp = dequeue(qp, rwlp, &more)) == NULL) 262*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mwaiters = 0; 263*7c478bd9Sstevel@tonic-gate else { 264*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 265*7c478bd9Sstevel@tonic-gate lwpid_t lwpid = ulwp->ul_lwpid; 266*7c478bd9Sstevel@tonic-gate 267*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mwaiters = (more? 1 : 0); 268*7c478bd9Sstevel@tonic-gate no_preempt(self); 269*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 270*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 271*7c478bd9Sstevel@tonic-gate preempt(self); 272*7c478bd9Sstevel@tonic-gate return (1); 273*7c478bd9Sstevel@tonic-gate } 274*7c478bd9Sstevel@tonic-gate } 275*7c478bd9Sstevel@tonic-gate } 276*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 277*7c478bd9Sstevel@tonic-gate return (0); 278*7c478bd9Sstevel@tonic-gate } 279*7c478bd9Sstevel@tonic-gate 280*7c478bd9Sstevel@tonic-gate /* 281*7c478bd9Sstevel@tonic-gate * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock, 282*7c478bd9Sstevel@tonic-gate * and trywrlock for process-shared (USYNC_PROCESS) rwlocks. 283*7c478bd9Sstevel@tonic-gate * 284*7c478bd9Sstevel@tonic-gate * Note: if the lock appears to be contended we call __lwp_rwlock_rdlock() 285*7c478bd9Sstevel@tonic-gate * or __lwp_rwlock_wrlock() holding the mutex. These return with the mutex 286*7c478bd9Sstevel@tonic-gate * released, and if they need to sleep will release the mutex first. In the 287*7c478bd9Sstevel@tonic-gate * event of a spurious wakeup, these will return EAGAIN (because it is much 288*7c478bd9Sstevel@tonic-gate * easier for us to re-acquire the mutex here). 289*7c478bd9Sstevel@tonic-gate */ 290*7c478bd9Sstevel@tonic-gate int 291*7c478bd9Sstevel@tonic-gate shared_rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr) 292*7c478bd9Sstevel@tonic-gate { 293*7c478bd9Sstevel@tonic-gate uint32_t *rwstate = (uint32_t *)&rwlp->readers; 294*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 295*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 296*7c478bd9Sstevel@tonic-gate int try_flag; 297*7c478bd9Sstevel@tonic-gate int error = 0; 298*7c478bd9Sstevel@tonic-gate 299*7c478bd9Sstevel@tonic-gate try_flag = (rd_wr & TRY_FLAG); 300*7c478bd9Sstevel@tonic-gate rd_wr &= ~TRY_FLAG; 301*7c478bd9Sstevel@tonic-gate ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK); 302*7c478bd9Sstevel@tonic-gate 303*7c478bd9Sstevel@tonic-gate if (!try_flag) { 304*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr); 305*7c478bd9Sstevel@tonic-gate } 306*7c478bd9Sstevel@tonic-gate 307*7c478bd9Sstevel@tonic-gate do { 308*7c478bd9Sstevel@tonic-gate if ((error = _private_mutex_lock(&rwlp->mutex)) != 0) 309*7c478bd9Sstevel@tonic-gate break; 310*7c478bd9Sstevel@tonic-gate 311*7c478bd9Sstevel@tonic-gate if (rd_wr == READ_LOCK) { 312*7c478bd9Sstevel@tonic-gate /* 313*7c478bd9Sstevel@tonic-gate * We are a reader. 314*7c478bd9Sstevel@tonic-gate */ 315*7c478bd9Sstevel@tonic-gate 316*7c478bd9Sstevel@tonic-gate if ((*rwstate & ~URW_READERS_MASK) == 0) { 317*7c478bd9Sstevel@tonic-gate (*rwstate)++; 318*7c478bd9Sstevel@tonic-gate (void) _private_mutex_unlock(&rwlp->mutex); 319*7c478bd9Sstevel@tonic-gate } else if (try_flag) { 320*7c478bd9Sstevel@tonic-gate if (*rwstate & URW_WRITE_LOCKED) { 321*7c478bd9Sstevel@tonic-gate error = EBUSY; 322*7c478bd9Sstevel@tonic-gate (void) _private_mutex_unlock( 323*7c478bd9Sstevel@tonic-gate &rwlp->mutex); 324*7c478bd9Sstevel@tonic-gate } else { 325*7c478bd9Sstevel@tonic-gate /* 326*7c478bd9Sstevel@tonic-gate * We have a higher priority than any 327*7c478bd9Sstevel@tonic-gate * queued waiters, or the waiters bit 328*7c478bd9Sstevel@tonic-gate * may be inaccurate. Only the kernel 329*7c478bd9Sstevel@tonic-gate * knows for sure. 330*7c478bd9Sstevel@tonic-gate */ 331*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 332*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 333*7c478bd9Sstevel@tonic-gate error = __lwp_rwlock_tryrdlock(rwlp); 334*7c478bd9Sstevel@tonic-gate } 335*7c478bd9Sstevel@tonic-gate } else { 336*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 337*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 338*7c478bd9Sstevel@tonic-gate error = __lwp_rwlock_rdlock(rwlp, tsp); 339*7c478bd9Sstevel@tonic-gate } 340*7c478bd9Sstevel@tonic-gate } else { 341*7c478bd9Sstevel@tonic-gate /* 342*7c478bd9Sstevel@tonic-gate * We are a writer. 343*7c478bd9Sstevel@tonic-gate */ 344*7c478bd9Sstevel@tonic-gate 345*7c478bd9Sstevel@tonic-gate if (*rwstate == 0) { 346*7c478bd9Sstevel@tonic-gate *rwstate = URW_WRITE_LOCKED; 347*7c478bd9Sstevel@tonic-gate (void) _private_mutex_unlock(&rwlp->mutex); 348*7c478bd9Sstevel@tonic-gate } else if (try_flag) { 349*7c478bd9Sstevel@tonic-gate if (*rwstate & URW_WRITE_LOCKED) { 350*7c478bd9Sstevel@tonic-gate error = EBUSY; 351*7c478bd9Sstevel@tonic-gate (void) _private_mutex_unlock( 352*7c478bd9Sstevel@tonic-gate &rwlp->mutex); 353*7c478bd9Sstevel@tonic-gate } else { 354*7c478bd9Sstevel@tonic-gate /* 355*7c478bd9Sstevel@tonic-gate * The waiters bit may be inaccurate. 356*7c478bd9Sstevel@tonic-gate * Only the kernel knows for sure. 357*7c478bd9Sstevel@tonic-gate */ 358*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 359*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 360*7c478bd9Sstevel@tonic-gate error = __lwp_rwlock_trywrlock(rwlp); 361*7c478bd9Sstevel@tonic-gate } 362*7c478bd9Sstevel@tonic-gate } else { 363*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 364*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 365*7c478bd9Sstevel@tonic-gate error = __lwp_rwlock_wrlock(rwlp, tsp); 366*7c478bd9Sstevel@tonic-gate } 367*7c478bd9Sstevel@tonic-gate } 368*7c478bd9Sstevel@tonic-gate } while (error == EAGAIN); 369*7c478bd9Sstevel@tonic-gate 370*7c478bd9Sstevel@tonic-gate if (error == 0) { 371*7c478bd9Sstevel@tonic-gate if (rd_wr == WRITE_LOCK) { 372*7c478bd9Sstevel@tonic-gate rwlp->rwlock_owner = (uintptr_t)self; 373*7c478bd9Sstevel@tonic-gate rwlp->rwlock_ownerpid = udp->pid; 374*7c478bd9Sstevel@tonic-gate } 375*7c478bd9Sstevel@tonic-gate if (!try_flag) { 376*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 1); 377*7c478bd9Sstevel@tonic-gate } 378*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr); 379*7c478bd9Sstevel@tonic-gate } else if (!try_flag) { 380*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 0); 381*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__error, rwlp, rd_wr, error); 382*7c478bd9Sstevel@tonic-gate } 383*7c478bd9Sstevel@tonic-gate return (error); 384*7c478bd9Sstevel@tonic-gate } 385*7c478bd9Sstevel@tonic-gate 386*7c478bd9Sstevel@tonic-gate /* 387*7c478bd9Sstevel@tonic-gate * Code for unlock of process-shared (USYNC_PROCESS) rwlocks. 388*7c478bd9Sstevel@tonic-gate * 389*7c478bd9Sstevel@tonic-gate * Note: if the lock appears to have waiters we call __lwp_rwlock_unlock() 390*7c478bd9Sstevel@tonic-gate * holding the mutex. This returns with the mutex still held (for us to 391*7c478bd9Sstevel@tonic-gate * release). 392*7c478bd9Sstevel@tonic-gate */ 393*7c478bd9Sstevel@tonic-gate int 394*7c478bd9Sstevel@tonic-gate shared_rwlock_unlock(rwlock_t *rwlp, int *waked) 395*7c478bd9Sstevel@tonic-gate { 396*7c478bd9Sstevel@tonic-gate uint32_t *rwstate = (uint32_t *)&rwlp->readers; 397*7c478bd9Sstevel@tonic-gate int error = 0; 398*7c478bd9Sstevel@tonic-gate 399*7c478bd9Sstevel@tonic-gate if ((error = _private_mutex_lock(&rwlp->mutex)) != 0) 400*7c478bd9Sstevel@tonic-gate return (error); 401*7c478bd9Sstevel@tonic-gate 402*7c478bd9Sstevel@tonic-gate /* Reset flag used to suggest caller yields. */ 403*7c478bd9Sstevel@tonic-gate *waked = 0; 404*7c478bd9Sstevel@tonic-gate 405*7c478bd9Sstevel@tonic-gate /* Our right to unlock was checked in __rw_unlock(). */ 406*7c478bd9Sstevel@tonic-gate if (*rwstate & URW_WRITE_LOCKED) { 407*7c478bd9Sstevel@tonic-gate rwlp->rwlock_owner = 0; 408*7c478bd9Sstevel@tonic-gate rwlp->rwlock_ownerpid = 0; 409*7c478bd9Sstevel@tonic-gate } 410*7c478bd9Sstevel@tonic-gate 411*7c478bd9Sstevel@tonic-gate if ((*rwstate & ~URW_READERS_MASK) == 0) { 412*7c478bd9Sstevel@tonic-gate /* Simple multiple readers, no waiters case. */ 413*7c478bd9Sstevel@tonic-gate if (*rwstate > 0) 414*7c478bd9Sstevel@tonic-gate (*rwstate)--; 415*7c478bd9Sstevel@tonic-gate } else if (!(*rwstate & URW_HAS_WAITERS)) { 416*7c478bd9Sstevel@tonic-gate /* Simple no waiters case (i.e. was write locked). */ 417*7c478bd9Sstevel@tonic-gate *rwstate = 0; 418*7c478bd9Sstevel@tonic-gate } else { 419*7c478bd9Sstevel@tonic-gate /* 420*7c478bd9Sstevel@tonic-gate * We appear to have waiters so we must call into the kernel. 421*7c478bd9Sstevel@tonic-gate * If there are waiters a full handoff will occur (rwstate 422*7c478bd9Sstevel@tonic-gate * will be updated, and one or more threads will be awoken). 423*7c478bd9Sstevel@tonic-gate */ 424*7c478bd9Sstevel@tonic-gate error = __lwp_rwlock_unlock(rwlp); 425*7c478bd9Sstevel@tonic-gate 426*7c478bd9Sstevel@tonic-gate /* Suggest caller yields. */ 427*7c478bd9Sstevel@tonic-gate *waked = 1; 428*7c478bd9Sstevel@tonic-gate } 429*7c478bd9Sstevel@tonic-gate 430*7c478bd9Sstevel@tonic-gate (void) _private_mutex_unlock(&rwlp->mutex); 431*7c478bd9Sstevel@tonic-gate 432*7c478bd9Sstevel@tonic-gate if (error) { 433*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__error, rwlp, 0, error); 434*7c478bd9Sstevel@tonic-gate } else { 435*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 436*7c478bd9Sstevel@tonic-gate } 437*7c478bd9Sstevel@tonic-gate 438*7c478bd9Sstevel@tonic-gate return (error); 439*7c478bd9Sstevel@tonic-gate } 440*7c478bd9Sstevel@tonic-gate 441*7c478bd9Sstevel@tonic-gate 442*7c478bd9Sstevel@tonic-gate /* 443*7c478bd9Sstevel@tonic-gate * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock, 444*7c478bd9Sstevel@tonic-gate * and trywrlock for process-private (USYNC_THREAD) rwlocks. 445*7c478bd9Sstevel@tonic-gate */ 446*7c478bd9Sstevel@tonic-gate int 447*7c478bd9Sstevel@tonic-gate rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr) 448*7c478bd9Sstevel@tonic-gate { 449*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 450*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 451*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 452*7c478bd9Sstevel@tonic-gate int try_flag; 453*7c478bd9Sstevel@tonic-gate int error = 0; 454*7c478bd9Sstevel@tonic-gate 455*7c478bd9Sstevel@tonic-gate try_flag = (rd_wr & TRY_FLAG); 456*7c478bd9Sstevel@tonic-gate rd_wr &= ~TRY_FLAG; 457*7c478bd9Sstevel@tonic-gate ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK); 458*7c478bd9Sstevel@tonic-gate 459*7c478bd9Sstevel@tonic-gate /* 460*7c478bd9Sstevel@tonic-gate * Optimize for the case of having only a single thread. 461*7c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 462*7c478bd9Sstevel@tonic-gate * We don't need the protection of queue_lock() in this case. 463*7c478bd9Sstevel@tonic-gate * We need to defer signals, however (the other form of concurrency). 464*7c478bd9Sstevel@tonic-gate */ 465*7c478bd9Sstevel@tonic-gate if (!self->ul_uberdata->uberflags.uf_mt) { 466*7c478bd9Sstevel@tonic-gate sigoff(self); 467*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_readers < 0 || 468*7c478bd9Sstevel@tonic-gate (rd_wr == WRITE_LOCK && rwlp->rwlock_readers != 0)) { 469*7c478bd9Sstevel@tonic-gate sigon(self); 470*7c478bd9Sstevel@tonic-gate if (try_flag) 471*7c478bd9Sstevel@tonic-gate return (EBUSY); 472*7c478bd9Sstevel@tonic-gate /* 473*7c478bd9Sstevel@tonic-gate * Sombody other than ourself owns the lock. (If we 474*7c478bd9Sstevel@tonic-gate * owned the lock, either for reading or writing, we 475*7c478bd9Sstevel@tonic-gate * would already have returned EDEADLK in our caller.) 476*7c478bd9Sstevel@tonic-gate * This can happen only in the child of fork1() when 477*7c478bd9Sstevel@tonic-gate * some now-defunct thread was holding the lock when 478*7c478bd9Sstevel@tonic-gate * the fork1() was executed by the current thread. 479*7c478bd9Sstevel@tonic-gate * In this case, we just fall into the long way 480*7c478bd9Sstevel@tonic-gate * to block, either forever or with a timeout. 481*7c478bd9Sstevel@tonic-gate */ 482*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNER(&rwlp->mutex) != self); 483*7c478bd9Sstevel@tonic-gate } else { 484*7c478bd9Sstevel@tonic-gate if (rd_wr == READ_LOCK) 485*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers++; 486*7c478bd9Sstevel@tonic-gate else { 487*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers = -1; 488*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKSET; 489*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = (uintptr_t)self; 490*7c478bd9Sstevel@tonic-gate } 491*7c478bd9Sstevel@tonic-gate sigon(self); 492*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr); 493*7c478bd9Sstevel@tonic-gate return (0); 494*7c478bd9Sstevel@tonic-gate } 495*7c478bd9Sstevel@tonic-gate } 496*7c478bd9Sstevel@tonic-gate 497*7c478bd9Sstevel@tonic-gate if (!try_flag) { 498*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr); 499*7c478bd9Sstevel@tonic-gate } 500*7c478bd9Sstevel@tonic-gate 501*7c478bd9Sstevel@tonic-gate /* 502*7c478bd9Sstevel@tonic-gate * Do it the long way. 503*7c478bd9Sstevel@tonic-gate */ 504*7c478bd9Sstevel@tonic-gate qp = queue_lock(rwlp, MX); 505*7c478bd9Sstevel@tonic-gate while (error == 0) { 506*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_readers < 0 || 507*7c478bd9Sstevel@tonic-gate (rd_wr == WRITE_LOCK && rwlp->rwlock_readers != 0)) 508*7c478bd9Sstevel@tonic-gate /* EMPTY */; /* somebody holds the lock */ 509*7c478bd9Sstevel@tonic-gate else if (!rwlp->rwlock_mwaiters) 510*7c478bd9Sstevel@tonic-gate break; /* no queued waiters */ 511*7c478bd9Sstevel@tonic-gate else if ((ulwp = queue_waiter(qp, rwlp)) == NULL) { 512*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mwaiters = 0; 513*7c478bd9Sstevel@tonic-gate break; /* no queued waiters */ 514*7c478bd9Sstevel@tonic-gate } else { 515*7c478bd9Sstevel@tonic-gate int our_pri = real_priority(self); 516*7c478bd9Sstevel@tonic-gate int his_pri = real_priority(ulwp); 517*7c478bd9Sstevel@tonic-gate 518*7c478bd9Sstevel@tonic-gate if (rd_wr == WRITE_LOCK) { 519*7c478bd9Sstevel@tonic-gate /* 520*7c478bd9Sstevel@tonic-gate * We defer to a queued thread that has 521*7c478bd9Sstevel@tonic-gate * a higher priority than ours. 522*7c478bd9Sstevel@tonic-gate */ 523*7c478bd9Sstevel@tonic-gate if (his_pri <= our_pri) 524*7c478bd9Sstevel@tonic-gate break; 525*7c478bd9Sstevel@tonic-gate } else { 526*7c478bd9Sstevel@tonic-gate /* 527*7c478bd9Sstevel@tonic-gate * We defer to a queued thread that has 528*7c478bd9Sstevel@tonic-gate * a higher priority than ours or that 529*7c478bd9Sstevel@tonic-gate * is a writer whose priority equals ours. 530*7c478bd9Sstevel@tonic-gate */ 531*7c478bd9Sstevel@tonic-gate if (his_pri < our_pri || 532*7c478bd9Sstevel@tonic-gate (his_pri == our_pri && !ulwp->ul_writer)) 533*7c478bd9Sstevel@tonic-gate break; 534*7c478bd9Sstevel@tonic-gate } 535*7c478bd9Sstevel@tonic-gate } 536*7c478bd9Sstevel@tonic-gate /* 537*7c478bd9Sstevel@tonic-gate * We are about to block. 538*7c478bd9Sstevel@tonic-gate * If we're doing a trylock, return EBUSY instead. 539*7c478bd9Sstevel@tonic-gate */ 540*7c478bd9Sstevel@tonic-gate if (try_flag) { 541*7c478bd9Sstevel@tonic-gate error = EBUSY; 542*7c478bd9Sstevel@tonic-gate break; 543*7c478bd9Sstevel@tonic-gate } 544*7c478bd9Sstevel@tonic-gate /* 545*7c478bd9Sstevel@tonic-gate * Enqueue writers ahead of readers of the 546*7c478bd9Sstevel@tonic-gate * same priority. 547*7c478bd9Sstevel@tonic-gate */ 548*7c478bd9Sstevel@tonic-gate self->ul_writer = rd_wr; /* *must* be 0 or 1 */ 549*7c478bd9Sstevel@tonic-gate enqueue(qp, self, rwlp, MX); 550*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mwaiters = 1; 551*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 552*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 553*7c478bd9Sstevel@tonic-gate if ((error = __lwp_park(tsp, 0)) == EINTR) 554*7c478bd9Sstevel@tonic-gate error = 0; 555*7c478bd9Sstevel@tonic-gate self->ul_writer = 0; 556*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 557*7c478bd9Sstevel@tonic-gate qp = queue_lock(rwlp, MX); 558*7c478bd9Sstevel@tonic-gate if (self->ul_sleepq) /* timeout or spurious wakeup */ 559*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mwaiters = dequeue_self(qp, rwlp); 560*7c478bd9Sstevel@tonic-gate } 561*7c478bd9Sstevel@tonic-gate 562*7c478bd9Sstevel@tonic-gate if (error == 0) { 563*7c478bd9Sstevel@tonic-gate if (rd_wr == READ_LOCK) 564*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers++; 565*7c478bd9Sstevel@tonic-gate else { 566*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers = -1; 567*7c478bd9Sstevel@tonic-gate /* make it look like we acquired the embedded mutex */ 568*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKSET; 569*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = (uintptr_t)self; 570*7c478bd9Sstevel@tonic-gate } 571*7c478bd9Sstevel@tonic-gate if (!try_flag) { 572*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 1); 573*7c478bd9Sstevel@tonic-gate } 574*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr); 575*7c478bd9Sstevel@tonic-gate } else if (!try_flag) { 576*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 0); 577*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__error, rwlp, rd_wr, error); 578*7c478bd9Sstevel@tonic-gate } 579*7c478bd9Sstevel@tonic-gate 580*7c478bd9Sstevel@tonic-gate (void) rw_queue_release(qp, rwlp); 581*7c478bd9Sstevel@tonic-gate 582*7c478bd9Sstevel@tonic-gate return (error); 583*7c478bd9Sstevel@tonic-gate } 584*7c478bd9Sstevel@tonic-gate 585*7c478bd9Sstevel@tonic-gate int 586*7c478bd9Sstevel@tonic-gate rw_rdlock_impl(rwlock_t *rwlp, timespec_t *tsp) 587*7c478bd9Sstevel@tonic-gate { 588*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 589*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 590*7c478bd9Sstevel@tonic-gate readlock_t *readlockp; 591*7c478bd9Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 592*7c478bd9Sstevel@tonic-gate int error; 593*7c478bd9Sstevel@tonic-gate 594*7c478bd9Sstevel@tonic-gate /* 595*7c478bd9Sstevel@tonic-gate * If we already hold a readers lock on this rwlock, 596*7c478bd9Sstevel@tonic-gate * just increment our reference count and return. 597*7c478bd9Sstevel@tonic-gate */ 598*7c478bd9Sstevel@tonic-gate readlockp = rwl_entry(rwlp); 599*7c478bd9Sstevel@tonic-gate if (readlockp->rd_count != 0) { 600*7c478bd9Sstevel@tonic-gate if (readlockp->rd_count == READ_LOCK_MAX) 601*7c478bd9Sstevel@tonic-gate return (EAGAIN); 602*7c478bd9Sstevel@tonic-gate readlockp->rd_count++; 603*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK); 604*7c478bd9Sstevel@tonic-gate return (0); 605*7c478bd9Sstevel@tonic-gate } 606*7c478bd9Sstevel@tonic-gate 607*7c478bd9Sstevel@tonic-gate /* 608*7c478bd9Sstevel@tonic-gate * If we hold the writer lock, bail out. 609*7c478bd9Sstevel@tonic-gate */ 610*7c478bd9Sstevel@tonic-gate if (rw_write_is_held(rwlp)) { 611*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection) 612*7c478bd9Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_rdlock", 613*7c478bd9Sstevel@tonic-gate "calling thread owns the writer lock"); 614*7c478bd9Sstevel@tonic-gate return (EDEADLK); 615*7c478bd9Sstevel@tonic-gate } 616*7c478bd9Sstevel@tonic-gate 617*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */ 618*7c478bd9Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, tsp, READ_LOCK); 619*7c478bd9Sstevel@tonic-gate else /* user-level */ 620*7c478bd9Sstevel@tonic-gate error = rwlock_lock(rwlp, tsp, READ_LOCK); 621*7c478bd9Sstevel@tonic-gate 622*7c478bd9Sstevel@tonic-gate if (error == 0) { 623*7c478bd9Sstevel@tonic-gate readlockp->rd_count = 1; 624*7c478bd9Sstevel@tonic-gate if (rwsp) 625*7c478bd9Sstevel@tonic-gate tdb_incr(rwsp->rw_rdlock); 626*7c478bd9Sstevel@tonic-gate } 627*7c478bd9Sstevel@tonic-gate 628*7c478bd9Sstevel@tonic-gate return (error); 629*7c478bd9Sstevel@tonic-gate } 630*7c478bd9Sstevel@tonic-gate 631*7c478bd9Sstevel@tonic-gate #pragma weak rw_rdlock = __rw_rdlock 632*7c478bd9Sstevel@tonic-gate #pragma weak _rw_rdlock = __rw_rdlock 633*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_rdlock = __rw_rdlock 634*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_rwlock_rdlock = __rw_rdlock 635*7c478bd9Sstevel@tonic-gate int 636*7c478bd9Sstevel@tonic-gate __rw_rdlock(rwlock_t *rwlp) 637*7c478bd9Sstevel@tonic-gate { 638*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 639*7c478bd9Sstevel@tonic-gate return (rw_rdlock_impl(rwlp, NULL)); 640*7c478bd9Sstevel@tonic-gate } 641*7c478bd9Sstevel@tonic-gate 642*7c478bd9Sstevel@tonic-gate void 643*7c478bd9Sstevel@tonic-gate lrw_rdlock(rwlock_t *rwlp) 644*7c478bd9Sstevel@tonic-gate { 645*7c478bd9Sstevel@tonic-gate enter_critical(curthread); 646*7c478bd9Sstevel@tonic-gate (void) rw_rdlock_impl(rwlp, NULL); 647*7c478bd9Sstevel@tonic-gate } 648*7c478bd9Sstevel@tonic-gate 649*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_reltimedrdlock_np = \ 650*7c478bd9Sstevel@tonic-gate _pthread_rwlock_reltimedrdlock_np 651*7c478bd9Sstevel@tonic-gate int 652*7c478bd9Sstevel@tonic-gate _pthread_rwlock_reltimedrdlock_np(rwlock_t *rwlp, const timespec_t *reltime) 653*7c478bd9Sstevel@tonic-gate { 654*7c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 655*7c478bd9Sstevel@tonic-gate int error; 656*7c478bd9Sstevel@tonic-gate 657*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 658*7c478bd9Sstevel@tonic-gate error = rw_rdlock_impl(rwlp, &tslocal); 659*7c478bd9Sstevel@tonic-gate if (error == ETIME) 660*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 661*7c478bd9Sstevel@tonic-gate return (error); 662*7c478bd9Sstevel@tonic-gate } 663*7c478bd9Sstevel@tonic-gate 664*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_timedrdlock = _pthread_rwlock_timedrdlock 665*7c478bd9Sstevel@tonic-gate int 666*7c478bd9Sstevel@tonic-gate _pthread_rwlock_timedrdlock(rwlock_t *rwlp, const timespec_t *abstime) 667*7c478bd9Sstevel@tonic-gate { 668*7c478bd9Sstevel@tonic-gate timespec_t tslocal; 669*7c478bd9Sstevel@tonic-gate int error; 670*7c478bd9Sstevel@tonic-gate 671*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 672*7c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 673*7c478bd9Sstevel@tonic-gate error = rw_rdlock_impl(rwlp, &tslocal); 674*7c478bd9Sstevel@tonic-gate if (error == ETIME) 675*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 676*7c478bd9Sstevel@tonic-gate return (error); 677*7c478bd9Sstevel@tonic-gate } 678*7c478bd9Sstevel@tonic-gate 679*7c478bd9Sstevel@tonic-gate int 680*7c478bd9Sstevel@tonic-gate rw_wrlock_impl(rwlock_t *rwlp, timespec_t *tsp) 681*7c478bd9Sstevel@tonic-gate { 682*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 683*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 684*7c478bd9Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 685*7c478bd9Sstevel@tonic-gate int error; 686*7c478bd9Sstevel@tonic-gate 687*7c478bd9Sstevel@tonic-gate /* 688*7c478bd9Sstevel@tonic-gate * If we hold a readers lock on this rwlock, bail out. 689*7c478bd9Sstevel@tonic-gate */ 690*7c478bd9Sstevel@tonic-gate if (rw_read_is_held(rwlp)) { 691*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection) 692*7c478bd9Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_wrlock", 693*7c478bd9Sstevel@tonic-gate "calling thread owns the readers lock"); 694*7c478bd9Sstevel@tonic-gate return (EDEADLK); 695*7c478bd9Sstevel@tonic-gate } 696*7c478bd9Sstevel@tonic-gate 697*7c478bd9Sstevel@tonic-gate /* 698*7c478bd9Sstevel@tonic-gate * If we hold the writer lock, bail out. 699*7c478bd9Sstevel@tonic-gate */ 700*7c478bd9Sstevel@tonic-gate if (rw_write_is_held(rwlp)) { 701*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection) 702*7c478bd9Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_wrlock", 703*7c478bd9Sstevel@tonic-gate "calling thread owns the writer lock"); 704*7c478bd9Sstevel@tonic-gate return (EDEADLK); 705*7c478bd9Sstevel@tonic-gate } 706*7c478bd9Sstevel@tonic-gate 707*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */ 708*7c478bd9Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, tsp, WRITE_LOCK); 709*7c478bd9Sstevel@tonic-gate } else { /* user-level */ 710*7c478bd9Sstevel@tonic-gate error = rwlock_lock(rwlp, tsp, WRITE_LOCK); 711*7c478bd9Sstevel@tonic-gate } 712*7c478bd9Sstevel@tonic-gate 713*7c478bd9Sstevel@tonic-gate if (error == 0 && rwsp) { 714*7c478bd9Sstevel@tonic-gate tdb_incr(rwsp->rw_wrlock); 715*7c478bd9Sstevel@tonic-gate rwsp->rw_wrlock_begin_hold = gethrtime(); 716*7c478bd9Sstevel@tonic-gate } 717*7c478bd9Sstevel@tonic-gate 718*7c478bd9Sstevel@tonic-gate return (error); 719*7c478bd9Sstevel@tonic-gate } 720*7c478bd9Sstevel@tonic-gate 721*7c478bd9Sstevel@tonic-gate #pragma weak rw_wrlock = __rw_wrlock 722*7c478bd9Sstevel@tonic-gate #pragma weak _rw_wrlock = __rw_wrlock 723*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_wrlock = __rw_wrlock 724*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_rwlock_wrlock = __rw_wrlock 725*7c478bd9Sstevel@tonic-gate int 726*7c478bd9Sstevel@tonic-gate __rw_wrlock(rwlock_t *rwlp) 727*7c478bd9Sstevel@tonic-gate { 728*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 729*7c478bd9Sstevel@tonic-gate return (rw_wrlock_impl(rwlp, NULL)); 730*7c478bd9Sstevel@tonic-gate } 731*7c478bd9Sstevel@tonic-gate 732*7c478bd9Sstevel@tonic-gate void 733*7c478bd9Sstevel@tonic-gate lrw_wrlock(rwlock_t *rwlp) 734*7c478bd9Sstevel@tonic-gate { 735*7c478bd9Sstevel@tonic-gate enter_critical(curthread); 736*7c478bd9Sstevel@tonic-gate (void) rw_wrlock_impl(rwlp, NULL); 737*7c478bd9Sstevel@tonic-gate } 738*7c478bd9Sstevel@tonic-gate 739*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_reltimedwrlock_np = \ 740*7c478bd9Sstevel@tonic-gate _pthread_rwlock_reltimedwrlock_np 741*7c478bd9Sstevel@tonic-gate int 742*7c478bd9Sstevel@tonic-gate _pthread_rwlock_reltimedwrlock_np(rwlock_t *rwlp, const timespec_t *reltime) 743*7c478bd9Sstevel@tonic-gate { 744*7c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 745*7c478bd9Sstevel@tonic-gate int error; 746*7c478bd9Sstevel@tonic-gate 747*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 748*7c478bd9Sstevel@tonic-gate error = rw_wrlock_impl(rwlp, &tslocal); 749*7c478bd9Sstevel@tonic-gate if (error == ETIME) 750*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 751*7c478bd9Sstevel@tonic-gate return (error); 752*7c478bd9Sstevel@tonic-gate } 753*7c478bd9Sstevel@tonic-gate 754*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_timedwrlock = _pthread_rwlock_timedwrlock 755*7c478bd9Sstevel@tonic-gate int 756*7c478bd9Sstevel@tonic-gate _pthread_rwlock_timedwrlock(rwlock_t *rwlp, const timespec_t *abstime) 757*7c478bd9Sstevel@tonic-gate { 758*7c478bd9Sstevel@tonic-gate timespec_t tslocal; 759*7c478bd9Sstevel@tonic-gate int error; 760*7c478bd9Sstevel@tonic-gate 761*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 762*7c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 763*7c478bd9Sstevel@tonic-gate error = rw_wrlock_impl(rwlp, &tslocal); 764*7c478bd9Sstevel@tonic-gate if (error == ETIME) 765*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 766*7c478bd9Sstevel@tonic-gate return (error); 767*7c478bd9Sstevel@tonic-gate } 768*7c478bd9Sstevel@tonic-gate 769*7c478bd9Sstevel@tonic-gate #pragma weak rw_tryrdlock = __rw_tryrdlock 770*7c478bd9Sstevel@tonic-gate #pragma weak _rw_tryrdlock = __rw_tryrdlock 771*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_tryrdlock = __rw_tryrdlock 772*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_rwlock_tryrdlock = __rw_tryrdlock 773*7c478bd9Sstevel@tonic-gate int 774*7c478bd9Sstevel@tonic-gate __rw_tryrdlock(rwlock_t *rwlp) 775*7c478bd9Sstevel@tonic-gate { 776*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 777*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 778*7c478bd9Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 779*7c478bd9Sstevel@tonic-gate readlock_t *readlockp; 780*7c478bd9Sstevel@tonic-gate int error; 781*7c478bd9Sstevel@tonic-gate 782*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 783*7c478bd9Sstevel@tonic-gate 784*7c478bd9Sstevel@tonic-gate if (rwsp) 785*7c478bd9Sstevel@tonic-gate tdb_incr(rwsp->rw_rdlock_try); 786*7c478bd9Sstevel@tonic-gate 787*7c478bd9Sstevel@tonic-gate /* 788*7c478bd9Sstevel@tonic-gate * If we already hold a readers lock on this rwlock, 789*7c478bd9Sstevel@tonic-gate * just increment our reference count and return. 790*7c478bd9Sstevel@tonic-gate */ 791*7c478bd9Sstevel@tonic-gate readlockp = rwl_entry(rwlp); 792*7c478bd9Sstevel@tonic-gate if (readlockp->rd_count != 0) { 793*7c478bd9Sstevel@tonic-gate if (readlockp->rd_count == READ_LOCK_MAX) 794*7c478bd9Sstevel@tonic-gate return (EAGAIN); 795*7c478bd9Sstevel@tonic-gate readlockp->rd_count++; 796*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK); 797*7c478bd9Sstevel@tonic-gate return (0); 798*7c478bd9Sstevel@tonic-gate } 799*7c478bd9Sstevel@tonic-gate 800*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */ 801*7c478bd9Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, NULL, READ_LOCK_TRY); 802*7c478bd9Sstevel@tonic-gate else /* user-level */ 803*7c478bd9Sstevel@tonic-gate error = rwlock_lock(rwlp, NULL, READ_LOCK_TRY); 804*7c478bd9Sstevel@tonic-gate 805*7c478bd9Sstevel@tonic-gate if (error == 0) 806*7c478bd9Sstevel@tonic-gate readlockp->rd_count = 1; 807*7c478bd9Sstevel@tonic-gate else if (rwsp) 808*7c478bd9Sstevel@tonic-gate tdb_incr(rwsp->rw_rdlock_try_fail); 809*7c478bd9Sstevel@tonic-gate 810*7c478bd9Sstevel@tonic-gate return (error); 811*7c478bd9Sstevel@tonic-gate } 812*7c478bd9Sstevel@tonic-gate 813*7c478bd9Sstevel@tonic-gate #pragma weak rw_trywrlock = __rw_trywrlock 814*7c478bd9Sstevel@tonic-gate #pragma weak _rw_trywrlock = __rw_trywrlock 815*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_trywrlock = __rw_trywrlock 816*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_rwlock_trywrlock = __rw_trywrlock 817*7c478bd9Sstevel@tonic-gate int 818*7c478bd9Sstevel@tonic-gate __rw_trywrlock(rwlock_t *rwlp) 819*7c478bd9Sstevel@tonic-gate { 820*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 821*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 822*7c478bd9Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 823*7c478bd9Sstevel@tonic-gate int error; 824*7c478bd9Sstevel@tonic-gate 825*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 826*7c478bd9Sstevel@tonic-gate 827*7c478bd9Sstevel@tonic-gate if (rwsp) 828*7c478bd9Sstevel@tonic-gate tdb_incr(rwsp->rw_wrlock_try); 829*7c478bd9Sstevel@tonic-gate 830*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */ 831*7c478bd9Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY); 832*7c478bd9Sstevel@tonic-gate } else { /* user-level */ 833*7c478bd9Sstevel@tonic-gate error = rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY); 834*7c478bd9Sstevel@tonic-gate } 835*7c478bd9Sstevel@tonic-gate if (rwsp) { 836*7c478bd9Sstevel@tonic-gate if (error) 837*7c478bd9Sstevel@tonic-gate tdb_incr(rwsp->rw_wrlock_try_fail); 838*7c478bd9Sstevel@tonic-gate else 839*7c478bd9Sstevel@tonic-gate rwsp->rw_wrlock_begin_hold = gethrtime(); 840*7c478bd9Sstevel@tonic-gate } 841*7c478bd9Sstevel@tonic-gate return (error); 842*7c478bd9Sstevel@tonic-gate } 843*7c478bd9Sstevel@tonic-gate 844*7c478bd9Sstevel@tonic-gate #pragma weak rw_unlock = __rw_unlock 845*7c478bd9Sstevel@tonic-gate #pragma weak _rw_unlock = __rw_unlock 846*7c478bd9Sstevel@tonic-gate #pragma weak pthread_rwlock_unlock = __rw_unlock 847*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_rwlock_unlock = __rw_unlock 848*7c478bd9Sstevel@tonic-gate int 849*7c478bd9Sstevel@tonic-gate __rw_unlock(rwlock_t *rwlp) 850*7c478bd9Sstevel@tonic-gate { 851*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 852*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 853*7c478bd9Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp; 854*7c478bd9Sstevel@tonic-gate int32_t lock_count; 855*7c478bd9Sstevel@tonic-gate int waked; 856*7c478bd9Sstevel@tonic-gate 857*7c478bd9Sstevel@tonic-gate /* fetch the lock count once; it may change underfoot */ 858*7c478bd9Sstevel@tonic-gate lock_count = rwlp->rwlock_readers; 859*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { 860*7c478bd9Sstevel@tonic-gate /* munge it from rwstate */ 861*7c478bd9Sstevel@tonic-gate if (lock_count & URW_WRITE_LOCKED) 862*7c478bd9Sstevel@tonic-gate lock_count = -1; 863*7c478bd9Sstevel@tonic-gate else 864*7c478bd9Sstevel@tonic-gate lock_count &= URW_READERS_MASK; 865*7c478bd9Sstevel@tonic-gate } 866*7c478bd9Sstevel@tonic-gate 867*7c478bd9Sstevel@tonic-gate if (lock_count < 0) { 868*7c478bd9Sstevel@tonic-gate /* 869*7c478bd9Sstevel@tonic-gate * Since the writer lock is held, we'd better be 870*7c478bd9Sstevel@tonic-gate * holding it, else we cannot legitimately be here. 871*7c478bd9Sstevel@tonic-gate */ 872*7c478bd9Sstevel@tonic-gate if (!rw_write_is_held(rwlp)) { 873*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection) 874*7c478bd9Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_unlock", 875*7c478bd9Sstevel@tonic-gate "writer lock held, " 876*7c478bd9Sstevel@tonic-gate "but not by the calling thread"); 877*7c478bd9Sstevel@tonic-gate return (EPERM); 878*7c478bd9Sstevel@tonic-gate } 879*7c478bd9Sstevel@tonic-gate if ((rwsp = RWLOCK_STATS(rwlp, udp)) != NULL) { 880*7c478bd9Sstevel@tonic-gate if (rwsp->rw_wrlock_begin_hold) 881*7c478bd9Sstevel@tonic-gate rwsp->rw_wrlock_hold_time += 882*7c478bd9Sstevel@tonic-gate gethrtime() - rwsp->rw_wrlock_begin_hold; 883*7c478bd9Sstevel@tonic-gate rwsp->rw_wrlock_begin_hold = 0; 884*7c478bd9Sstevel@tonic-gate } 885*7c478bd9Sstevel@tonic-gate } else if (lock_count > 0) { 886*7c478bd9Sstevel@tonic-gate /* 887*7c478bd9Sstevel@tonic-gate * A readers lock is held; if we don't hold one, bail out. 888*7c478bd9Sstevel@tonic-gate */ 889*7c478bd9Sstevel@tonic-gate readlock_t *readlockp = rwl_entry(rwlp); 890*7c478bd9Sstevel@tonic-gate if (readlockp->rd_count == 0) { 891*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection) 892*7c478bd9Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_unlock", 893*7c478bd9Sstevel@tonic-gate "readers lock held, " 894*7c478bd9Sstevel@tonic-gate "but not by the calling thread"); 895*7c478bd9Sstevel@tonic-gate return (EPERM); 896*7c478bd9Sstevel@tonic-gate } 897*7c478bd9Sstevel@tonic-gate /* 898*7c478bd9Sstevel@tonic-gate * If we hold more than one readers lock on this rwlock, 899*7c478bd9Sstevel@tonic-gate * just decrement our reference count and return. 900*7c478bd9Sstevel@tonic-gate */ 901*7c478bd9Sstevel@tonic-gate if (--readlockp->rd_count != 0) { 902*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 903*7c478bd9Sstevel@tonic-gate return (0); 904*7c478bd9Sstevel@tonic-gate } 905*7c478bd9Sstevel@tonic-gate } else { 906*7c478bd9Sstevel@tonic-gate /* 907*7c478bd9Sstevel@tonic-gate * This is a usage error. 908*7c478bd9Sstevel@tonic-gate * No thread should release an unowned lock. 909*7c478bd9Sstevel@tonic-gate */ 910*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection) 911*7c478bd9Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_unlock", "lock not owned"); 912*7c478bd9Sstevel@tonic-gate return (EPERM); 913*7c478bd9Sstevel@tonic-gate } 914*7c478bd9Sstevel@tonic-gate 915*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */ 916*7c478bd9Sstevel@tonic-gate (void) shared_rwlock_unlock(rwlp, &waked); 917*7c478bd9Sstevel@tonic-gate } else if (!udp->uberflags.uf_mt) { /* single threaded */ 918*7c478bd9Sstevel@tonic-gate /* 919*7c478bd9Sstevel@tonic-gate * In the case of having only a single thread, we don't 920*7c478bd9Sstevel@tonic-gate * need the protection of queue_lock() (this parallels 921*7c478bd9Sstevel@tonic-gate * the optimization made in rwlock_lock(), above). 922*7c478bd9Sstevel@tonic-gate * As in rwlock_lock(), we need to defer signals. 923*7c478bd9Sstevel@tonic-gate */ 924*7c478bd9Sstevel@tonic-gate sigoff(self); 925*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_readers > 0) { 926*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers--; 927*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 928*7c478bd9Sstevel@tonic-gate } else { 929*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers = 0; 930*7c478bd9Sstevel@tonic-gate /* make it look like we released the embedded mutex */ 931*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 932*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKCLEAR; 933*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, WRITE_LOCK); 934*7c478bd9Sstevel@tonic-gate } 935*7c478bd9Sstevel@tonic-gate sigon(self); 936*7c478bd9Sstevel@tonic-gate waked = 0; 937*7c478bd9Sstevel@tonic-gate } else { /* multithreaded */ 938*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 939*7c478bd9Sstevel@tonic-gate 940*7c478bd9Sstevel@tonic-gate qp = queue_lock(rwlp, MX); 941*7c478bd9Sstevel@tonic-gate if (rwlp->rwlock_readers > 0) { 942*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers--; 943*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 944*7c478bd9Sstevel@tonic-gate } else { 945*7c478bd9Sstevel@tonic-gate rwlp->rwlock_readers = 0; 946*7c478bd9Sstevel@tonic-gate /* make it look like we released the embedded mutex */ 947*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 948*7c478bd9Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKCLEAR; 949*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, WRITE_LOCK); 950*7c478bd9Sstevel@tonic-gate } 951*7c478bd9Sstevel@tonic-gate waked = rw_queue_release(qp, rwlp); 952*7c478bd9Sstevel@tonic-gate } 953*7c478bd9Sstevel@tonic-gate 954*7c478bd9Sstevel@tonic-gate /* 955*7c478bd9Sstevel@tonic-gate * Yield to the thread we just waked up, just in case we might 956*7c478bd9Sstevel@tonic-gate * be about to grab the rwlock again immediately upon return. 957*7c478bd9Sstevel@tonic-gate * This is pretty weak but it helps on a uniprocessor and also 958*7c478bd9Sstevel@tonic-gate * when cpu affinity has assigned both ourself and the other 959*7c478bd9Sstevel@tonic-gate * thread to the same CPU. Note that lwp_yield() will yield 960*7c478bd9Sstevel@tonic-gate * the processor only if the writer is at the same or higher 961*7c478bd9Sstevel@tonic-gate * priority than ourself. This provides more balanced program 962*7c478bd9Sstevel@tonic-gate * behavior; it doesn't guarantee acquisition of the lock by 963*7c478bd9Sstevel@tonic-gate * the pending writer. 964*7c478bd9Sstevel@tonic-gate */ 965*7c478bd9Sstevel@tonic-gate if (waked) 966*7c478bd9Sstevel@tonic-gate lwp_yield(); 967*7c478bd9Sstevel@tonic-gate return (0); 968*7c478bd9Sstevel@tonic-gate } 969*7c478bd9Sstevel@tonic-gate 970*7c478bd9Sstevel@tonic-gate void 971*7c478bd9Sstevel@tonic-gate lrw_unlock(rwlock_t *rwlp) 972*7c478bd9Sstevel@tonic-gate { 973*7c478bd9Sstevel@tonic-gate (void) __rw_unlock(rwlp); 974*7c478bd9Sstevel@tonic-gate exit_critical(curthread); 975*7c478bd9Sstevel@tonic-gate } 976