Lines Matching refs:lp
215 rw_panic(char *msg, rwlock_impl_t *lp) in rw_panic() argument
220 if (atomic_cas_ptr(&panic_rwlock_addr, NULL, lp) == NULL) in rw_panic()
221 panic_rwlock = *lp; in rw_panic()
224 msg, (void *)lp, panic_rwlock.rw_wwwh, (void *)curthread); in rw_panic()
237 rwlock_impl_t *lp = (rwlock_impl_t *)rwlp; in rw_destroy() local
239 if (lp->rw_wwwh != 0) { in rw_destroy()
240 if ((lp->rw_wwwh & RW_DOUBLE_LOCK) == RW_DOUBLE_LOCK) in rw_destroy()
241 rw_panic("rw_destroy: lock already destroyed", lp); in rw_destroy()
243 rw_panic("rw_destroy: lock still active", lp); in rw_destroy()
246 lp->rw_wwwh = RW_DOUBLE_LOCK; in rw_destroy()
253 rw_locked(rwlock_impl_t *lp, krw_t rw) in rw_locked() argument
255 uintptr_t old = lp->rw_wwwh; in rw_locked()
277 rw_enter_sleep(rwlock_impl_t *lp, krw_t rw) in rw_enter_sleep() argument
300 if (((old = lp->rw_wwwh) & lock_busy) == 0) { in rw_enter_sleep()
301 if (casip(&lp->rw_wwwh, old, old + lock_value) != old) { in rw_enter_sleep()
319 rw_panic("rw_enter: bad rwlock", lp); in rw_enter_sleep()
324 rw_panic("recursive rw_enter", lp); in rw_enter_sleep()
328 ts = turnstile_lookup(lp); in rw_enter_sleep()
331 if (((old = lp->rw_wwwh) & lock_busy) == 0) in rw_enter_sleep()
334 } while (old != new && casip(&lp->rw_wwwh, old, new) != old); in rw_enter_sleep()
340 turnstile_exit(lp); in rw_enter_sleep()
348 ASSERT(lp->rw_wwwh & lock_wait); in rw_enter_sleep()
349 ASSERT(lp->rw_wwwh & RW_LOCKED); in rw_enter_sleep()
355 (void) turnstile_block(ts, TS_READER_Q, lp, in rw_enter_sleep()
359 (void) turnstile_block(ts, TS_WRITER_Q, lp, in rw_enter_sleep()
364 LOCKSTAT_RECORD4(LS_RW_ENTER_BLOCK, lp, sleep_time, rw, in rw_enter_sleep()
375 ASSERT(rw_locked(lp, rw)); in rw_enter_sleep()
379 LOCKSTAT_RECORD(LS_RW_ENTER_ACQUIRE, lp, rw); in rw_enter_sleep()
411 rw_exit_wakeup(rwlock_impl_t *lp) in rw_exit_wakeup() argument
422 old = lp->rw_wwwh; in rw_exit_wakeup()
425 rw_panic("rw_exit: not owner", lp); in rw_exit_wakeup()
426 lp->rw_wwwh = 0; in rw_exit_wakeup()
432 rw_panic("rw_exit: lock not held", lp); in rw_exit_wakeup()
443 old = lp->rw_wwwh; in rw_exit_wakeup()
446 if (casip(&lp->rw_wwwh, old, new) != old) { in rw_exit_wakeup()
474 if (casip(&lp->rw_wwwh, old, new) != old) in rw_exit_wakeup()
481 ts = turnstile_lookup(lp); in rw_exit_wakeup()
496 lp->rw_wwwh = new; in rw_exit_wakeup()
512 ASSERT(lp->rw_wwwh & RW_WRITE_WANTED); in rw_exit_wakeup()
518 lp->rw_wwwh = new; in rw_exit_wakeup()
527 LOCKSTAT_RECORD(LS_RW_EXIT_RELEASE, lp, RW_READER); in rw_exit_wakeup()
529 LOCKSTAT_RECORD(LS_RW_EXIT_RELEASE, lp, RW_WRITER); in rw_exit_wakeup()
536 rwlock_impl_t *lp = (rwlock_impl_t *)rwlp; in rw_tryenter() local
544 if ((old = lp->rw_wwwh) & (rw == RW_READER ? in rw_tryenter()
549 if (casip(&lp->rw_wwwh, old, old + RW_READ_LOCK) == old) in rw_tryenter()
560 LOCKSTAT_RECORD(LS_RW_TRYENTER_ACQUIRE, lp, rw); in rw_tryenter()
562 if (casip(&lp->rw_wwwh, 0, RW_WRITE_LOCK(curthread)) != 0) in rw_tryenter()
564 LOCKSTAT_RECORD(LS_RW_TRYENTER_ACQUIRE, lp, rw); in rw_tryenter()
566 ASSERT(rw_locked(lp, rw)); in rw_tryenter()
574 rwlock_impl_t *lp = (rwlock_impl_t *)rwlp; in rw_downgrade() local
579 if ((lp->rw_wwwh & RW_OWNER) != (uintptr_t)curthread) { in rw_downgrade()
580 rw_panic("rw_downgrade: not owner", lp); in rw_downgrade()
584 if (atomic_add_ip_nv(&lp->rw_wwwh, in rw_downgrade()
586 turnstile_t *ts = turnstile_lookup(lp); in rw_downgrade()
592 atomic_add_ip(&lp->rw_wwwh, delta); in rw_downgrade()
596 ASSERT(rw_locked(lp, RW_READER)); in rw_downgrade()
597 LOCKSTAT_RECORD0(LS_RW_DOWNGRADE_DOWNGRADE, lp); in rw_downgrade()
603 rwlock_impl_t *lp = (rwlock_impl_t *)rwlp; in rw_tryupgrade() local
606 ASSERT(rw_locked(lp, RW_READER)); in rw_tryupgrade()
609 if (((old = lp->rw_wwwh) & ~RW_HAS_WAITERS) != RW_READ_LOCK) in rw_tryupgrade()
612 } while (casip(&lp->rw_wwwh, old, new) != old); in rw_tryupgrade()
616 LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, lp); in rw_tryupgrade()
617 ASSERT(rw_locked(lp, RW_WRITER)); in rw_tryupgrade()