Lines Matching refs:rw
138 #define rw_wowner(rw) lv_rw_wowner(RW_READ_VALUE(rw)) argument
144 #define rw_recursed(rw) ((rw)->rw_recurse != 0) argument
149 #define rw_wlocked(rw) (rw_wowner((rw)) == curthread) argument
156 #define rw_owner(rw) rw_wowner(rw) argument
172 struct rwlock *rw; in lock_rw() local
174 rw = (struct rwlock *)lock; in lock_rw()
176 rw_rlock(rw); in lock_rw()
178 rw_wlock(rw); in lock_rw()
184 struct rwlock *rw; in trylock_rw() local
186 rw = (struct rwlock *)lock; in trylock_rw()
188 return (rw_try_rlock(rw)); in trylock_rw()
190 return (rw_try_wlock(rw)); in trylock_rw()
196 struct rwlock *rw; in unlock_rw() local
198 rw = (struct rwlock *)lock; in unlock_rw()
199 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED); in unlock_rw()
200 if (rw->rw_lock & RW_LOCK_READ) { in unlock_rw()
201 rw_runlock(rw); in unlock_rw()
204 rw_wunlock(rw); in unlock_rw()
213 const struct rwlock *rw = (const struct rwlock *)lock; in owner_rw() local
214 uintptr_t x = rw->rw_lock; in owner_rw()
216 *owner = rw_wowner(rw); in owner_rw()
225 struct rwlock *rw; in _rw_init_flags() local
228 rw = rwlock2rw(c); in _rw_init_flags()
232 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock, in _rw_init_flags()
234 &rw->rw_lock)); in _rw_init_flags()
250 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags); in _rw_init_flags()
251 rw->rw_lock = RW_UNLOCKED; in _rw_init_flags()
252 rw->rw_recurse = 0; in _rw_init_flags()
258 struct rwlock *rw; in _rw_destroy() local
260 rw = rwlock2rw(c); in _rw_destroy()
262 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw)); in _rw_destroy()
263 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw)); in _rw_destroy()
264 rw->rw_lock = RW_DESTROYED; in _rw_destroy()
265 lock_destroy(&rw->lock_object); in _rw_destroy()
288 struct rwlock *rw; in _rw_wlock_cookie() local
291 rw = rwlock2rw(c); in _rw_wlock_cookie()
296 curthread, rw, file, line)); in _rw_wlock_cookie()
297 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_wlock_cookie()
298 ("rw_wlock() of destroyed rwlock %p @ %s:%d", rw, file, line)); in _rw_wlock_cookie()
299 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, in _rw_wlock_cookie()
303 if (!_rw_write_lock_fetch(rw, &v, tid)) in _rw_wlock_cookie()
304 _rw_wlock_hard(rw, v, file, line); in _rw_wlock_cookie()
306 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, in _rw_wlock_cookie()
309 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); in _rw_wlock_cookie()
310 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); in _rw_wlock_cookie()
315 __rw_try_wlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF) in __rw_try_wlock_int()
329 curthread, rw, file, line)); in __rw_try_wlock_int()
330 KASSERT(rw->rw_lock != RW_DESTROYED, in __rw_try_wlock_int()
331 ("rw_try_wlock() of destroyed rwlock %p @ %s:%d", rw, file, line)); in __rw_try_wlock_int()
337 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid)) in __rw_try_wlock_int()
341 if (v == tid && (rw->lock_object.lo_flags & LO_RECURSABLE)) { in __rw_try_wlock_int()
342 rw->rw_recurse++; in __rw_try_wlock_int()
343 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); in __rw_try_wlock_int()
350 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); in __rw_try_wlock_int()
352 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, in __rw_try_wlock_int()
356 rw, 0, 0, file, line, LOCKSTAT_WRITER); in __rw_try_wlock_int()
365 struct rwlock *rw; in __rw_try_wlock() local
367 rw = rwlock2rw(c); in __rw_try_wlock()
368 return (__rw_try_wlock_int(rw LOCK_FILE_LINE_ARG)); in __rw_try_wlock()
374 struct rwlock *rw; in _rw_wunlock_cookie() local
376 rw = rwlock2rw(c); in _rw_wunlock_cookie()
378 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_wunlock_cookie()
379 ("rw_wunlock() of destroyed rwlock %p @ %s:%d", rw, file, line)); in _rw_wunlock_cookie()
381 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); in _rw_wunlock_cookie()
382 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, in _rw_wunlock_cookie()
386 _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); in _rw_wunlock_cookie()
388 __rw_wunlock(rw, curthread, file, line); in _rw_wunlock_cookie()
414 __rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, bool fp in __rw_rlock_try() argument
429 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp, in __rw_rlock_try()
431 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_rlock_try()
434 rw, (void *)*vp, in __rw_rlock_try()
444 __rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v in __rw_rlock_hard() argument
472 if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG)) in __rw_rlock_hard()
475 all_time -= lockstat_nsecs(&rw->lock_object); in __rw_rlock_hard()
495 lock_profile_obtain_lock_failed(&rw->lock_object, false, in __rw_rlock_hard()
498 THREAD_CONTENDS_ON_LOCK(&rw->lock_object); in __rw_rlock_hard()
501 if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG)) in __rw_rlock_hard()
516 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_rlock_hard()
519 __func__, rw, owner); in __rw_rlock_hard()
522 "lockname:\"%s\"", rw->lock_object.lo_name); in __rw_rlock_hard()
525 v = RW_READ_VALUE(rw); in __rw_rlock_hard()
536 v = RW_READ_VALUE(rw); in __rw_rlock_hard()
543 rw->lock_object.lo_name); in __rw_rlock_hard()
547 v = RW_READ_VALUE(rw); in __rw_rlock_hard()
573 ts = turnstile_trywait(&rw->lock_object); in __rw_rlock_hard()
579 v = RW_READ_VALUE(rw); in __rw_rlock_hard()
617 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v, in __rw_rlock_hard()
620 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_rlock_hard()
622 __func__, rw); in __rw_rlock_hard()
629 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_rlock_hard()
631 rw); in __rw_rlock_hard()
633 sleep_time -= lockstat_nsecs(&rw->lock_object); in __rw_rlock_hard()
635 MPASS(owner == rw_owner(rw)); in __rw_rlock_hard()
638 sleep_time += lockstat_nsecs(&rw->lock_object); in __rw_rlock_hard()
641 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_rlock_hard()
643 __func__, rw); in __rw_rlock_hard()
644 v = RW_READ_VALUE(rw); in __rw_rlock_hard()
646 THREAD_CONTENTION_DONE(&rw->lock_object); in __rw_rlock_hard()
652 all_time += lockstat_nsecs(&rw->lock_object); in __rw_rlock_hard()
654 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, in __rw_rlock_hard()
660 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, in __rw_rlock_hard()
670 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, in __rw_rlock_hard()
675 __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF) in __rw_rlock_int()
685 td, rw, file, line)); in __rw_rlock_int()
686 KASSERT(rw->rw_lock != RW_DESTROYED, in __rw_rlock_int()
687 ("rw_rlock() of destroyed rwlock %p @ %s:%d", rw, file, line)); in __rw_rlock_int()
688 KASSERT(rw_wowner(rw) != td, in __rw_rlock_int()
690 rw, file, line)); in __rw_rlock_int()
691 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); in __rw_rlock_int()
693 v = RW_READ_VALUE(rw); in __rw_rlock_int()
695 !__rw_rlock_try(rw, td, &v, true LOCK_FILE_LINE_ARG))) in __rw_rlock_int()
696 __rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG); in __rw_rlock_int()
698 lock_profile_obtain_lock_success(&rw->lock_object, false, 0, 0, in __rw_rlock_int()
701 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); in __rw_rlock_int()
702 WITNESS_LOCK(&rw->lock_object, 0, file, line); in __rw_rlock_int()
709 struct rwlock *rw; in __rw_rlock() local
711 rw = rwlock2rw(c); in __rw_rlock()
712 __rw_rlock_int(rw LOCK_FILE_LINE_ARG); in __rw_rlock()
716 __rw_try_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF) in __rw_try_rlock_int()
725 curthread, rw, file, line)); in __rw_try_rlock_int()
727 x = rw->rw_lock; in __rw_try_rlock_int()
729 KASSERT(rw->rw_lock != RW_DESTROYED, in __rw_try_rlock_int()
730 ("rw_try_rlock() of destroyed rwlock %p @ %s:%d", rw, file, in __rw_try_rlock_int()
734 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &x, x + RW_ONE_READER)) { in __rw_try_rlock_int()
735 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, in __rw_try_rlock_int()
737 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); in __rw_try_rlock_int()
739 rw, 0, 0, file, line, LOCKSTAT_READER); in __rw_try_rlock_int()
746 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); in __rw_try_rlock_int()
753 struct rwlock *rw; in __rw_try_rlock() local
755 rw = rwlock2rw(c); in __rw_try_rlock()
756 return (__rw_try_rlock_int(rw LOCK_FILE_LINE_ARG)); in __rw_try_rlock()
760 __rw_runlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp) in __rw_runlock_try() argument
765 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp, in __rw_runlock_try()
767 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_runlock_try()
770 __func__, rw, (void *)*vp, in __rw_runlock_try()
783 __rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v in __rw_runlock_hard() argument
792 if (__rw_runlock_try(rw, td, &v)) in __rw_runlock_hard()
799 turnstile_chain_lock(&rw->lock_object); in __rw_runlock_hard()
800 v = RW_READ_VALUE(rw); in __rw_runlock_hard()
802 if (__rw_runlock_try(rw, td, &v)) in __rw_runlock_hard()
830 if (!atomic_fcmpset_rel_ptr(&rw->rw_lock, &v, setv)) in __rw_runlock_hard()
832 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_runlock_hard()
834 __func__, rw); in __rw_runlock_hard()
843 ts = turnstile_lookup(&rw->lock_object); in __rw_runlock_hard()
850 turnstile_chain_unlock(&rw->lock_object); in __rw_runlock_hard()
852 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER); in __rw_runlock_hard()
856 _rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF) in _rw_runlock_cookie_int()
861 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_runlock_cookie_int()
862 ("rw_runlock() of destroyed rwlock %p @ %s:%d", rw, file, line)); in _rw_runlock_cookie_int()
863 __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line); in _rw_runlock_cookie_int()
864 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); in _rw_runlock_cookie_int()
865 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); in _rw_runlock_cookie_int()
868 v = RW_READ_VALUE(rw); in _rw_runlock_cookie_int()
871 !__rw_runlock_try(rw, td, &v))) in _rw_runlock_cookie_int()
872 __rw_runlock_hard(rw, td, v LOCK_FILE_LINE_ARG); in _rw_runlock_cookie_int()
874 lock_profile_release_lock(&rw->lock_object, false); in _rw_runlock_cookie_int()
882 struct rwlock *rw; in _rw_runlock_cookie() local
884 rw = rwlock2rw(c); in _rw_runlock_cookie()
885 _rw_runlock_cookie_int(rw LOCK_FILE_LINE_ARG); in _rw_runlock_cookie()
914 struct rwlock *rw; in __rw_wlock_hard() local
943 rw = rwlock2rw(c); in __rw_wlock_hard()
948 if (_rw_write_lock_fetch(rw, &v, tid)) in __rw_wlock_hard()
953 all_time -= lockstat_nsecs(&rw->lock_object); in __rw_wlock_hard()
966 v = RW_READ_VALUE(rw); in __rw_wlock_hard()
969 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, in __rw_wlock_hard()
971 __func__, rw, file, line)); in __rw_wlock_hard()
972 rw->rw_recurse++; in __rw_wlock_hard()
973 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); in __rw_wlock_hard()
974 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wlock_hard()
975 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); in __rw_wlock_hard()
979 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wlock_hard()
981 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); in __rw_wlock_hard()
992 lock_profile_obtain_lock_failed(&rw->lock_object, false, in __rw_wlock_hard()
995 THREAD_CONTENDS_ON_LOCK(&rw->lock_object); in __rw_wlock_hard()
999 if (_rw_write_lock_fetch(rw, &v, tid)) in __rw_wlock_hard()
1009 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid)) in __rw_wlock_hard()
1025 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wlock_hard()
1027 __func__, rw, owner); in __rw_wlock_hard()
1030 rw->lock_object.lo_name); in __rw_wlock_hard()
1033 v = RW_READ_VALUE(rw); in __rw_wlock_hard()
1049 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v, in __rw_wlock_hard()
1060 rw->lock_object.lo_name); in __rw_wlock_hard()
1064 v = RW_READ_VALUE(rw); in __rw_wlock_hard()
1083 ts = turnstile_trywait(&rw->lock_object); in __rw_wlock_hard()
1084 v = RW_READ_VALUE(rw); in __rw_wlock_hard()
1118 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid | setv)) { in __rw_wlock_hard()
1134 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v, setv)) in __rw_wlock_hard()
1148 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v, in __rw_wlock_hard()
1151 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wlock_hard()
1153 __func__, rw); in __rw_wlock_hard()
1162 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wlock_hard()
1164 rw); in __rw_wlock_hard()
1166 sleep_time -= lockstat_nsecs(&rw->lock_object); in __rw_wlock_hard()
1168 MPASS(owner == rw_owner(rw)); in __rw_wlock_hard()
1171 sleep_time += lockstat_nsecs(&rw->lock_object); in __rw_wlock_hard()
1174 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wlock_hard()
1176 __func__, rw); in __rw_wlock_hard()
1180 v = RW_READ_VALUE(rw); in __rw_wlock_hard()
1182 THREAD_CONTENTION_DONE(&rw->lock_object); in __rw_wlock_hard()
1194 all_time += lockstat_nsecs(&rw->lock_object); in __rw_wlock_hard()
1196 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, in __rw_wlock_hard()
1202 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, in __rw_wlock_hard()
1207 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, in __rw_wlock_hard()
1220 struct rwlock *rw; in __rw_wunlock_hard() local
1229 rw = rwlock2rw(c); in __rw_wunlock_hard()
1231 v = RW_READ_VALUE(rw); in __rw_wunlock_hard()
1234 if (--(rw->rw_recurse) == 0) in __rw_wunlock_hard()
1235 atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); in __rw_wunlock_hard()
1236 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wunlock_hard()
1237 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); in __rw_wunlock_hard()
1241 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_WRITER); in __rw_wunlock_hard()
1242 if (v == tid && _rw_write_unlock(rw, tid)) in __rw_wunlock_hard()
1245 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), in __rw_wunlock_hard()
1248 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wunlock_hard()
1249 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); in __rw_wunlock_hard()
1251 turnstile_chain_lock(&rw->lock_object); in __rw_wunlock_hard()
1270 v = RW_READ_VALUE(rw); in __rw_wunlock_hard()
1276 atomic_store_rel_ptr(&rw->rw_lock, setv); in __rw_wunlock_hard()
1279 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in __rw_wunlock_hard()
1280 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, in __rw_wunlock_hard()
1283 ts = turnstile_lookup(&rw->lock_object); in __rw_wunlock_hard()
1287 turnstile_chain_unlock(&rw->lock_object); in __rw_wunlock_hard()
1296 __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF) in __rw_try_upgrade_int()
1305 KASSERT(rw->rw_lock != RW_DESTROYED, in __rw_try_upgrade_int()
1306 ("rw_try_upgrade() of destroyed rwlock %p @ %s:%d", rw, file, in __rw_try_upgrade_int()
1308 __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line); in __rw_try_upgrade_int()
1319 v = RW_READ_VALUE(rw); in __rw_try_upgrade_int()
1324 success = atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid); in __rw_try_upgrade_int()
1333 ts = turnstile_trywait(&rw->lock_object); in __rw_try_upgrade_int()
1334 v = RW_READ_VALUE(rw); in __rw_try_upgrade_int()
1347 success = atomic_fcmpset_ptr(&rw->rw_lock, &v, setv); in __rw_try_upgrade_int()
1357 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); in __rw_try_upgrade_int()
1360 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, in __rw_try_upgrade_int()
1362 LOCKSTAT_RECORD0(rw__upgrade, rw); in __rw_try_upgrade_int()
1370 struct rwlock *rw; in __rw_try_upgrade() local
1372 rw = rwlock2rw(c); in __rw_try_upgrade()
1373 return (__rw_try_upgrade_int(rw LOCK_FILE_LINE_ARG)); in __rw_try_upgrade()
1380 __rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF) in __rw_downgrade_int()
1389 KASSERT(rw->rw_lock != RW_DESTROYED, in __rw_downgrade_int()
1390 ("rw_downgrade() of destroyed rwlock %p @ %s:%d", rw, file, line)); in __rw_downgrade_int()
1391 __rw_assert(&rw->rw_lock, RA_WLOCKED | RA_NOTRECURSED, file, line); in __rw_downgrade_int()
1393 if (rw_recursed(rw)) in __rw_downgrade_int()
1397 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); in __rw_downgrade_int()
1405 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) in __rw_downgrade_int()
1412 turnstile_chain_lock(&rw->lock_object); in __rw_downgrade_int()
1413 v = rw->rw_lock & RW_LOCK_WAITERS; in __rw_downgrade_int()
1422 ts = turnstile_lookup(&rw->lock_object); in __rw_downgrade_int()
1426 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); in __rw_downgrade_int()
1436 turnstile_chain_unlock(&rw->lock_object); in __rw_downgrade_int()
1439 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); in __rw_downgrade_int()
1440 LOCKSTAT_RECORD0(rw__downgrade, rw); in __rw_downgrade_int()
1446 struct rwlock *rw; in __rw_downgrade() local
1448 rw = rwlock2rw(c); in __rw_downgrade()
1449 __rw_downgrade_int(rw LOCK_FILE_LINE_ARG); in __rw_downgrade()
1465 const struct rwlock *rw; in __rw_assert() local
1470 rw = rwlock2rw(c); in __rw_assert()
1480 witness_assert(&rw->lock_object, what, file, line); in __rw_assert()
1487 if (rw->rw_lock == RW_UNLOCKED || in __rw_assert()
1488 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED || in __rw_assert()
1489 rw_wowner(rw) != curthread))) in __rw_assert()
1491 rw->lock_object.lo_name, (what & RA_RLOCKED) ? in __rw_assert()
1494 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) { in __rw_assert()
1495 if (rw_recursed(rw)) { in __rw_assert()
1498 rw->lock_object.lo_name, file, in __rw_assert()
1502 rw->lock_object.lo_name, file, line); in __rw_assert()
1509 if (rw_wowner(rw) != curthread) in __rw_assert()
1511 rw->lock_object.lo_name, file, line); in __rw_assert()
1512 if (rw_recursed(rw)) { in __rw_assert()
1515 rw->lock_object.lo_name, file, line); in __rw_assert()
1518 rw->lock_object.lo_name, file, line); in __rw_assert()
1522 witness_assert(&rw->lock_object, what, file, line); in __rw_assert()
1528 if (rw_wowner(rw) == curthread) in __rw_assert()
1530 rw->lock_object.lo_name, file, line); in __rw_assert()
1544 const struct rwlock *rw; in db_show_rwlock() local
1547 rw = (const struct rwlock *)lock; in db_show_rwlock()
1550 if (rw->rw_lock == RW_UNLOCKED) in db_show_rwlock()
1552 else if (rw->rw_lock == RW_DESTROYED) { in db_show_rwlock()
1555 } else if (rw->rw_lock & RW_LOCK_READ) in db_show_rwlock()
1557 (uintmax_t)(RW_READERS(rw->rw_lock))); in db_show_rwlock()
1559 td = rw_wowner(rw); in db_show_rwlock()
1562 if (rw_recursed(rw)) in db_show_rwlock()
1563 db_printf(" recursed: %u\n", rw->rw_recurse); in db_show_rwlock()
1566 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { in db_show_rwlock()