Lines Matching refs:rm
66 #define rm_destroyed(rm) \ argument
67 (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
124 struct rmlock *rm; in lock_rm() local
127 rm = (struct rmlock *)lock; in lock_rm()
129 rm_wlock(rm); in lock_rm()
132 rm_rlock(rm, tracker); in lock_rm()
141 struct rmlock *rm; in unlock_rm() local
146 rm = (struct rmlock *)lock; in unlock_rm()
149 rm_assert(rm, RA_LOCKED | RA_NOTRECURSED); in unlock_rm()
150 if (rm_wowned(rm)) in unlock_rm()
151 rm_wunlock(rm); in unlock_rm()
164 if ((tracker->rmp_rmlock == rm) && in unlock_rm()
173 rm_runlock(rm, tracker); in unlock_rm()
182 const struct rmlock *rm; in owner_rm() local
185 rm = (const struct rmlock *)lock; in owner_rm()
186 lc = LOCK_CLASS(&rm->rm_wlock_object); in owner_rm()
187 return (lc->lc_owner(&rm->rm_wlock_object, owner)); in owner_rm()
223 rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm, in rm_trackers_present() argument
234 if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td)) in rm_trackers_present()
259 struct rmlock *rm = arg; in rm_cleanIPI() local
267 if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) { in rm_cleanIPI()
270 LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, in rm_cleanIPI()
278 rm_init_flags(struct rmlock *rm, const char *name, int opts) in rm_init_flags() argument
292 rm->rm_writecpus = all_cpus; in rm_init_flags()
293 LIST_INIT(&rm->rm_activeReaders); in rm_init_flags()
298 sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", in rm_init_flags()
303 mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", in rm_init_flags()
306 lock_init(&rm->lock_object, lc, name, NULL, liflags); in rm_init_flags()
310 rm_init(struct rmlock *rm, const char *name) in rm_init() argument
313 rm_init_flags(rm, name, 0); in rm_init()
317 rm_destroy(struct rmlock *rm) in rm_destroy() argument
320 rm_assert(rm, RA_UNLOCKED); in rm_destroy()
321 LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED; in rm_destroy()
322 if (rm->lock_object.lo_flags & LO_SLEEPABLE) in rm_destroy()
323 sx_destroy(&rm->rm_lock_sx); in rm_destroy()
325 mtx_destroy(&rm->rm_lock_mtx); in rm_destroy()
326 lock_destroy(&rm->lock_object); in rm_destroy()
330 rm_wowned(const struct rmlock *rm) in rm_wowned() argument
333 if (rm->lock_object.lo_flags & LO_SLEEPABLE) in rm_wowned()
334 return (sx_xlocked(&rm->rm_lock_sx)); in rm_wowned()
336 return (mtx_owned(&rm->rm_lock_mtx)); in rm_wowned()
349 _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) in _rm_rlock_hard() argument
357 if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) { in _rm_rlock_hard()
381 if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) { in _rm_rlock_hard()
386 if (rm_trackers_present(pc, rm, curthread) != 0) { in _rm_rlock_hard()
388 LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, in _rm_rlock_hard()
402 if (rm->lock_object.lo_flags & LO_SLEEPABLE) { in _rm_rlock_hard()
403 if (!sx_try_xlock(&rm->rm_lock_sx)) in _rm_rlock_hard()
406 if (!mtx_trylock(&rm->rm_lock_mtx)) in _rm_rlock_hard()
410 if (rm->lock_object.lo_flags & LO_SLEEPABLE) { in _rm_rlock_hard()
412 sx_xlock(&rm->rm_lock_sx); in _rm_rlock_hard()
415 mtx_lock(&rm->rm_lock_mtx); in _rm_rlock_hard()
420 CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus); in _rm_rlock_hard()
425 if (rm->lock_object.lo_flags & LO_SLEEPABLE) in _rm_rlock_hard()
426 sx_xunlock(&rm->rm_lock_sx); in _rm_rlock_hard()
428 mtx_unlock(&rm->rm_lock_mtx); in _rm_rlock_hard()
434 _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) in _rm_rlock() argument
444 tracker->rmp_rmlock = rm; in _rm_rlock()
446 if (rm->lock_object.lo_flags & LO_SLEEPABLE) in _rm_rlock()
464 CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))) in _rm_rlock()
468 return _rm_rlock_hard(rm, tracker, trylock); in _rm_rlock()
487 struct rmlock *rm; in _rm_unlock_hard() local
490 rm = tracker->rmp_rmlock; in _rm_unlock_hard()
492 turnstile_chain_lock(&rm->lock_object); in _rm_unlock_hard()
495 ts = turnstile_lookup(&rm->lock_object); in _rm_unlock_hard()
499 turnstile_chain_unlock(&rm->lock_object); in _rm_unlock_hard()
505 _rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker) in _rm_runlock() argument
523 if (rm->lock_object.lo_flags & LO_SLEEPABLE) in _rm_runlock()
533 _rm_wlock(struct rmlock *rm) in _rm_wlock() argument
542 if (rm->lock_object.lo_flags & LO_SLEEPABLE) in _rm_wlock()
543 sx_xlock(&rm->rm_lock_sx); in _rm_wlock()
545 mtx_lock(&rm->rm_lock_mtx); in _rm_wlock()
547 if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) { in _rm_wlock()
550 CPU_ANDNOT(&readcpus, &readcpus, &rm->rm_writecpus); in _rm_wlock()
551 rm->rm_writecpus = all_cpus; in _rm_wlock()
562 rm); in _rm_wlock()
565 rm_cleanIPI(rm); in _rm_wlock()
569 while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) { in _rm_wlock()
570 ts = turnstile_trywait(&rm->lock_object); in _rm_wlock()
582 _rm_wunlock(struct rmlock *rm) in _rm_wunlock() argument
585 if (rm->lock_object.lo_flags & LO_SLEEPABLE) in _rm_wunlock()
586 sx_xunlock(&rm->rm_lock_sx); in _rm_wunlock()
588 mtx_unlock(&rm->rm_lock_mtx); in _rm_wunlock()
594 _rm_wlock_debug(struct rmlock *rm, const char *file, int line) in _rm_wlock_debug() argument
602 curthread, rm, file, line)); in _rm_wlock_debug()
603 KASSERT(!rm_destroyed(rm), in _rm_wlock_debug()
604 ("rm_wlock() of destroyed rmlock %p @ %s:%d", rm, file, line)); in _rm_wlock_debug()
605 _rm_assert(rm, RA_UNLOCKED, file, line); in _rm_wlock_debug()
607 WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, in _rm_wlock_debug()
610 _rm_wlock(rm); in _rm_wlock_debug()
612 LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line); in _rm_wlock_debug()
613 WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); in _rm_wlock_debug()
618 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line) in _rm_wunlock_debug() argument
624 KASSERT(!rm_destroyed(rm), in _rm_wunlock_debug()
625 ("rm_wunlock() of destroyed rmlock %p @ %s:%d", rm, file, line)); in _rm_wunlock_debug()
626 _rm_assert(rm, RA_WLOCKED, file, line); in _rm_wunlock_debug()
627 WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); in _rm_wunlock_debug()
628 LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line); in _rm_wunlock_debug()
629 _rm_wunlock(rm); in _rm_wunlock_debug()
634 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, in _rm_rlock_debug() argument
642 if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) { in _rm_rlock_debug()
644 KASSERT(rm_trackers_present(get_pcpu(), rm, in _rm_rlock_debug()
647 rm, file, line)); in _rm_rlock_debug()
653 curthread, rm, file, line)); in _rm_rlock_debug()
654 KASSERT(!rm_destroyed(rm), in _rm_rlock_debug()
655 ("rm_rlock() of destroyed rmlock %p @ %s:%d", rm, file, line)); in _rm_rlock_debug()
657 KASSERT(!rm_wowned(rm), in _rm_rlock_debug()
659 rm->lock_object.lo_name, file, line)); in _rm_rlock_debug()
660 WITNESS_CHECKORDER(&rm->lock_object, in _rm_rlock_debug()
664 if (_rm_rlock(rm, tracker, trylock)) { in _rm_rlock_debug()
666 LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file, in _rm_rlock_debug()
669 LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, in _rm_rlock_debug()
671 WITNESS_LOCK(&rm->lock_object, LOP_NOSLEEP, file, line); in _rm_rlock_debug()
675 LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line); in _rm_rlock_debug()
681 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, in _rm_runlock_debug() argument
688 KASSERT(!rm_destroyed(rm), in _rm_runlock_debug()
689 ("rm_runlock() of destroyed rmlock %p @ %s:%d", rm, file, line)); in _rm_runlock_debug()
690 _rm_assert(rm, RA_RLOCKED, file, line); in _rm_runlock_debug()
691 WITNESS_UNLOCK(&rm->lock_object, 0, file, line); in _rm_runlock_debug()
692 LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line); in _rm_runlock_debug()
693 _rm_runlock(rm, tracker); in _rm_runlock_debug()
704 _rm_wlock_debug(struct rmlock *rm, const char *file, int line) in _rm_wlock_debug() argument
707 _rm_wlock(rm); in _rm_wlock_debug()
711 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line) in _rm_wunlock_debug() argument
714 _rm_wunlock(rm); in _rm_wunlock_debug()
718 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, in _rm_rlock_debug() argument
722 return _rm_rlock(rm, tracker, trylock); in _rm_rlock_debug()
726 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, in _rm_runlock_debug() argument
730 _rm_runlock(rm, tracker); in _rm_runlock_debug()
746 _rm_assert(const struct rmlock *rm, int what, const char *file, int line) in _rm_assert() argument
763 if (rm_wowned(rm)) { in _rm_assert()
766 rm->lock_object.lo_name, file, line); in _rm_assert()
769 rm->lock_object.lo_name, file, line); in _rm_assert()
774 count = rm_trackers_present(get_pcpu(), rm, curthread); in _rm_assert()
779 rm->lock_object.lo_name, (what & RA_RLOCKED) ? in _rm_assert()
784 rm->lock_object.lo_name, file, line); in _rm_assert()
787 rm->lock_object.lo_name, file, line); in _rm_assert()
790 if (!rm_wowned(rm)) in _rm_assert()
792 rm->lock_object.lo_name, file, line); in _rm_assert()
795 if (rm_wowned(rm)) in _rm_assert()
797 rm->lock_object.lo_name, file, line); in _rm_assert()
800 count = rm_trackers_present(get_pcpu(), rm, curthread); in _rm_assert()
805 rm->lock_object.lo_name, file, line); in _rm_assert()
837 const struct rmlock *rm; in db_show_rm() local
841 rm = (const struct rmlock *)lock; in db_show_rm()
843 ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus)); in db_show_rm()
850 if (tr->rmp_rmlock == rm) in db_show_rm()
854 LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry) in db_show_rm()
856 lc = LOCK_CLASS(&rm->rm_wlock_object); in db_show_rm()
858 lc->lc_ddb_show(&rm->rm_wlock_object); in db_show_rm()