Lines Matching refs:lock
86 rangelock_cheat_drain(struct rangelock *lock) in rangelock_cheat_drain() argument
92 v = atomic_load_ptr(&lock->head); in rangelock_cheat_drain()
95 sleepq_add(&lock->head, NULL, "ranged1", 0, 0); in rangelock_cheat_drain()
96 sleepq_wait(&lock->head, PRI_USER); in rangelock_cheat_drain()
97 sleepq_lock(&lock->head); in rangelock_cheat_drain()
99 sleepq_release(&lock->head); in rangelock_cheat_drain()
104 rangelock_cheat_lock(struct rangelock *lock, int locktype, bool trylock, in rangelock_cheat_lock() argument
109 v = atomic_load_ptr(&lock->head); in rangelock_cheat_lock()
118 sleepq_lock(&lock->head); in rangelock_cheat_lock()
120 rangelock_cheat_drain(lock); in rangelock_cheat_lock()
133 sleepq_lock(&lock->head); in rangelock_cheat_lock()
134 if (atomic_fcmpset_rel_ptr(&lock->head, &v, in rangelock_cheat_lock()
137 sleepq_release(&lock->head); in rangelock_cheat_lock()
142 if (atomic_fcmpset_acq_ptr(&lock->head, &v, in rangelock_cheat_lock()
162 sleepq_lock(&lock->head); in rangelock_cheat_lock()
163 if (atomic_fcmpset_rel_ptr(&lock->head, &v, in rangelock_cheat_lock()
166 sleepq_release(&lock->head); in rangelock_cheat_lock()
170 if (atomic_fcmpset_acq_ptr(&lock->head, &v, in rangelock_cheat_lock()
189 rangelock_cheat_unlock(struct rangelock *lock, void *cookie) in rangelock_cheat_unlock() argument
193 v = atomic_load_ptr(&lock->head); in rangelock_cheat_unlock()
210 if (atomic_fcmpset_rel_ptr(&lock->head, in rangelock_cheat_unlock()
214 sleepq_lock(&lock->head); in rangelock_cheat_unlock()
215 if (atomic_fcmpset_rel_ptr(&lock->head, in rangelock_cheat_unlock()
218 &lock->head, in rangelock_cheat_unlock()
220 sleepq_release(&lock->head); in rangelock_cheat_unlock()
223 sleepq_release(&lock->head); in rangelock_cheat_unlock()
227 if (atomic_fcmpset_rel_ptr(&lock->head, &v, in rangelock_cheat_unlock()
237 sleepq_lock(&lock->head); in rangelock_cheat_unlock()
238 atomic_store_ptr(&lock->head, 0); in rangelock_cheat_unlock()
239 sleepq_broadcast(&lock->head, in rangelock_cheat_unlock()
241 sleepq_release(&lock->head); in rangelock_cheat_unlock()
244 if (atomic_fcmpset_ptr(&lock->head, &v, in rangelock_cheat_unlock()
258 rangelock_cheat_destroy(struct rangelock *lock) in rangelock_cheat_destroy() argument
262 v = atomic_load_ptr(&lock->head); in rangelock_cheat_destroy()
300 static void rangelock_noncheating_destroy(struct rangelock *lock);
330 rangelock_init(struct rangelock *lock) in rangelock_init() argument
332 lock->sleepers = false; in rangelock_init()
333 atomic_store_ptr(&lock->head, rangelock_cheat ? RL_CHEAT_CHEATING : 0); in rangelock_init()
337 rangelock_destroy(struct rangelock *lock) in rangelock_destroy() argument
339 MPASS(!lock->sleepers); in rangelock_destroy()
340 if (!rangelock_cheat_destroy(lock)) in rangelock_destroy()
341 rangelock_noncheating_destroy(lock); in rangelock_destroy()
342 DEBUG_POISON_POINTER(*(void **)&lock->head); in rangelock_destroy()
401 rangelock_unlock_int(struct rangelock *lock, struct rl_q_entry *e) in rangelock_unlock_int() argument
405 MPASS(lock != NULL && e != NULL); in rangelock_unlock_int()
410 sleepers = lock->sleepers; in rangelock_unlock_int()
411 lock->sleepers = false; in rangelock_unlock_int()
413 sleepq_broadcast(&lock->sleepers, SLEEPQ_SLEEP, 0, 0); in rangelock_unlock_int()
417 rangelock_unlock(struct rangelock *lock, void *cookie) in rangelock_unlock() argument
419 if (rangelock_cheat_unlock(lock, cookie)) in rangelock_unlock()
422 sleepq_lock(&lock->sleepers); in rangelock_unlock()
423 rangelock_unlock_int(lock, cookie); in rangelock_unlock()
424 sleepq_release(&lock->sleepers); in rangelock_unlock()
454 rl_insert_sleep(struct rangelock *lock) in rl_insert_sleep() argument
458 lock->sleepers = true; in rl_insert_sleep()
459 sleepq_add(&lock->sleepers, NULL, "rangelk", 0, 0); in rl_insert_sleep()
460 sleepq_wait(&lock->sleepers, PRI_USER); in rl_insert_sleep()
475 rangelock_noncheating_destroy(struct rangelock *lock) in rangelock_noncheating_destroy() argument
482 prev = (struct rl_q_entry **)&lock->head; in rangelock_noncheating_destroy()
508 sleepq_lock(&lock->sleepers); in rangelock_noncheating_destroy()
510 rl_insert_sleep(lock); in rangelock_noncheating_destroy()
525 rl_r_validate(struct rangelock *lock, struct rl_q_entry *e, bool trylock, in rl_r_validate() argument
554 sleepq_lock(&lock->sleepers); in rl_r_validate()
556 sleepq_release(&lock->sleepers); in rl_r_validate()
559 rangelock_unlock_int(lock, e); in rl_r_validate()
561 sleepq_release(&lock->sleepers); in rl_r_validate()
564 rl_insert_sleep(lock); in rl_r_validate()
571 rl_w_validate(struct rangelock *lock, struct rl_q_entry *e, in rl_w_validate() argument
577 prev = (struct rl_q_entry **)&lock->head; in rl_w_validate()
599 sleepq_lock(&lock->sleepers); in rl_w_validate()
603 sleepq_release(&lock->sleepers); in rl_w_validate()
606 rangelock_unlock_int(lock, e); in rl_w_validate()
608 sleepq_release(&lock->sleepers); in rl_w_validate()
611 rl_insert_sleep(lock); in rl_w_validate()
617 rl_insert(struct rangelock *lock, struct rl_q_entry *e, bool trylock, in rl_insert() argument
624 prev = (struct rl_q_entry **)&lock->head; in rl_insert()
656 sleepq_lock(&lock->sleepers); in rl_insert()
659 sleepq_release(&lock->sleepers); in rl_insert()
663 sleepq_release(&lock->sleepers); in rl_insert()
666 rl_insert_sleep(lock); in rl_insert()
674 rl_r_validate(lock, e, trylock, free) : in rl_insert()
675 rl_w_validate(lock, e, trylock, free)); in rl_insert()
685 rangelock_lock_int(struct rangelock *lock, bool trylock, vm_ooffset_t start, in rangelock_lock_int() argument
692 if (rangelock_cheat_lock(lock, locktype, trylock, &cookie)) in rangelock_lock_int()
698 res = rl_insert(lock, e, trylock, &free); in rangelock_lock_int()
712 rangelock_rlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) in rangelock_rlock() argument
714 return (rangelock_lock_int(lock, false, start, end, RL_LOCK_READ)); in rangelock_rlock()
718 rangelock_tryrlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) in rangelock_tryrlock() argument
720 return (rangelock_lock_int(lock, true, start, end, RL_LOCK_READ)); in rangelock_tryrlock()
724 rangelock_wlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) in rangelock_wlock() argument
726 return (rangelock_lock_int(lock, false, start, end, RL_LOCK_WRITE)); in rangelock_wlock()
730 rangelock_trywlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) in rangelock_trywlock() argument
732 return (rangelock_lock_int(lock, true, start, end, RL_LOCK_WRITE)); in rangelock_trywlock()
741 rangelock_may_recurse(struct rangelock *lock) in rangelock_may_recurse() argument
745 v = atomic_load_ptr(&lock->head); in rangelock_may_recurse()
749 sleepq_lock(&lock->head); in rangelock_may_recurse()
752 sleepq_release(&lock->head); in rangelock_may_recurse()
760 if (atomic_fcmpset_ptr(&lock->head, &v, x) != 0) { in rangelock_may_recurse()
761 rangelock_cheat_drain(lock); in rangelock_may_recurse()
769 if (atomic_fcmpset_ptr(&lock->head, &v, x) != 0) { in rangelock_may_recurse()
770 sleepq_release(&lock->head); in rangelock_may_recurse()
789 struct rangelock *lock; in DB_SHOW_COMMAND() local
798 lock = (struct rangelock *)addr; in DB_SHOW_COMMAND()
799 db_printf("rangelock %p sleepers %d\n", lock, lock->sleepers); in DB_SHOW_COMMAND()
800 v = lock->head; in DB_SHOW_COMMAND()
805 for (e = (struct rl_q_entry *)(lock->head);;) { in DB_SHOW_COMMAND()