36 #include <sys/lock.h>
64  * Lock is in cheat mode when RL_CHEAT_CHEATING bit is set in the
65  * lock->head.  Special cookies are returned in this mode, and
86 rangelock_cheat_drain(struct rangelock *lock)  in rangelock_cheat_drain()  argument
92 		v = atomic_load_ptr(&lock->head);  in rangelock_cheat_drain()
95 		sleepq_add(&lock->head, NULL, "ranged1", 0, 0);  in rangelock_cheat_drain()
96 		sleepq_wait(&lock->head, PRI_USER);  in rangelock_cheat_drain()
97 		sleepq_lock(&lock->head);  in rangelock_cheat_drain()
99 	sleepq_release(&lock->head);  in rangelock_cheat_drain()
104 rangelock_cheat_lock(struct rangelock *lock, int locktype, bool trylock,  in rangelock_cheat_lock()  argument
109 	v = atomic_load_ptr(&lock->head);  in rangelock_cheat_lock()
118 		sleepq_lock(&lock->head);  in rangelock_cheat_lock()
120 		rangelock_cheat_drain(lock);  in rangelock_cheat_lock()
133 				sleepq_lock(&lock->head);  in rangelock_cheat_lock()
134 				if (atomic_fcmpset_rel_ptr(&lock->head, &v,  in rangelock_cheat_lock()
137 				sleepq_release(&lock->head);  in rangelock_cheat_lock()
142 				if (atomic_fcmpset_acq_ptr(&lock->head, &v,  in rangelock_cheat_lock()
162 				sleepq_lock(&lock->head);  in rangelock_cheat_lock()
163 				if (atomic_fcmpset_rel_ptr(&lock->head, &v,  in rangelock_cheat_lock()
166 				sleepq_release(&lock->head);  in rangelock_cheat_lock()
170 				if (atomic_fcmpset_acq_ptr(&lock->head, &v,  in rangelock_cheat_lock()
189 rangelock_cheat_unlock(struct rangelock *lock, void *cookie)  in rangelock_cheat_unlock()  argument
193 	v = atomic_load_ptr(&lock->head);  in rangelock_cheat_unlock()
210 					if (atomic_fcmpset_rel_ptr(&lock->head,  in rangelock_cheat_unlock()
214 					sleepq_lock(&lock->head);  in rangelock_cheat_unlock()
215 					if (atomic_fcmpset_rel_ptr(&lock->head,  in rangelock_cheat_unlock()
218 						    &lock->head,  in rangelock_cheat_unlock()
220 						sleepq_release(&lock->head);  in rangelock_cheat_unlock()
223 					sleepq_release(&lock->head);  in rangelock_cheat_unlock()
227 				if (atomic_fcmpset_rel_ptr(&lock->head, &v,  in rangelock_cheat_unlock()
237 				sleepq_lock(&lock->head);  in rangelock_cheat_unlock()
238 				atomic_store_ptr(&lock->head, 0);  in rangelock_cheat_unlock()
239 				sleepq_broadcast(&lock->head,  in rangelock_cheat_unlock()
241 				sleepq_release(&lock->head);  in rangelock_cheat_unlock()
244 				if (atomic_fcmpset_ptr(&lock->head, &v,  in rangelock_cheat_unlock()
258 rangelock_cheat_destroy(struct rangelock *lock)  in rangelock_cheat_destroy()  argument
262 	v = atomic_load_ptr(&lock->head);  in rangelock_cheat_destroy()
280  * rl_q_next links all granted ranges in the lock.  We cannot free an
300 static void rangelock_noncheating_destroy(struct rangelock *lock);
330 rangelock_init(struct rangelock *lock)  in rangelock_init()  argument
332 	lock->sleepers = false;  in rangelock_init()
333 	atomic_store_ptr(&lock->head, rangelock_cheat ? RL_CHEAT_CHEATING : 0);  in rangelock_init()
337 rangelock_destroy(struct rangelock *lock)  in rangelock_destroy()  argument
339 	MPASS(!lock->sleepers);  in rangelock_destroy()
340 	if (!rangelock_cheat_destroy(lock))  in rangelock_destroy()
341 		rangelock_noncheating_destroy(lock);  in rangelock_destroy()
342 	DEBUG_POISON_POINTER(*(void **)&lock->head);  in rangelock_destroy()
401 rangelock_unlock_int(struct rangelock *lock, struct rl_q_entry *e)  in rangelock_unlock_int()  argument
405 	MPASS(lock != NULL && e != NULL);  in rangelock_unlock_int()
410 	sleepers = lock->sleepers;  in rangelock_unlock_int()
411 	lock->sleepers = false;  in rangelock_unlock_int()
413 		sleepq_broadcast(&lock->sleepers, SLEEPQ_SLEEP, 0, 0);  in rangelock_unlock_int()
417 rangelock_unlock(struct rangelock *lock, void *cookie)  in rangelock_unlock()  argument
419 	if (rangelock_cheat_unlock(lock, cookie))  in rangelock_unlock()
422 	sleepq_lock(&lock->sleepers);  in rangelock_unlock()
423 	rangelock_unlock_int(lock, cookie);  in rangelock_unlock()
424 	sleepq_release(&lock->sleepers);  in rangelock_unlock()
432  *          0 if e1 and e2 overlap and at least one lock is writer
454 rl_insert_sleep(struct rangelock *lock)  in rl_insert_sleep()  argument
458 	lock->sleepers = true;  in rl_insert_sleep()
459 	sleepq_add(&lock->sleepers, NULL, "rangelk", 0, 0);  in rl_insert_sleep()
460 	sleepq_wait(&lock->sleepers, PRI_USER);  in rl_insert_sleep()
479 rangelock_noncheating_destroy(struct rangelock *lock)  in rangelock_noncheating_destroy()  argument
486 	prev = (struct rl_q_entry **)&lock->head;  in rangelock_noncheating_destroy()
512 		sleepq_lock(&lock->sleepers);  in rangelock_noncheating_destroy()
514 			rl_insert_sleep(lock);  in rangelock_noncheating_destroy()
530  * Handle a possible lock conflict between cur and e.  "inserted" is true if e
534 rl_conflict(struct rangelock *lock, struct rl_q_entry *cur, struct rl_q_entry *e,  in rl_conflict()  argument
537 	sleepq_lock(&lock->sleepers);  in rl_conflict()
539 		sleepq_release(&lock->sleepers);  in rl_conflict()
554 		rangelock_unlock_int(lock, e);  in rl_conflict()
556 		sleepq_release(&lock->sleepers);  in rl_conflict()
564 	rl_insert_sleep(lock);  in rl_conflict()
573 rl_r_validate(struct rangelock *lock, struct rl_q_entry *e, bool trylock,  in rl_r_validate()  argument
603 		res = rl_conflict(lock, cur, e, trylock, true);  in rl_r_validate()
614 rl_w_validate(struct rangelock *lock, struct rl_q_entry *e,  in rl_w_validate()  argument
621 	prev = (struct rl_q_entry **)&lock->head;  in rl_w_validate()
644 		res = rl_conflict(lock, cur, e, trylock, true);  in rl_w_validate()
651 rl_insert(struct rangelock *lock, struct rl_q_entry *e, bool trylock,  in rl_insert()  argument
658 	prev = (struct rl_q_entry **)&lock->head;  in rl_insert()
692 			res = rl_conflict(lock, cur, e, trylock, false);  in rl_insert()
708 				    rl_r_validate(lock, e, trylock, free) :  in rl_insert()
709 				    rl_w_validate(lock, e, trylock, free));  in rl_insert()
719 rangelock_lock_int(struct rangelock *lock, bool trylock, vm_ooffset_t start,  in rangelock_lock_int()  argument
726 	if (rangelock_cheat_lock(lock, locktype, trylock, &cookie))  in rangelock_lock_int()
732 		res = rl_insert(lock, e, trylock, &free);  in rangelock_lock_int()
748 rangelock_rlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)  in rangelock_rlock()  argument
750 	return (rangelock_lock_int(lock, false, start, end, RL_LOCK_READ));  in rangelock_rlock()
754 rangelock_tryrlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)  in rangelock_tryrlock()  argument
756 	return (rangelock_lock_int(lock, true, start, end, RL_LOCK_READ));  in rangelock_tryrlock()
760 rangelock_wlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)  in rangelock_wlock()  argument
762 	return (rangelock_lock_int(lock, false, start, end, RL_LOCK_WRITE));  in rangelock_wlock()
766 rangelock_trywlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)  in rangelock_trywlock()  argument
768 	return (rangelock_lock_int(lock, true, start, end, RL_LOCK_WRITE));  in rangelock_trywlock()
773  * same lock simultaneously, switch to the non-cheat mode.  Cheat mode
777 rangelock_may_recurse(struct rangelock *lock)  in rangelock_may_recurse()  argument
781 	v = atomic_load_ptr(&lock->head);  in rangelock_may_recurse()
785 	sleepq_lock(&lock->head);  in rangelock_may_recurse()
788 			sleepq_release(&lock->head);  in rangelock_may_recurse()
796 			if (atomic_fcmpset_ptr(&lock->head, &v, x) != 0) {  in rangelock_may_recurse()
797 				rangelock_cheat_drain(lock);  in rangelock_may_recurse()
805 		if (atomic_fcmpset_ptr(&lock->head, &v, x) != 0) {  in rangelock_may_recurse()
806 			sleepq_release(&lock->head);  in rangelock_may_recurse()
825 	struct rangelock *lock;  in DB_SHOW_COMMAND()  local
834 	lock = (struct rangelock *)addr;  in DB_SHOW_COMMAND()
835 	db_printf("rangelock %p sleepers %d\n", lock, lock->sleepers);  in DB_SHOW_COMMAND()
836 	v = lock->head;  in DB_SHOW_COMMAND()
841 	for (e = (struct rl_q_entry *)(lock->head);;) {  in DB_SHOW_COMMAND()