84 rrn_find(rrwlock_t *rrl)  in rrn_find()  argument
88 	if (zfs_refcount_count(&rrl->rr_linked_rcount) == 0)  in rrn_find()
92 		if (rn->rn_rrl == rrl)  in rrn_find()
102 rrn_add(rrwlock_t *rrl, void *tag)  in rrn_add()  argument
107 	rn->rn_rrl = rrl;  in rrn_add()
118 rrn_find_and_remove(rrwlock_t *rrl, void *tag)  in rrn_find_and_remove()  argument
123 	if (zfs_refcount_count(&rrl->rr_linked_rcount) == 0)  in rrn_find_and_remove()
127 		if (rn->rn_rrl == rrl && rn->rn_tag == tag) {  in rrn_find_and_remove()
141 rrw_init(rrwlock_t *rrl, boolean_t track_all)  in rrw_init()  argument
143 	mutex_init(&rrl->rr_lock, NULL, MUTEX_DEFAULT, NULL);  in rrw_init()
144 	cv_init(&rrl->rr_cv, NULL, CV_DEFAULT, NULL);  in rrw_init()
145 	rrl->rr_writer = NULL;  in rrw_init()
146 	zfs_refcount_create(&rrl->rr_anon_rcount);  in rrw_init()
147 	zfs_refcount_create(&rrl->rr_linked_rcount);  in rrw_init()
148 	rrl->rr_writer_wanted = B_FALSE;  in rrw_init()
149 	rrl->rr_track_all = track_all;  in rrw_init()
153 rrw_destroy(rrwlock_t *rrl)  in rrw_destroy()  argument
155 	mutex_destroy(&rrl->rr_lock);  in rrw_destroy()
156 	cv_destroy(&rrl->rr_cv);  in rrw_destroy()
157 	ASSERT(rrl->rr_writer == NULL);  in rrw_destroy()
158 	zfs_refcount_destroy(&rrl->rr_anon_rcount);  in rrw_destroy()
159 	zfs_refcount_destroy(&rrl->rr_linked_rcount);  in rrw_destroy()
163 rrw_enter_read_impl(rrwlock_t *rrl, boolean_t prio, void *tag)  in rrw_enter_read_impl()  argument
165 	mutex_enter(&rrl->rr_lock);  in rrw_enter_read_impl()
167 	if (rrl->rr_writer == NULL && !rrl->rr_writer_wanted &&  in rrw_enter_read_impl()
168 	    !rrl->rr_track_all) {  in rrw_enter_read_impl()
169 		rrl->rr_anon_rcount.rc_count++;  in rrw_enter_read_impl()
170 		mutex_exit(&rrl->rr_lock);  in rrw_enter_read_impl()
175 	ASSERT(rrl->rr_writer != curthread);  in rrw_enter_read_impl()
176 	ASSERT(zfs_refcount_count(&rrl->rr_anon_rcount) >= 0);  in rrw_enter_read_impl()
178 	while (rrl->rr_writer != NULL || (rrl->rr_writer_wanted &&  in rrw_enter_read_impl()
179 	    zfs_refcount_is_zero(&rrl->rr_anon_rcount) && !prio &&  in rrw_enter_read_impl()
180 	    rrn_find(rrl) == NULL))  in rrw_enter_read_impl()
181 		cv_wait(&rrl->rr_cv, &rrl->rr_lock);  in rrw_enter_read_impl()
183 	if (rrl->rr_writer_wanted || rrl->rr_track_all) {  in rrw_enter_read_impl()
185 		rrn_add(rrl, tag);  in rrw_enter_read_impl()
186 		(void) zfs_refcount_add(&rrl->rr_linked_rcount, tag);  in rrw_enter_read_impl()
188 		(void) zfs_refcount_add(&rrl->rr_anon_rcount, tag);  in rrw_enter_read_impl()
190 	ASSERT(rrl->rr_writer == NULL);  in rrw_enter_read_impl()
191 	mutex_exit(&rrl->rr_lock);  in rrw_enter_read_impl()
195 rrw_enter_read(rrwlock_t *rrl, void *tag)  in rrw_enter_read()  argument
197 	rrw_enter_read_impl(rrl, B_FALSE, tag);  in rrw_enter_read()
207 rrw_enter_read_prio(rrwlock_t *rrl, void *tag)  in rrw_enter_read_prio()  argument
209 	rrw_enter_read_impl(rrl, B_TRUE, tag);  in rrw_enter_read_prio()
214 rrw_enter_write(rrwlock_t *rrl)  in rrw_enter_write()  argument
216 	mutex_enter(&rrl->rr_lock);  in rrw_enter_write()
217 	ASSERT(rrl->rr_writer != curthread);  in rrw_enter_write()
219 	while (zfs_refcount_count(&rrl->rr_anon_rcount) > 0 ||  in rrw_enter_write()
220 	    zfs_refcount_count(&rrl->rr_linked_rcount) > 0 ||  in rrw_enter_write()
221 	    rrl->rr_writer != NULL) {  in rrw_enter_write()
222 		rrl->rr_writer_wanted = B_TRUE;  in rrw_enter_write()
223 		cv_wait(&rrl->rr_cv, &rrl->rr_lock);  in rrw_enter_write()
225 	rrl->rr_writer_wanted = B_FALSE;  in rrw_enter_write()
226 	rrl->rr_writer = curthread;  in rrw_enter_write()
227 	mutex_exit(&rrl->rr_lock);  in rrw_enter_write()
231 rrw_enter(rrwlock_t *rrl, krw_t rw, void *tag)  in rrw_enter()  argument
234 		rrw_enter_read(rrl, tag);  in rrw_enter()
236 		rrw_enter_write(rrl);  in rrw_enter()
240 rrw_exit(rrwlock_t *rrl, void *tag)  in rrw_exit()  argument
242 	mutex_enter(&rrl->rr_lock);  in rrw_exit()
244 	if (!rrl->rr_writer && rrl->rr_linked_rcount.rc_count == 0) {  in rrw_exit()
245 		rrl->rr_anon_rcount.rc_count--;  in rrw_exit()
246 		if (rrl->rr_anon_rcount.rc_count == 0)  in rrw_exit()
247 			cv_broadcast(&rrl->rr_cv);  in rrw_exit()
248 		mutex_exit(&rrl->rr_lock);  in rrw_exit()
253 	ASSERT(!zfs_refcount_is_zero(&rrl->rr_anon_rcount) ||  in rrw_exit()
254 	    !zfs_refcount_is_zero(&rrl->rr_linked_rcount) ||  in rrw_exit()
255 	    rrl->rr_writer != NULL);  in rrw_exit()
257 	if (rrl->rr_writer == NULL) {  in rrw_exit()
259 		if (rrn_find_and_remove(rrl, tag)) {  in rrw_exit()
261 			    &rrl->rr_linked_rcount, tag);  in rrw_exit()
263 			ASSERT(!rrl->rr_track_all);  in rrw_exit()
264 			count = zfs_refcount_remove(&rrl->rr_anon_rcount, tag);  in rrw_exit()
267 			cv_broadcast(&rrl->rr_cv);  in rrw_exit()
269 		ASSERT(rrl->rr_writer == curthread);  in rrw_exit()
270 		ASSERT(zfs_refcount_is_zero(&rrl->rr_anon_rcount) &&  in rrw_exit()
271 		    zfs_refcount_is_zero(&rrl->rr_linked_rcount));  in rrw_exit()
272 		rrl->rr_writer = NULL;  in rrw_exit()
273 		cv_broadcast(&rrl->rr_cv);  in rrw_exit()
275 	mutex_exit(&rrl->rr_lock);  in rrw_exit()
284 rrw_held(rrwlock_t *rrl, krw_t rw)  in rrw_held()  argument
288 	mutex_enter(&rrl->rr_lock);  in rrw_held()
290 		held = (rrl->rr_writer == curthread);  in rrw_held()
292 		held = (!zfs_refcount_is_zero(&rrl->rr_anon_rcount) ||  in rrw_held()
293 		    rrn_find(rrl) != NULL);  in rrw_held()
295 	mutex_exit(&rrl->rr_lock);  in rrw_held()
324 rrm_init(rrmlock_t *rrl, boolean_t track_all)  in rrm_init()  argument
329 		rrw_init(&rrl->locks[i], track_all);  in rrm_init()
333 rrm_destroy(rrmlock_t *rrl)  in rrm_destroy()  argument
338 		rrw_destroy(&rrl->locks[i]);  in rrm_destroy()
342 rrm_enter(rrmlock_t *rrl, krw_t rw, void *tag)  in rrm_enter()  argument
345 		rrm_enter_read(rrl, tag);  in rrm_enter()
347 		rrm_enter_write(rrl);  in rrm_enter()
361 rrm_enter_read(rrmlock_t *rrl, void *tag)  in rrm_enter_read()  argument
363 	rrw_enter_read(&rrl->locks[RRM_TD_LOCK()], tag);  in rrm_enter_read()
367 rrm_enter_write(rrmlock_t *rrl)  in rrm_enter_write()  argument
372 		rrw_enter_write(&rrl->locks[i]);  in rrm_enter_write()
376 rrm_exit(rrmlock_t *rrl, void *tag)  in rrm_exit()  argument
380 	if (rrl->locks[0].rr_writer == curthread) {  in rrm_exit()
382 			rrw_exit(&rrl->locks[i], tag);  in rrm_exit()
384 		rrw_exit(&rrl->locks[RRM_TD_LOCK()], tag);  in rrm_exit()
389 rrm_held(rrmlock_t *rrl, krw_t rw)  in rrm_held()  argument
392 		return (rrw_held(&rrl->locks[0], rw));  in rrm_held()
394 		return (rrw_held(&rrl->locks[RRM_TD_LOCK()], rw));  in rrm_held()