46 nvme_rwlock_block_ns_rdlock(nvme_t *nvme, nvme_namespace_t *ns)  in nvme_rwlock_block_ns_rdlock()  argument
48 	return (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||  in nvme_rwlock_block_ns_rdlock()
63 nvme_rwlock_block_ns_wrlock(nvme_t *nvme, nvme_namespace_t *ns)  in nvme_rwlock_block_ns_wrlock()  argument
65 	return (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||  in nvme_rwlock_block_ns_wrlock()
76 nvme_rwlock_block_ctrl_rdlock(nvme_t *nvme)  in nvme_rwlock_block_ctrl_rdlock()  argument
78 	return (nvme_rwlock_wr_or_pend(&nvme->n_lock));  in nvme_rwlock_block_ctrl_rdlock()
95 nvme_rwlock_block_ctrl_wrlock(nvme_t *nvme)  in nvme_rwlock_block_ctrl_wrlock()  argument
97 	if (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||  in nvme_rwlock_block_ctrl_wrlock()
98 	    list_is_empty(&nvme->n_lock.nl_readers) == 0) {  in nvme_rwlock_block_ctrl_wrlock()
102 	for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {  in nvme_rwlock_block_ctrl_wrlock()
103 		nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);  in nvme_rwlock_block_ctrl_wrlock()
121 nvme_rwlock_handoff_ctrl_wrlock(nvme_t *nvme)  in nvme_rwlock_handoff_ctrl_wrlock()  argument
124 	ASSERT3P(nvme->n_lock.nl_writer, ==, NULL);  in nvme_rwlock_handoff_ctrl_wrlock()
126 	if (list_is_empty(&nvme->n_lock.nl_readers) == 0) {  in nvme_rwlock_handoff_ctrl_wrlock()
130 	for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {  in nvme_rwlock_handoff_ctrl_wrlock()
131 		nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);  in nvme_rwlock_handoff_ctrl_wrlock()
147 nvme_rwlock_handoff_ns_wrlock(nvme_t *nvme, nvme_namespace_t *ns)  in nvme_rwlock_handoff_ns_wrlock()  argument
149 	if (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||  in nvme_rwlock_handoff_ns_wrlock()
150 	    list_is_empty(&nvme->n_lock.nl_readers) == 0) {  in nvme_rwlock_handoff_ns_wrlock()
275 nvme_rwlock_wakeup(nvme_t *nvme)  in nvme_rwlock_wakeup()  argument
277 	nvme_lock_t *ctrl_lock = &nvme->n_lock;  in nvme_rwlock_wakeup()
298 		if (!nvme_rwlock_handoff_ctrl_wrlock(nvme))  in nvme_rwlock_wakeup()
313 	for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {  in nvme_rwlock_wakeup()
314 		nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);  in nvme_rwlock_wakeup()
320 			if (!nvme_rwlock_handoff_ns_wrlock(nvme, ns))  in nvme_rwlock_wakeup()
355 	nvme_t *const nvme = info->nli_nvme;  in nvme_rwunlock()  local
358 	VERIFY(MUTEX_HELD(&nvme->n_minor_mutex));  in nvme_rwunlock()
380 	nvme_rwlock_wakeup(nvme);  in nvme_rwunlock()
463 	nvme_t *const nvme = minor->nm_ctrl;  in nvme_rwlock()  local
473 	VERIFY(MUTEX_HELD(&nvme->n_minor_mutex));  in nvme_rwlock()
477 		lock = &nvme->n_lock;  in nvme_rwlock()
480 			waiters = nvme_rwlock_block_ctrl_rdlock(nvme);  in nvme_rwlock()
482 			waiters = nvme_rwlock_block_ctrl_wrlock(nvme);  in nvme_rwlock()
490 		ns = nvme_nsid2ns(nvme, nsid);  in nvme_rwlock()
495 			waiters = nvme_rwlock_block_ns_rdlock(nvme, ns);  in nvme_rwlock()
497 			waiters = nvme_rwlock_block_ns_wrlock(nvme, ns);  in nvme_rwlock()
557 		if (cv_wait_sig(&minor->nm_cv, &nvme->n_minor_mutex) == 0) {  in nvme_rwlock()
597 nvme_rwlock_ctrl_dead_cleanup_one(nvme_t *nvme, nvme_minor_lock_info_t *info)  in nvme_rwlock_ctrl_dead_cleanup_one()  argument
607 	nvme_rwlock_signal_one(info, nvme->n_dead_status);  in nvme_rwlock_ctrl_dead_cleanup_one()
630 	nvme_t *nvme = arg;  in nvme_rwlock_ctrl_dead()  local
631 	nvme_lock_t *ctrl_lock = &nvme->n_lock;  in nvme_rwlock_ctrl_dead()
634 	mutex_enter(&nvme->n_minor_mutex);  in nvme_rwlock_ctrl_dead()
635 	if ((nvme->n_progress & NVME_NS_INIT) == 0) {  in nvme_rwlock_ctrl_dead()
636 		mutex_exit(&nvme->n_minor_mutex);  in nvme_rwlock_ctrl_dead()
640 	for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {  in nvme_rwlock_ctrl_dead()
641 		nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);  in nvme_rwlock_ctrl_dead()
646 			nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);  in nvme_rwlock_ctrl_dead()
651 			nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);  in nvme_rwlock_ctrl_dead()
656 		nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);  in nvme_rwlock_ctrl_dead()
661 		nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);  in nvme_rwlock_ctrl_dead()
663 	mutex_exit(&nvme->n_minor_mutex);  in nvme_rwlock_ctrl_dead()