531 static struct upimutex *
534 	struct upimutex *upip;  in upi_get()
546 upi_chain_add(upib_t *upibp, struct upimutex *upimutex)  in upi_chain_add()  argument
556 	upimutex->upi_nextchain = upibp->upib_first;  in upi_chain_add()
557 	upibp->upib_first = upimutex;  in upi_chain_add()
561 upi_chain_del(upib_t *upibp, struct upimutex *upimutex)  in upi_chain_del()  argument
563 	struct upimutex **prev;  in upi_chain_del()
568 	while (*prev != upimutex) {  in upi_chain_del()
571 	*prev = upimutex->upi_nextchain;  in upi_chain_del()
572 	upimutex->upi_nextchain = NULL;  in upi_chain_del()
580 upi_mylist_add(struct upimutex *upimutex)  in upi_mylist_add()  argument
589 	upimutex->upi_nextowned = t->t_upimutex;  in upi_mylist_add()
590 	t->t_upimutex = upimutex;  in upi_mylist_add()
600 upi_mylist_del(struct upimutex *upimutex)  in upi_mylist_del()  argument
603 	struct upimutex **prev;  in upi_mylist_del()
611 	while (*prev != upimutex) {  in upi_mylist_del()
614 	*prev = upimutex->upi_nextowned;  in upi_mylist_del()
615 	upimutex->upi_nextowned = NULL;  in upi_mylist_del()
633 static struct upimutex *
638 	struct upimutex *upimutex;  in lwp_upimutex_owned()  local
646 	upimutex = upi_get(upibp, &lwpchan);  in lwp_upimutex_owned()
647 	if (upimutex == NULL || upimutex->upi_owner != curthread) {  in lwp_upimutex_owned()
652 	return (upimutex);  in lwp_upimutex_owned()
660 upimutex_unlock(struct upimutex *upimutex, uint16_t flag)  in upimutex_unlock()  argument
666 	upi_mylist_del(upimutex);  in upimutex_unlock()
667 	upibp = upimutex->upi_upibp;  in upimutex_unlock()
669 	if (upimutex->upi_waiter != 0) { /* if waiters */  in upimutex_unlock()
670 		ts = turnstile_lookup(upimutex);  in upimutex_unlock()
674 			upimutex->upi_owner = newowner;  in upimutex_unlock()
676 				upimutex->upi_waiter = 0;  in upimutex_unlock()
690 			turnstile_exit(upimutex);  in upimutex_unlock()
699 	upi_chain_del(upimutex->upi_upibp, upimutex);  in upimutex_unlock()
701 	kmem_free(upimutex, sizeof (upimutex_t));  in upimutex_unlock()
712 	volatile struct upimutex *upimutex = NULL;  in lwp_upimutex_lock()  local
719 			upimutex_unlock((upimutex_t *)upimutex, 0);  in lwp_upimutex_lock()
731 	upimutex = upi_get(upibp, &lwpchan);  in lwp_upimutex_lock()
732 	if (upimutex == NULL)  {  in lwp_upimutex_lock()
734 		upimutex = kmem_zalloc(sizeof (upimutex_t), KM_SLEEP);  in lwp_upimutex_lock()
735 		upi_chain_add(upibp, (upimutex_t *)upimutex);  in lwp_upimutex_lock()
736 		upimutex->upi_owner = curthread; /* grab lock */  in lwp_upimutex_lock()
737 		upimutex->upi_upibp = upibp;  in lwp_upimutex_lock()
738 		upimutex->upi_vaddr = lp;  in lwp_upimutex_lock()
739 		upimutex->upi_lwpchan = lwpchan;  in lwp_upimutex_lock()
741 		nupinest = upi_mylist_add((upimutex_t *)upimutex);  in lwp_upimutex_lock()
746 			upimutex_unlock((upimutex_t *)upimutex, flag);  in lwp_upimutex_lock()
759 			upimutex_unlock((upimutex_t *)upimutex, flag);  in lwp_upimutex_lock()
777 	ASSERT(upimutex->upi_owner != NULL);  in lwp_upimutex_lock()
778 	if (upimutex->upi_owner == curthread) {  in lwp_upimutex_lock()
843 	ts = turnstile_lookup((upimutex_t *)upimutex);  in lwp_upimutex_lock()
844 	upimutex->upi_waiter = 1;  in lwp_upimutex_lock()
845 	error = turnstile_block(ts, TS_WRITER_Q, (upimutex_t *)upimutex,  in lwp_upimutex_lock()
858 		    (upimutex = lwp_upimutex_owned(lp, type))) {  in lwp_upimutex_lock()
863 			(void) upi_mylist_add((upimutex_t *)upimutex);  in lwp_upimutex_lock()
864 			upimutex_unlock((upimutex_t *)upimutex, 0);  in lwp_upimutex_lock()
871 		    (error == EDEADLK && !upi_owned((upimutex_t *)upimutex)));  in lwp_upimutex_lock()
876 		ASSERT(lwp_upimutex_owned(lp, type) == upimutex);  in lwp_upimutex_lock()
877 		nupinest = upi_mylist_add((upimutex_t *)upimutex);  in lwp_upimutex_lock()
910 			upimutex_unlock((upimutex_t *)upimutex, flag);  in lwp_upimutex_lock()
978 	volatile struct upimutex *upimutex = NULL;  in lwp_upimutex_unlock()  local
983 			upimutex_unlock((upimutex_t *)upimutex, 0);  in lwp_upimutex_unlock()
994 	upimutex = upi_get(upibp, &lwpchan);  in lwp_upimutex_unlock()
1000 	if (upimutex == NULL || upimutex->upi_owner != curthread) {  in lwp_upimutex_unlock()
1017 	upimutex_unlock((upimutex_t *)upimutex, flag);  in lwp_upimutex_unlock()
1105 	struct upimutex *upip;  in upimutex_cleanup()
2869 	volatile struct upimutex *upimutex = NULL;  in lwp_mutex_cleanup()  local
2882 			upimutex_unlock((upimutex_t *)upimutex, 0);  in lwp_mutex_cleanup()
2895 		upimutex = upi_get(upibp, &lwpchan);  in lwp_mutex_cleanup()
2896 		if (upimutex == NULL || upimutex->upi_owner != curthread) {  in lwp_mutex_cleanup()
2904 		upimutex_unlock((upimutex_t *)upimutex, flag);  in lwp_mutex_cleanup()