241 mutex_panic(char *msg, mutex_impl_t *lp)  in mutex_panic()  argument
246 	if (atomic_cas_ptr(&panic_mutex_addr, NULL, lp) == NULL)  in mutex_panic()
247 		panic_mutex = *lp;  in mutex_panic()
250 	    msg, (void *)lp, (void *)MUTEX_OWNER(&panic_mutex),  in mutex_panic()
342 mutex_vector_enter(mutex_impl_t *lp)  in mutex_vector_enter()  argument
350 	volatile mutex_impl_t *vlp = (volatile mutex_impl_t *)lp;  in mutex_vector_enter()
356 	if (MUTEX_TYPE_SPIN(lp)) {  in mutex_vector_enter()
357 		lock_set_spl(&lp->m_spin.m_spinlock, lp->m_spin.m_minspl,  in mutex_vector_enter()
358 		    &lp->m_spin.m_oldspl);  in mutex_vector_enter()
362 	if (!MUTEX_TYPE_ADAPTIVE(lp)) {  in mutex_vector_enter()
363 		mutex_panic("mutex_enter: bad mutex", lp);  in mutex_vector_enter()
376 			mutex_panic("mutex_enter: adaptive at high PIL", lp);  in mutex_vector_enter()
392 			if (mutex_adaptive_tryenter(lp)) {  in mutex_vector_enter()
411 			mutex_panic("recursive mutex_enter", lp);  in mutex_vector_enter()
420 		if (mutex_owner_running(lp) != NULL)  {  in mutex_vector_enter()
428 		ts = turnstile_lookup(lp);  in mutex_vector_enter()
429 		MUTEX_SET_WAITERS(lp);  in mutex_vector_enter()
436 		if (mutex_owner_running(lp) != NULL) {  in mutex_vector_enter()
437 			turnstile_exit(lp);  in mutex_vector_enter()
447 			(void) turnstile_block(ts, TS_WRITER_Q, lp,  in mutex_vector_enter()
453 			turnstile_exit(lp);  in mutex_vector_enter()
457 	ASSERT(MUTEX_OWNER(lp) == curthread);  in mutex_vector_enter()
464 		LOCKSTAT_RECORD(LS_MUTEX_ENTER_BLOCK, lp, sleep_time);  in mutex_vector_enter()
469 		LOCKSTAT_RECORD_TIME(LS_MUTEX_ENTER_SPIN, lp,  in mutex_vector_enter()
473 	LOCKSTAT_RECORD0(LS_MUTEX_ENTER_ACQUIRE, lp);  in mutex_vector_enter()
481 mutex_vector_tryenter(mutex_impl_t *lp)  in mutex_vector_tryenter()  argument
485 	if (MUTEX_TYPE_ADAPTIVE(lp))  in mutex_vector_tryenter()
488 	if (!MUTEX_TYPE_SPIN(lp)) {  in mutex_vector_tryenter()
489 		mutex_panic("mutex_tryenter: bad mutex", lp);  in mutex_vector_tryenter()
493 	s = splr(lp->m_spin.m_minspl);  in mutex_vector_tryenter()
494 	if (lock_try(&lp->m_spin.m_spinlock)) {  in mutex_vector_tryenter()
495 		lp->m_spin.m_oldspl = (ushort_t)s;  in mutex_vector_tryenter()
507 mutex_vector_exit(mutex_impl_t *lp)  in mutex_vector_exit()  argument
511 	if (MUTEX_TYPE_SPIN(lp)) {  in mutex_vector_exit()
512 		lock_clear_splx(&lp->m_spin.m_spinlock, lp->m_spin.m_oldspl);  in mutex_vector_exit()
516 	if (MUTEX_OWNER(lp) != curthread) {  in mutex_vector_exit()
517 		mutex_panic("mutex_exit: not owner", lp);  in mutex_vector_exit()
521 	ts = turnstile_lookup(lp);  in mutex_vector_exit()
522 	MUTEX_CLEAR_LOCK_AND_WAITERS(lp);  in mutex_vector_exit()
524 		turnstile_exit(lp);  in mutex_vector_exit()
527 	LOCKSTAT_RECORD0(LS_MUTEX_EXIT_RELEASE, lp);  in mutex_vector_exit()
533 	const mutex_impl_t *lp = (const mutex_impl_t *)mp;  in mutex_owned()  local
538 	if (MUTEX_TYPE_ADAPTIVE(lp))  in mutex_owned()
539 		return (MUTEX_OWNER(lp) == curthread);  in mutex_owned()
540 	return (LOCK_HELD(&lp->m_spin.m_spinlock));  in mutex_owned()
546 	const mutex_impl_t *lp = (const mutex_impl_t *)mp;  in mutex_owner()  local
549 	if (MUTEX_TYPE_ADAPTIVE(lp) && (t = MUTEX_OWNER(lp)) != MUTEX_NO_OWNER)  in mutex_owner()
568 	mutex_impl_t *lp = (mutex_impl_t *)mp;  in mutex_init()  local
574 		MUTEX_SET_TYPE(lp, MUTEX_SPIN);  in mutex_init()
575 		LOCK_INIT_CLEAR(&lp->m_spin.m_spinlock);  in mutex_init()
576 		LOCK_INIT_HELD(&lp->m_spin.m_dummylock);  in mutex_init()
577 		lp->m_spin.m_minspl = (int)(intptr_t)ibc;  in mutex_init()
582 		if (((uintptr_t)lp & (uintptr_t)(MUTEX_ALIGN - 1)) &&  in mutex_init()
600 			    (void *)lp, MUTEX_ALIGN,  in mutex_init()
612 		MUTEX_SET_TYPE(lp, MUTEX_ADAPTIVE);  in mutex_init()
613 		MUTEX_CLEAR_LOCK_AND_WAITERS(lp);  in mutex_init()
620 	mutex_impl_t *lp = (mutex_impl_t *)mp;  in mutex_destroy()  local
622 	if (lp->m_owner == 0 && !MUTEX_HAS_WAITERS(lp)) {  in mutex_destroy()
623 		MUTEX_DESTROY(lp);  in mutex_destroy()
624 	} else if (MUTEX_TYPE_SPIN(lp)) {  in mutex_destroy()
625 		LOCKSTAT_RECORD0(LS_MUTEX_DESTROY_RELEASE, lp);  in mutex_destroy()
626 		MUTEX_DESTROY(lp);  in mutex_destroy()
627 	} else if (MUTEX_TYPE_ADAPTIVE(lp)) {  in mutex_destroy()
628 		LOCKSTAT_RECORD0(LS_MUTEX_DESTROY_RELEASE, lp);  in mutex_destroy()
629 		if (MUTEX_OWNER(lp) != curthread)  in mutex_destroy()
630 			mutex_panic("mutex_destroy: not owner", lp);  in mutex_destroy()
631 		if (MUTEX_HAS_WAITERS(lp)) {  in mutex_destroy()
632 			turnstile_t *ts = turnstile_lookup(lp);  in mutex_destroy()
633 			turnstile_exit(lp);  in mutex_destroy()
635 				mutex_panic("mutex_destroy: has waiters", lp);  in mutex_destroy()
637 		MUTEX_DESTROY(lp);  in mutex_destroy()
639 		mutex_panic("mutex_destroy: bad mutex", lp);  in mutex_destroy()
647 lock_set_spin(lock_t *lp)  in lock_set_spin()  argument
657 		panic("lock_set: %p lock held and only one CPU", (void *)lp);  in lock_set_spin()
661 	while (LOCK_HELD(lp) || !lock_spin_try(lp)) {  in lock_set_spin()
675 	LOCKSTAT_RECORD_TIME(LS_LOCK_SET_SPIN, lp, spin_time);  in lock_set_spin()
677 	LOCKSTAT_RECORD0(LS_LOCK_SET_ACQUIRE, lp);  in lock_set_spin()
681 lock_set_spl_spin(lock_t *lp, int new_pil, ushort_t *old_pil_addr, int old_pil)  in lock_set_spl_spin()  argument
692 		    (void *)lp);  in lock_set_spl_spin()
700 		while (LOCK_HELD(lp)) {  in lock_set_spl_spin()
716 	} while (!lock_spin_try(lp));  in lock_set_spl_spin()
720 	LOCKSTAT_RECORD_TIME(LS_LOCK_SET_SPL_SPIN, lp, spin_time);  in lock_set_spl_spin()
722 	LOCKSTAT_RECORD0(LS_LOCK_SET_SPL_ACQUIRE, lp);  in lock_set_spl_spin()