Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
75 #include <machine/cpu.h>
90 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
96 mtx_assert(&uc->uc_lock, MA_OWNED); \
97 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); \
104 * Don't propagate time-sharing priority, there is a security reason,
105 * a user can simply introduce PI-mutex, let thread A lock the mutex,
109 * if it is using 100%CPU, this is unfair to other processes.
112 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
113 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
114 PRI_MAX_TIMESHARE : (td)->td_user_pri)
120 #define UMTX_SHIFTS (__WORD_BIT - 9)
226 mtx_lock(&uc->uc_lock); in sysctl_debug_umtx_chains_peaks()
227 tot += uc->max_length; in sysctl_debug_umtx_chains_peaks()
228 mtx_unlock(&uc->uc_lock); in sysctl_debug_umtx_chains_peaks()
238 mtx_lock(&uc->uc_lock); in sysctl_debug_umtx_chains_peaks()
239 whole = uc->max_length * 100; in sysctl_debug_umtx_chains_peaks()
240 mtx_unlock(&uc->uc_lock); in sysctl_debug_umtx_chains_peaks()
297 if (error != 0 || req->newptr == NULL) in sysctl_debug_umtx_chains_clear()
304 mtx_lock(&uc->uc_lock); in sysctl_debug_umtx_chains_clear()
305 uc->length = 0; in sysctl_debug_umtx_chains_clear()
306 uc->max_length = 0; in sysctl_debug_umtx_chains_clear()
307 mtx_unlock(&uc->uc_lock); in sysctl_debug_umtx_chains_clear()
360 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, in umtxq_alloc()
362 TAILQ_INIT(&uq->uq_spare_queue->head); in umtxq_alloc()
363 TAILQ_INIT(&uq->uq_pi_contested); in umtxq_alloc()
364 uq->uq_inherited_pri = PRI_MAX; in umtxq_alloc()
372 MPASS(uq->uq_spare_queue != NULL); in umtxq_free()
373 free(uq->uq_spare_queue, M_UMTX); in umtxq_free()
382 n = (uintptr_t)key->info.both.a + key->info.both.b; in umtxq_hash()
383 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; in umtxq_hash()
390 if (key->type <= TYPE_SEM) in umtxq_getchain()
391 return (&umtxq_chains[1][key->hash]); in umtxq_getchain()
392 return (&umtxq_chains[0][key->hash]); in umtxq_getchain()
405 mtx_assert(&uc->uc_lock, MA_OWNED); in umtxq_busy()
406 if (uc->uc_busy) { in umtxq_busy()
412 while (uc->uc_busy && --count > 0) in umtxq_busy()
418 while (uc->uc_busy) { in umtxq_busy()
419 uc->uc_waiters++; in umtxq_busy()
420 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); in umtxq_busy()
421 uc->uc_waiters--; in umtxq_busy()
424 uc->uc_busy = 1; in umtxq_busy()
436 mtx_assert(&uc->uc_lock, MA_OWNED); in umtxq_unbusy()
437 KASSERT(uc->uc_busy != 0, ("not busy")); in umtxq_unbusy()
438 uc->uc_busy = 0; in umtxq_unbusy()
439 if (uc->uc_waiters) in umtxq_unbusy()
460 LIST_FOREACH(uh, &uc->uc_queue[q], link) { in umtxq_queue_lookup()
461 if (umtx_key_match(&uh->key, key)) in umtxq_queue_lookup()
474 uc = umtxq_getchain(&uq->uq_key); in umtxq_insert_queue()
476 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue")); in umtxq_insert_queue()
477 uh = umtxq_queue_lookup(&uq->uq_key, q); in umtxq_insert_queue()
479 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link); in umtxq_insert_queue()
481 uh = uq->uq_spare_queue; in umtxq_insert_queue()
482 uh->key = uq->uq_key; in umtxq_insert_queue()
483 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link); in umtxq_insert_queue()
485 uc->length++; in umtxq_insert_queue()
486 if (uc->length > uc->max_length) { in umtxq_insert_queue()
487 uc->max_length = uc->length; in umtxq_insert_queue()
488 if (uc->max_length > max_length) in umtxq_insert_queue()
489 max_length = uc->max_length; in umtxq_insert_queue()
493 uq->uq_spare_queue = NULL; in umtxq_insert_queue()
495 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link); in umtxq_insert_queue()
496 uh->length++; in umtxq_insert_queue()
497 uq->uq_flags |= UQF_UMTXQ; in umtxq_insert_queue()
498 uq->uq_cur_queue = uh; in umtxq_insert_queue()
508 uc = umtxq_getchain(&uq->uq_key); in umtxq_remove_queue()
510 if (uq->uq_flags & UQF_UMTXQ) { in umtxq_remove_queue()
511 uh = uq->uq_cur_queue; in umtxq_remove_queue()
512 TAILQ_REMOVE(&uh->head, uq, uq_link); in umtxq_remove_queue()
513 uh->length--; in umtxq_remove_queue()
514 uq->uq_flags &= ~UQF_UMTXQ; in umtxq_remove_queue()
515 if (TAILQ_EMPTY(&uh->head)) { in umtxq_remove_queue()
516 KASSERT(uh->length == 0, in umtxq_remove_queue()
519 uc->length--; in umtxq_remove_queue()
523 uh = LIST_FIRST(&uc->uc_spare_queue); in umtxq_remove_queue()
527 uq->uq_spare_queue = uh; in umtxq_remove_queue()
528 uq->uq_cur_queue = NULL; in umtxq_remove_queue()
543 return (uh->length); in umtxq_count()
560 *first = TAILQ_FIRST(&uh->head); in umtxq_count_pi()
561 return (uh->length); in umtxq_count_pi()
567 * Wake up threads waiting on an userland object by a bit mask.
581 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) { in umtxq_signal_mask()
582 if ((uq->uq_bitset & bitset) == 0) in umtxq_signal_mask()
593 * Wake up threads waiting on an userland object.
607 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) { in umtxq_signal_queue()
618 * Wake up specified thread.
624 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key)); in umtxq_signal_thread()
630 * Wake up a maximum of n_wake threads that are waiting on an userland
650 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) { in umtxq_requeue()
656 uq->uq_key = *key2; in umtxq_requeue()
658 if (ret - n_wake == n_requeue) in umtxq_requeue()
679 timo->clockid = clockid; in umtx_abs_timeout_init()
681 timo->is_abs_real = false; in umtx_abs_timeout_init()
682 kern_clock_gettime(curthread, timo->clockid, &timo->cur); in umtx_abs_timeout_init()
683 timespecadd(&timo->cur, timeout, &timo->end); in umtx_abs_timeout_init()
685 timo->end = *timeout; in umtx_abs_timeout_init()
686 timo->is_abs_real = clockid == CLOCK_REALTIME || in umtx_abs_timeout_init()
698 umtx_abs_timeout_init(timo, umtxtime->_clockid, in umtx_abs_timeout_init2()
699 (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); in umtx_abs_timeout_init2()
707 mint = curproc->p_umtx_min_timeout; in umtx_abs_timeout_enforce_min()
723 switch (timo->clockid) { in umtx_abs_timeout_getsbt()
736 timespec2bintime(&timo->end, &bt); in umtx_abs_timeout_getsbt()
737 switch (timo->clockid) { in umtx_abs_timeout_getsbt()
758 * avoid firing multiple timer events in non-periodic in umtx_abs_timeout_getsbt()
759 * timer mode. in umtx_abs_timeout_getsbt()
761 switch (timo->clockid) { in umtx_abs_timeout_getsbt()
767 *sbt += tc_tick_sbt - rem; in umtx_abs_timeout_getsbt()
772 *sbt += SBT_1S - rem; in umtx_abs_timeout_getsbt()
784 kern_clock_gettime(curthread, timo->clockid, &timo->cur); in umtx_abs_timeout_getsbt()
785 if (timespeccmp(&timo->end, &timo->cur, <=)) in umtx_abs_timeout_getsbt()
787 timespecsub(&timo->end, &timo->cur, &tts); in umtx_abs_timeout_getsbt()
819 uc = umtxq_getchain(&uq->uq_key); in umtxq_sleep()
822 if (!(uq->uq_flags & UQF_UMTXQ)) { in umtxq_sleep()
827 if (timo->is_abs_real) in umtxq_sleep()
828 curthread->td_rtcgen = in umtxq_sleep()
834 error = msleep_sbt(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, in umtxq_sleep()
836 uc = umtxq_getchain(&uq->uq_key); in umtxq_sleep()
837 mtx_lock(&uc->uc_lock); in umtxq_sleep()
846 curthread->td_rtcgen = 0; in umtxq_sleep()
863 key->type = type; in umtx_key_get()
865 key->shared = 0; in umtx_key_get()
866 key->info.private.vs = td->td_proc->p_vmspace; in umtx_key_get()
867 key->info.private.addr = (uintptr_t)addr; in umtx_key_get()
870 map = &td->td_proc->p_vmspace->vm_map; in umtx_key_get()
872 &entry, &key->info.shared.object, &pindex, &prot, in umtx_key_get()
879 VM_INHERIT_SHARE == entry->inheritance)) { in umtx_key_get()
880 key->shared = 1; in umtx_key_get()
881 key->info.shared.offset = (vm_offset_t)addr - in umtx_key_get()
882 entry->start + entry->offset; in umtx_key_get()
883 vm_object_reference(key->info.shared.object); in umtx_key_get()
885 key->shared = 0; in umtx_key_get()
886 key->info.private.vs = td->td_proc->p_vmspace; in umtx_key_get()
887 key->info.private.addr = (uintptr_t)addr; in umtx_key_get()
902 if (key->shared) in umtx_key_release()
903 vm_object_deallocate(key->info.shared.object); in umtx_key_release()
920 uq = td->td_umtxq; in do_lock_umtx()
932 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id); in do_lock_umtx()
939 if (owner == -1) in do_lock_umtx()
944 owner = casuword(&umtx->u_owner, in do_lock_umtx()
951 if (owner == -1) in do_lock_umtx()
970 AUTO_SHARE, &uq->uq_key)) != 0) in do_lock_umtx()
973 umtxq_lock(&uq->uq_key); in do_lock_umtx()
974 umtxq_busy(&uq->uq_key); in do_lock_umtx()
976 umtxq_unbusy(&uq->uq_key); in do_lock_umtx()
977 umtxq_unlock(&uq->uq_key); in do_lock_umtx()
985 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED); in do_lock_umtx()
988 if (old == -1) { in do_lock_umtx()
989 umtxq_lock(&uq->uq_key); in do_lock_umtx()
991 umtxq_unlock(&uq->uq_key); in do_lock_umtx()
992 umtx_key_release(&uq->uq_key); in do_lock_umtx()
1001 umtxq_lock(&uq->uq_key); in do_lock_umtx()
1006 umtxq_unlock(&uq->uq_key); in do_lock_umtx()
1007 umtx_key_release(&uq->uq_key); in do_lock_umtx()
1018 /* Timed-locking is not restarted. */ in do_lock_umtx()
1040 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner)); in do_unlock_umtx()
1041 if (owner == -1) in do_unlock_umtx()
1049 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED); in do_unlock_umtx()
1050 if (old == -1) in do_unlock_umtx()
1072 old = casuword(&umtx->u_owner, owner, in do_unlock_umtx()
1079 if (old == -1) in do_unlock_umtx()
1101 uq = td->td_umtxq; in do_lock_umtx32()
1121 if (owner == -1) in do_lock_umtx32()
1132 if (owner == -1) in do_lock_umtx32()
1151 AUTO_SHARE, &uq->uq_key)) != 0) in do_lock_umtx32()
1154 umtxq_lock(&uq->uq_key); in do_lock_umtx32()
1155 umtxq_busy(&uq->uq_key); in do_lock_umtx32()
1157 umtxq_unbusy(&uq->uq_key); in do_lock_umtx32()
1158 umtxq_unlock(&uq->uq_key); in do_lock_umtx32()
1169 if (old == -1) { in do_lock_umtx32()
1170 umtxq_lock(&uq->uq_key); in do_lock_umtx32()
1172 umtxq_unlock(&uq->uq_key); in do_lock_umtx32()
1173 umtx_key_release(&uq->uq_key); in do_lock_umtx32()
1182 umtxq_lock(&uq->uq_key); in do_lock_umtx32()
1187 umtxq_unlock(&uq->uq_key); in do_lock_umtx32()
1188 umtx_key_release(&uq->uq_key); in do_lock_umtx32()
1199 /* Timed-locking is not restarted. */ in do_lock_umtx32()
1222 if (owner == -1) in do_unlock_umtx32()
1231 if (old == -1) in do_unlock_umtx32()
1260 if (old == -1) in do_unlock_umtx32()
1282 uq = td->td_umtxq; in do_wait()
1284 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) in do_wait()
1290 umtxq_lock(&uq->uq_key); in do_wait()
1292 umtxq_unlock(&uq->uq_key); in do_wait()
1304 umtxq_lock(&uq->uq_key); in do_wait()
1309 if ((uq->uq_flags & UQF_UMTXQ) == 0) in do_wait()
1313 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) { in do_wait()
1316 umtxq_unlock(&uq->uq_key); in do_wait()
1317 umtx_key_release(&uq->uq_key); in do_wait()
1324 * Wake up threads sleeping on the specified address.
1354 id = td->td_tid; in do_lock_normal()
1355 uq = td->td_umtxq; in do_lock_normal()
1365 rv = fueword32(&m->m_owner, &owner); in do_lock_normal()
1366 if (rv == -1) in do_lock_normal()
1382 rv = casueword32(&m->m_owner, in do_lock_normal()
1385 if (rv == -1) in do_lock_normal()
1404 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, in do_lock_normal()
1407 if (rv == -1) in do_lock_normal()
1422 rv = casueword32(&m->m_owner, in do_lock_normal()
1426 if (rv == -1) in do_lock_normal()
1462 GET_SHARE(flags), &uq->uq_key)) != 0) in do_lock_normal()
1465 umtxq_lock(&uq->uq_key); in do_lock_normal()
1466 umtxq_busy(&uq->uq_key); in do_lock_normal()
1468 umtxq_unlock(&uq->uq_key); in do_lock_normal()
1476 rv = casueword32(&m->m_owner, owner, &old, in do_lock_normal()
1480 if (rv == -1 || rv == 1) { in do_lock_normal()
1481 umtxq_lock(&uq->uq_key); in do_lock_normal()
1483 umtxq_unbusy(&uq->uq_key); in do_lock_normal()
1484 umtxq_unlock(&uq->uq_key); in do_lock_normal()
1485 umtx_key_release(&uq->uq_key); in do_lock_normal()
1486 if (rv == -1) in do_lock_normal()
1501 umtxq_lock(&uq->uq_key); in do_lock_normal()
1502 umtxq_unbusy(&uq->uq_key); in do_lock_normal()
1507 umtxq_unlock(&uq->uq_key); in do_lock_normal()
1508 umtx_key_release(&uq->uq_key); in do_lock_normal()
1527 id = td->td_tid; in do_unlock_normal()
1533 error = fueword32(&m->m_owner, &owner); in do_unlock_normal()
1534 if (error == -1) in do_unlock_normal()
1542 error = casueword32(&m->m_owner, owner, &old, newlock); in do_unlock_normal()
1543 if (error == -1) in do_unlock_normal()
1572 error = casueword32(&m->m_owner, owner, &old, newlock); in do_unlock_normal()
1578 if (error == -1) in do_unlock_normal()
1592 * Check if the mutex is available and wake up a waiter,
1605 error = fueword32(&m->m_owner, &owner); in do_wake_umutex()
1606 if (error == -1) in do_wake_umutex()
1613 error = fueword32(&m->m_flags, &flags); in do_wake_umutex()
1614 if (error == -1) in do_wake_umutex()
1629 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, in do_wake_umutex()
1631 if (error == -1) { in do_wake_umutex()
1700 error = fueword32(&m->m_owner, &owner); in do_wake2_umutex()
1701 if (error == -1) in do_wake2_umutex()
1711 error = casueword32(&m->m_owner, owner, &old, in do_wake2_umutex()
1713 if (error == -1) { in do_wake2_umutex()
1743 TAILQ_INIT(&pi->pi_blocked); in umtx_pi_alloc()
1752 atomic_add_int(&umtx_pi_allocated, -1); in umtx_pi_free()
1769 uq = td->td_umtxq; in umtx_pi_adjust_thread()
1778 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || in umtx_pi_adjust_thread()
1779 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { in umtx_pi_adjust_thread()
1784 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); in umtx_pi_adjust_thread()
1785 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { in umtx_pi_adjust_thread()
1786 td1 = uq1->uq_thread; in umtx_pi_adjust_thread()
1787 MPASS(td1->td_proc->p_magic == P_MAGIC); in umtx_pi_adjust_thread()
1793 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); in umtx_pi_adjust_thread()
1805 if (pi->pi_owner == NULL) in umtx_pi_next()
1807 uq_owner = pi->pi_owner->td_umtxq; in umtx_pi_next()
1810 return (uq_owner->uq_pi_blocked); in umtx_pi_next()
1814 * Floyd's Cycle-Finding Algorithm.
1854 uq = td->td_umtxq; in umtx_propagate_priority()
1855 pi = uq->uq_pi_blocked; in umtx_propagate_priority()
1862 td = pi->pi_owner; in umtx_propagate_priority()
1866 MPASS(td->td_proc != NULL); in umtx_propagate_priority()
1867 MPASS(td->td_proc->p_magic == P_MAGIC); in umtx_propagate_priority()
1870 if (td->td_lend_user_pri > pri) in umtx_propagate_priority()
1881 uq = td->td_umtxq; in umtx_propagate_priority()
1882 pi = uq->uq_pi_blocked; in umtx_propagate_priority()
1905 while (pi != NULL && pi->pi_owner != NULL) { in umtx_repropagate_priority()
1907 uq_owner = pi->pi_owner->td_umtxq; in umtx_repropagate_priority()
1909 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { in umtx_repropagate_priority()
1910 uq = TAILQ_FIRST(&pi2->pi_blocked); in umtx_repropagate_priority()
1912 if (pri > UPRI(uq->uq_thread)) in umtx_repropagate_priority()
1913 pri = UPRI(uq->uq_thread); in umtx_repropagate_priority()
1917 if (pri > uq_owner->uq_inherited_pri) in umtx_repropagate_priority()
1918 pri = uq_owner->uq_inherited_pri; in umtx_repropagate_priority()
1919 thread_lock(pi->pi_owner); in umtx_repropagate_priority()
1920 sched_lend_user_prio(pi->pi_owner, pri); in umtx_repropagate_priority()
1921 thread_unlock(pi->pi_owner); in umtx_repropagate_priority()
1922 if ((pi = uq_owner->uq_pi_blocked) != NULL) in umtx_repropagate_priority()
1923 umtx_pi_adjust_thread(pi, uq_owner->uq_thread); in umtx_repropagate_priority()
1935 uq_owner = owner->td_umtxq; in umtx_pi_setowner()
1937 MPASS(pi->pi_owner == NULL); in umtx_pi_setowner()
1938 pi->pi_owner = owner; in umtx_pi_setowner()
1939 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); in umtx_pi_setowner()
1950 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link); in umtx_pi_disown()
1951 pi->pi_owner = NULL; in umtx_pi_disown()
1964 if (pi->pi_owner == owner) { in umtx_pi_claim()
1969 if (pi->pi_owner != NULL) { in umtx_pi_claim()
1977 uq = TAILQ_FIRST(&pi->pi_blocked); in umtx_pi_claim()
1979 pri = UPRI(uq->uq_thread); in umtx_pi_claim()
1999 uq = td->td_umtxq; in umtx_pi_adjust()
2004 pi = uq->uq_pi_blocked; in umtx_pi_adjust()
2025 uc = umtxq_getchain(&pi->pi_key); in umtxq_sleep_pi()
2028 td = uq->uq_thread; in umtxq_sleep_pi()
2030 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key)); in umtxq_sleep_pi()
2031 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); in umtxq_sleep_pi()
2034 if (pi->pi_owner == NULL) { in umtxq_sleep_pi()
2036 td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid); in umtxq_sleep_pi()
2039 if (pi->pi_owner == NULL) in umtxq_sleep_pi()
2041 PROC_UNLOCK(td1->td_proc); in umtxq_sleep_pi()
2045 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { in umtxq_sleep_pi()
2046 pri = UPRI(uq1->uq_thread); in umtxq_sleep_pi()
2054 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); in umtxq_sleep_pi()
2056 uq->uq_pi_blocked = pi; in umtxq_sleep_pi()
2058 td->td_flags |= TDF_UPIBLOCKED; in umtxq_sleep_pi()
2062 umtxq_unbusy(&uq->uq_key); in umtxq_sleep_pi()
2068 uq->uq_pi_blocked = NULL; in umtxq_sleep_pi()
2070 td->td_flags &= ~TDF_UPIBLOCKED; in umtxq_sleep_pi()
2072 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); in umtxq_sleep_pi()
2075 umtxq_unlock(&uq->uq_key); in umtxq_sleep_pi()
2087 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key)); in umtx_pi_ref()
2088 pi->pi_refcount++; in umtx_pi_ref()
2100 uc = umtxq_getchain(&pi->pi_key); in umtx_pi_unref()
2102 KASSERT(pi->pi_refcount > 0, ("invalid reference count")); in umtx_pi_unref()
2103 if (--pi->pi_refcount == 0) { in umtx_pi_unref()
2105 if (pi->pi_owner != NULL) in umtx_pi_unref()
2107 KASSERT(TAILQ_EMPTY(&pi->pi_blocked), in umtx_pi_unref()
2110 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); in umtx_pi_unref()
2127 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { in umtx_pi_lookup()
2128 if (umtx_key_match(&pi->pi_key, key)) { in umtx_pi_lookup()
2143 uc = umtxq_getchain(&pi->pi_key); in umtx_pi_insert()
2145 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); in umtx_pi_insert()
2162 pi = uq_first->uq_pi_blocked; in umtx_pi_drop()
2164 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { in umtx_pi_drop()
2169 uq_me = td->td_umtxq; in umtx_pi_drop()
2170 if (pi->pi_owner == td) in umtx_pi_drop()
2173 uq_first = TAILQ_FIRST(&pi->pi_blocked); in umtx_pi_drop()
2175 (uq_first->uq_flags & UQF_UMTXQ) == 0) { in umtx_pi_drop()
2179 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { in umtx_pi_drop()
2180 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); in umtx_pi_drop()
2182 if (pri > UPRI(uq_first2->uq_thread)) in umtx_pi_drop()
2183 pri = UPRI(uq_first2->uq_thread); in umtx_pi_drop()
2207 if (pi->pi_owner == td) in umtx_pi_drop()
2228 id = td->td_tid; in do_lock_pi()
2229 uq = td->td_umtxq; in do_lock_pi()
2233 &uq->uq_key)) != 0) in do_lock_pi()
2239 umtxq_lock(&uq->uq_key); in do_lock_pi()
2240 pi = umtx_pi_lookup(&uq->uq_key); in do_lock_pi()
2244 umtxq_unlock(&uq->uq_key); in do_lock_pi()
2246 umtxq_lock(&uq->uq_key); in do_lock_pi()
2247 pi = umtx_pi_lookup(&uq->uq_key); in do_lock_pi()
2254 new_pi->pi_key = uq->uq_key; in do_lock_pi()
2260 umtxq_unlock(&uq->uq_key); in do_lock_pi()
2270 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); in do_lock_pi()
2272 if (rv == -1) { in do_lock_pi()
2312 rv = casueword32(&m->m_owner, owner, &owner, in do_lock_pi()
2315 if (rv == -1) { in do_lock_pi()
2335 umtxq_lock(&uq->uq_key); in do_lock_pi()
2336 umtxq_busy(&uq->uq_key); in do_lock_pi()
2338 umtxq_unbusy(&uq->uq_key); in do_lock_pi()
2339 umtxq_unlock(&uq->uq_key); in do_lock_pi()
2347 (void)casuword32(&m->m_owner, in do_lock_pi()
2372 umtxq_lock(&uq->uq_key); in do_lock_pi()
2373 umtxq_busy(&uq->uq_key); in do_lock_pi()
2374 umtxq_unlock(&uq->uq_key); in do_lock_pi()
2382 rv = casueword32(&m->m_owner, owner, &old, owner | in do_lock_pi()
2386 if (rv == -1) { in do_lock_pi()
2387 umtxq_unbusy_unlocked(&uq->uq_key); in do_lock_pi()
2392 umtxq_unbusy_unlocked(&uq->uq_key); in do_lock_pi()
2406 umtxq_lock(&uq->uq_key); in do_lock_pi()
2421 umtxq_lock(&uq->uq_key); in do_lock_pi()
2423 umtxq_unlock(&uq->uq_key); in do_lock_pi()
2425 umtx_key_release(&uq->uq_key); in do_lock_pi()
2439 id = td->td_tid; in do_unlock_pi()
2445 error = fueword32(&m->m_owner, &owner); in do_unlock_pi()
2446 if (error == -1) in do_unlock_pi()
2456 error = casueword32(&m->m_owner, owner, &old, new_owner); in do_unlock_pi()
2457 if (error == -1) in do_unlock_pi()
2497 error = casueword32(&m->m_owner, owner, &old, new_owner); in do_unlock_pi()
2505 if (error == -1) in do_unlock_pi()
2527 id = td->td_tid; in do_lock_pp()
2528 uq = td->td_umtxq; in do_lock_pp()
2531 &uq->uq_key)) != 0) in do_lock_pp()
2539 old_inherited_pri = uq->uq_inherited_pri; in do_lock_pp()
2540 umtxq_lock(&uq->uq_key); in do_lock_pp()
2541 umtxq_busy(&uq->uq_key); in do_lock_pp()
2542 umtxq_unlock(&uq->uq_key); in do_lock_pp()
2544 rv = fueword32(&m->m_ceilings[0], &ceiling); in do_lock_pp()
2545 if (rv == -1) { in do_lock_pp()
2549 ceiling = RTP_PRIO_MAX - ceiling; in do_lock_pp()
2556 if (td->td_base_user_pri < new_pri) { in do_lock_pp()
2562 if (new_pri < uq->uq_inherited_pri) { in do_lock_pp()
2563 uq->uq_inherited_pri = new_pri; in do_lock_pp()
2572 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, in do_lock_pp()
2575 if (rv == -1) { in do_lock_pp()
2586 rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD, in do_lock_pp()
2588 if (rv == -1) { in do_lock_pp()
2609 umtxq_unbusy_unlocked(&uq->uq_key); in do_lock_pp()
2628 umtxq_lock(&uq->uq_key); in do_lock_pp()
2630 umtxq_unbusy(&uq->uq_key); in do_lock_pp()
2634 umtxq_unlock(&uq->uq_key); in do_lock_pp()
2637 uq->uq_inherited_pri = old_inherited_pri; in do_lock_pp()
2639 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { in do_lock_pp()
2640 uq2 = TAILQ_FIRST(&pi->pi_blocked); in do_lock_pp()
2642 if (pri > UPRI(uq2->uq_thread)) in do_lock_pp()
2643 pri = UPRI(uq2->uq_thread); in do_lock_pp()
2646 if (pri > uq->uq_inherited_pri) in do_lock_pp()
2647 pri = uq->uq_inherited_pri; in do_lock_pp()
2656 uq->uq_inherited_pri = old_inherited_pri; in do_lock_pp()
2658 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { in do_lock_pp()
2659 uq2 = TAILQ_FIRST(&pi->pi_blocked); in do_lock_pp()
2661 if (pri > UPRI(uq2->uq_thread)) in do_lock_pp()
2662 pri = UPRI(uq2->uq_thread); in do_lock_pp()
2665 if (pri > uq->uq_inherited_pri) in do_lock_pp()
2666 pri = uq->uq_inherited_pri; in do_lock_pp()
2674 umtxq_unbusy_unlocked(&uq->uq_key); in do_lock_pp()
2675 umtx_key_release(&uq->uq_key); in do_lock_pp()
2692 id = td->td_tid; in do_unlock_pp()
2693 uq = td->td_umtxq; in do_unlock_pp()
2699 error = fueword32(&m->m_owner, &owner); in do_unlock_pp()
2700 if (error == -1) in do_unlock_pp()
2706 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); in do_unlock_pp()
2710 if (rceiling == -1) in do_unlock_pp()
2713 rceiling = RTP_PRIO_MAX - rceiling; in do_unlock_pp()
2732 error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) | in do_unlock_pp()
2741 if (error == -1) in do_unlock_pp()
2746 uq->uq_inherited_pri = new_inherited_pri; in do_unlock_pp()
2748 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { in do_unlock_pp()
2749 uq2 = TAILQ_FIRST(&pi->pi_blocked); in do_unlock_pp()
2751 if (pri > UPRI(uq2->uq_thread)) in do_unlock_pp()
2752 pri = UPRI(uq2->uq_thread); in do_unlock_pp()
2755 if (pri > uq->uq_inherited_pri) in do_unlock_pp()
2756 pri = uq->uq_inherited_pri; in do_unlock_pp()
2774 error = fueword32(&m->m_flags, &flags); in do_set_ceiling()
2775 if (error == -1) in do_set_ceiling()
2781 id = td->td_tid; in do_set_ceiling()
2782 uq = td->td_umtxq; in do_set_ceiling()
2785 &uq->uq_key)) != 0) in do_set_ceiling()
2788 umtxq_lock(&uq->uq_key); in do_set_ceiling()
2789 umtxq_busy(&uq->uq_key); in do_set_ceiling()
2790 umtxq_unlock(&uq->uq_key); in do_set_ceiling()
2792 rv = fueword32(&m->m_ceilings[0], &save_ceiling); in do_set_ceiling()
2793 if (rv == -1) { in do_set_ceiling()
2798 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, in do_set_ceiling()
2800 if (rv == -1) { in do_set_ceiling()
2807 rv = suword32(&m->m_ceilings[0], ceiling); in do_set_ceiling()
2808 rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED); in do_set_ceiling()
2814 rv = suword32(&m->m_ceilings[0], ceiling); in do_set_ceiling()
2839 umtxq_lock(&uq->uq_key); in do_set_ceiling()
2841 umtxq_unbusy(&uq->uq_key); in do_set_ceiling()
2844 umtxq_unlock(&uq->uq_key); in do_set_ceiling()
2846 umtxq_lock(&uq->uq_key); in do_set_ceiling()
2848 umtxq_signal(&uq->uq_key, INT_MAX); in do_set_ceiling()
2849 umtxq_unbusy(&uq->uq_key); in do_set_ceiling()
2850 umtxq_unlock(&uq->uq_key); in do_set_ceiling()
2851 umtx_key_release(&uq->uq_key); in do_set_ceiling()
2869 error = fueword32(&m->m_flags, &flags); in do_lock_umutex()
2870 if (error == -1) in do_lock_umutex()
2890 /* Timed-locking is not restarted. */ in do_lock_umutex()
2906 error = fueword32(&m->m_flags, &flags); in do_unlock_umutex()
2907 if (error == -1) in do_unlock_umutex()
2931 uq = td->td_umtxq; in do_cv_wait()
2932 error = fueword32(&cv->c_flags, &flags); in do_cv_wait()
2933 if (error == -1) in do_cv_wait()
2935 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); in do_cv_wait()
2940 error = fueword32(&cv->c_clockid, &clockid); in do_cv_wait()
2941 if (error == -1) { in do_cv_wait()
2942 umtx_key_release(&uq->uq_key); in do_cv_wait()
2948 umtx_key_release(&uq->uq_key); in do_cv_wait()
2955 umtxq_lock(&uq->uq_key); in do_cv_wait()
2956 umtxq_busy(&uq->uq_key); in do_cv_wait()
2958 umtxq_unlock(&uq->uq_key); in do_cv_wait()
2964 error = fueword32(&cv->c_has_waiters, &hasw); in do_cv_wait()
2966 error = suword32(&cv->c_has_waiters, 1); in do_cv_wait()
2968 umtxq_lock(&uq->uq_key); in do_cv_wait()
2970 umtxq_unbusy(&uq->uq_key); in do_cv_wait()
2975 umtxq_unbusy_unlocked(&uq->uq_key); in do_cv_wait()
2983 umtxq_lock(&uq->uq_key); in do_cv_wait()
2989 if ((uq->uq_flags & UQF_UMTXQ) == 0) in do_cv_wait()
2997 umtxq_busy(&uq->uq_key); in do_cv_wait()
2998 if ((uq->uq_flags & UQF_UMTXQ) != 0) { in do_cv_wait()
2999 int oldlen = uq->uq_cur_queue->length; in do_cv_wait()
3002 umtxq_unlock(&uq->uq_key); in do_cv_wait()
3003 if (suword32(&cv->c_has_waiters, 0) != 0 && in do_cv_wait()
3006 umtxq_lock(&uq->uq_key); in do_cv_wait()
3009 umtxq_unbusy(&uq->uq_key); in do_cv_wait()
3014 umtxq_unlock(&uq->uq_key); in do_cv_wait()
3015 umtx_key_release(&uq->uq_key); in do_cv_wait()
3029 error = fueword32(&cv->c_flags, &flags); in do_cv_signal()
3030 if (error == -1) in do_cv_signal()
3040 error = suword32(&cv->c_has_waiters, 0); in do_cv_signal()
3041 if (error == -1) in do_cv_signal()
3058 error = fueword32(&cv->c_flags, &flags); in do_cv_broadcast()
3059 if (error == -1) in do_cv_broadcast()
3069 error = suword32(&cv->c_has_waiters, 0); in do_cv_broadcast()
3070 if (error == -1) in do_cv_broadcast()
3090 uq = td->td_umtxq; in do_rw_rdlock()
3091 error = fueword32(&rwlock->rw_flags, &flags); in do_rw_rdlock()
3092 if (error == -1) in do_rw_rdlock()
3094 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); in do_rw_rdlock()
3106 rv = fueword32(&rwlock->rw_state, &state); in do_rw_rdlock()
3107 if (rv == -1) { in do_rw_rdlock()
3108 umtx_key_release(&uq->uq_key); in do_rw_rdlock()
3116 umtx_key_release(&uq->uq_key); in do_rw_rdlock()
3119 rv = casueword32(&rwlock->rw_state, state, in do_rw_rdlock()
3121 if (rv == -1) { in do_rw_rdlock()
3122 umtx_key_release(&uq->uq_key); in do_rw_rdlock()
3127 umtx_key_release(&uq->uq_key); in do_rw_rdlock()
3140 umtxq_lock(&uq->uq_key); in do_rw_rdlock()
3141 umtxq_busy(&uq->uq_key); in do_rw_rdlock()
3142 umtxq_unlock(&uq->uq_key); in do_rw_rdlock()
3145 * re-read the state, in case it changed between the try-lock above in do_rw_rdlock()
3148 rv = fueword32(&rwlock->rw_state, &state); in do_rw_rdlock()
3149 if (rv == -1) in do_rw_rdlock()
3155 rv = casueword32(&rwlock->rw_state, state, in do_rw_rdlock()
3157 if (rv == -1) { in do_rw_rdlock()
3171 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_rdlock()
3177 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_rdlock()
3189 rv = fueword32(&rwlock->rw_blocked_readers, in do_rw_rdlock()
3192 rv = suword32(&rwlock->rw_blocked_readers, in do_rw_rdlock()
3194 if (rv == -1) { in do_rw_rdlock()
3195 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_rdlock()
3201 umtxq_lock(&uq->uq_key); in do_rw_rdlock()
3203 umtxq_unbusy(&uq->uq_key); in do_rw_rdlock()
3208 umtxq_busy(&uq->uq_key); in do_rw_rdlock()
3210 umtxq_unlock(&uq->uq_key); in do_rw_rdlock()
3213 rv = fueword32(&rwlock->rw_state, &state); in do_rw_rdlock()
3214 if (rv == -1) { in do_rw_rdlock()
3221 rv = fueword32(&rwlock->rw_blocked_readers, in do_rw_rdlock()
3224 rv = suword32(&rwlock->rw_blocked_readers, in do_rw_rdlock()
3225 blocked_readers - 1); in do_rw_rdlock()
3226 if (rv == -1) { in do_rw_rdlock()
3227 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_rdlock()
3232 rv = fueword32(&rwlock->rw_state, &state); in do_rw_rdlock()
3233 if (rv == -1) { in do_rw_rdlock()
3234 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_rdlock()
3239 rv = casueword32(&rwlock->rw_state, state, in do_rw_rdlock()
3241 if (rv == -1) { in do_rw_rdlock()
3259 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_rdlock()
3263 umtx_key_release(&uq->uq_key); in do_rw_rdlock()
3280 uq = td->td_umtxq; in do_rw_wrlock()
3281 error = fueword32(&rwlock->rw_flags, &flags); in do_rw_wrlock()
3282 if (error == -1) in do_rw_wrlock()
3284 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); in do_rw_wrlock()
3293 rv = fueword32(&rwlock->rw_state, &state); in do_rw_wrlock()
3294 if (rv == -1) { in do_rw_wrlock()
3295 umtx_key_release(&uq->uq_key); in do_rw_wrlock()
3300 rv = casueword32(&rwlock->rw_state, state, in do_rw_wrlock()
3302 if (rv == -1) { in do_rw_wrlock()
3303 umtx_key_release(&uq->uq_key); in do_rw_wrlock()
3308 umtx_key_release(&uq->uq_key); in do_rw_wrlock()
3321 umtxq_lock(&uq->uq_key); in do_rw_wrlock()
3322 umtxq_busy(&uq->uq_key); in do_rw_wrlock()
3323 umtxq_signal_queue(&uq->uq_key, INT_MAX, in do_rw_wrlock()
3325 umtxq_unbusy(&uq->uq_key); in do_rw_wrlock()
3326 umtxq_unlock(&uq->uq_key); in do_rw_wrlock()
3333 umtxq_lock(&uq->uq_key); in do_rw_wrlock()
3334 umtxq_busy(&uq->uq_key); in do_rw_wrlock()
3335 umtxq_unlock(&uq->uq_key); in do_rw_wrlock()
3338 * Re-read the state, in case it changed between the in do_rw_wrlock()
3339 * try-lock above and the check below. in do_rw_wrlock()
3341 rv = fueword32(&rwlock->rw_state, &state); in do_rw_wrlock()
3342 if (rv == -1) in do_rw_wrlock()
3348 rv = casueword32(&rwlock->rw_state, state, in do_rw_wrlock()
3350 if (rv == -1) { in do_rw_wrlock()
3364 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_wrlock()
3370 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_wrlock()
3377 rv = fueword32(&rwlock->rw_blocked_writers, in do_rw_wrlock()
3380 rv = suword32(&rwlock->rw_blocked_writers, in do_rw_wrlock()
3382 if (rv == -1) { in do_rw_wrlock()
3383 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_wrlock()
3390 umtxq_lock(&uq->uq_key); in do_rw_wrlock()
3392 umtxq_unbusy(&uq->uq_key); in do_rw_wrlock()
3397 umtxq_busy(&uq->uq_key); in do_rw_wrlock()
3399 umtxq_unlock(&uq->uq_key); in do_rw_wrlock()
3402 rv = fueword32(&rwlock->rw_state, &state); in do_rw_wrlock()
3403 if (rv == -1) { in do_rw_wrlock()
3409 rv = fueword32(&rwlock->rw_blocked_writers, in do_rw_wrlock()
3412 rv = suword32(&rwlock->rw_blocked_writers, in do_rw_wrlock()
3413 blocked_writers - 1); in do_rw_wrlock()
3414 if (rv == -1) { in do_rw_wrlock()
3415 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_wrlock()
3420 rv = fueword32(&rwlock->rw_state, &state); in do_rw_wrlock()
3421 if (rv == -1) { in do_rw_wrlock()
3422 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_wrlock()
3427 rv = casueword32(&rwlock->rw_state, state, in do_rw_wrlock()
3429 if (rv == -1) { in do_rw_wrlock()
3450 rv = fueword32(&rwlock->rw_blocked_readers, in do_rw_wrlock()
3452 if (rv == -1) { in do_rw_wrlock()
3453 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_wrlock()
3460 umtxq_unbusy_unlocked(&uq->uq_key); in do_rw_wrlock()
3463 umtx_key_release(&uq->uq_key); in do_rw_wrlock()
3477 uq = td->td_umtxq; in do_rw_unlock()
3478 error = fueword32(&rwlock->rw_flags, &flags); in do_rw_unlock()
3479 if (error == -1) in do_rw_unlock()
3481 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); in do_rw_unlock()
3485 error = fueword32(&rwlock->rw_state, &state); in do_rw_unlock()
3486 if (error == -1) { in do_rw_unlock()
3492 rv = casueword32(&rwlock->rw_state, state, in do_rw_unlock()
3494 if (rv == -1) { in do_rw_unlock()
3512 rv = casueword32(&rwlock->rw_state, state, in do_rw_unlock()
3513 &oldstate, state - 1); in do_rw_unlock()
3514 if (rv == -1) { in do_rw_unlock()
3556 umtxq_lock(&uq->uq_key); in do_rw_unlock()
3557 umtxq_busy(&uq->uq_key); in do_rw_unlock()
3558 umtxq_signal_queue(&uq->uq_key, count, q); in do_rw_unlock()
3559 umtxq_unbusy(&uq->uq_key); in do_rw_unlock()
3560 umtxq_unlock(&uq->uq_key); in do_rw_unlock()
3563 umtx_key_release(&uq->uq_key); in do_rw_unlock()
3576 uq = td->td_umtxq; in do_sem_wait()
3577 error = fueword32(&sem->_flags, &flags); in do_sem_wait()
3578 if (error == -1) in do_sem_wait()
3580 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); in do_sem_wait()
3588 umtxq_lock(&uq->uq_key); in do_sem_wait()
3589 umtxq_busy(&uq->uq_key); in do_sem_wait()
3591 umtxq_unlock(&uq->uq_key); in do_sem_wait()
3592 rv = casueword32(&sem->_has_waiters, 0, &count1, 1); in do_sem_wait()
3593 if (rv != -1) in do_sem_wait()
3594 rv1 = fueword32(&sem->_count, &count); in do_sem_wait()
3595 if (rv == -1 || rv1 == -1 || count != 0 || (rv == 1 && count1 == 0)) { in do_sem_wait()
3597 rv = suword32(&sem->_has_waiters, 0); in do_sem_wait()
3598 umtxq_lock(&uq->uq_key); in do_sem_wait()
3599 umtxq_unbusy(&uq->uq_key); in do_sem_wait()
3601 umtxq_unlock(&uq->uq_key); in do_sem_wait()
3602 if (rv == -1 || rv1 == -1) { in do_sem_wait()
3617 umtxq_lock(&uq->uq_key); in do_sem_wait()
3618 umtxq_unbusy(&uq->uq_key); in do_sem_wait()
3622 if ((uq->uq_flags & UQF_UMTXQ) == 0) in do_sem_wait()
3626 /* A relative timeout cannot be restarted. */ in do_sem_wait()
3628 (timeout->_flags & UMTX_ABSTIME) == 0) in do_sem_wait()
3631 umtxq_unlock(&uq->uq_key); in do_sem_wait()
3633 umtx_key_release(&uq->uq_key); in do_sem_wait()
3647 error = fueword32(&sem->_flags, &flags); in do_sem_wake()
3648 if (error == -1) in do_sem_wake()
3663 error = suword32(&sem->_has_waiters, 0); in do_sem_wake()
3665 if (error == -1) in do_sem_wake()
3685 uq = td->td_umtxq; in do_sem2_wait()
3686 flags = fuword32(&sem->_flags); in do_sem2_wait()
3691 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); in do_sem2_wait()
3694 umtxq_lock(&uq->uq_key); in do_sem2_wait()
3695 umtxq_busy(&uq->uq_key); in do_sem2_wait()
3697 umtxq_unlock(&uq->uq_key); in do_sem2_wait()
3698 rv = fueword32(&sem->_count, &count); in do_sem2_wait()
3699 if (rv == -1) { in do_sem2_wait()
3700 umtxq_lock(&uq->uq_key); in do_sem2_wait()
3701 umtxq_unbusy(&uq->uq_key); in do_sem2_wait()
3703 umtxq_unlock(&uq->uq_key); in do_sem2_wait()
3704 umtx_key_release(&uq->uq_key); in do_sem2_wait()
3709 umtxq_lock(&uq->uq_key); in do_sem2_wait()
3710 umtxq_unbusy(&uq->uq_key); in do_sem2_wait()
3712 umtxq_unlock(&uq->uq_key); in do_sem2_wait()
3713 umtx_key_release(&uq->uq_key); in do_sem2_wait()
3718 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); in do_sem2_wait()
3721 umtxq_lock(&uq->uq_key); in do_sem2_wait()
3722 umtxq_unbusy(&uq->uq_key); in do_sem2_wait()
3724 umtxq_unlock(&uq->uq_key); in do_sem2_wait()
3725 umtx_key_release(&uq->uq_key); in do_sem2_wait()
3726 if (rv == -1) in do_sem2_wait()
3733 umtxq_lock(&uq->uq_key); in do_sem2_wait()
3734 umtxq_unbusy(&uq->uq_key); in do_sem2_wait()
3738 if ((uq->uq_flags & UQF_UMTXQ) == 0) in do_sem2_wait()
3742 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) { in do_sem2_wait()
3743 /* A relative timeout cannot be restarted. */ in do_sem2_wait()
3750 &timeout->_timeout); in do_sem2_wait()
3754 umtxq_unlock(&uq->uq_key); in do_sem2_wait()
3755 umtx_key_release(&uq->uq_key); in do_sem2_wait()
3769 rv = fueword32(&sem->_flags, &flags); in do_sem2_wake()
3770 if (rv == -1) in do_sem2_wake()
3784 rv = fueword32(&sem->_count, &count); in do_sem2_wake()
3785 while (rv != -1 && count & USEM_HAS_WAITERS) { in do_sem2_wake()
3786 rv = casueword32(&sem->_count, count, &count, in do_sem2_wake()
3794 if (rv == -1) in do_sem2_wake()
3814 return (do_lock_umtx(td, uap->umtx, td->td_tid, 0)); in freebsd10__umtx_lock()
3821 return (do_unlock_umtx(td, uap->umtx, td->td_tid)); in freebsd10__umtx_unlock()
3843 if (size <= sizeof(tp->_timeout)) { in umtx_copyin_umtx_time()
3844 tp->_clockid = CLOCK_REALTIME; in umtx_copyin_umtx_time()
3845 tp->_flags = 0; in umtx_copyin_umtx_time()
3846 error = copyin(uaddr, &tp->_timeout, sizeof(tp->_timeout)); in umtx_copyin_umtx_time()
3851 if (!timespecvalid_interval(&tp->_timeout)) in umtx_copyin_umtx_time()
3871 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time) in umtx_copyout_timeout()
3890 if (uap->uaddr2 == NULL) in __umtx_op_lock_umtx()
3893 error = ops->copyin_timeout(uap->uaddr2, &timeout); in __umtx_op_lock_umtx()
3899 if (ops->compat32) in __umtx_op_lock_umtx()
3900 return (do_lock_umtx32(td, uap->obj, uap->val, ts)); in __umtx_op_lock_umtx()
3902 return (do_lock_umtx(td, uap->obj, uap->val, ts)); in __umtx_op_lock_umtx()
3910 if (ops->compat32) in __umtx_op_unlock_umtx()
3911 return (do_unlock_umtx32(td, uap->obj, uap->val)); in __umtx_op_unlock_umtx()
3913 return (do_unlock_umtx(td, uap->obj, uap->val)); in __umtx_op_unlock_umtx()
3933 if (uap->uaddr2 == NULL) in __umtx_op_wait()
3936 error = ops->copyin_umtx_time( in __umtx_op_wait()
3937 uap->uaddr2, (size_t)uap->uaddr1, &timeout); in __umtx_op_wait()
3942 return (do_wait(td, uap->obj, uap->val, tm_p, ops->compat32, 0)); in __umtx_op_wait()
3952 if (uap->uaddr2 == NULL) in __umtx_op_wait_uint()
3955 error = ops->copyin_umtx_time( in __umtx_op_wait_uint()
3956 uap->uaddr2, (size_t)uap->uaddr1, &timeout); in __umtx_op_wait_uint()
3961 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); in __umtx_op_wait_uint()
3971 if (uap->uaddr2 == NULL) in __umtx_op_wait_uint_private()
3974 error = ops->copyin_umtx_time( in __umtx_op_wait_uint_private()
3975 uap->uaddr2, (size_t)uap->uaddr1, &timeout); in __umtx_op_wait_uint_private()
3980 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); in __umtx_op_wait_uint_private()
3988 return (kern_umtx_wake(td, uap->obj, uap->val, 0)); in __umtx_op_wake()
3998 upp = (char **)uap->obj; in __umtx_op_nwake_private_native()
4000 for (count = uap->val, pos = 0; count > 0; count -= tocopy, in __umtx_op_nwake_private_native()
4020 upp = (uint32_t *)uap->obj; in __umtx_op_nwake_private_compat32()
4022 for (count = uap->val, pos = 0; count > 0; count -= tocopy, in __umtx_op_nwake_private_compat32()
4042 if (ops->compat32) in __umtx_op_nwake_private()
4052 return (kern_umtx_wake(td, uap->obj, uap->val, 1)); in __umtx_op_wake_private()
4063 if (uap->uaddr2 == NULL) in __umtx_op_lock_umutex()
4066 error = ops->copyin_umtx_time( in __umtx_op_lock_umutex()
4067 uap->uaddr2, (size_t)uap->uaddr1, &timeout); in __umtx_op_lock_umutex()
4072 return (do_lock_umutex(td, uap->obj, tm_p, 0)); in __umtx_op_lock_umutex()
4080 return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY)); in __umtx_op_trylock_umutex()
4091 if (uap->uaddr2 == NULL) in __umtx_op_wait_umutex()
4094 error = ops->copyin_umtx_time( in __umtx_op_wait_umutex()
4095 uap->uaddr2, (size_t)uap->uaddr1, &timeout); in __umtx_op_wait_umutex()
4100 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); in __umtx_op_wait_umutex()
4108 return (do_wake_umutex(td, uap->obj)); in __umtx_op_wake_umutex()
4116 return (do_unlock_umutex(td, uap->obj, false)); in __umtx_op_unlock_umutex()
4124 return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1)); in __umtx_op_set_ceiling()
4135 if (uap->uaddr2 == NULL) in __umtx_op_cv_wait()
4138 error = ops->copyin_timeout(uap->uaddr2, &timeout); in __umtx_op_cv_wait()
4143 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); in __umtx_op_cv_wait()
4151 return (do_cv_signal(td, uap->obj)); in __umtx_op_cv_signal()
4159 return (do_cv_broadcast(td, uap->obj)); in __umtx_op_cv_broadcast()
4170 if (uap->uaddr2 == NULL) { in __umtx_op_rw_rdlock()
4171 error = do_rw_rdlock(td, uap->obj, uap->val, 0); in __umtx_op_rw_rdlock()
4173 error = ops->copyin_umtx_time(uap->uaddr2, in __umtx_op_rw_rdlock()
4174 (size_t)uap->uaddr1, &timeout); in __umtx_op_rw_rdlock()
4177 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); in __umtx_op_rw_rdlock()
4190 if (uap->uaddr2 == NULL) { in __umtx_op_rw_wrlock()
4191 error = do_rw_wrlock(td, uap->obj, 0); in __umtx_op_rw_wrlock()
4193 error = ops->copyin_umtx_time(uap->uaddr2, in __umtx_op_rw_wrlock()
4194 (size_t)uap->uaddr1, &timeout); in __umtx_op_rw_wrlock()
4198 error = do_rw_wrlock(td, uap->obj, &timeout); in __umtx_op_rw_wrlock()
4208 return (do_rw_unlock(td, uap->obj)); in __umtx_op_rw_unlock()
4220 if (uap->uaddr2 == NULL) in __umtx_op_sem_wait()
4223 error = ops->copyin_umtx_time( in __umtx_op_sem_wait()
4224 uap->uaddr2, (size_t)uap->uaddr1, &timeout); in __umtx_op_sem_wait()
4229 return (do_sem_wait(td, uap->obj, tm_p)); in __umtx_op_sem_wait()
4237 return (do_sem_wake(td, uap->obj)); in __umtx_op_sem_wake()
4246 return (do_wake2_umutex(td, uap->obj, uap->val)); in __umtx_op_wake2_umutex()
4258 if (uap->uaddr2 == NULL) { in __umtx_op_sem2_wait()
4262 uasize = (size_t)uap->uaddr1; in __umtx_op_sem2_wait()
4263 error = ops->copyin_umtx_time(uap->uaddr2, uasize, &timeout); in __umtx_op_sem2_wait()
4268 error = do_sem2_wait(td, uap->obj, tm_p); in __umtx_op_sem2_wait()
4269 if (error == EINTR && uap->uaddr2 != NULL && in __umtx_op_sem2_wait()
4271 uasize >= ops->umtx_time_sz + ops->timespec_sz) { in __umtx_op_sem2_wait()
4272 error = ops->copyout_timeout( in __umtx_op_sem2_wait()
4273 (void *)((uintptr_t)uap->uaddr2 + ops->umtx_time_sz), in __umtx_op_sem2_wait()
4274 uasize - ops->umtx_time_sz, &timeout._timeout); in __umtx_op_sem2_wait()
4288 return (do_sem2_wake(td, uap->obj)); in __umtx_op_sem2_wake()
4292 ((struct umtx_shm_obj_list *)(&(o)->umtx_data))
4350 KASSERT(key->shared, ("umtx_p_find_rg: private key")); in umtx_shm_find_reg_locked()
4352 reg_head = &umtx_shm_registry[key->hash]; in umtx_shm_find_reg_locked()
4354 KASSERT(reg->ushm_key.shared, in umtx_shm_find_reg_locked()
4355 ("non-shared key on reg %p %d", reg, reg->ushm_key.shared)); in umtx_shm_find_reg_locked()
4356 if (reg->ushm_key.info.shared.object == in umtx_shm_find_reg_locked()
4357 key->info.shared.object && in umtx_shm_find_reg_locked()
4358 reg->ushm_key.info.shared.offset == in umtx_shm_find_reg_locked()
4359 key->info.shared.offset) { in umtx_shm_find_reg_locked()
4360 KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM")); in umtx_shm_find_reg_locked()
4361 KASSERT(reg->ushm_refcnt != 0, in umtx_shm_find_reg_locked()
4363 KASSERT((reg->ushm_flags & USHMF_LINKED) != 0, in umtx_shm_find_reg_locked()
4371 if (__predict_false(reg->ushm_refcnt == UINT_MAX)) in umtx_shm_find_reg_locked()
4373 reg->ushm_refcnt++; in umtx_shm_find_reg_locked()
4399 chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0); in umtx_shm_free_reg()
4400 crfree(reg->ushm_cred); in umtx_shm_free_reg()
4401 shm_drop(reg->ushm_obj); in umtx_shm_free_reg()
4409 KASSERT(reg->ushm_refcnt != 0, ("ushm_reg %p refcnt 0", reg)); in umtx_shm_unref_reg_locked()
4412 if ((reg->ushm_flags & USHMF_LINKED) == 0) in umtx_shm_unref_reg_locked()
4419 TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], reg, in umtx_shm_unref_reg_locked()
4422 reg->ushm_flags &= ~USHMF_LINKED; in umtx_shm_unref_reg_locked()
4425 reg->ushm_refcnt--; in umtx_shm_unref_reg_locked()
4426 return (reg->ushm_refcnt == 0); in umtx_shm_unref_reg_locked()
4438 * shared-memory VM object in presence of concurrent callers in umtx_shm_unref_reg()
4442 object = reg->ushm_obj->shm_object; in umtx_shm_unref_reg()
4504 cred = td->td_ucred; in umtx_shm_create_reg()
4505 if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP))) in umtx_shm_create_reg()
4508 bcopy(key, ®->ushm_key, sizeof(*key)); in umtx_shm_create_reg()
4509 reg->ushm_obj = shm_alloc(td->td_ucred, O_RDWR, false); in umtx_shm_create_reg()
4510 reg->ushm_cred = crhold(cred); in umtx_shm_create_reg()
4511 error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE); in umtx_shm_create_reg()
4517 /* Re-lookup as 'umtx_shm_lock' has been temporarily released. */ in umtx_shm_create_reg()
4532 TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link); in umtx_shm_create_reg()
4533 LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg, in umtx_shm_create_reg()
4535 reg->ushm_flags = USHMF_LINKED; in umtx_shm_create_reg()
4542 reg->ushm_refcnt = 2; in umtx_shm_create_reg()
4559 map = &td->td_proc->p_vmspace->vm_map; in umtx_shm_alive()
4567 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0; in umtx_shm_alive()
4600 KASSERT(key.shared == 1, ("non-shared key")); in umtx_shm()
4613 error = mac_posixshm_check_open(td->td_ucred, in umtx_shm()
4614 reg->ushm_obj, FFLAGS(O_RDWR)); in umtx_shm()
4617 error = shm_access(reg->ushm_obj, td->td_ucred, in umtx_shm()
4623 shm_hold(reg->ushm_obj); in umtx_shm()
4624 finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj, in umtx_shm()
4626 td->td_retval[0] = fd; in umtx_shm()
4639 return (umtx_shm(td, uap->uaddr1, uap->val)); in __umtx_op_shm()
4649 if (ops->compat32) { in __umtx_op_robust_lists()
4650 if ((td->td_pflags2 & TDP2_COMPAT32RB) == 0 && in __umtx_op_robust_lists()
4651 (td->td_rb_list != 0 || td->td_rbp_list != 0 || in __umtx_op_robust_lists()
4652 td->td_rb_inact != 0)) in __umtx_op_robust_lists()
4654 } else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) { in __umtx_op_robust_lists()
4659 error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb); in __umtx_op_robust_lists()
4663 if (ops->compat32) in __umtx_op_robust_lists()
4664 td->td_pflags2 |= TDP2_COMPAT32RB; in __umtx_op_robust_lists()
4666 td->td_rb_list = rb.robust_list_offset; in __umtx_op_robust_lists()
4667 td->td_rbp_list = rb.robust_priv_list_offset; in __umtx_op_robust_lists()
4668 td->td_rb_inact = rb.robust_inact_offset; in __umtx_op_robust_lists()
4679 val = sbttons(td->td_proc->p_umtx_min_timeout); in __umtx_op_get_min_timeout()
4680 if (ops->compat32) { in __umtx_op_get_min_timeout()
4682 error = copyout(&val1, uap->uaddr1, sizeof(val1)); in __umtx_op_get_min_timeout()
4684 error = copyout(&val, uap->uaddr1, sizeof(val)); in __umtx_op_get_min_timeout()
4693 if (uap->val < 0) in __umtx_op_set_min_timeout()
4695 td->td_proc->p_umtx_min_timeout = nstosbt(uap->val); in __umtx_op_set_min_timeout()
4701 * Provide the standard 32-bit definitions for x86, since native/compat32 use a
4702 * 32-bit time_t there. Other architectures just need the i386 definitions
4721 /* 32-bit architectures can emulate i386, so define these almost everywhere. */
4803 .tv_sec = tsp->tv_sec, in umtx_copyout_timeouti386()
4804 .tv_nsec = tsp->tv_nsec, in umtx_copyout_timeouti386()
4808 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time) in umtx_copyout_timeouti386()
4864 .tv_sec = tsp->tv_sec, in umtx_copyout_timeoutx32()
4865 .tv_nsec = tsp->tv_nsec, in umtx_copyout_timeoutx32()
4869 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time) in umtx_copyout_timeoutx32()
4947 /* i386 can emulate other 32-bit archs, too! */
4993 if ((uap->op & (UMTX_OP__32BIT | UMTX_OP__I386)) != 0) { in sys__umtx_op()
4994 if ((uap->op & UMTX_OP__I386) != 0) in sys__umtx_op()
5001 if ((uap->op & UMTX_OP__I386) != 0) in sys__umtx_op()
5005 if ((uap->op & UMTX_OP__32BIT) != 0) in sys__umtx_op()
5008 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1, in sys__umtx_op()
5009 uap->uaddr2, umtx_ops)); in sys__umtx_op()
5018 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL)); in freebsd10_freebsd32__umtx_lock()
5025 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid)); in freebsd10_freebsd32__umtx_unlock()
5033 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1, in freebsd32__umtx_op()
5034 uap->uaddr2, &umtx_native_ops32)); in freebsd32__umtx_op()
5042 td->td_umtxq = umtxq_alloc(); in umtx_thread_init()
5043 td->td_umtxq->uq_thread = td; in umtx_thread_init()
5050 umtxq_free(td->td_umtxq); in umtx_thread_fini()
5061 uq = td->td_umtxq; in umtx_thread_alloc()
5062 uq->uq_inherited_pri = PRI_MAX; in umtx_thread_alloc()
5064 KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); in umtx_thread_alloc()
5065 KASSERT(uq->uq_thread == td, ("uq_thread != td")); in umtx_thread_alloc()
5066 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); in umtx_thread_alloc()
5067 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); in umtx_thread_alloc()
5083 KASSERT((p->p_flag & P_HADTHREADS) == 0 || in umtx_exec()
5084 (p->p_flag & P_STOPPED_SINGLE) != 0, in umtx_exec()
5085 ("curproc must be single-threaded")); in umtx_exec()
5092 ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)), in umtx_exec()
5095 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0; in umtx_exec()
5098 p->p_umtx_min_timeout = 0; in umtx_exec()
5142 *rb_list = m->m_rb_lnk; in umtx_read_rb_list()
5153 KASSERT(td->td_proc == curproc, ("need current vmspace")); in umtx_handle_rb()
5161 if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid) in umtx_handle_rb()
5188 td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb); in umtx_cleanup_rb_list()
5192 td->td_proc->p_comm, td->td_proc->p_pid, name, error); in umtx_cleanup_rb_list()
5210 uq = td->td_umtxq; in umtx_thread_cleanup()
5212 if (uq->uq_inherited_pri != PRI_MAX || in umtx_thread_cleanup()
5213 !TAILQ_EMPTY(&uq->uq_pi_contested)) { in umtx_thread_cleanup()
5215 uq->uq_inherited_pri = PRI_MAX; in umtx_thread_cleanup()
5216 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { in umtx_thread_cleanup()
5217 pi->pi_owner = NULL; in umtx_thread_cleanup()
5218 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); in umtx_thread_cleanup()
5225 compat32 = (td->td_pflags2 & TDP2_COMPAT32RB) != 0; in umtx_thread_cleanup()
5226 td->td_pflags2 &= ~TDP2_COMPAT32RB; in umtx_thread_cleanup()
5228 if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0) in umtx_thread_cleanup()
5236 rb_inact = td->td_rb_inact; in umtx_thread_cleanup()
5239 umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, "", compat32); in umtx_thread_cleanup()
5240 umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv ", compat32); in umtx_thread_cleanup()