Lines Matching +full:waking +full:- +full:up
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
65 _Static_assert((PRILASTFLAG * 2) - 1 <= USHRT_MAX,
78 #define TD_SLOCKS_INC(td) ((td)->td_lk_slocks++)
79 #define TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--)
86 #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
87 #define STACK_SAVE(lk) stack_save(&(lk)->lk_stack)
88 #define STACK_ZERO(lk) stack_zero(&(lk)->lk_stack)
92 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
95 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
103 while (_i--) \
127 if ((curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||
128 (curthread->td_pflags & TDP_DEADLKTREAT))
195 class->lc_unlock(ilk);
206 LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
207 WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
217 WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
218 LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
230 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
231 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
242 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
245 LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
281 class->lc_unlock(ilk);
283 if (lk->lk_exslpfail < USHRT_MAX)
284 lk->lk_exslpfail++;
287 sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
290 sleepq_set_timeout(&lk->lock_object, timo);
296 error = sleepq_timedwait_sig(&lk->lock_object, pri);
298 error = sleepq_timedwait(&lk->lock_object, pri);
300 error = sleepq_wait_sig(&lk->lock_object, pri);
302 sleepq_wait(&lk->lock_object, pri);
326 sleepq_lock(&lk->lock_object);
334 * order to avoid deadlock with shared runners up.
337 * queue by giving them precedence and cleaning up the
345 realexslp = sleepq_sleepcnt(&lk->lock_object,
348 if (lk->lk_exslpfail != USHRT_MAX && lk->lk_exslpfail < realexslp) {
349 lk->lk_exslpfail = 0;
353 lk->lk_exslpfail = 0;
358 "%s: %p waking up threads on the exclusive queue",
360 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
369 * it up anyway.
371 lk->lk_exslpfail = 0;
376 sleepq_release(&lk->lock_object);
381 if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
385 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
388 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
389 sleepq_release(&lk->lock_object);
432 ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
434 &lk->lk_lock));
453 lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
454 lk->lk_lock = LK_UNLOCKED;
455 lk->lk_recurse = 0;
456 lk->lk_exslpfail = 0;
457 lk->lk_timo = timo;
458 lk->lk_pri = pri;
471 lk->lock_object.lo_flags &= ~LK_NOSHARE;
479 lk->lock_object.lo_flags |= LK_NOSHARE;
487 lk->lock_object.lo_flags |= LO_RECURSABLE;
495 lk->lock_object.lo_flags &= ~LO_RECURSABLE;
502 KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
503 KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
504 KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
505 lock_destroy(&lk->lock_object);
520 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
534 if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
535 *xp - LK_ONE_SHARER))
600 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
622 lock_profile_obtain_lock_failed(&lk->lock_object, false,
635 * If the lock is expected to not sleep just give up
649 sleepq_lock(&lk->lock_object);
658 sleepq_release(&lk->lock_object);
667 if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
676 iwmesg = lk->lock_object.lo_name;
677 ipri = lk->lk_pri;
678 itimo = lk->lk_timo;
680 iwmesg = lwa->iwmesg;
681 ipri = lwa->ipri;
682 itimo = lwa->itimo;
691 sleep_time -= lockstat_nsecs(&lk->lock_object);
696 sleep_time += lockstat_nsecs(&lk->lock_object);
786 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
796 (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
799 * give up and return.
810 class->lc_unlock(ilk);
816 atomic_set_ptr(&lk->lk_lock, LK_WRITER_RECURSED);
817 lk->lk_recurse++;
819 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
820 lk->lk_recurse, file, line);
821 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
833 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x, tid))
838 lock_profile_obtain_lock_failed(&lk->lock_object, false,
850 * If the lock is expected to not sleep just give up
864 sleepq_lock(&lk->lock_object);
873 sleepq_release(&lk->lock_object);
889 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
891 sleepq_release(&lk->lock_object);
905 if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
914 iwmesg = lk->lock_object.lo_name;
915 ipri = lk->lk_pri;
916 itimo = lk->lk_timo;
918 iwmesg = lwa->iwmesg;
919 ipri = lwa->ipri;
920 itimo = lwa->itimo;
929 sleep_time -= lockstat_nsecs(&lk->lock_object);
934 sleep_time += lockstat_nsecs(&lk->lock_object);
993 if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &v,
994 v - LK_ONE_SHARER)) {
1009 if (atomic_fcmpset_ptr(&lk->lk_lock, &v, setv)) {
1010 LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
1012 WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
1045 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
1047 if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
1061 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1066 atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
1084 class->lc_unlock(ilk);
1119 * Fix-up the tid var if the lock has been disowned.
1130 lk->lk_recurse--;
1131 if (lk->lk_recurse == 0)
1132 atomic_clear_ptr(&lk->lk_lock, LK_WRITER_RECURSED);
1139 if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
1142 sleepq_lock(&lk->lock_object);
1149 * shared runners up.
1153 * and cleaning up the exclusive waiters bit anyway.
1162 realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
1164 if (lk->lk_exslpfail != USHRT_MAX && lk->lk_exslpfail < realexslp) {
1165 lk->lk_exslpfail = 0;
1169 lk->lk_exslpfail = 0;
1174 "%s: %p waking up threads on the exclusive queue",
1176 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
1185 * on, so clean it up anyway.
1187 lk->lk_exslpfail = 0;
1191 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
1194 atomic_store_rel_ptr(&lk->lk_lock, v);
1195 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
1196 sleepq_release(&lk->lock_object);
1221 MPASS((lk->lock_object.lo_flags & LK_NOSHARE) == 0);
1224 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
1244 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1247 if (atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
1278 if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
1309 iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
1310 ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
1311 itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
1318 KASSERT((op & (op - 1)) == 0,
1329 lk->lock_object.lo_name, file, line));
1333 if (lk->lock_object.lo_flags & LK_NOSHARE) {
1344 class->lc_unlock(ilk);
1362 WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
1369 class->lc_unlock(ilk);
1382 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1387 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
1404 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1414 class->lc_unlock(ilk);
1420 if (lk->lk_lock == LK_UNLOCKED &&
1421 atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
1427 lock_profile_obtain_lock_failed(&lk->lock_object, false,
1431 * If the lock is expected to not sleep just give up
1445 sleepq_lock(&lk->lock_object);
1453 sleepq_release(&lk->lock_object);
1465 * giving them precedence and cleaning up the
1484 * counts on, so clean it up anyway.
1487 lk->lk_exslpfail = 0;
1493 sleepq_sleepcnt(&lk->lock_object,
1495 if (lk->lk_exslpfail >= realexslp) {
1496 lk->lk_exslpfail = 0;
1504 "%s: %p waking up threads on the exclusive queue",
1507 &lk->lock_object,
1512 lk->lk_exslpfail = 0;
1514 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
1515 sleepq_release(&lk->lock_object);
1519 "%s: %p waking up all threads on the %s queue",
1522 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
1526 * If shared waiters have been woken up we need
1532 for (v = lk->lk_lock;
1534 v = lk->lk_lock)
1544 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
1546 sleepq_release(&lk->lock_object);
1559 class->lc_unlock(ilk);
1563 sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
1565 sleepq_wait(&lk->lock_object, ipri & PRIMASK);
1572 lock_profile_obtain_lock_success(&lk->lock_object,
1574 LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
1575 lk->lk_recurse, file, line);
1576 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
1584 class->lc_unlock(ilk);
1589 class->lc_unlock(ilk);
1615 if (LK_HOLDER(lk->lk_lock) != tid)
1617 lock_profile_release_lock(&lk->lock_object, false);
1619 LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
1620 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
1631 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1644 if (lk->lk_lock == LK_UNLOCKED)
1645 printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
1646 else if (lk->lk_lock & LK_SHARE)
1648 lk->lock_object.lo_name,
1649 (uintmax_t)LK_SHARERS(lk->lk_lock));
1654 lk->lock_object.lo_name);
1657 "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
1658 td, td->td_proc->p_pid, td->td_proc->p_comm,
1659 td->td_tid);
1662 x = lk->lk_lock;
1726 if (slocked || (lk->lk_lock & LK_SHARE)) {
1727 witness_assert(&lk->lock_object, what, file, line);
1731 if (lk->lk_lock == LK_UNLOCKED ||
1732 ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
1735 lk->lock_object.lo_name, slocked ? "share" : "",
1738 if ((lk->lk_lock & LK_SHARE) == 0) {
1742 lk->lock_object.lo_name, file,
1746 lk->lock_object.lo_name, file, line);
1754 lk->lock_object.lo_name, file, line);
1758 lk->lock_object.lo_name, file, line);
1761 lk->lock_object.lo_name, file, line);
1766 lk->lock_object.lo_name, file, line);
1781 lk = td->td_wchan;
1783 if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
1785 db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
1786 if (lk->lk_lock & LK_SHARE)
1788 (uintmax_t)LK_SHARERS(lk->lk_lock));
1805 if (lk->lk_lock == LK_UNLOCKED)
1807 else if (lk->lk_lock & LK_SHARE)
1808 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
1815 td->td_tid, td->td_proc->p_pid,
1816 td->td_proc->p_comm);
1818 db_printf(" recursed: %d\n", lk->lk_recurse);
1821 switch (lk->lk_lock & LK_ALL_WAITERS) {
1835 if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)