Lines Matching +full:de +full:- +full:asserts
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
79 * Asserts below verify the stability of struct thread and struct proc
185 if (nthreads_new >= maxthread - 100) { in thread_count_inc_try()
186 if (priv_check_cred(curthread->td_ucred, PRIV_MAXPROC) != 0 || in thread_count_inc_try()
214 curthread->td_ucred->cr_ruid, curproc->p_pid); in thread_count_inc()
247 if (tid == -1) { in tid_alloc()
251 KASSERT(tid != -1, ("unexpectedly ran out of IDs")); in tid_alloc()
267 tid = rtid - NO_PID; in tid_free_locked()
306 tb->n = 0; in tidbatch_prep()
313 KASSERT(tb->n < nitems(tb->tab), in tidbatch_add()
314 ("%s: count too high %d", __func__, tb->n)); in tidbatch_add()
315 tb->tab[tb->n] = td->td_tid; in tidbatch_add()
316 tb->n++; in tidbatch_add()
323 KASSERT(tb->n <= nitems(tb->tab), in tidbatch_process()
324 ("%s: count too high %d", __func__, tb->n)); in tidbatch_process()
325 if (tb->n == nitems(tb->tab)) { in tidbatch_process()
326 tid_free_batch(tb->tab, tb->n); in tidbatch_process()
327 tb->n = 0; in tidbatch_process()
335 KASSERT(tb->n <= nitems(tb->tab), in tidbatch_final()
336 ("%s: count too high %d", __func__, tb->n)); in tidbatch_final()
337 if (tb->n != 0) { in tidbatch_final()
338 tid_free_batch(tb->tab, tb->n); in tidbatch_final()
353 tb->n = 0; in tdcountbatch_prep()
360 tb->n++; in tdcountbatch_add()
367 if (tb->n == 32) { in tdcountbatch_process()
368 thread_count_sub(tb->n); in tdcountbatch_process()
369 tb->n = 0; in tdcountbatch_process()
377 if (tb->n != 0) { in tdcountbatch_final()
378 thread_count_sub(tb->n); in tdcountbatch_final()
392 td->td_lastcpu = td->td_oncpu = NOCPU; in thread_ctor()
399 td->td_critnest = 1; in thread_ctor()
400 td->td_lend_user_pri = PRI_MAX; in thread_ctor()
408 MPASS(td->td_sel == NULL); in thread_ctor()
455 * Initialize type-stable parts of a thread (when newly created).
464 td->td_allocdomain = vm_phys_domain(vtophys(td)); in thread_init()
465 td->td_sleepqueue = sleepq_alloc(); in thread_init()
466 td->td_turnstile = turnstile_alloc(); in thread_init()
469 td->td_kstack = 0; in thread_init()
470 td->td_sel = NULL; in thread_init()
475 * Tear down type-stable parts of a thread (just before being discarded).
484 turnstile_free(td->td_turnstile); in thread_fini()
485 sleepq_free(td->td_sleepqueue); in thread_fini()
487 MPASS(td->td_sel == NULL); in thread_fini()
501 TAILQ_INIT(&p->p_threads); /* all threads in proc */ in proc_linkup0()
509 sigqueue_init(&p->p_sigqueue, p); in proc_linkup()
510 p->p_ksi = ksiginfo_alloc(M_WAITOK); in proc_linkup()
511 if (p->p_ksi != NULL) { in proc_linkup()
513 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; in proc_linkup()
515 LIST_INIT(&p->p_mqnotifier); in proc_linkup()
516 p->p_numthreads = 0; in proc_linkup()
525 p = td->td_proc; in ast_suspend()
549 * Note that other factors may make the de facto limit much lower. in threadinit()
580 UMA_ALIGN_CACHE_AND_MASK(32 - 1), UMA_ZONE_NOFREE); in threadinit()
584 tidhashlock--; in threadinit()
606 tdd = &thread_domain_data[td->td_allocdomain]; in thread_zombie()
607 ztd = atomic_load_ptr(&tdd->tdd_zombies); in thread_zombie()
609 td->td_zombie = ztd; in thread_zombie()
610 if (atomic_fcmpset_rel_ptr((uintptr_t *)&tdd->tdd_zombies, in thread_zombie()
623 atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); in thread_stash()
643 if (tdd->tdd_zombies == NULL) in thread_reap_domain()
646 itd = (struct thread *)atomic_swap_ptr((uintptr_t *)&tdd->tdd_zombies, in thread_reap_domain()
655 tdd->tdd_reapticks = ticks; in thread_reap_domain()
663 ntd = itd->td_zombie; in thread_reap_domain()
736 lticks = tdd->tdd_reapticks; in thread_reap_callout_cb()
737 if (tdd->tdd_zombies != NULL && in thread_reap_callout_cb()
738 (u_int)(cticks - lticks) > 5 * hz) { in thread_reap_callout_cb()
794 KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); in thread_alloc()
801 td->td_tid = tid; in thread_alloc()
802 bzero(&td->td_sa.args, sizeof(td->td_sa.args)); in thread_alloc()
813 if (td->td_kstack == 0 || td->td_kstack_pages != pages) { in thread_recycle()
814 if (td->td_kstack != 0) in thread_recycle()
833 if (td->td_cpuset) in thread_free_batched()
834 cpuset_rel(td->td_cpuset); in thread_free_batched()
835 td->td_cpuset = NULL; in thread_free_batched()
837 if (td->td_kstack != 0) in thread_free_batched()
839 callout_drain(&td->td_slpcallout); in thread_free_batched()
843 td->td_tid = -1; in thread_free_batched()
854 tid = td->td_tid; in thread_free()
865 newtd->td_realucred = crcowget(p->p_ucred); in thread_cow_get_proc()
866 newtd->td_ucred = newtd->td_realucred; in thread_cow_get_proc()
867 newtd->td_limit = lim_hold(p->p_limit); in thread_cow_get_proc()
868 newtd->td_cowgen = p->p_cowgen; in thread_cow_get_proc()
875 MPASS(td->td_realucred == td->td_ucred); in thread_cow_get()
876 newtd->td_realucred = crcowget(td->td_realucred); in thread_cow_get()
877 newtd->td_ucred = newtd->td_realucred; in thread_cow_get()
878 newtd->td_limit = lim_hold(td->td_limit); in thread_cow_get()
879 newtd->td_cowgen = td->td_cowgen; in thread_cow_get()
886 if (td->td_realucred != NULL) in thread_cow_free()
888 if (td->td_limit != NULL) in thread_cow_free()
889 lim_free(td->td_limit); in thread_cow_free()
899 p = td->td_proc; in thread_cow_update()
903 td->td_cowgen = p->p_cowgen; in thread_cow_update()
916 p = td->td_proc; in thread_cow_synced()
918 MPASS(td->td_cowgen != p->p_cowgen); in thread_cow_synced()
919 MPASS(td->td_ucred == p->p_ucred); in thread_cow_synced()
920 MPASS(td->td_limit == p->p_limit); in thread_cow_synced()
921 td->td_cowgen = p->p_cowgen; in thread_cow_synced()
942 p = td->td_proc; in thread_exit()
950 (long)p->p_pid, td->td_name); in thread_exit()
952 KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); in thread_exit()
953 MPASS(td->td_realucred == td->td_ucred); in thread_exit()
970 if (p->p_flag & P_HADTHREADS) { in thread_exit()
971 if (p->p_numthreads > 1) { in thread_exit()
972 atomic_add_int(&td->td_proc->p_exitthreads, 1); in thread_exit()
983 if (p->p_numthreads == p->p_suspcount) { in thread_exit()
984 thread_lock(p->p_singlethread); in thread_exit()
985 thread_unsuspend_one(p->p_singlethread, in thread_exit()
1003 if (PMC_PROC_IS_USING_PMCS(td->td_proc)) { in thread_exit()
1021 runtime = new_switchtime - PCPU_GET(switchtime); in thread_exit()
1022 td->td_runtime += runtime; in thread_exit()
1023 td->td_incruntime += runtime; in thread_exit()
1029 td->td_ru.ru_nvcsw++; in thread_exit()
1031 rucollect(&p->p_ru, &td->td_ru); in thread_exit()
1054 KASSERT(p->p_numthreads == 1, ("multiple threads in thread_wait()")); in thread_wait()
1055 KASSERT(p->p_exitthreads == 0, ("p_exitthreads leaking")); in thread_wait()
1061 cpuset_rel(td->td_cpuset); in thread_wait()
1062 td->td_cpuset = NULL; in thread_wait()
1065 callout_drain(&td->td_slpcallout); in thread_wait()
1084 td->td_proc = p; in thread_link()
1085 td->td_flags = TDF_INMEM; in thread_link()
1087 LIST_INIT(&td->td_contested); in thread_link()
1088 LIST_INIT(&td->td_lprof[0]); in thread_link()
1089 LIST_INIT(&td->td_lprof[1]); in thread_link()
1091 SLIST_INIT(&td->td_epochs); in thread_link()
1093 sigqueue_init(&td->td_sigqueue, p); in thread_link()
1094 callout_init(&td->td_slpcallout, 1); in thread_link()
1095 TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist); in thread_link()
1096 p->p_numthreads++; in thread_link()
1106 struct proc *p = td->td_proc; in thread_unlink()
1110 MPASS(SLIST_EMPTY(&td->td_epochs)); in thread_unlink()
1113 TAILQ_REMOVE(&p->p_threads, td, td_plist); in thread_unlink()
1114 p->p_numthreads--; in thread_unlink()
1127 remaining = p->p_numthreads; in calc_remaining()
1129 remaining = p->p_numthreads - p->p_boundary_count; in calc_remaining()
1131 remaining = p->p_numthreads - p->p_suspcount; in calc_remaining()
1171 (td2->td_flags & TDF_BOUNDARY) == 0) { in weed_inhib()
1188 * un-suspend. in weed_inhib()
1191 (td2->td_flags & TDF_ALLPROCSUSP) == 0) { in weed_inhib()
1197 td2->td_flags |= TDF_ALLPROCSUSP; in weed_inhib()
1209 * Enforce single-threading.
1233 * If allowing non-ALLPROC singlethreading for non-curproc in thread_single()
1235 * adjusted to also account for td->td_proc != p. For now in thread_single()
1238 KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) || in thread_single()
1239 (mode != SINGLE_ALLPROC && td->td_proc == p), in thread_single()
1240 ("mode %d proc %p curproc %p", mode, p, td->td_proc)); in thread_single()
1249 while ((p->p_flag & P_STOPPED_SINGLE) != 0) { in thread_single()
1250 if ((p->p_flag2 & P2_WEXIT) != 0) in thread_single()
1252 msleep(&p->p_flag, &p->p_mtx, PCATCH, "thrsgl", 0); in thread_single()
1254 if ((p->p_flag & (P_STOPPED_SIG | P_TRACED)) != 0 || in thread_single()
1255 (p->p_flag2 & P2_WEXIT) != 0) in thread_single()
1257 } else if ((p->p_flag & P_HADTHREADS) == 0) in thread_single()
1259 if (p->p_singlethread != NULL && p->p_singlethread != td) in thread_single()
1263 p->p_flag |= P_SINGLE_EXIT; in thread_single()
1264 p->p_flag &= ~P_SINGLE_BOUNDARY; in thread_single()
1266 p->p_flag &= ~P_SINGLE_EXIT; in thread_single()
1268 p->p_flag |= P_SINGLE_BOUNDARY; in thread_single()
1270 p->p_flag &= ~P_SINGLE_BOUNDARY; in thread_single()
1273 p->p_flag |= P_TOTAL_STOP; in thread_single()
1274 p->p_flag |= P_STOPPED_SINGLE; in thread_single()
1276 p->p_singlethread = td; in thread_single()
1318 KASSERT(p->p_numthreads == 1, ("Unthreading with >1 threads")); in thread_single()
1319 p->p_singlethread = NULL; in thread_single()
1320 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS); in thread_single()
1325 while (p->p_exitthreads != 0) { in thread_single()
1349 KASSERT((td2->td_flags & TDF_BOUNDARY) != 0, in thread_single()
1367 p = td->td_proc; in thread_suspend_check_needed()
1369 return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 && in thread_suspend_check_needed()
1370 (td->td_dbgflags & TDB_SUSPEND) != 0)); in thread_suspend_check_needed()
1376 * single-thread event (e.g. fork).
1391 *---------------+--------------------+---------------------
1394 *---------------+--------------------+---------------------
1413 p = td->td_proc; in thread_suspend_check()
1418 KASSERT(p->p_singlethread != NULL, in thread_suspend_check()
1422 * single-threading. Single threader need not stop. in thread_suspend_check()
1423 * It is safe to access p->p_singlethread unlocked in thread_suspend_check()
1426 if (p->p_singlethread == td) in thread_suspend_check()
1429 if ((p->p_flag & P_SINGLE_EXIT) && return_instead) in thread_suspend_check()
1434 (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) in thread_suspend_check()
1440 if ((td->td_flags & TDF_SBDRY) != 0) { in thread_suspend_check()
1443 KASSERT((td->td_flags & (TDF_SEINTR | TDF_SERESTART)) != in thread_suspend_check()
1462 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { in thread_suspend_check()
1469 if (__predict_false(p->p_sysent->sv_thread_detach != NULL)) in thread_suspend_check()
1470 (p->p_sysent->sv_thread_detach)(td); in thread_suspend_check()
1479 if (p->p_numthreads == p->p_suspcount + 1) { in thread_suspend_check()
1480 thread_lock(p->p_singlethread); in thread_suspend_check()
1481 thread_unsuspend_one(p->p_singlethread, p, in thread_suspend_check()
1493 p->p_boundary_count++; in thread_suspend_check()
1494 td->td_flags |= TDF_BOUNDARY; in thread_suspend_check()
1514 * returned non-zero. On the other hand, retrying the whole lock
1534 p = td->td_proc; in thread_check_susp()
1536 if (p->p_flag & P_SINGLE_EXIT) in thread_check_susp()
1539 ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) in thread_check_susp()
1556 if (p == td->td_proc) { in thread_suspend_switch()
1558 p->p_suspcount++; in thread_suspend_switch()
1578 p = td->td_proc; in thread_suspend_one()
1582 p->p_suspcount++; in thread_suspend_one()
1595 td->td_flags &= ~TDF_ALLPROCSUSP; in thread_unsuspend_one()
1596 if (td->td_proc == p) { in thread_unsuspend_one()
1598 p->p_suspcount--; in thread_unsuspend_one()
1599 if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) { in thread_unsuspend_one()
1600 td->td_flags &= ~TDF_BOUNDARY; in thread_unsuspend_one()
1601 p->p_boundary_count--; in thread_unsuspend_one()
1612 p = td->td_proc; in thread_run_flash()
1625 MPASS(p->p_suspcount > 0); in thread_run_flash()
1626 p->p_suspcount--; in thread_run_flash()
1650 p->p_numthreads == p->p_suspcount) { in thread_unsuspend()
1653 * threading request. Now we've downgraded to single-threaded, in thread_unsuspend()
1656 if (p->p_singlethread->td_proc == p) { in thread_unsuspend()
1657 thread_lock(p->p_singlethread); in thread_unsuspend()
1658 thread_unsuspend_one(p->p_singlethread, p, false); in thread_unsuspend()
1675 KASSERT((mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) || in thread_single_end()
1676 (mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0), in thread_single_end()
1678 KASSERT(mode == SINGLE_ALLPROC || p->p_singlethread == curthread, in thread_single_end()
1680 curthread, p->p_singlethread)); in thread_single_end()
1682 (p->p_flag & P_SINGLE_BOUNDARY) != 0, in thread_single_end()
1683 ("mis-matched SINGLE_BOUNDARY flags %x", p->p_flag)); in thread_single_end()
1684 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY | in thread_single_end()
1687 p->p_singlethread = NULL; in thread_single_end()
1695 if (p->p_numthreads != remain_for_mode(mode) && !P_SHOULDSTOP(p)) { in thread_single_end()
1705 p->p_boundary_count == 0, in thread_single_end()
1707 p->p_pid, p, p->p_flag, p->p_boundary_count)); in thread_single_end()
1709 wakeup(&p->p_flag); in thread_single_end()
1715 * thread exit establishes proc -> tidhash lock ordering, but lookup
1718 * The problem is worked around by relying on type-safety of both
1720 * - tidhash-locked lookup which saves both thread and proc pointers
1721 * - proc-locked verification that the found thread still matches
1736 if (td->td_tid != tid) { in tdfind_hash()
1740 p = td->td_proc; in tdfind_hash()
1741 if (pid != -1 && p->p_pid != pid) { in tdfind_hash()
1748 LIST_INSERT_HEAD(TIDHASH(td->td_tid), in tdfind_hash()
1773 if (td->td_tid == tid) { in tdfind()
1774 if (pid != -1 && td->td_proc->p_pid != pid) in tdfind()
1776 PROC_LOCK(td->td_proc); in tdfind()
1784 if (td->td_tid != tid) { in tdfind()
1788 if (td->td_proc != p) { in tdfind()
1792 if (p->p_state == PRS_NEW) { in tdfind()
1803 rw_wlock(TIDHASHLOCK(td->td_tid)); in tidhash_add()
1804 LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); in tidhash_add()
1805 rw_wunlock(TIDHASHLOCK(td->td_tid)); in tidhash_add()
1812 rw_wlock(TIDHASHLOCK(td->td_tid)); in tidhash_remove()
1814 rw_wunlock(TIDHASHLOCK(td->td_tid)); in tidhash_remove()