Lines Matching refs:td
148 static void thread_unsuspend_one(struct thread *td, struct proc *p,
150 static void thread_free_batched(struct thread *td);
306 tidbatch_add(struct tidbatch *tb, struct thread *td) in tidbatch_add() argument
311 tb->tab[tb->n] = td->td_tid; in tidbatch_add()
353 tdcountbatch_add(struct tdcountbatch *tb, struct thread *td __unused) in tdcountbatch_add()
384 struct thread *td; in thread_ctor() local
386 td = (struct thread *)mem; in thread_ctor()
387 TD_SET_STATE(td, TDS_INACTIVE); in thread_ctor()
388 td->td_lastcpu = td->td_oncpu = NOCPU; in thread_ctor()
395 td->td_critnest = 1; in thread_ctor()
396 td->td_lend_user_pri = PRI_MAX; in thread_ctor()
398 audit_thread_alloc(td); in thread_ctor()
401 kdtrace_thread_ctor(td); in thread_ctor()
403 umtx_thread_alloc(td); in thread_ctor()
404 MPASS(td->td_sel == NULL); in thread_ctor()
414 struct thread *td; in thread_dtor() local
416 td = (struct thread *)mem; in thread_dtor()
420 switch (TD_GET_STATE(td)) { in thread_dtor()
439 audit_thread_free(td); in thread_dtor()
442 kdtrace_thread_dtor(td); in thread_dtor()
445 osd_thread_exit(td); in thread_dtor()
446 ast_kclear(td); in thread_dtor()
447 seltdfini(td); in thread_dtor()
456 struct thread *td; in thread_init() local
458 td = (struct thread *)mem; in thread_init()
460 td->td_allocdomain = vm_phys_domain(vtophys(td)); in thread_init()
461 td->td_sleepqueue = sleepq_alloc(); in thread_init()
462 td->td_turnstile = turnstile_alloc(); in thread_init()
463 EVENTHANDLER_DIRECT_INVOKE(thread_init, td); in thread_init()
464 umtx_thread_init(td); in thread_init()
465 td->td_kstack = 0; in thread_init()
466 td->td_sel = NULL; in thread_init()
476 struct thread *td; in thread_fini() local
478 td = (struct thread *)mem; in thread_fini()
479 EVENTHANDLER_DIRECT_INVOKE(thread_fini, td); in thread_fini()
480 turnstile_free(td->td_turnstile); in thread_fini()
481 sleepq_free(td->td_sleepqueue); in thread_fini()
482 umtx_thread_fini(td); in thread_fini()
483 MPASS(td->td_sel == NULL); in thread_fini()
495 proc_linkup0(struct proc *p, struct thread *td) in proc_linkup0() argument
498 proc_linkup(p, td); in proc_linkup0()
502 proc_linkup(struct proc *p, struct thread *td) in proc_linkup() argument
513 thread_link(td, p); in proc_linkup()
517 ast_suspend(struct thread *td, int tda __unused) in ast_suspend() argument
521 p = td->td_proc; in ast_suspend()
597 thread_zombie(struct thread *td) in thread_zombie() argument
602 tdd = &thread_domain_data[td->td_allocdomain]; in thread_zombie()
605 td->td_zombie = ztd; in thread_zombie()
607 (uintptr_t *)&ztd, (uintptr_t)td)) in thread_zombie()
617 thread_stash(struct thread *td) in thread_stash() argument
619 atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); in thread_stash()
620 thread_zombie(td); in thread_stash()
781 struct thread *td; in thread_alloc() local
789 td = uma_zalloc(thread_zone, M_WAITOK); in thread_alloc()
790 KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); in thread_alloc()
791 if (!vm_thread_new(td, pages)) { in thread_alloc()
792 uma_zfree(thread_zone, td); in thread_alloc()
797 td->td_tid = tid; in thread_alloc()
798 bzero(&td->td_sa.args, sizeof(td->td_sa.args)); in thread_alloc()
799 kasan_thread_alloc(td); in thread_alloc()
800 kmsan_thread_alloc(td); in thread_alloc()
801 cpu_thread_alloc(td); in thread_alloc()
802 EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td); in thread_alloc()
803 return (td); in thread_alloc()
807 thread_recycle(struct thread *td, int pages) in thread_recycle() argument
809 if (td->td_kstack == 0 || td->td_kstack_pages != pages) { in thread_recycle()
810 if (td->td_kstack != 0) in thread_recycle()
811 vm_thread_dispose(td); in thread_recycle()
812 if (!vm_thread_new(td, pages)) in thread_recycle()
814 cpu_thread_alloc(td); in thread_recycle()
816 kasan_thread_alloc(td); in thread_recycle()
817 kmsan_thread_alloc(td); in thread_recycle()
825 thread_free_batched(struct thread *td) in thread_free_batched() argument
828 lock_profile_thread_exit(td); in thread_free_batched()
829 if (td->td_cpuset) in thread_free_batched()
830 cpuset_rel(td->td_cpuset); in thread_free_batched()
831 td->td_cpuset = NULL; in thread_free_batched()
832 cpu_thread_free(td); in thread_free_batched()
833 if (td->td_kstack != 0) in thread_free_batched()
834 vm_thread_dispose(td); in thread_free_batched()
835 callout_drain(&td->td_slpcallout); in thread_free_batched()
839 td->td_tid = -1; in thread_free_batched()
840 kmsan_thread_free(td); in thread_free_batched()
841 uma_zfree(thread_zone, td); in thread_free_batched()
845 thread_free(struct thread *td) in thread_free() argument
849 EVENTHANDLER_DIRECT_INVOKE(thread_dtor, td); in thread_free()
850 tid = td->td_tid; in thread_free()
851 thread_free_batched(td); in thread_free()
868 thread_cow_get(struct thread *newtd, struct thread *td) in thread_cow_get() argument
871 MPASS(td->td_realucred == td->td_ucred); in thread_cow_get()
872 newtd->td_realucred = crcowget(td->td_realucred); in thread_cow_get()
874 newtd->td_limit = lim_hold(td->td_limit); in thread_cow_get()
875 newtd->td_cowgen = td->td_cowgen; in thread_cow_get()
879 thread_cow_free(struct thread *td) in thread_cow_free() argument
882 if (td->td_realucred != NULL) in thread_cow_free()
883 crcowfree(td); in thread_cow_free()
884 if (td->td_limit != NULL) in thread_cow_free()
885 lim_free(td->td_limit); in thread_cow_free()
889 thread_cow_update(struct thread *td) in thread_cow_update() argument
895 p = td->td_proc; in thread_cow_update()
899 td->td_cowgen = p->p_cowgen; in thread_cow_update()
908 thread_cow_synced(struct thread *td) in thread_cow_synced() argument
912 p = td->td_proc; in thread_cow_synced()
914 MPASS(td->td_cowgen != p->p_cowgen); in thread_cow_synced()
915 MPASS(td->td_ucred == p->p_ucred); in thread_cow_synced()
916 MPASS(td->td_limit == p->p_limit); in thread_cow_synced()
917 td->td_cowgen = p->p_cowgen; in thread_cow_synced()
933 struct thread *td; in thread_exit() local
937 td = curthread; in thread_exit()
938 p = td->td_proc; in thread_exit()
945 CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, in thread_exit()
946 (long)p->p_pid, td->td_name); in thread_exit()
948 KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); in thread_exit()
949 MPASS(td->td_realucred == td->td_ucred); in thread_exit()
956 cpu_thread_exit(td); in thread_exit()
968 atomic_add_int(&td->td_proc->p_exitthreads, 1); in thread_exit()
969 thread_unlink(td); in thread_exit()
971 sched_exit_thread(td2, td); in thread_exit()
986 PCPU_SET(deadthread, td); in thread_exit()
999 if (PMC_PROC_IS_USING_PMCS(td->td_proc)) { in thread_exit()
1000 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); in thread_exit()
1001 PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT, NULL); in thread_exit()
1003 PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT_LOG, NULL); in thread_exit()
1007 thread_lock(td); in thread_exit()
1013 td->td_runtime += runtime; in thread_exit()
1014 td->td_incruntime += runtime; in thread_exit()
1020 td->td_ru.ru_nvcsw++; in thread_exit()
1021 ruxagg_locked(p, td); in thread_exit()
1022 rucollect(&p->p_ru, &td->td_ru); in thread_exit()
1025 TD_SET_STATE(td, TDS_INACTIVE); in thread_exit()
1027 witness_thread_exit(td); in thread_exit()
1029 CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); in thread_exit()
1030 sched_throw(td); in thread_exit()
1042 struct thread *td; in thread_wait() local
1047 td = FIRST_THREAD_IN_PROC(p); in thread_wait()
1049 thread_lock(td); in thread_wait()
1050 thread_unlock(td); in thread_wait()
1051 lock_profile_thread_exit(td); in thread_wait()
1052 cpuset_rel(td->td_cpuset); in thread_wait()
1053 td->td_cpuset = NULL; in thread_wait()
1054 cpu_thread_clean(td); in thread_wait()
1055 thread_cow_free(td); in thread_wait()
1056 callout_drain(&td->td_slpcallout); in thread_wait()
1066 thread_link(struct thread *td, struct proc *p) in thread_link() argument
1074 TD_SET_STATE(td, TDS_INACTIVE); in thread_link()
1075 td->td_proc = p; in thread_link()
1076 td->td_flags = TDF_INMEM; in thread_link()
1078 LIST_INIT(&td->td_contested); in thread_link()
1079 LIST_INIT(&td->td_lprof[0]); in thread_link()
1080 LIST_INIT(&td->td_lprof[1]); in thread_link()
1082 SLIST_INIT(&td->td_epochs); in thread_link()
1084 sigqueue_init(&td->td_sigqueue, p); in thread_link()
1085 callout_init(&td->td_slpcallout, 1); in thread_link()
1086 TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist); in thread_link()
1095 thread_unlink(struct thread *td) in thread_unlink() argument
1097 struct proc *p = td->td_proc; in thread_unlink()
1101 MPASS(SLIST_EMPTY(&td->td_epochs)); in thread_unlink()
1104 TAILQ_REMOVE(&p->p_threads, td, td_plist); in thread_unlink()
1215 struct thread *td; in thread_single() local
1219 td = curthread; in thread_single()
1229 KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) || in thread_single()
1230 (mode != SINGLE_ALLPROC && td->td_proc == p), in thread_single()
1231 ("mode %d proc %p curproc %p", mode, p, td->td_proc)); in thread_single()
1250 if (p->p_singlethread != NULL && p->p_singlethread != td) in thread_single()
1267 p->p_singlethread = td; in thread_single()
1273 if (td2 == td) in thread_single()
1300 thread_suspend_switch(td, p); in thread_single()
1319 sched_relinquish(td); in thread_single()
1337 if (td2 == td) in thread_single()
1355 struct thread *td; in thread_suspend_check_needed() local
1357 td = curthread; in thread_suspend_check_needed()
1358 p = td->td_proc; in thread_suspend_check_needed()
1361 (td->td_dbgflags & TDB_SUSPEND) != 0)); in thread_suspend_check_needed()
1400 struct thread *td; in thread_suspend_check() local
1403 td = curthread; in thread_suspend_check()
1404 p = td->td_proc; in thread_suspend_check()
1417 if (p->p_singlethread == td) in thread_suspend_check()
1431 if ((td->td_flags & TDF_SBDRY) != 0) { in thread_suspend_check()
1434 KASSERT((td->td_flags & (TDF_SEINTR | TDF_SERESTART)) != in thread_suspend_check()
1437 return (TD_SBDRY_INTR(td) ? TD_SBDRY_ERRNO(td) : 0); in thread_suspend_check()
1445 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { in thread_suspend_check()
1453 (p->p_sysent->sv_thread_detach)(td); in thread_suspend_check()
1454 umtx_thread_exit(td); in thread_suspend_check()
1455 kern_thr_exit(td); in thread_suspend_check()
1469 thread_lock(td); in thread_suspend_check()
1474 thread_suspend_one(td); in thread_suspend_check()
1477 td->td_flags |= TDF_BOUNDARY; in thread_suspend_check()
1506 thread_check_susp(struct thread *td, bool sleep) in thread_check_susp() argument
1515 if (!td_ast_pending(td, TDA_SUSPEND)) in thread_check_susp()
1518 p = td->td_proc; in thread_check_susp()
1523 ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) in thread_check_susp()
1530 thread_suspend_switch(struct thread *td, struct proc *p) in thread_suspend_switch() argument
1533 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); in thread_suspend_switch()
1540 if (p == td->td_proc) { in thread_suspend_switch()
1545 thread_lock(td); in thread_suspend_switch()
1546 ast_unsched_locked(td, TDA_SUSPEND); in thread_suspend_switch()
1547 TD_SET_SUSPENDED(td); in thread_suspend_switch()
1548 sched_sleep(td, 0); in thread_suspend_switch()
1558 thread_suspend_one(struct thread *td) in thread_suspend_one() argument
1562 p = td->td_proc; in thread_suspend_one()
1564 THREAD_LOCK_ASSERT(td, MA_OWNED); in thread_suspend_one()
1565 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); in thread_suspend_one()
1567 ast_unsched_locked(td, TDA_SUSPEND); in thread_suspend_one()
1568 TD_SET_SUSPENDED(td); in thread_suspend_one()
1569 sched_sleep(td, 0); in thread_suspend_one()
1573 thread_unsuspend_one(struct thread *td, struct proc *p, bool boundary) in thread_unsuspend_one() argument
1576 THREAD_LOCK_ASSERT(td, MA_OWNED); in thread_unsuspend_one()
1577 KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); in thread_unsuspend_one()
1578 TD_CLR_SUSPENDED(td); in thread_unsuspend_one()
1579 td->td_flags &= ~TDF_ALLPROCSUSP; in thread_unsuspend_one()
1580 if (td->td_proc == p) { in thread_unsuspend_one()
1583 if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) { in thread_unsuspend_one()
1584 td->td_flags &= ~TDF_BOUNDARY; in thread_unsuspend_one()
1588 setrunnable(td, 0); in thread_unsuspend_one()
1592 thread_run_flash(struct thread *td) in thread_run_flash() argument
1596 p = td->td_proc; in thread_run_flash()
1599 if (TD_ON_SLEEPQ(td)) in thread_run_flash()
1600 sleepq_remove_nested(td); in thread_run_flash()
1602 thread_lock(td); in thread_run_flash()
1604 THREAD_LOCK_ASSERT(td, MA_OWNED); in thread_run_flash()
1605 KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); in thread_run_flash()
1607 TD_CLR_SUSPENDED(td); in thread_run_flash()
1612 setrunnable(td, 0); in thread_run_flash()
1621 struct thread *td; in thread_unsuspend() local
1626 FOREACH_THREAD_IN_PROC(p, td) { in thread_unsuspend()
1627 thread_lock(td); in thread_unsuspend()
1628 if (TD_IS_SUSPENDED(td)) in thread_unsuspend()
1629 thread_unsuspend_one(td, p, true); in thread_unsuspend()
1631 thread_unlock(td); in thread_unsuspend()
1653 struct thread *td; in thread_single_end() local
1680 FOREACH_THREAD_IN_PROC(p, td) { in thread_single_end()
1681 thread_lock(td); in thread_single_end()
1682 if (TD_IS_SUSPENDED(td)) in thread_single_end()
1683 thread_unsuspend_one(td, p, true); in thread_single_end()
1685 thread_unlock(td); in thread_single_end()
1710 struct thread *td; in tdfind_hash() local
1717 LIST_FOREACH(td, TIDHASH(tid), td_hash) { in tdfind_hash()
1718 if (td->td_tid != tid) { in tdfind_hash()
1722 p = td->td_proc; in tdfind_hash()
1724 td = NULL; in tdfind_hash()
1729 LIST_REMOVE(td, td_hash); in tdfind_hash()
1730 LIST_INSERT_HEAD(TIDHASH(td->td_tid), in tdfind_hash()
1731 td, td_hash); in tdfind_hash()
1741 if (td == NULL) in tdfind_hash()
1744 *tdp = td; in tdfind_hash()
1752 struct thread *td; in tdfind() local
1754 td = curthread; in tdfind()
1755 if (td->td_tid == tid) { in tdfind()
1756 if (pid != -1 && td->td_proc->p_pid != pid) in tdfind()
1758 PROC_LOCK(td->td_proc); in tdfind()
1759 return (td); in tdfind()
1763 if (!tdfind_hash(tid, pid, &p, &td)) in tdfind()
1766 if (td->td_tid != tid) { in tdfind()
1770 if (td->td_proc != p) { in tdfind()
1778 return (td); in tdfind()
1783 tidhash_add(struct thread *td) in tidhash_add() argument
1785 rw_wlock(TIDHASHLOCK(td->td_tid)); in tidhash_add()
1786 LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); in tidhash_add()
1787 rw_wunlock(TIDHASHLOCK(td->td_tid)); in tidhash_add()
1791 tidhash_remove(struct thread *td) in tidhash_remove() argument
1794 rw_wlock(TIDHASHLOCK(td->td_tid)); in tidhash_remove()
1795 LIST_REMOVE(td, td_hash); in tidhash_remove()
1796 rw_wunlock(TIDHASHLOCK(td->td_tid)); in tidhash_remove()