xref: /freebsd/sys/kern/kern_thread.c (revision 5c274b362256b1e0f680dacd9ef5a78bb21fedcd)
19454b2d8SWarner Losh /*-
28a36da99SPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
38a36da99SPedro F. Giffuni  *
444990b8cSJulian Elischer  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
544990b8cSJulian Elischer  *  All rights reserved.
644990b8cSJulian Elischer  *
744990b8cSJulian Elischer  * Redistribution and use in source and binary forms, with or without
844990b8cSJulian Elischer  * modification, are permitted provided that the following conditions
944990b8cSJulian Elischer  * are met:
1044990b8cSJulian Elischer  * 1. Redistributions of source code must retain the above copyright
1144990b8cSJulian Elischer  *    notice(s), this list of conditions and the following disclaimer as
1244990b8cSJulian Elischer  *    the first lines of this file unmodified other than the possible
1344990b8cSJulian Elischer  *    addition of one or more copyright notices.
1444990b8cSJulian Elischer  * 2. Redistributions in binary form must reproduce the above copyright
1544990b8cSJulian Elischer  *    notice(s), this list of conditions and the following disclaimer in the
1644990b8cSJulian Elischer  *    documentation and/or other materials provided with the distribution.
1744990b8cSJulian Elischer  *
1844990b8cSJulian Elischer  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
1944990b8cSJulian Elischer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
2044990b8cSJulian Elischer  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
2144990b8cSJulian Elischer  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
2244990b8cSJulian Elischer  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
2344990b8cSJulian Elischer  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
2444990b8cSJulian Elischer  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
2544990b8cSJulian Elischer  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2644990b8cSJulian Elischer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2744990b8cSJulian Elischer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
2844990b8cSJulian Elischer  * DAMAGE.
2944990b8cSJulian Elischer  */
3044990b8cSJulian Elischer 
313d06b4b3SAttilio Rao #include "opt_witness.h"
3216d95d4fSJoseph Koshy #include "opt_hwpmc_hooks.h"
333d06b4b3SAttilio Rao 
34677b542eSDavid E. O'Brien #include <sys/cdefs.h>
35677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
36677b542eSDavid E. O'Brien 
3744990b8cSJulian Elischer #include <sys/param.h>
3844990b8cSJulian Elischer #include <sys/systm.h>
3944990b8cSJulian Elischer #include <sys/kernel.h>
4044990b8cSJulian Elischer #include <sys/lock.h>
415dda15adSMark Johnston #include <sys/msan.h>
4244990b8cSJulian Elischer #include <sys/mutex.h>
4344990b8cSJulian Elischer #include <sys/proc.h>
4435bb59edSMateusz Guzik #include <sys/bitstring.h>
456febf180SGleb Smirnoff #include <sys/epoch.h>
468f0e9130SKonstantin Belousov #include <sys/rangelock.h>
47e170bfdaSDavid Xu #include <sys/resourcevar.h>
48b3e9e682SRyan Stone #include <sys/sdt.h>
4994e0a4cdSJulian Elischer #include <sys/smp.h>
50de028f5aSJeff Roberson #include <sys/sched.h>
5144f3b092SJohn Baldwin #include <sys/sleepqueue.h>
52ace8398dSJeff Roberson #include <sys/selinfo.h>
53d1e7a4a5SJohn Baldwin #include <sys/syscallsubr.h>
54598f2b81SMateusz Guzik #include <sys/dtrace_bsd.h>
5591d1786fSDmitry Chagin #include <sys/sysent.h>
56961a7b24SJohn Baldwin #include <sys/turnstile.h>
57d116b9f1SMateusz Guzik #include <sys/taskqueue.h>
5844990b8cSJulian Elischer #include <sys/ktr.h>
59cf7d9a8cSDavid Xu #include <sys/rwlock.h>
60af29f399SDmitry Chagin #include <sys/umtxvar.h>
619ed01c32SGleb Smirnoff #include <sys/vmmeter.h>
62d7f687fcSJeff Roberson #include <sys/cpuset.h>
6316d95d4fSJoseph Koshy #ifdef	HWPMC_HOOKS
6416d95d4fSJoseph Koshy #include <sys/pmckern.h>
6516d95d4fSJoseph Koshy #endif
661bd3cf5dSMateusz Guzik #include <sys/priv.h>
6744990b8cSJulian Elischer 
68911b84b0SRobert Watson #include <security/audit/audit.h>
69911b84b0SRobert Watson 
70d116b9f1SMateusz Guzik #include <vm/pmap.h>
7144990b8cSJulian Elischer #include <vm/vm.h>
7249a2507bSAlan Cox #include <vm/vm_extern.h>
7344990b8cSJulian Elischer #include <vm/uma.h>
74d116b9f1SMateusz Guzik #include <vm/vm_phys.h>
75b209f889SRandall Stewart #include <sys/eventhandler.h>
7602fb42b0SPeter Wemm 
77acd9f517SKonstantin Belousov /*
78acd9f517SKonstantin Belousov  * Asserts below verify the stability of struct thread and struct proc
79acd9f517SKonstantin Belousov  * layout, as exposed by KBI to modules.  On head, the KBI is allowed
80acd9f517SKonstantin Belousov  * to drift, change to the structures must be accompanied by the
81acd9f517SKonstantin Belousov  * assert update.
82acd9f517SKonstantin Belousov  *
83acd9f517SKonstantin Belousov  * On the stable branches after KBI freeze, conditions must not be
84acd9f517SKonstantin Belousov  * violated.  Typically new fields are moved to the end of the
85acd9f517SKonstantin Belousov  * structures.
86acd9f517SKonstantin Belousov  */
87acd9f517SKonstantin Belousov #ifdef __amd64__
88a422084aSMark Johnston _Static_assert(offsetof(struct thread, td_flags) == 0x108,
89acd9f517SKonstantin Belousov     "struct thread KBI td_flags");
90c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_pflags) == 0x114,
91acd9f517SKonstantin Belousov     "struct thread KBI td_pflags");
92c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_frame) == 0x4b0,
93acd9f517SKonstantin Belousov     "struct thread KBI td_frame");
94c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_emuldata) == 0x6c0,
95acd9f517SKonstantin Belousov     "struct thread KBI td_emuldata");
9685078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_flag) == 0xb8,
97acd9f517SKonstantin Belousov     "struct proc KBI p_flag");
9885078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_pid) == 0xc4,
99acd9f517SKonstantin Belousov     "struct proc KBI p_pid");
100351d5f7fSKonstantin Belousov _Static_assert(offsetof(struct proc, p_filemon) == 0x3c8,
101acd9f517SKonstantin Belousov     "struct proc KBI p_filemon");
1025e5675cbSKonstantin Belousov _Static_assert(offsetof(struct proc, p_comm) == 0x3e0,
103acd9f517SKonstantin Belousov     "struct proc KBI p_comm");
104351d5f7fSKonstantin Belousov _Static_assert(offsetof(struct proc, p_emuldata) == 0x4c8,
105acd9f517SKonstantin Belousov     "struct proc KBI p_emuldata");
106acd9f517SKonstantin Belousov #endif
107acd9f517SKonstantin Belousov #ifdef __i386__
108a422084aSMark Johnston _Static_assert(offsetof(struct thread, td_flags) == 0x9c,
109acd9f517SKonstantin Belousov     "struct thread KBI td_flags");
110c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_pflags) == 0xa8,
111acd9f517SKonstantin Belousov     "struct thread KBI td_pflags");
112c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_frame) == 0x30c,
113acd9f517SKonstantin Belousov     "struct thread KBI td_frame");
114c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_emuldata) == 0x350,
115acd9f517SKonstantin Belousov     "struct thread KBI td_emuldata");
11685078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_flag) == 0x6c,
117acd9f517SKonstantin Belousov     "struct proc KBI p_flag");
11885078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_pid) == 0x78,
119acd9f517SKonstantin Belousov     "struct proc KBI p_pid");
1204d675b80SKonstantin Belousov _Static_assert(offsetof(struct proc, p_filemon) == 0x270,
121acd9f517SKonstantin Belousov     "struct proc KBI p_filemon");
1225e5675cbSKonstantin Belousov _Static_assert(offsetof(struct proc, p_comm) == 0x284,
123acd9f517SKonstantin Belousov     "struct proc KBI p_comm");
1245e5675cbSKonstantin Belousov _Static_assert(offsetof(struct proc, p_emuldata) == 0x310,
125acd9f517SKonstantin Belousov     "struct proc KBI p_emuldata");
126acd9f517SKonstantin Belousov #endif
127acd9f517SKonstantin Belousov 
128b3e9e682SRyan Stone SDT_PROVIDER_DECLARE(proc);
129d9fae5abSAndriy Gapon SDT_PROBE_DEFINE(proc, , , lwp__exit);
130b3e9e682SRyan Stone 
1318460a577SJohn Birrell /*
1328460a577SJohn Birrell  * thread related storage.
1338460a577SJohn Birrell  */
13444990b8cSJulian Elischer static uma_zone_t thread_zone;
13544990b8cSJulian Elischer 
136d116b9f1SMateusz Guzik struct thread_domain_data {
137d116b9f1SMateusz Guzik 	struct thread	*tdd_zombies;
138d116b9f1SMateusz Guzik 	int		tdd_reapticks;
139d116b9f1SMateusz Guzik } __aligned(CACHE_LINE_SIZE);
140d116b9f1SMateusz Guzik 
141d116b9f1SMateusz Guzik static struct thread_domain_data thread_domain_data[MAXMEMDOM];
142d116b9f1SMateusz Guzik 
143d116b9f1SMateusz Guzik static struct task	thread_reap_task;
144d116b9f1SMateusz Guzik static struct callout  	thread_reap_callout;
14544990b8cSJulian Elischer 
146ff8fbcffSJeff Roberson static void thread_zombie(struct thread *);
147b83e94beSMateusz Guzik static void thread_reap(void);
148d116b9f1SMateusz Guzik static void thread_reap_all(void);
149d116b9f1SMateusz Guzik static void thread_reap_task_cb(void *, int);
150d116b9f1SMateusz Guzik static void thread_reap_callout_cb(void *);
15184cdea97SKonstantin Belousov static int thread_unsuspend_one(struct thread *td, struct proc *p,
15284cdea97SKonstantin Belousov     bool boundary);
153755341dfSMateusz Guzik static void thread_free_batched(struct thread *td);
154ff8fbcffSJeff Roberson 
155d1ca25beSMateusz Guzik static __exclusive_cache_line struct mtx tid_lock;
156934e7e5eSMateusz Guzik static bitstr_t *tid_bitmap;
15735bb59edSMateusz Guzik 
158cf7d9a8cSDavid Xu static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash");
159cf7d9a8cSDavid Xu 
1601bd3cf5dSMateusz Guzik static int maxthread;
1611bd3cf5dSMateusz Guzik SYSCTL_INT(_kern, OID_AUTO, maxthread, CTLFLAG_RDTUN,
1621bd3cf5dSMateusz Guzik     &maxthread, 0, "Maximum number of threads");
1631bd3cf5dSMateusz Guzik 
16462dbc992SMateusz Guzik static __exclusive_cache_line int nthreads;
1651bd3cf5dSMateusz Guzik 
166aae3547bSMateusz Guzik static LIST_HEAD(tidhashhead, thread) *tidhashtbl;
167aae3547bSMateusz Guzik static u_long	tidhash;
16826007fe3SMateusz Guzik static u_long	tidhashlock;
16926007fe3SMateusz Guzik static struct	rwlock *tidhashtbl_lock;
170aae3547bSMateusz Guzik #define	TIDHASH(tid)		(&tidhashtbl[(tid) & tidhash])
17126007fe3SMateusz Guzik #define	TIDHASHLOCK(tid)	(&tidhashtbl_lock[(tid) & tidhashlock])
172cf7d9a8cSDavid Xu 
1732ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_ctor);
1742ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_dtor);
1752ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_init);
1762ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_fini);
1772ca45184SMatt Joras 
17862dbc992SMateusz Guzik static bool
179d116b9f1SMateusz Guzik thread_count_inc_try(void)
180ec6ea5e8SDavid Xu {
18162dbc992SMateusz Guzik 	int nthreads_new;
182ec6ea5e8SDavid Xu 
18362dbc992SMateusz Guzik 	nthreads_new = atomic_fetchadd_int(&nthreads, 1) + 1;
18462dbc992SMateusz Guzik 	if (nthreads_new >= maxthread - 100) {
1851bd3cf5dSMateusz Guzik 		if (priv_check_cred(curthread->td_ucred, PRIV_MAXPROC) != 0 ||
18662dbc992SMateusz Guzik 		    nthreads_new >= maxthread) {
18762dbc992SMateusz Guzik 			atomic_subtract_int(&nthreads, 1);
188d116b9f1SMateusz Guzik 			return (false);
189d116b9f1SMateusz Guzik 		}
190d116b9f1SMateusz Guzik 	}
191d116b9f1SMateusz Guzik 	return (true);
192d116b9f1SMateusz Guzik }
193d116b9f1SMateusz Guzik 
194d116b9f1SMateusz Guzik static bool
195d116b9f1SMateusz Guzik thread_count_inc(void)
196d116b9f1SMateusz Guzik {
197d116b9f1SMateusz Guzik 	static struct timeval lastfail;
198d116b9f1SMateusz Guzik 	static int curfail;
199d116b9f1SMateusz Guzik 
200d116b9f1SMateusz Guzik 	thread_reap();
201d116b9f1SMateusz Guzik 	if (thread_count_inc_try()) {
202d116b9f1SMateusz Guzik 		return (true);
203d116b9f1SMateusz Guzik 	}
204d116b9f1SMateusz Guzik 
205d116b9f1SMateusz Guzik 	thread_reap_all();
206d116b9f1SMateusz Guzik 	if (thread_count_inc_try()) {
207d116b9f1SMateusz Guzik 		return (true);
208d116b9f1SMateusz Guzik 	}
209d116b9f1SMateusz Guzik 
2101bd3cf5dSMateusz Guzik 	if (ppsratecheck(&lastfail, &curfail, 1)) {
2111bd3cf5dSMateusz Guzik 		printf("maxthread limit exceeded by uid %u "
2121bd3cf5dSMateusz Guzik 		    "(pid %d); consider increasing kern.maxthread\n",
2131bd3cf5dSMateusz Guzik 		    curthread->td_ucred->cr_ruid, curproc->p_pid);
2141bd3cf5dSMateusz Guzik 	}
21562dbc992SMateusz Guzik 	return (false);
2161bd3cf5dSMateusz Guzik }
2171bd3cf5dSMateusz Guzik 
21862dbc992SMateusz Guzik static void
21962dbc992SMateusz Guzik thread_count_sub(int n)
22062dbc992SMateusz Guzik {
22162dbc992SMateusz Guzik 
22262dbc992SMateusz Guzik 	atomic_subtract_int(&nthreads, n);
22362dbc992SMateusz Guzik }
22462dbc992SMateusz Guzik 
22562dbc992SMateusz Guzik static void
22662dbc992SMateusz Guzik thread_count_dec(void)
22762dbc992SMateusz Guzik {
22862dbc992SMateusz Guzik 
22962dbc992SMateusz Guzik 	thread_count_sub(1);
23062dbc992SMateusz Guzik }
23162dbc992SMateusz Guzik 
23262dbc992SMateusz Guzik static lwpid_t
23362dbc992SMateusz Guzik tid_alloc(void)
23462dbc992SMateusz Guzik {
23562dbc992SMateusz Guzik 	static lwpid_t trytid;
23662dbc992SMateusz Guzik 	lwpid_t tid;
23762dbc992SMateusz Guzik 
23862dbc992SMateusz Guzik 	mtx_lock(&tid_lock);
23935bb59edSMateusz Guzik 	/*
24035bb59edSMateusz Guzik 	 * It is an invariant that the bitmap is big enough to hold maxthread
24135bb59edSMateusz Guzik 	 * IDs. If we got to this point there has to be at least one free.
24235bb59edSMateusz Guzik 	 */
24335bb59edSMateusz Guzik 	if (trytid >= maxthread)
24435bb59edSMateusz Guzik 		trytid = 0;
24535bb59edSMateusz Guzik 	bit_ffc_at(tid_bitmap, trytid, maxthread, &tid);
24635bb59edSMateusz Guzik 	if (tid == -1) {
24735bb59edSMateusz Guzik 		KASSERT(trytid != 0, ("unexpectedly ran out of IDs"));
24835bb59edSMateusz Guzik 		trytid = 0;
24935bb59edSMateusz Guzik 		bit_ffc_at(tid_bitmap, trytid, maxthread, &tid);
25035bb59edSMateusz Guzik 		KASSERT(tid != -1, ("unexpectedly ran out of IDs"));
251ec6ea5e8SDavid Xu 	}
25235bb59edSMateusz Guzik 	bit_set(tid_bitmap, tid);
253934e7e5eSMateusz Guzik 	trytid = tid + 1;
254ec6ea5e8SDavid Xu 	mtx_unlock(&tid_lock);
25535bb59edSMateusz Guzik 	return (tid + NO_PID);
256ec6ea5e8SDavid Xu }
257ec6ea5e8SDavid Xu 
258ec6ea5e8SDavid Xu static void
259755341dfSMateusz Guzik tid_free_locked(lwpid_t rtid)
260ec6ea5e8SDavid Xu {
26135bb59edSMateusz Guzik 	lwpid_t tid;
262ec6ea5e8SDavid Xu 
263755341dfSMateusz Guzik 	mtx_assert(&tid_lock, MA_OWNED);
26435bb59edSMateusz Guzik 	KASSERT(rtid >= NO_PID,
26535bb59edSMateusz Guzik 	    ("%s: invalid tid %d\n", __func__, rtid));
26635bb59edSMateusz Guzik 	tid = rtid - NO_PID;
26735bb59edSMateusz Guzik 	KASSERT(bit_test(tid_bitmap, tid) != 0,
26835bb59edSMateusz Guzik 	    ("thread ID %d not allocated\n", rtid));
26935bb59edSMateusz Guzik 	bit_clear(tid_bitmap, tid);
270755341dfSMateusz Guzik }
271755341dfSMateusz Guzik 
272755341dfSMateusz Guzik static void
273755341dfSMateusz Guzik tid_free(lwpid_t rtid)
274755341dfSMateusz Guzik {
275755341dfSMateusz Guzik 
276755341dfSMateusz Guzik 	mtx_lock(&tid_lock);
277755341dfSMateusz Guzik 	tid_free_locked(rtid);
278755341dfSMateusz Guzik 	mtx_unlock(&tid_lock);
279755341dfSMateusz Guzik }
280755341dfSMateusz Guzik 
281755341dfSMateusz Guzik static void
282755341dfSMateusz Guzik tid_free_batch(lwpid_t *batch, int n)
283755341dfSMateusz Guzik {
284755341dfSMateusz Guzik 	int i;
285755341dfSMateusz Guzik 
286755341dfSMateusz Guzik 	mtx_lock(&tid_lock);
287755341dfSMateusz Guzik 	for (i = 0; i < n; i++) {
288755341dfSMateusz Guzik 		tid_free_locked(batch[i]);
289755341dfSMateusz Guzik 	}
290ec6ea5e8SDavid Xu 	mtx_unlock(&tid_lock);
291ec6ea5e8SDavid Xu }
292ec6ea5e8SDavid Xu 
293fdcac928SMarcel Moolenaar /*
2945ef7b7a0SMateusz Guzik  * Batching for thread reapping.
2955ef7b7a0SMateusz Guzik  */
2965ef7b7a0SMateusz Guzik struct tidbatch {
2975ef7b7a0SMateusz Guzik 	lwpid_t tab[16];
2985ef7b7a0SMateusz Guzik 	int n;
2995ef7b7a0SMateusz Guzik };
3005ef7b7a0SMateusz Guzik 
3015ef7b7a0SMateusz Guzik static void
3025ef7b7a0SMateusz Guzik tidbatch_prep(struct tidbatch *tb)
3035ef7b7a0SMateusz Guzik {
3045ef7b7a0SMateusz Guzik 
3055ef7b7a0SMateusz Guzik 	tb->n = 0;
3065ef7b7a0SMateusz Guzik }
3075ef7b7a0SMateusz Guzik 
3085ef7b7a0SMateusz Guzik static void
3095ef7b7a0SMateusz Guzik tidbatch_add(struct tidbatch *tb, struct thread *td)
3105ef7b7a0SMateusz Guzik {
3115ef7b7a0SMateusz Guzik 
3125ef7b7a0SMateusz Guzik 	KASSERT(tb->n < nitems(tb->tab),
3135ef7b7a0SMateusz Guzik 	    ("%s: count too high %d", __func__, tb->n));
3145ef7b7a0SMateusz Guzik 	tb->tab[tb->n] = td->td_tid;
3155ef7b7a0SMateusz Guzik 	tb->n++;
3165ef7b7a0SMateusz Guzik }
3175ef7b7a0SMateusz Guzik 
3185ef7b7a0SMateusz Guzik static void
3195ef7b7a0SMateusz Guzik tidbatch_process(struct tidbatch *tb)
3205ef7b7a0SMateusz Guzik {
3215ef7b7a0SMateusz Guzik 
3225ef7b7a0SMateusz Guzik 	KASSERT(tb->n <= nitems(tb->tab),
3235ef7b7a0SMateusz Guzik 	    ("%s: count too high %d", __func__, tb->n));
3245ef7b7a0SMateusz Guzik 	if (tb->n == nitems(tb->tab)) {
3255ef7b7a0SMateusz Guzik 		tid_free_batch(tb->tab, tb->n);
3265ef7b7a0SMateusz Guzik 		tb->n = 0;
3275ef7b7a0SMateusz Guzik 	}
3285ef7b7a0SMateusz Guzik }
3295ef7b7a0SMateusz Guzik 
3305ef7b7a0SMateusz Guzik static void
3315ef7b7a0SMateusz Guzik tidbatch_final(struct tidbatch *tb)
3325ef7b7a0SMateusz Guzik {
3335ef7b7a0SMateusz Guzik 
3345ef7b7a0SMateusz Guzik 	KASSERT(tb->n <= nitems(tb->tab),
3355ef7b7a0SMateusz Guzik 	    ("%s: count too high %d", __func__, tb->n));
3365ef7b7a0SMateusz Guzik 	if (tb->n != 0) {
3375ef7b7a0SMateusz Guzik 		tid_free_batch(tb->tab, tb->n);
3385ef7b7a0SMateusz Guzik 	}
3395ef7b7a0SMateusz Guzik }
3405ef7b7a0SMateusz Guzik 
3415ef7b7a0SMateusz Guzik /*
342696058c3SJulian Elischer  * Prepare a thread for use.
34344990b8cSJulian Elischer  */
344b23f72e9SBrian Feldman static int
345b23f72e9SBrian Feldman thread_ctor(void *mem, int size, void *arg, int flags)
34644990b8cSJulian Elischer {
34744990b8cSJulian Elischer 	struct thread	*td;
34844990b8cSJulian Elischer 
34944990b8cSJulian Elischer 	td = (struct thread *)mem;
350fa2528acSAlex Richardson 	TD_SET_STATE(td, TDS_INACTIVE);
35194dd54b9SKonstantin Belousov 	td->td_lastcpu = td->td_oncpu = NOCPU;
3526c27c603SJuli Mallett 
3536c27c603SJuli Mallett 	/*
3546c27c603SJuli Mallett 	 * Note that td_critnest begins life as 1 because the thread is not
3556c27c603SJuli Mallett 	 * running and is thereby implicitly waiting to be on the receiving
356a54e85fdSJeff Roberson 	 * end of a context switch.
3576c27c603SJuli Mallett 	 */
358139b7550SJohn Baldwin 	td->td_critnest = 1;
359acbe332aSDavid Xu 	td->td_lend_user_pri = PRI_MAX;
360911b84b0SRobert Watson #ifdef AUDIT
361911b84b0SRobert Watson 	audit_thread_alloc(td);
362911b84b0SRobert Watson #endif
363598f2b81SMateusz Guzik #ifdef KDTRACE_HOOKS
364598f2b81SMateusz Guzik 	kdtrace_thread_ctor(td);
365598f2b81SMateusz Guzik #endif
366d10183d9SDavid Xu 	umtx_thread_alloc(td);
36719d3e47dSMateusz Guzik 	MPASS(td->td_sel == NULL);
368b23f72e9SBrian Feldman 	return (0);
36944990b8cSJulian Elischer }
37044990b8cSJulian Elischer 
37144990b8cSJulian Elischer /*
37244990b8cSJulian Elischer  * Reclaim a thread after use.
37344990b8cSJulian Elischer  */
37444990b8cSJulian Elischer static void
37544990b8cSJulian Elischer thread_dtor(void *mem, int size, void *arg)
37644990b8cSJulian Elischer {
37744990b8cSJulian Elischer 	struct thread *td;
37844990b8cSJulian Elischer 
37944990b8cSJulian Elischer 	td = (struct thread *)mem;
38044990b8cSJulian Elischer 
38144990b8cSJulian Elischer #ifdef INVARIANTS
38244990b8cSJulian Elischer 	/* Verify that this thread is in a safe state to free. */
383fa2528acSAlex Richardson 	switch (TD_GET_STATE(td)) {
38471fad9fdSJulian Elischer 	case TDS_INHIBITED:
38571fad9fdSJulian Elischer 	case TDS_RUNNING:
38671fad9fdSJulian Elischer 	case TDS_CAN_RUN:
38744990b8cSJulian Elischer 	case TDS_RUNQ:
38844990b8cSJulian Elischer 		/*
38944990b8cSJulian Elischer 		 * We must never unlink a thread that is in one of
39044990b8cSJulian Elischer 		 * these states, because it is currently active.
39144990b8cSJulian Elischer 		 */
39244990b8cSJulian Elischer 		panic("bad state for thread unlinking");
39344990b8cSJulian Elischer 		/* NOTREACHED */
39471fad9fdSJulian Elischer 	case TDS_INACTIVE:
39544990b8cSJulian Elischer 		break;
39644990b8cSJulian Elischer 	default:
39744990b8cSJulian Elischer 		panic("bad thread state");
39844990b8cSJulian Elischer 		/* NOTREACHED */
39944990b8cSJulian Elischer 	}
40044990b8cSJulian Elischer #endif
4016e8525ceSRobert Watson #ifdef AUDIT
4026e8525ceSRobert Watson 	audit_thread_free(td);
4036e8525ceSRobert Watson #endif
404598f2b81SMateusz Guzik #ifdef KDTRACE_HOOKS
405598f2b81SMateusz Guzik 	kdtrace_thread_dtor(td);
406598f2b81SMateusz Guzik #endif
4071ba4a712SPawel Jakub Dawidek 	/* Free all OSD associated to this thread. */
4081ba4a712SPawel Jakub Dawidek 	osd_thread_exit(td);
409c6d31b83SKonstantin Belousov 	ast_kclear(td);
41019d3e47dSMateusz Guzik 	seltdfini(td);
41144990b8cSJulian Elischer }
41244990b8cSJulian Elischer 
41344990b8cSJulian Elischer /*
41444990b8cSJulian Elischer  * Initialize type-stable parts of a thread (when newly created).
41544990b8cSJulian Elischer  */
416b23f72e9SBrian Feldman static int
417b23f72e9SBrian Feldman thread_init(void *mem, int size, int flags)
41844990b8cSJulian Elischer {
41944990b8cSJulian Elischer 	struct thread *td;
42044990b8cSJulian Elischer 
42144990b8cSJulian Elischer 	td = (struct thread *)mem;
422247aba24SMarcel Moolenaar 
423b83e94beSMateusz Guzik 	td->td_allocdomain = vm_phys_domain(vtophys(td));
42444f3b092SJohn Baldwin 	td->td_sleepqueue = sleepq_alloc();
425961a7b24SJohn Baldwin 	td->td_turnstile = turnstile_alloc();
4268f0e9130SKonstantin Belousov 	td->td_rlqe = NULL;
4272ca45184SMatt Joras 	EVENTHANDLER_DIRECT_INVOKE(thread_init, td);
428d10183d9SDavid Xu 	umtx_thread_init(td);
42989b57fcfSKonstantin Belousov 	td->td_kstack = 0;
430ad8b1d85SKonstantin Belousov 	td->td_sel = NULL;
431b23f72e9SBrian Feldman 	return (0);
43244990b8cSJulian Elischer }
43344990b8cSJulian Elischer 
43444990b8cSJulian Elischer /*
43544990b8cSJulian Elischer  * Tear down type-stable parts of a thread (just before being discarded).
43644990b8cSJulian Elischer  */
43744990b8cSJulian Elischer static void
43844990b8cSJulian Elischer thread_fini(void *mem, int size)
43944990b8cSJulian Elischer {
44044990b8cSJulian Elischer 	struct thread *td;
44144990b8cSJulian Elischer 
44244990b8cSJulian Elischer 	td = (struct thread *)mem;
4432ca45184SMatt Joras 	EVENTHANDLER_DIRECT_INVOKE(thread_fini, td);
4448f0e9130SKonstantin Belousov 	rlqentry_free(td->td_rlqe);
445961a7b24SJohn Baldwin 	turnstile_free(td->td_turnstile);
44644f3b092SJohn Baldwin 	sleepq_free(td->td_sleepqueue);
447d10183d9SDavid Xu 	umtx_thread_fini(td);
44819d3e47dSMateusz Guzik 	MPASS(td->td_sel == NULL);
44944990b8cSJulian Elischer }
4505215b187SJeff Roberson 
4515c8329edSJulian Elischer /*
4525215b187SJeff Roberson  * For a newly created process,
4535215b187SJeff Roberson  * link up all the structures and its initial threads etc.
454ed062c8dSJulian Elischer  * called from:
455e7d939bdSMarcel Moolenaar  * {arch}/{arch}/machdep.c   {arch}_init(), init386() etc.
456ed062c8dSJulian Elischer  * proc_dtor() (should go away)
457ed062c8dSJulian Elischer  * proc_init()
4585c8329edSJulian Elischer  */
4595c8329edSJulian Elischer void
46089b57fcfSKonstantin Belousov proc_linkup0(struct proc *p, struct thread *td)
46189b57fcfSKonstantin Belousov {
46289b57fcfSKonstantin Belousov 	TAILQ_INIT(&p->p_threads);	     /* all threads in proc */
46389b57fcfSKonstantin Belousov 	proc_linkup(p, td);
46489b57fcfSKonstantin Belousov }
46589b57fcfSKonstantin Belousov 
46689b57fcfSKonstantin Belousov void
4678460a577SJohn Birrell proc_linkup(struct proc *p, struct thread *td)
4685c8329edSJulian Elischer {
469a54e85fdSJeff Roberson 
4709104847fSDavid Xu 	sigqueue_init(&p->p_sigqueue, p);
471cc29f221SKonstantin Belousov 	p->p_ksi = ksiginfo_alloc(M_WAITOK);
472ebceaf6dSDavid Xu 	if (p->p_ksi != NULL) {
4735c474517SDavid Xu 		/* XXX p_ksi may be null if ksiginfo zone is not ready */
474ebceaf6dSDavid Xu 		p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
475ebceaf6dSDavid Xu 	}
476b2f92ef9SDavid Xu 	LIST_INIT(&p->p_mqnotifier);
4775c8329edSJulian Elischer 	p->p_numthreads = 0;
4788460a577SJohn Birrell 	thread_link(td, p);
4795c8329edSJulian Elischer }
4805c8329edSJulian Elischer 
481c6d31b83SKonstantin Belousov static void
482c6d31b83SKonstantin Belousov ast_suspend(struct thread *td, int tda __unused)
483c6d31b83SKonstantin Belousov {
484c6d31b83SKonstantin Belousov 	struct proc *p;
485c6d31b83SKonstantin Belousov 
486c6d31b83SKonstantin Belousov 	p = td->td_proc;
487c6d31b83SKonstantin Belousov 	/*
488c6d31b83SKonstantin Belousov 	 * We need to check to see if we have to exit or wait due to a
489c6d31b83SKonstantin Belousov 	 * single threading requirement or some other STOP condition.
490c6d31b83SKonstantin Belousov 	 */
491c6d31b83SKonstantin Belousov 	PROC_LOCK(p);
492c6d31b83SKonstantin Belousov 	thread_suspend_check(0);
493c6d31b83SKonstantin Belousov 	PROC_UNLOCK(p);
494c6d31b83SKonstantin Belousov }
495c6d31b83SKonstantin Belousov 
4961bd3cf5dSMateusz Guzik extern int max_threads_per_proc;
4971bd3cf5dSMateusz Guzik 
4985c8329edSJulian Elischer /*
49944990b8cSJulian Elischer  * Initialize global thread allocation resources.
50044990b8cSJulian Elischer  */
50144990b8cSJulian Elischer void
50244990b8cSJulian Elischer threadinit(void)
50344990b8cSJulian Elischer {
50426007fe3SMateusz Guzik 	u_long i;
505cf31cadeSMateusz Guzik 	lwpid_t tid0;
50644990b8cSJulian Elischer 
5071bd3cf5dSMateusz Guzik 	/*
5081bd3cf5dSMateusz Guzik 	 * Place an upper limit on threads which can be allocated.
5091bd3cf5dSMateusz Guzik 	 *
5101bd3cf5dSMateusz Guzik 	 * Note that other factors may make the de facto limit much lower.
5111bd3cf5dSMateusz Guzik 	 *
5121bd3cf5dSMateusz Guzik 	 * Platform limits are somewhat arbitrary but deemed "more than good
5131bd3cf5dSMateusz Guzik 	 * enough" for the foreseable future.
5141bd3cf5dSMateusz Guzik 	 */
5151bd3cf5dSMateusz Guzik 	if (maxthread == 0) {
5161bd3cf5dSMateusz Guzik #ifdef _LP64
5171bd3cf5dSMateusz Guzik 		maxthread = MIN(maxproc * max_threads_per_proc, 1000000);
5181bd3cf5dSMateusz Guzik #else
5191bd3cf5dSMateusz Guzik 		maxthread = MIN(maxproc * max_threads_per_proc, 100000);
5201bd3cf5dSMateusz Guzik #endif
5211bd3cf5dSMateusz Guzik 	}
5221bd3cf5dSMateusz Guzik 
5231ea7a6f8SPoul-Henning Kamp 	mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
52435bb59edSMateusz Guzik 	tid_bitmap = bit_alloc(maxthread, M_TIDHASH, M_WAITOK);
52562dbc992SMateusz Guzik 	/*
52662dbc992SMateusz Guzik 	 * Handle thread0.
52762dbc992SMateusz Guzik 	 */
52862dbc992SMateusz Guzik 	thread_count_inc();
529cf31cadeSMateusz Guzik 	tid0 = tid_alloc();
530cf31cadeSMateusz Guzik 	if (tid0 != THREAD0_TID)
531cf31cadeSMateusz Guzik 		panic("tid0 %d != %d\n", tid0, THREAD0_TID);
5321ea7a6f8SPoul-Henning Kamp 
533de028f5aSJeff Roberson 	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
53444990b8cSJulian Elischer 	    thread_ctor, thread_dtor, thread_init, thread_fini,
5352c10be9eSMark Johnston 	    32 - 1, UMA_ZONE_NOFREE);
536cf7d9a8cSDavid Xu 	tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash);
53726007fe3SMateusz Guzik 	tidhashlock = (tidhash + 1) / 64;
53826007fe3SMateusz Guzik 	if (tidhashlock > 0)
53926007fe3SMateusz Guzik 		tidhashlock--;
54026007fe3SMateusz Guzik 	tidhashtbl_lock = malloc(sizeof(*tidhashtbl_lock) * (tidhashlock + 1),
54126007fe3SMateusz Guzik 	    M_TIDHASH, M_WAITOK | M_ZERO);
54226007fe3SMateusz Guzik 	for (i = 0; i < tidhashlock + 1; i++)
54326007fe3SMateusz Guzik 		rw_init(&tidhashtbl_lock[i], "tidhash");
544d116b9f1SMateusz Guzik 
545d116b9f1SMateusz Guzik 	TASK_INIT(&thread_reap_task, 0, thread_reap_task_cb, NULL);
546d116b9f1SMateusz Guzik 	callout_init(&thread_reap_callout, 1);
547845d7797SKonstantin Belousov 	callout_reset(&thread_reap_callout, 5 * hz,
548845d7797SKonstantin Belousov 	    thread_reap_callout_cb, NULL);
549c6d31b83SKonstantin Belousov 	ast_register(TDA_SUSPEND, ASTR_ASTF_REQUIRED, 0, ast_suspend);
55044990b8cSJulian Elischer }
55144990b8cSJulian Elischer 
55244990b8cSJulian Elischer /*
553ff8fbcffSJeff Roberson  * Place an unused thread on the zombie list.
55444990b8cSJulian Elischer  */
55544990b8cSJulian Elischer void
556ff8fbcffSJeff Roberson thread_zombie(struct thread *td)
55744990b8cSJulian Elischer {
558d116b9f1SMateusz Guzik 	struct thread_domain_data *tdd;
559c5315f51SMateusz Guzik 	struct thread *ztd;
560c5315f51SMateusz Guzik 
561a9568cd2SMateusz Guzik 	tdd = &thread_domain_data[td->td_allocdomain];
562d116b9f1SMateusz Guzik 	ztd = atomic_load_ptr(&tdd->tdd_zombies);
563c5315f51SMateusz Guzik 	for (;;) {
564c5315f51SMateusz Guzik 		td->td_zombie = ztd;
565d116b9f1SMateusz Guzik 		if (atomic_fcmpset_rel_ptr((uintptr_t *)&tdd->tdd_zombies,
566c5315f51SMateusz Guzik 		    (uintptr_t *)&ztd, (uintptr_t)td))
567c5315f51SMateusz Guzik 			break;
568c5315f51SMateusz Guzik 		continue;
569c5315f51SMateusz Guzik 	}
57044990b8cSJulian Elischer }
57144990b8cSJulian Elischer 
5725c8329edSJulian Elischer /*
573ff8fbcffSJeff Roberson  * Release a thread that has exited after cpu_throw().
574ff8fbcffSJeff Roberson  */
575ff8fbcffSJeff Roberson void
576ff8fbcffSJeff Roberson thread_stash(struct thread *td)
577ff8fbcffSJeff Roberson {
578ff8fbcffSJeff Roberson 	atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
579ff8fbcffSJeff Roberson 	thread_zombie(td);
580ff8fbcffSJeff Roberson }
581ff8fbcffSJeff Roberson 
582ff8fbcffSJeff Roberson /*
583d116b9f1SMateusz Guzik  * Reap zombies from passed domain.
58444990b8cSJulian Elischer  */
585d116b9f1SMateusz Guzik static void
586d116b9f1SMateusz Guzik thread_reap_domain(struct thread_domain_data *tdd)
58744990b8cSJulian Elischer {
588c5315f51SMateusz Guzik 	struct thread *itd, *ntd;
5895ef7b7a0SMateusz Guzik 	struct tidbatch tidbatch;
590f34a2f56SMateusz Guzik 	struct credbatch credbatch;
5915ef7b7a0SMateusz Guzik 	int tdcount;
592fb8ab680SMateusz Guzik 	struct plimit *lim;
593fb8ab680SMateusz Guzik 	int limcount;
59444990b8cSJulian Elischer 
59544990b8cSJulian Elischer 	/*
596c5315f51SMateusz Guzik 	 * Reading upfront is pessimal if followed by concurrent atomic_swap,
597c5315f51SMateusz Guzik 	 * but most of the time the list is empty.
59844990b8cSJulian Elischer 	 */
599d116b9f1SMateusz Guzik 	if (tdd->tdd_zombies == NULL)
600c5315f51SMateusz Guzik 		return;
601c5315f51SMateusz Guzik 
602d116b9f1SMateusz Guzik 	itd = (struct thread *)atomic_swap_ptr((uintptr_t *)&tdd->tdd_zombies,
603c5315f51SMateusz Guzik 	    (uintptr_t)NULL);
6045ef7b7a0SMateusz Guzik 	if (itd == NULL)
6055ef7b7a0SMateusz Guzik 		return;
6065ef7b7a0SMateusz Guzik 
607d116b9f1SMateusz Guzik 	/*
608d116b9f1SMateusz Guzik 	 * Multiple CPUs can get here, the race is fine as ticks is only
609d116b9f1SMateusz Guzik 	 * advisory.
610d116b9f1SMateusz Guzik 	 */
611d116b9f1SMateusz Guzik 	tdd->tdd_reapticks = ticks;
612d116b9f1SMateusz Guzik 
6135ef7b7a0SMateusz Guzik 	tidbatch_prep(&tidbatch);
614f34a2f56SMateusz Guzik 	credbatch_prep(&credbatch);
6155ef7b7a0SMateusz Guzik 	tdcount = 0;
616fb8ab680SMateusz Guzik 	lim = NULL;
617fb8ab680SMateusz Guzik 	limcount = 0;
618d116b9f1SMateusz Guzik 
619c5315f51SMateusz Guzik 	while (itd != NULL) {
620c5315f51SMateusz Guzik 		ntd = itd->td_zombie;
6215ef7b7a0SMateusz Guzik 		EVENTHANDLER_DIRECT_INVOKE(thread_dtor, itd);
6225ef7b7a0SMateusz Guzik 		tidbatch_add(&tidbatch, itd);
623f34a2f56SMateusz Guzik 		credbatch_add(&credbatch, itd);
624fb8ab680SMateusz Guzik 		MPASS(itd->td_limit != NULL);
625fb8ab680SMateusz Guzik 		if (lim != itd->td_limit) {
626fb8ab680SMateusz Guzik 			if (limcount != 0) {
627fb8ab680SMateusz Guzik 				lim_freen(lim, limcount);
628fb8ab680SMateusz Guzik 				limcount = 0;
629fb8ab680SMateusz Guzik 			}
630fb8ab680SMateusz Guzik 		}
631fb8ab680SMateusz Guzik 		lim = itd->td_limit;
632fb8ab680SMateusz Guzik 		limcount++;
633755341dfSMateusz Guzik 		thread_free_batched(itd);
6345ef7b7a0SMateusz Guzik 		tidbatch_process(&tidbatch);
635f34a2f56SMateusz Guzik 		credbatch_process(&credbatch);
6365ef7b7a0SMateusz Guzik 		tdcount++;
6375ef7b7a0SMateusz Guzik 		if (tdcount == 32) {
6385ef7b7a0SMateusz Guzik 			thread_count_sub(tdcount);
6395ef7b7a0SMateusz Guzik 			tdcount = 0;
640755341dfSMateusz Guzik 		}
641c5315f51SMateusz Guzik 		itd = ntd;
64244990b8cSJulian Elischer 	}
643755341dfSMateusz Guzik 
6445ef7b7a0SMateusz Guzik 	tidbatch_final(&tidbatch);
645f34a2f56SMateusz Guzik 	credbatch_final(&credbatch);
6465ef7b7a0SMateusz Guzik 	if (tdcount != 0) {
6475ef7b7a0SMateusz Guzik 		thread_count_sub(tdcount);
648755341dfSMateusz Guzik 	}
649fb8ab680SMateusz Guzik 	MPASS(limcount != 0);
650fb8ab680SMateusz Guzik 	lim_freen(lim, limcount);
651ed062c8dSJulian Elischer }
65244990b8cSJulian Elischer 
6534f0db5e0SJulian Elischer /*
654d116b9f1SMateusz Guzik  * Reap zombies from all domains.
655d116b9f1SMateusz Guzik  */
656d116b9f1SMateusz Guzik static void
657d116b9f1SMateusz Guzik thread_reap_all(void)
658d116b9f1SMateusz Guzik {
659d116b9f1SMateusz Guzik 	struct thread_domain_data *tdd;
660d116b9f1SMateusz Guzik 	int i, domain;
661d116b9f1SMateusz Guzik 
662d116b9f1SMateusz Guzik 	domain = PCPU_GET(domain);
663d116b9f1SMateusz Guzik 	for (i = 0; i < vm_ndomains; i++) {
664d116b9f1SMateusz Guzik 		tdd = &thread_domain_data[(i + domain) % vm_ndomains];
665d116b9f1SMateusz Guzik 		thread_reap_domain(tdd);
666d116b9f1SMateusz Guzik 	}
667d116b9f1SMateusz Guzik }
668d116b9f1SMateusz Guzik 
669d116b9f1SMateusz Guzik /*
670d116b9f1SMateusz Guzik  * Reap zombies from local domain.
671d116b9f1SMateusz Guzik  */
672b83e94beSMateusz Guzik static void
673d116b9f1SMateusz Guzik thread_reap(void)
674d116b9f1SMateusz Guzik {
675d116b9f1SMateusz Guzik 	struct thread_domain_data *tdd;
676d116b9f1SMateusz Guzik 	int domain;
677d116b9f1SMateusz Guzik 
678d116b9f1SMateusz Guzik 	domain = PCPU_GET(domain);
679d116b9f1SMateusz Guzik 	tdd = &thread_domain_data[domain];
680d116b9f1SMateusz Guzik 
681d116b9f1SMateusz Guzik 	thread_reap_domain(tdd);
682d116b9f1SMateusz Guzik }
683d116b9f1SMateusz Guzik 
684d116b9f1SMateusz Guzik static void
685d116b9f1SMateusz Guzik thread_reap_task_cb(void *arg __unused, int pending __unused)
686d116b9f1SMateusz Guzik {
687d116b9f1SMateusz Guzik 
688d116b9f1SMateusz Guzik 	thread_reap_all();
689d116b9f1SMateusz Guzik }
690d116b9f1SMateusz Guzik 
691d116b9f1SMateusz Guzik static void
692d116b9f1SMateusz Guzik thread_reap_callout_cb(void *arg __unused)
693d116b9f1SMateusz Guzik {
694d116b9f1SMateusz Guzik 	struct thread_domain_data *tdd;
695d116b9f1SMateusz Guzik 	int i, cticks, lticks;
696d116b9f1SMateusz Guzik 	bool wantreap;
697d116b9f1SMateusz Guzik 
698d116b9f1SMateusz Guzik 	wantreap = false;
699d116b9f1SMateusz Guzik 	cticks = atomic_load_int(&ticks);
700d116b9f1SMateusz Guzik 	for (i = 0; i < vm_ndomains; i++) {
701d116b9f1SMateusz Guzik 		tdd = &thread_domain_data[i];
702d116b9f1SMateusz Guzik 		lticks = tdd->tdd_reapticks;
703d116b9f1SMateusz Guzik 		if (tdd->tdd_zombies != NULL &&
704d116b9f1SMateusz Guzik 		    (u_int)(cticks - lticks) > 5 * hz) {
705d116b9f1SMateusz Guzik 			wantreap = true;
706d116b9f1SMateusz Guzik 			break;
707d116b9f1SMateusz Guzik 		}
708d116b9f1SMateusz Guzik 	}
709d116b9f1SMateusz Guzik 
710d116b9f1SMateusz Guzik 	if (wantreap)
711d116b9f1SMateusz Guzik 		taskqueue_enqueue(taskqueue_thread, &thread_reap_task);
712845d7797SKonstantin Belousov 	callout_reset(&thread_reap_callout, 5 * hz,
713845d7797SKonstantin Belousov 	    thread_reap_callout_cb, NULL);
714d116b9f1SMateusz Guzik }
715d116b9f1SMateusz Guzik 
716d116b9f1SMateusz Guzik /*
717f62c7e54SKonstantin Belousov  * Calling this function guarantees that any thread that exited before
718f62c7e54SKonstantin Belousov  * the call is reaped when the function returns.  By 'exited' we mean
719f62c7e54SKonstantin Belousov  * a thread removed from the process linkage with thread_unlink().
720f62c7e54SKonstantin Belousov  * Practically this means that caller must lock/unlock corresponding
721f62c7e54SKonstantin Belousov  * process lock before the call, to synchronize with thread_exit().
722f62c7e54SKonstantin Belousov  */
723f62c7e54SKonstantin Belousov void
724f62c7e54SKonstantin Belousov thread_reap_barrier(void)
725f62c7e54SKonstantin Belousov {
726f62c7e54SKonstantin Belousov 	struct task *t;
727f62c7e54SKonstantin Belousov 
728f62c7e54SKonstantin Belousov 	/*
729f62c7e54SKonstantin Belousov 	 * First do context switches to each CPU to ensure that all
730f62c7e54SKonstantin Belousov 	 * PCPU pc_deadthreads are moved to zombie list.
731f62c7e54SKonstantin Belousov 	 */
732f62c7e54SKonstantin Belousov 	quiesce_all_cpus("", PDROP);
733f62c7e54SKonstantin Belousov 
734f62c7e54SKonstantin Belousov 	/*
735f62c7e54SKonstantin Belousov 	 * Second, fire the task in the same thread as normal
736f62c7e54SKonstantin Belousov 	 * thread_reap() is done, to serialize reaping.
737f62c7e54SKonstantin Belousov 	 */
738f62c7e54SKonstantin Belousov 	t = malloc(sizeof(*t), M_TEMP, M_WAITOK);
739f62c7e54SKonstantin Belousov 	TASK_INIT(t, 0, thread_reap_task_cb, t);
740f62c7e54SKonstantin Belousov 	taskqueue_enqueue(taskqueue_thread, t);
741f62c7e54SKonstantin Belousov 	taskqueue_drain(taskqueue_thread, t);
742f62c7e54SKonstantin Belousov 	free(t, M_TEMP);
743f62c7e54SKonstantin Belousov }
744f62c7e54SKonstantin Belousov 
745f62c7e54SKonstantin Belousov /*
74644990b8cSJulian Elischer  * Allocate a thread.
74744990b8cSJulian Elischer  */
74844990b8cSJulian Elischer struct thread *
7498a945d10SKonstantin Belousov thread_alloc(int pages)
75044990b8cSJulian Elischer {
75189b57fcfSKonstantin Belousov 	struct thread *td;
7521bd3cf5dSMateusz Guzik 	lwpid_t tid;
7538460a577SJohn Birrell 
75462dbc992SMateusz Guzik 	if (!thread_count_inc()) {
7551bd3cf5dSMateusz Guzik 		return (NULL);
7561bd3cf5dSMateusz Guzik 	}
7571bd3cf5dSMateusz Guzik 
75862dbc992SMateusz Guzik 	tid = tid_alloc();
7591bd3cf5dSMateusz Guzik 	td = uma_zalloc(thread_zone, M_WAITOK);
76089b57fcfSKonstantin Belousov 	KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
7618a945d10SKonstantin Belousov 	if (!vm_thread_new(td, pages)) {
76289b57fcfSKonstantin Belousov 		uma_zfree(thread_zone, td);
7631bd3cf5dSMateusz Guzik 		tid_free(tid);
76462dbc992SMateusz Guzik 		thread_count_dec();
76589b57fcfSKonstantin Belousov 		return (NULL);
76689b57fcfSKonstantin Belousov 	}
7671bd3cf5dSMateusz Guzik 	td->td_tid = tid;
768f575573cSKonstantin Belousov 	bzero(&td->td_sa.args, sizeof(td->td_sa.args));
7695dda15adSMark Johnston 	kmsan_thread_alloc(td);
7700c3967e7SMarcel Moolenaar 	cpu_thread_alloc(td);
7711bd3cf5dSMateusz Guzik 	EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td);
77289b57fcfSKonstantin Belousov 	return (td);
77344990b8cSJulian Elischer }
77444990b8cSJulian Elischer 
7758a945d10SKonstantin Belousov int
7768a945d10SKonstantin Belousov thread_alloc_stack(struct thread *td, int pages)
7778a945d10SKonstantin Belousov {
7788a945d10SKonstantin Belousov 
7798a945d10SKonstantin Belousov 	KASSERT(td->td_kstack == 0,
7808a945d10SKonstantin Belousov 	    ("thread_alloc_stack called on a thread with kstack"));
7818a945d10SKonstantin Belousov 	if (!vm_thread_new(td, pages))
7828a945d10SKonstantin Belousov 		return (0);
7838a945d10SKonstantin Belousov 	cpu_thread_alloc(td);
7848a945d10SKonstantin Belousov 	return (1);
7858a945d10SKonstantin Belousov }
7864f0db5e0SJulian Elischer 
7874f0db5e0SJulian Elischer /*
78844990b8cSJulian Elischer  * Deallocate a thread.
78944990b8cSJulian Elischer  */
790755341dfSMateusz Guzik static void
791755341dfSMateusz Guzik thread_free_batched(struct thread *td)
79244990b8cSJulian Elischer {
7932e6b8de4SJeff Roberson 
7942e6b8de4SJeff Roberson 	lock_profile_thread_exit(td);
79545aea8deSJeff Roberson 	if (td->td_cpuset)
796d7f687fcSJeff Roberson 		cpuset_rel(td->td_cpuset);
797d7f687fcSJeff Roberson 	td->td_cpuset = NULL;
7980c3967e7SMarcel Moolenaar 	cpu_thread_free(td);
79989b57fcfSKonstantin Belousov 	if (td->td_kstack != 0)
80089b57fcfSKonstantin Belousov 		vm_thread_dispose(td);
8012d19b736SKonstantin Belousov 	callout_drain(&td->td_slpcallout);
802755341dfSMateusz Guzik 	/*
803755341dfSMateusz Guzik 	 * Freeing handled by the caller.
804755341dfSMateusz Guzik 	 */
8051bd3cf5dSMateusz Guzik 	td->td_tid = -1;
8065dda15adSMark Johnston 	kmsan_thread_free(td);
80744990b8cSJulian Elischer 	uma_zfree(thread_zone, td);
80844990b8cSJulian Elischer }
80944990b8cSJulian Elischer 
8104ea6a9a2SMateusz Guzik void
811755341dfSMateusz Guzik thread_free(struct thread *td)
812755341dfSMateusz Guzik {
813755341dfSMateusz Guzik 	lwpid_t tid;
814755341dfSMateusz Guzik 
8155ef7b7a0SMateusz Guzik 	EVENTHANDLER_DIRECT_INVOKE(thread_dtor, td);
816755341dfSMateusz Guzik 	tid = td->td_tid;
817755341dfSMateusz Guzik 	thread_free_batched(td);
818755341dfSMateusz Guzik 	tid_free(tid);
81962dbc992SMateusz Guzik 	thread_count_dec();
820755341dfSMateusz Guzik }
821755341dfSMateusz Guzik 
822755341dfSMateusz Guzik void
8234ea6a9a2SMateusz Guzik thread_cow_get_proc(struct thread *newtd, struct proc *p)
8244ea6a9a2SMateusz Guzik {
8254ea6a9a2SMateusz Guzik 
8264ea6a9a2SMateusz Guzik 	PROC_LOCK_ASSERT(p, MA_OWNED);
8271724c563SMateusz Guzik 	newtd->td_realucred = crcowget(p->p_ucred);
8281724c563SMateusz Guzik 	newtd->td_ucred = newtd->td_realucred;
829f6f6d240SMateusz Guzik 	newtd->td_limit = lim_hold(p->p_limit);
8304ea6a9a2SMateusz Guzik 	newtd->td_cowgen = p->p_cowgen;
8314ea6a9a2SMateusz Guzik }
8324ea6a9a2SMateusz Guzik 
8334ea6a9a2SMateusz Guzik void
8344ea6a9a2SMateusz Guzik thread_cow_get(struct thread *newtd, struct thread *td)
8354ea6a9a2SMateusz Guzik {
8364ea6a9a2SMateusz Guzik 
8371724c563SMateusz Guzik 	MPASS(td->td_realucred == td->td_ucred);
8381724c563SMateusz Guzik 	newtd->td_realucred = crcowget(td->td_realucred);
8391724c563SMateusz Guzik 	newtd->td_ucred = newtd->td_realucred;
840f6f6d240SMateusz Guzik 	newtd->td_limit = lim_hold(td->td_limit);
8414ea6a9a2SMateusz Guzik 	newtd->td_cowgen = td->td_cowgen;
8424ea6a9a2SMateusz Guzik }
8434ea6a9a2SMateusz Guzik 
8444ea6a9a2SMateusz Guzik void
8454ea6a9a2SMateusz Guzik thread_cow_free(struct thread *td)
8464ea6a9a2SMateusz Guzik {
8474ea6a9a2SMateusz Guzik 
8481724c563SMateusz Guzik 	if (td->td_realucred != NULL)
8491724c563SMateusz Guzik 		crcowfree(td);
850cd672ca6SMateusz Guzik 	if (td->td_limit != NULL)
851f6f6d240SMateusz Guzik 		lim_free(td->td_limit);
8524ea6a9a2SMateusz Guzik }
8534ea6a9a2SMateusz Guzik 
8544ea6a9a2SMateusz Guzik void
8554ea6a9a2SMateusz Guzik thread_cow_update(struct thread *td)
8564ea6a9a2SMateusz Guzik {
8574ea6a9a2SMateusz Guzik 	struct proc *p;
858cd672ca6SMateusz Guzik 	struct ucred *oldcred;
859cd672ca6SMateusz Guzik 	struct plimit *oldlimit;
8604ea6a9a2SMateusz Guzik 
8614ea6a9a2SMateusz Guzik 	p = td->td_proc;
8624ea6a9a2SMateusz Guzik 	PROC_LOCK(p);
8631724c563SMateusz Guzik 	oldcred = crcowsync();
8648a0cb04dSMateusz Guzik 	oldlimit = lim_cowsync();
8654ea6a9a2SMateusz Guzik 	td->td_cowgen = p->p_cowgen;
8664ea6a9a2SMateusz Guzik 	PROC_UNLOCK(p);
867cd672ca6SMateusz Guzik 	if (oldcred != NULL)
868cd672ca6SMateusz Guzik 		crfree(oldcred);
869cd672ca6SMateusz Guzik 	if (oldlimit != NULL)
870cd672ca6SMateusz Guzik 		lim_free(oldlimit);
8714ea6a9a2SMateusz Guzik }
8724ea6a9a2SMateusz Guzik 
87332114b63SMateusz Guzik void
87432114b63SMateusz Guzik thread_cow_synced(struct thread *td)
87532114b63SMateusz Guzik {
87632114b63SMateusz Guzik 	struct proc *p;
87732114b63SMateusz Guzik 
87832114b63SMateusz Guzik 	p = td->td_proc;
87932114b63SMateusz Guzik 	PROC_LOCK_ASSERT(p, MA_OWNED);
88032114b63SMateusz Guzik 	MPASS(td->td_cowgen != p->p_cowgen);
88132114b63SMateusz Guzik 	MPASS(td->td_ucred == p->p_ucred);
88232114b63SMateusz Guzik 	MPASS(td->td_limit == p->p_limit);
88332114b63SMateusz Guzik 	td->td_cowgen = p->p_cowgen;
88432114b63SMateusz Guzik }
88532114b63SMateusz Guzik 
88644990b8cSJulian Elischer /*
88744990b8cSJulian Elischer  * Discard the current thread and exit from its context.
88894e0a4cdSJulian Elischer  * Always called with scheduler locked.
88944990b8cSJulian Elischer  *
89044990b8cSJulian Elischer  * Because we can't free a thread while we're operating under its context,
891696058c3SJulian Elischer  * push the current thread into our CPU's deadthread holder. This means
892696058c3SJulian Elischer  * we needn't worry about someone else grabbing our context before we
8936617724cSJeff Roberson  * do a cpu_throw().
89444990b8cSJulian Elischer  */
89544990b8cSJulian Elischer void
89644990b8cSJulian Elischer thread_exit(void)
89744990b8cSJulian Elischer {
8987e3a96eaSJohn Baldwin 	uint64_t runtime, new_switchtime;
89944990b8cSJulian Elischer 	struct thread *td;
9001c4bcd05SJeff Roberson 	struct thread *td2;
90144990b8cSJulian Elischer 	struct proc *p;
9027847a9daSJohn Baldwin 	int wakeup_swapper;
90344990b8cSJulian Elischer 
90444990b8cSJulian Elischer 	td = curthread;
90544990b8cSJulian Elischer 	p = td->td_proc;
90644990b8cSJulian Elischer 
907a54e85fdSJeff Roberson 	PROC_SLOCK_ASSERT(p, MA_OWNED);
908ed062c8dSJulian Elischer 	mtx_assert(&Giant, MA_NOTOWNED);
909a54e85fdSJeff Roberson 
91044990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
911ed062c8dSJulian Elischer 	KASSERT(p != NULL, ("thread exiting without a process"));
912cc701b73SRobert Watson 	CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
913e01eafefSJulian Elischer 	    (long)p->p_pid, td->td_name);
9146c9271a9SAndriy Gapon 	SDT_PROBE0(proc, , , lwp__exit);
9159104847fSDavid Xu 	KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
916936c24faSMateusz Guzik 	MPASS(td->td_realucred == td->td_ucred);
91744990b8cSJulian Elischer 
918ed062c8dSJulian Elischer 	/*
919ed062c8dSJulian Elischer 	 * drop FPU & debug register state storage, or any other
920ed062c8dSJulian Elischer 	 * architecture specific resources that
921ed062c8dSJulian Elischer 	 * would not be on a new untouched process.
922ed062c8dSJulian Elischer 	 */
923bd07998eSKonstantin Belousov 	cpu_thread_exit(td);
92444990b8cSJulian Elischer 
925ed062c8dSJulian Elischer 	/*
9261faf202eSJulian Elischer 	 * The last thread is left attached to the process
9271faf202eSJulian Elischer 	 * So that the whole bundle gets recycled. Skip
928ed062c8dSJulian Elischer 	 * all this stuff if we never had threads.
929ed062c8dSJulian Elischer 	 * EXIT clears all sign of other threads when
930ed062c8dSJulian Elischer 	 * it goes to single threading, so the last thread always
931ed062c8dSJulian Elischer 	 * takes the short path.
9321faf202eSJulian Elischer 	 */
933ed062c8dSJulian Elischer 	if (p->p_flag & P_HADTHREADS) {
9341faf202eSJulian Elischer 		if (p->p_numthreads > 1) {
935fd229b5bSKonstantin Belousov 			atomic_add_int(&td->td_proc->p_exitthreads, 1);
936d3a0bd78SJulian Elischer 			thread_unlink(td);
9371c4bcd05SJeff Roberson 			td2 = FIRST_THREAD_IN_PROC(p);
9381c4bcd05SJeff Roberson 			sched_exit_thread(td2, td);
939ed062c8dSJulian Elischer 
940ed062c8dSJulian Elischer 			/*
94144990b8cSJulian Elischer 			 * The test below is NOT true if we are the
9429182554aSKonstantin Belousov 			 * sole exiting thread. P_STOPPED_SINGLE is unset
94344990b8cSJulian Elischer 			 * in exit1() after it is the only survivor.
94444990b8cSJulian Elischer 			 */
9451279572aSDavid Xu 			if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
94644990b8cSJulian Elischer 				if (p->p_numthreads == p->p_suspcount) {
947a54e85fdSJeff Roberson 					thread_lock(p->p_singlethread);
9487847a9daSJohn Baldwin 					wakeup_swapper = thread_unsuspend_one(
94984cdea97SKonstantin Belousov 						p->p_singlethread, p, false);
9507847a9daSJohn Baldwin 					if (wakeup_swapper)
9517847a9daSJohn Baldwin 						kick_proc0();
95244990b8cSJulian Elischer 				}
95344990b8cSJulian Elischer 			}
95448bfcdddSJulian Elischer 
955696058c3SJulian Elischer 			PCPU_SET(deadthread, td);
9561faf202eSJulian Elischer 		} else {
957ed062c8dSJulian Elischer 			/*
958ed062c8dSJulian Elischer 			 * The last thread is exiting.. but not through exit()
959ed062c8dSJulian Elischer 			 */
960ed062c8dSJulian Elischer 			panic ("thread_exit: Last thread exiting on its own");
961ed062c8dSJulian Elischer 		}
9621faf202eSJulian Elischer 	}
96316d95d4fSJoseph Koshy #ifdef	HWPMC_HOOKS
96416d95d4fSJoseph Koshy 	/*
96516d95d4fSJoseph Koshy 	 * If this thread is part of a process that is being tracked by hwpmc(4),
96616d95d4fSJoseph Koshy 	 * inform the module of the thread's impending exit.
96716d95d4fSJoseph Koshy 	 */
9686161b98cSMatt Macy 	if (PMC_PROC_IS_USING_PMCS(td->td_proc)) {
96916d95d4fSJoseph Koshy 		PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
9706161b98cSMatt Macy 		PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT, NULL);
971ebfaf69cSMatt Macy 	} else if (PMC_SYSTEM_SAMPLING_ACTIVE())
972ebfaf69cSMatt Macy 		PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT_LOG, NULL);
97316d95d4fSJoseph Koshy #endif
974a54e85fdSJeff Roberson 	PROC_UNLOCK(p);
9755c7bebf9SKonstantin Belousov 	PROC_STATLOCK(p);
9765c7bebf9SKonstantin Belousov 	thread_lock(td);
9775c7bebf9SKonstantin Belousov 	PROC_SUNLOCK(p);
9787e3a96eaSJohn Baldwin 
9797e3a96eaSJohn Baldwin 	/* Do the same timestamp bookkeeping that mi_switch() would do. */
9807e3a96eaSJohn Baldwin 	new_switchtime = cpu_ticks();
9817e3a96eaSJohn Baldwin 	runtime = new_switchtime - PCPU_GET(switchtime);
9827e3a96eaSJohn Baldwin 	td->td_runtime += runtime;
9837e3a96eaSJohn Baldwin 	td->td_incruntime += runtime;
9847e3a96eaSJohn Baldwin 	PCPU_SET(switchtime, new_switchtime);
9857e3a96eaSJohn Baldwin 	PCPU_SET(switchticks, ticks);
98683c9dea1SGleb Smirnoff 	VM_CNT_INC(v_swtch);
9877e3a96eaSJohn Baldwin 
9887e3a96eaSJohn Baldwin 	/* Save our resource usage in our process. */
9897e3a96eaSJohn Baldwin 	td->td_ru.ru_nvcsw++;
99061a74c5cSJeff Roberson 	ruxagg_locked(p, td);
9917e3a96eaSJohn Baldwin 	rucollect(&p->p_ru, &td->td_ru);
9925c7bebf9SKonstantin Belousov 	PROC_STATUNLOCK(p);
9937e3a96eaSJohn Baldwin 
994fa2528acSAlex Richardson 	TD_SET_STATE(td, TDS_INACTIVE);
9953d06b4b3SAttilio Rao #ifdef WITNESS
9963d06b4b3SAttilio Rao 	witness_thread_exit(td);
9973d06b4b3SAttilio Rao #endif
998732d9528SJulian Elischer 	CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
999a54e85fdSJeff Roberson 	sched_throw(td);
1000cc66ebe2SPeter Wemm 	panic("I'm a teapot!");
100144990b8cSJulian Elischer 	/* NOTREACHED */
100244990b8cSJulian Elischer }
100344990b8cSJulian Elischer 
100444990b8cSJulian Elischer /*
1005696058c3SJulian Elischer  * Do any thread specific cleanups that may be needed in wait()
100637814395SPeter Wemm  * called with Giant, proc and schedlock not held.
1007696058c3SJulian Elischer  */
1008696058c3SJulian Elischer void
1009696058c3SJulian Elischer thread_wait(struct proc *p)
1010696058c3SJulian Elischer {
1011696058c3SJulian Elischer 	struct thread *td;
1012696058c3SJulian Elischer 
101337814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
1014624bf9e1SKonstantin Belousov 	KASSERT(p->p_numthreads == 1, ("multiple threads in thread_wait()"));
1015624bf9e1SKonstantin Belousov 	KASSERT(p->p_exitthreads == 0, ("p_exitthreads leaking"));
1016ff8fbcffSJeff Roberson 	td = FIRST_THREAD_IN_PROC(p);
1017ff8fbcffSJeff Roberson 	/* Lock the last thread so we spin until it exits cpu_throw(). */
1018ff8fbcffSJeff Roberson 	thread_lock(td);
1019ff8fbcffSJeff Roberson 	thread_unlock(td);
10202e6b8de4SJeff Roberson 	lock_profile_thread_exit(td);
1021d7f687fcSJeff Roberson 	cpuset_rel(td->td_cpuset);
1022d7f687fcSJeff Roberson 	td->td_cpuset = NULL;
1023696058c3SJulian Elischer 	cpu_thread_clean(td);
10244ea6a9a2SMateusz Guzik 	thread_cow_free(td);
10252d19b736SKonstantin Belousov 	callout_drain(&td->td_slpcallout);
1026696058c3SJulian Elischer 	thread_reap();	/* check for zombie threads etc. */
1027696058c3SJulian Elischer }
1028696058c3SJulian Elischer 
1029696058c3SJulian Elischer /*
103044990b8cSJulian Elischer  * Link a thread to a process.
10311faf202eSJulian Elischer  * set up anything that needs to be initialized for it to
10321faf202eSJulian Elischer  * be used by the process.
103344990b8cSJulian Elischer  */
103444990b8cSJulian Elischer void
10358460a577SJohn Birrell thread_link(struct thread *td, struct proc *p)
103644990b8cSJulian Elischer {
103744990b8cSJulian Elischer 
1038a54e85fdSJeff Roberson 	/*
1039a54e85fdSJeff Roberson 	 * XXX This can't be enabled because it's called for proc0 before
1040374ae2a3SJeff Roberson 	 * its lock has been created.
1041374ae2a3SJeff Roberson 	 * PROC_LOCK_ASSERT(p, MA_OWNED);
1042a54e85fdSJeff Roberson 	 */
1043fa2528acSAlex Richardson 	TD_SET_STATE(td, TDS_INACTIVE);
104444990b8cSJulian Elischer 	td->td_proc     = p;
1045b61ce5b0SJeff Roberson 	td->td_flags    = TDF_INMEM;
104644990b8cSJulian Elischer 
10471faf202eSJulian Elischer 	LIST_INIT(&td->td_contested);
1048eea4f254SJeff Roberson 	LIST_INIT(&td->td_lprof[0]);
1049eea4f254SJeff Roberson 	LIST_INIT(&td->td_lprof[1]);
1050f6eccf96SGleb Smirnoff #ifdef EPOCH_TRACE
1051dd902d01SGleb Smirnoff 	SLIST_INIT(&td->td_epochs);
1052f6eccf96SGleb Smirnoff #endif
10539104847fSDavid Xu 	sigqueue_init(&td->td_sigqueue, p);
1054fd90e2edSJung-uk Kim 	callout_init(&td->td_slpcallout, 1);
105566d8df9dSDaniel Eischen 	TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist);
105644990b8cSJulian Elischer 	p->p_numthreads++;
105744990b8cSJulian Elischer }
105844990b8cSJulian Elischer 
1059ed062c8dSJulian Elischer /*
1060ed062c8dSJulian Elischer  * Called from:
1061ed062c8dSJulian Elischer  *  thread_exit()
1062ed062c8dSJulian Elischer  */
1063d3a0bd78SJulian Elischer void
1064d3a0bd78SJulian Elischer thread_unlink(struct thread *td)
1065d3a0bd78SJulian Elischer {
1066d3a0bd78SJulian Elischer 	struct proc *p = td->td_proc;
1067d3a0bd78SJulian Elischer 
1068374ae2a3SJeff Roberson 	PROC_LOCK_ASSERT(p, MA_OWNED);
1069f6eccf96SGleb Smirnoff #ifdef EPOCH_TRACE
1070dd902d01SGleb Smirnoff 	MPASS(SLIST_EMPTY(&td->td_epochs));
1071f6eccf96SGleb Smirnoff #endif
1072dd902d01SGleb Smirnoff 
1073d3a0bd78SJulian Elischer 	TAILQ_REMOVE(&p->p_threads, td, td_plist);
1074d3a0bd78SJulian Elischer 	p->p_numthreads--;
1075d3a0bd78SJulian Elischer 	/* could clear a few other things here */
10768460a577SJohn Birrell 	/* Must  NOT clear links to proc! */
10775c8329edSJulian Elischer }
10785c8329edSJulian Elischer 
107979799053SKonstantin Belousov static int
108079799053SKonstantin Belousov calc_remaining(struct proc *p, int mode)
108179799053SKonstantin Belousov {
108279799053SKonstantin Belousov 	int remaining;
108379799053SKonstantin Belousov 
10847b519077SKonstantin Belousov 	PROC_LOCK_ASSERT(p, MA_OWNED);
10857b519077SKonstantin Belousov 	PROC_SLOCK_ASSERT(p, MA_OWNED);
108679799053SKonstantin Belousov 	if (mode == SINGLE_EXIT)
108779799053SKonstantin Belousov 		remaining = p->p_numthreads;
108879799053SKonstantin Belousov 	else if (mode == SINGLE_BOUNDARY)
108979799053SKonstantin Belousov 		remaining = p->p_numthreads - p->p_boundary_count;
10906ddcc233SKonstantin Belousov 	else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC)
109179799053SKonstantin Belousov 		remaining = p->p_numthreads - p->p_suspcount;
109279799053SKonstantin Belousov 	else
109379799053SKonstantin Belousov 		panic("calc_remaining: wrong mode %d", mode);
109479799053SKonstantin Belousov 	return (remaining);
109579799053SKonstantin Belousov }
109679799053SKonstantin Belousov 
109707a9368aSKonstantin Belousov static int
109807a9368aSKonstantin Belousov remain_for_mode(int mode)
109907a9368aSKonstantin Belousov {
110007a9368aSKonstantin Belousov 
11016ddcc233SKonstantin Belousov 	return (mode == SINGLE_ALLPROC ? 0 : 1);
110207a9368aSKonstantin Belousov }
110307a9368aSKonstantin Belousov 
110407a9368aSKonstantin Belousov static int
110507a9368aSKonstantin Belousov weed_inhib(int mode, struct thread *td2, struct proc *p)
110607a9368aSKonstantin Belousov {
110707a9368aSKonstantin Belousov 	int wakeup_swapper;
110807a9368aSKonstantin Belousov 
110907a9368aSKonstantin Belousov 	PROC_LOCK_ASSERT(p, MA_OWNED);
111007a9368aSKonstantin Belousov 	PROC_SLOCK_ASSERT(p, MA_OWNED);
111107a9368aSKonstantin Belousov 	THREAD_LOCK_ASSERT(td2, MA_OWNED);
111207a9368aSKonstantin Belousov 
111307a9368aSKonstantin Belousov 	wakeup_swapper = 0;
111461a74c5cSJeff Roberson 
111561a74c5cSJeff Roberson 	/*
111661a74c5cSJeff Roberson 	 * Since the thread lock is dropped by the scheduler we have
111761a74c5cSJeff Roberson 	 * to retry to check for races.
111861a74c5cSJeff Roberson 	 */
111961a74c5cSJeff Roberson restart:
112007a9368aSKonstantin Belousov 	switch (mode) {
112107a9368aSKonstantin Belousov 	case SINGLE_EXIT:
112261a74c5cSJeff Roberson 		if (TD_IS_SUSPENDED(td2)) {
112384cdea97SKonstantin Belousov 			wakeup_swapper |= thread_unsuspend_one(td2, p, true);
112461a74c5cSJeff Roberson 			thread_lock(td2);
112561a74c5cSJeff Roberson 			goto restart;
112661a74c5cSJeff Roberson 		}
112761a74c5cSJeff Roberson 		if (TD_CAN_ABORT(td2)) {
112807a9368aSKonstantin Belousov 			wakeup_swapper |= sleepq_abort(td2, EINTR);
112961a74c5cSJeff Roberson 			return (wakeup_swapper);
113061a74c5cSJeff Roberson 		}
113107a9368aSKonstantin Belousov 		break;
113207a9368aSKonstantin Belousov 	case SINGLE_BOUNDARY:
113307a9368aSKonstantin Belousov 	case SINGLE_NO_EXIT:
113461a74c5cSJeff Roberson 		if (TD_IS_SUSPENDED(td2) &&
113561a74c5cSJeff Roberson 		    (td2->td_flags & TDF_BOUNDARY) == 0) {
113684cdea97SKonstantin Belousov 			wakeup_swapper |= thread_unsuspend_one(td2, p, false);
113761a74c5cSJeff Roberson 			thread_lock(td2);
113861a74c5cSJeff Roberson 			goto restart;
113961a74c5cSJeff Roberson 		}
114061a74c5cSJeff Roberson 		if (TD_CAN_ABORT(td2)) {
114107a9368aSKonstantin Belousov 			wakeup_swapper |= sleepq_abort(td2, ERESTART);
114261a74c5cSJeff Roberson 			return (wakeup_swapper);
114361a74c5cSJeff Roberson 		}
1144917dd390SKonstantin Belousov 		break;
11456ddcc233SKonstantin Belousov 	case SINGLE_ALLPROC:
11466ddcc233SKonstantin Belousov 		/*
11476ddcc233SKonstantin Belousov 		 * ALLPROC suspend tries to avoid spurious EINTR for
11486ddcc233SKonstantin Belousov 		 * threads sleeping interruptable, by suspending the
11496ddcc233SKonstantin Belousov 		 * thread directly, similarly to sig_suspend_threads().
1150dd883e9aSKonstantin Belousov 		 * Since such sleep is not neccessary performed at the user
1151dd883e9aSKonstantin Belousov 		 * boundary, TDF_ALLPROCSUSP is used to avoid immediate
1152dd883e9aSKonstantin Belousov 		 * un-suspend.
11536ddcc233SKonstantin Belousov 		 */
1154*5c274b36SBrooks Davis 		if (TD_IS_SUSPENDED(td2) &&
1155*5c274b36SBrooks Davis 		    (td2->td_flags & TDF_ALLPROCSUSP) == 0) {
115684cdea97SKonstantin Belousov 			wakeup_swapper |= thread_unsuspend_one(td2, p, false);
115761a74c5cSJeff Roberson 			thread_lock(td2);
115861a74c5cSJeff Roberson 			goto restart;
115961a74c5cSJeff Roberson 		}
116061a74c5cSJeff Roberson 		if (TD_CAN_ABORT(td2)) {
11616ddcc233SKonstantin Belousov 			td2->td_flags |= TDF_ALLPROCSUSP;
11626ddcc233SKonstantin Belousov 			wakeup_swapper |= sleepq_abort(td2, ERESTART);
116361a74c5cSJeff Roberson 			return (wakeup_swapper);
11646ddcc233SKonstantin Belousov 		}
116507a9368aSKonstantin Belousov 		break;
116661a74c5cSJeff Roberson 	default:
116761a74c5cSJeff Roberson 		break;
116807a9368aSKonstantin Belousov 	}
116961a74c5cSJeff Roberson 	thread_unlock(td2);
117007a9368aSKonstantin Belousov 	return (wakeup_swapper);
117107a9368aSKonstantin Belousov }
117207a9368aSKonstantin Belousov 
11735215b187SJeff Roberson /*
117444990b8cSJulian Elischer  * Enforce single-threading.
117544990b8cSJulian Elischer  *
117644990b8cSJulian Elischer  * Returns 1 if the caller must abort (another thread is waiting to
117744990b8cSJulian Elischer  * exit the process or similar). Process is locked!
117844990b8cSJulian Elischer  * Returns 0 when you are successfully the only thread running.
117944990b8cSJulian Elischer  * A process has successfully single threaded in the suspend mode when
118044990b8cSJulian Elischer  * There are no threads in user mode. Threads in the kernel must be
118144990b8cSJulian Elischer  * allowed to continue until they get to the user boundary. They may even
118244990b8cSJulian Elischer  * copy out their return values and data before suspending. They may however be
1183e2668f55SMaxim Konovalov  * accelerated in reaching the user boundary as we will wake up
118444990b8cSJulian Elischer  * any sleeping threads that are interruptable. (PCATCH).
118544990b8cSJulian Elischer  */
118644990b8cSJulian Elischer int
11876ddcc233SKonstantin Belousov thread_single(struct proc *p, int mode)
118844990b8cSJulian Elischer {
118944990b8cSJulian Elischer 	struct thread *td;
119044990b8cSJulian Elischer 	struct thread *td2;
1191da7bbd2cSJohn Baldwin 	int remaining, wakeup_swapper;
119244990b8cSJulian Elischer 
119344990b8cSJulian Elischer 	td = curthread;
11946ddcc233SKonstantin Belousov 	KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
11956ddcc233SKonstantin Belousov 	    mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
11966ddcc233SKonstantin Belousov 	    ("invalid mode %d", mode));
11976ddcc233SKonstantin Belousov 	/*
11986ddcc233SKonstantin Belousov 	 * If allowing non-ALLPROC singlethreading for non-curproc
11996ddcc233SKonstantin Belousov 	 * callers, calc_remaining() and remain_for_mode() should be
12006ddcc233SKonstantin Belousov 	 * adjusted to also account for td->td_proc != p.  For now
12016ddcc233SKonstantin Belousov 	 * this is not implemented because it is not used.
12026ddcc233SKonstantin Belousov 	 */
12036ddcc233SKonstantin Belousov 	KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) ||
12046ddcc233SKonstantin Belousov 	    (mode != SINGLE_ALLPROC && td->td_proc == p),
12056ddcc233SKonstantin Belousov 	    ("mode %d proc %p curproc %p", mode, p, td->td_proc));
120637814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
120744990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
120844990b8cSJulian Elischer 
1209d7a9e6e7SKonstantin Belousov 	/*
1210d7a9e6e7SKonstantin Belousov 	 * Is someone already single threading?
1211d7a9e6e7SKonstantin Belousov 	 * Or may be singlethreading is not needed at all.
1212d7a9e6e7SKonstantin Belousov 	 */
1213d7a9e6e7SKonstantin Belousov 	if (mode == SINGLE_ALLPROC) {
1214d7a9e6e7SKonstantin Belousov 		while ((p->p_flag & P_STOPPED_SINGLE) != 0) {
1215d7a9e6e7SKonstantin Belousov 			if ((p->p_flag2 & P2_WEXIT) != 0)
1216d7a9e6e7SKonstantin Belousov 				return (1);
1217d7a9e6e7SKonstantin Belousov 			msleep(&p->p_flag, &p->p_mtx, PCATCH, "thrsgl", 0);
1218d7a9e6e7SKonstantin Belousov 		}
1219d7a9e6e7SKonstantin Belousov 	} else if ((p->p_flag & P_HADTHREADS) == 0)
122044990b8cSJulian Elischer 		return (0);
1221906ac69dSDavid Xu 	if (p->p_singlethread != NULL && p->p_singlethread != td)
122244990b8cSJulian Elischer 		return (1);
122344990b8cSJulian Elischer 
1224906ac69dSDavid Xu 	if (mode == SINGLE_EXIT) {
1225906ac69dSDavid Xu 		p->p_flag |= P_SINGLE_EXIT;
1226906ac69dSDavid Xu 		p->p_flag &= ~P_SINGLE_BOUNDARY;
1227906ac69dSDavid Xu 	} else {
1228906ac69dSDavid Xu 		p->p_flag &= ~P_SINGLE_EXIT;
1229906ac69dSDavid Xu 		if (mode == SINGLE_BOUNDARY)
1230906ac69dSDavid Xu 			p->p_flag |= P_SINGLE_BOUNDARY;
1231906ac69dSDavid Xu 		else
1232906ac69dSDavid Xu 			p->p_flag &= ~P_SINGLE_BOUNDARY;
1233906ac69dSDavid Xu 	}
1234f829268bSKonstantin Belousov 	if (mode == SINGLE_ALLPROC)
12356ddcc233SKonstantin Belousov 		p->p_flag |= P_TOTAL_STOP;
12361279572aSDavid Xu 	p->p_flag |= P_STOPPED_SINGLE;
12377b4a950aSDavid Xu 	PROC_SLOCK(p);
1238112afcb2SJohn Baldwin 	p->p_singlethread = td;
123979799053SKonstantin Belousov 	remaining = calc_remaining(p, mode);
124007a9368aSKonstantin Belousov 	while (remaining != remain_for_mode(mode)) {
1241bf1a3220SDavid Xu 		if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
1242bf1a3220SDavid Xu 			goto stopme;
1243da7bbd2cSJohn Baldwin 		wakeup_swapper = 0;
124444990b8cSJulian Elischer 		FOREACH_THREAD_IN_PROC(p, td2) {
124544990b8cSJulian Elischer 			if (td2 == td)
124644990b8cSJulian Elischer 				continue;
1247a54e85fdSJeff Roberson 			thread_lock(td2);
1248c6d31b83SKonstantin Belousov 			ast_sched_locked(td2, TDA_SUSPEND);
12496ddcc233SKonstantin Belousov 			if (TD_IS_INHIBITED(td2)) {
125007a9368aSKonstantin Belousov 				wakeup_swapper |= weed_inhib(mode, td2, p);
1251d8267df7SDavid Xu #ifdef SMP
1252b9009b17SKonstantin Belousov 			} else if (TD_IS_RUNNING(td2)) {
1253d8267df7SDavid Xu 				forward_signal(td2);
125461a74c5cSJeff Roberson 				thread_unlock(td2);
1255d8267df7SDavid Xu #endif
125661a74c5cSJeff Roberson 			} else
1257a54e85fdSJeff Roberson 				thread_unlock(td2);
12589d102777SJulian Elischer 		}
1259da7bbd2cSJohn Baldwin 		if (wakeup_swapper)
1260da7bbd2cSJohn Baldwin 			kick_proc0();
126179799053SKonstantin Belousov 		remaining = calc_remaining(p, mode);
1262ec008e96SDavid Xu 
12639d102777SJulian Elischer 		/*
12649d102777SJulian Elischer 		 * Maybe we suspended some threads.. was it enough?
12659d102777SJulian Elischer 		 */
126607a9368aSKonstantin Belousov 		if (remaining == remain_for_mode(mode))
12679d102777SJulian Elischer 			break;
12689d102777SJulian Elischer 
1269bf1a3220SDavid Xu stopme:
127044990b8cSJulian Elischer 		/*
127144990b8cSJulian Elischer 		 * Wake us up when everyone else has suspended.
1272e3b9bf71SJulian Elischer 		 * In the mean time we suspend as well.
127344990b8cSJulian Elischer 		 */
12746ddcc233SKonstantin Belousov 		thread_suspend_switch(td, p);
127579799053SKonstantin Belousov 		remaining = calc_remaining(p, mode);
127644990b8cSJulian Elischer 	}
1277906ac69dSDavid Xu 	if (mode == SINGLE_EXIT) {
127891599697SJulian Elischer 		/*
12798626a0ddSKonstantin Belousov 		 * Convert the process to an unthreaded process.  The
12808626a0ddSKonstantin Belousov 		 * SINGLE_EXIT is called by exit1() or execve(), in
12818626a0ddSKonstantin Belousov 		 * both cases other threads must be retired.
128291599697SJulian Elischer 		 */
12838626a0ddSKonstantin Belousov 		KASSERT(p->p_numthreads == 1, ("Unthreading with >1 threads"));
1284ed062c8dSJulian Elischer 		p->p_singlethread = NULL;
12858626a0ddSKonstantin Belousov 		p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS);
1286fd229b5bSKonstantin Belousov 
1287fd229b5bSKonstantin Belousov 		/*
1288fd229b5bSKonstantin Belousov 		 * Wait for any remaining threads to exit cpu_throw().
1289fd229b5bSKonstantin Belousov 		 */
1290fd229b5bSKonstantin Belousov 		while (p->p_exitthreads != 0) {
1291fd229b5bSKonstantin Belousov 			PROC_SUNLOCK(p);
1292fd229b5bSKonstantin Belousov 			PROC_UNLOCK(p);
1293fd229b5bSKonstantin Belousov 			sched_relinquish(td);
1294fd229b5bSKonstantin Belousov 			PROC_LOCK(p);
1295fd229b5bSKonstantin Belousov 			PROC_SLOCK(p);
1296fd229b5bSKonstantin Belousov 		}
1297ac437c07SKonstantin Belousov 	} else if (mode == SINGLE_BOUNDARY) {
1298ac437c07SKonstantin Belousov 		/*
1299ac437c07SKonstantin Belousov 		 * Wait until all suspended threads are removed from
1300ac437c07SKonstantin Belousov 		 * the processors.  The thread_suspend_check()
1301ac437c07SKonstantin Belousov 		 * increments p_boundary_count while it is still
1302ac437c07SKonstantin Belousov 		 * running, which makes it possible for the execve()
1303ac437c07SKonstantin Belousov 		 * to destroy vmspace while our other threads are
1304ac437c07SKonstantin Belousov 		 * still using the address space.
1305ac437c07SKonstantin Belousov 		 *
1306ac437c07SKonstantin Belousov 		 * We lock the thread, which is only allowed to
1307ac437c07SKonstantin Belousov 		 * succeed after context switch code finished using
1308ac437c07SKonstantin Belousov 		 * the address space.
1309ac437c07SKonstantin Belousov 		 */
1310ac437c07SKonstantin Belousov 		FOREACH_THREAD_IN_PROC(p, td2) {
1311ac437c07SKonstantin Belousov 			if (td2 == td)
1312ac437c07SKonstantin Belousov 				continue;
1313ac437c07SKonstantin Belousov 			thread_lock(td2);
1314ac437c07SKonstantin Belousov 			KASSERT((td2->td_flags & TDF_BOUNDARY) != 0,
1315ac437c07SKonstantin Belousov 			    ("td %p not on boundary", td2));
1316ac437c07SKonstantin Belousov 			KASSERT(TD_IS_SUSPENDED(td2),
1317ac437c07SKonstantin Belousov 			    ("td %p is not suspended", td2));
1318ac437c07SKonstantin Belousov 			thread_unlock(td2);
1319ac437c07SKonstantin Belousov 		}
132091599697SJulian Elischer 	}
13217b4a950aSDavid Xu 	PROC_SUNLOCK(p);
132244990b8cSJulian Elischer 	return (0);
132344990b8cSJulian Elischer }
132444990b8cSJulian Elischer 
13258638fe7bSKonstantin Belousov bool
13268638fe7bSKonstantin Belousov thread_suspend_check_needed(void)
13278638fe7bSKonstantin Belousov {
13288638fe7bSKonstantin Belousov 	struct proc *p;
13298638fe7bSKonstantin Belousov 	struct thread *td;
13308638fe7bSKonstantin Belousov 
13318638fe7bSKonstantin Belousov 	td = curthread;
13328638fe7bSKonstantin Belousov 	p = td->td_proc;
13338638fe7bSKonstantin Belousov 	PROC_LOCK_ASSERT(p, MA_OWNED);
13348638fe7bSKonstantin Belousov 	return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 &&
13358638fe7bSKonstantin Belousov 	    (td->td_dbgflags & TDB_SUSPEND) != 0));
13368638fe7bSKonstantin Belousov }
13378638fe7bSKonstantin Belousov 
133844990b8cSJulian Elischer /*
133944990b8cSJulian Elischer  * Called in from locations that can safely check to see
134044990b8cSJulian Elischer  * whether we have to suspend or at least throttle for a
134144990b8cSJulian Elischer  * single-thread event (e.g. fork).
134244990b8cSJulian Elischer  *
134344990b8cSJulian Elischer  * Such locations include userret().
134444990b8cSJulian Elischer  * If the "return_instead" argument is non zero, the thread must be able to
134544990b8cSJulian Elischer  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
134644990b8cSJulian Elischer  *
134744990b8cSJulian Elischer  * The 'return_instead' argument tells the function if it may do a
134844990b8cSJulian Elischer  * thread_exit() or suspend, or whether the caller must abort and back
134944990b8cSJulian Elischer  * out instead.
135044990b8cSJulian Elischer  *
135144990b8cSJulian Elischer  * If the thread that set the single_threading request has set the
135244990b8cSJulian Elischer  * P_SINGLE_EXIT bit in the process flags then this call will never return
135344990b8cSJulian Elischer  * if 'return_instead' is false, but will exit.
135444990b8cSJulian Elischer  *
135544990b8cSJulian Elischer  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
135644990b8cSJulian Elischer  *---------------+--------------------+---------------------
135744990b8cSJulian Elischer  *       0       | returns 0          |   returns 0 or 1
1358353374b5SJohn Baldwin  *               | when ST ends       |   immediately
135944990b8cSJulian Elischer  *---------------+--------------------+---------------------
136044990b8cSJulian Elischer  *       1       | thread exits       |   returns 1
1361353374b5SJohn Baldwin  *               |                    |  immediately
136244990b8cSJulian Elischer  * 0 = thread_exit() or suspension ok,
136344990b8cSJulian Elischer  * other = return error instead of stopping the thread.
136444990b8cSJulian Elischer  *
136544990b8cSJulian Elischer  * While a full suspension is under effect, even a single threading
136644990b8cSJulian Elischer  * thread would be suspended if it made this call (but it shouldn't).
136744990b8cSJulian Elischer  * This call should only be made from places where
136844990b8cSJulian Elischer  * thread_exit() would be safe as that may be the outcome unless
136944990b8cSJulian Elischer  * return_instead is set.
137044990b8cSJulian Elischer  */
137144990b8cSJulian Elischer int
137244990b8cSJulian Elischer thread_suspend_check(int return_instead)
137344990b8cSJulian Elischer {
1374ecafb24bSJuli Mallett 	struct thread *td;
1375ecafb24bSJuli Mallett 	struct proc *p;
137646e47c4fSKonstantin Belousov 	int wakeup_swapper;
137744990b8cSJulian Elischer 
137844990b8cSJulian Elischer 	td = curthread;
137944990b8cSJulian Elischer 	p = td->td_proc;
138037814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
138144990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
13828638fe7bSKonstantin Belousov 	while (thread_suspend_check_needed()) {
13831279572aSDavid Xu 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
138444990b8cSJulian Elischer 			KASSERT(p->p_singlethread != NULL,
138544990b8cSJulian Elischer 			    ("singlethread not set"));
138644990b8cSJulian Elischer 			/*
1387e3b9bf71SJulian Elischer 			 * The only suspension in action is a
1388e3b9bf71SJulian Elischer 			 * single-threading. Single threader need not stop.
1389bd07998eSKonstantin Belousov 			 * It is safe to access p->p_singlethread unlocked
1390bd07998eSKonstantin Belousov 			 * because it can only be set to our address by us.
139144990b8cSJulian Elischer 			 */
1392e3b9bf71SJulian Elischer 			if (p->p_singlethread == td)
139344990b8cSJulian Elischer 				return (0);	/* Exempt from stopping. */
139444990b8cSJulian Elischer 		}
139545a4bfa1SDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
139694f0972bSDavid Xu 			return (EINTR);
139744990b8cSJulian Elischer 
1398906ac69dSDavid Xu 		/* Should we goto user boundary if we didn't come from there? */
1399906ac69dSDavid Xu 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
1400906ac69dSDavid Xu 		    (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
140194f0972bSDavid Xu 			return (ERESTART);
1402906ac69dSDavid Xu 
140344990b8cSJulian Elischer 		/*
14043077f938SKonstantin Belousov 		 * Ignore suspend requests if they are deferred.
1405d071a6faSJohn Baldwin 		 */
14063077f938SKonstantin Belousov 		if ((td->td_flags & TDF_SBDRY) != 0) {
1407d071a6faSJohn Baldwin 			KASSERT(return_instead,
1408d071a6faSJohn Baldwin 			    ("TDF_SBDRY set for unsafe thread_suspend_check"));
140946e47c4fSKonstantin Belousov 			KASSERT((td->td_flags & (TDF_SEINTR | TDF_SERESTART)) !=
141046e47c4fSKonstantin Belousov 			    (TDF_SEINTR | TDF_SERESTART),
141146e47c4fSKonstantin Belousov 			    ("both TDF_SEINTR and TDF_SERESTART"));
141246e47c4fSKonstantin Belousov 			return (TD_SBDRY_INTR(td) ? TD_SBDRY_ERRNO(td) : 0);
1413d071a6faSJohn Baldwin 		}
1414d071a6faSJohn Baldwin 
1415d071a6faSJohn Baldwin 		/*
141644990b8cSJulian Elischer 		 * If the process is waiting for us to exit,
141744990b8cSJulian Elischer 		 * this thread should just suicide.
14181279572aSDavid Xu 		 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
141944990b8cSJulian Elischer 		 */
1420cf7d9a8cSDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) {
1421cf7d9a8cSDavid Xu 			PROC_UNLOCK(p);
142291d1786fSDmitry Chagin 
142391d1786fSDmitry Chagin 			/*
142491d1786fSDmitry Chagin 			 * Allow Linux emulation layer to do some work
142591d1786fSDmitry Chagin 			 * before thread suicide.
142691d1786fSDmitry Chagin 			 */
142791d1786fSDmitry Chagin 			if (__predict_false(p->p_sysent->sv_thread_detach != NULL))
142891d1786fSDmitry Chagin 				(p->p_sysent->sv_thread_detach)(td);
14292a339d9eSKonstantin Belousov 			umtx_thread_exit(td);
1430d1e7a4a5SJohn Baldwin 			kern_thr_exit(td);
1431d1e7a4a5SJohn Baldwin 			panic("stopped thread did not exit");
1432cf7d9a8cSDavid Xu 		}
143321ecd1e9SDavid Xu 
143421ecd1e9SDavid Xu 		PROC_SLOCK(p);
143521ecd1e9SDavid Xu 		thread_stopped(p);
1436a54e85fdSJeff Roberson 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
1437a54e85fdSJeff Roberson 			if (p->p_numthreads == p->p_suspcount + 1) {
1438a54e85fdSJeff Roberson 				thread_lock(p->p_singlethread);
143984cdea97SKonstantin Belousov 				wakeup_swapper = thread_unsuspend_one(
144084cdea97SKonstantin Belousov 				    p->p_singlethread, p, false);
14417847a9daSJohn Baldwin 				if (wakeup_swapper)
14427847a9daSJohn Baldwin 					kick_proc0();
1443a54e85fdSJeff Roberson 			}
1444a54e85fdSJeff Roberson 		}
14453f9be10eSDavid Xu 		PROC_UNLOCK(p);
14467b4a950aSDavid Xu 		thread_lock(td);
144744990b8cSJulian Elischer 		/*
144844990b8cSJulian Elischer 		 * When a thread suspends, it just
1449ad1e7d28SJulian Elischer 		 * gets taken off all queues.
145044990b8cSJulian Elischer 		 */
145171fad9fdSJulian Elischer 		thread_suspend_one(td);
1452906ac69dSDavid Xu 		if (return_instead == 0) {
1453906ac69dSDavid Xu 			p->p_boundary_count++;
1454906ac69dSDavid Xu 			td->td_flags |= TDF_BOUNDARY;
1455cf19bf91SJulian Elischer 		}
14567b4a950aSDavid Xu 		PROC_SUNLOCK(p);
1457686bcb5cSJeff Roberson 		mi_switch(SW_INVOL | SWT_SUSPEND);
145844990b8cSJulian Elischer 		PROC_LOCK(p);
145944990b8cSJulian Elischer 	}
146044990b8cSJulian Elischer 	return (0);
146144990b8cSJulian Elischer }
146244990b8cSJulian Elischer 
1463478ca4b0SKonstantin Belousov /*
1464478ca4b0SKonstantin Belousov  * Check for possible stops and suspensions while executing a
1465478ca4b0SKonstantin Belousov  * casueword or similar transiently failing operation.
1466478ca4b0SKonstantin Belousov  *
1467478ca4b0SKonstantin Belousov  * The sleep argument controls whether the function can handle a stop
1468478ca4b0SKonstantin Belousov  * request itself or it should return ERESTART and the request is
1469478ca4b0SKonstantin Belousov  * proceed at the kernel/user boundary in ast.
1470478ca4b0SKonstantin Belousov  *
1471478ca4b0SKonstantin Belousov  * Typically, when retrying due to casueword(9) failure (rv == 1), we
1472478ca4b0SKonstantin Belousov  * should handle the stop requests there, with exception of cases when
1473478ca4b0SKonstantin Belousov  * the thread owns a kernel resource, for instance busied the umtx
1474300b525dSKonstantin Belousov  * key, or when functions return immediately if thread_check_susp()
1475478ca4b0SKonstantin Belousov  * returned non-zero.  On the other hand, retrying the whole lock
1476478ca4b0SKonstantin Belousov  * operation, we better not stop there but delegate the handling to
1477478ca4b0SKonstantin Belousov  * ast.
1478478ca4b0SKonstantin Belousov  *
1479478ca4b0SKonstantin Belousov  * If the request is for thread termination P_SINGLE_EXIT, we cannot
1480478ca4b0SKonstantin Belousov  * handle it at all, and simply return EINTR.
1481478ca4b0SKonstantin Belousov  */
1482478ca4b0SKonstantin Belousov int
1483478ca4b0SKonstantin Belousov thread_check_susp(struct thread *td, bool sleep)
1484478ca4b0SKonstantin Belousov {
1485478ca4b0SKonstantin Belousov 	struct proc *p;
1486478ca4b0SKonstantin Belousov 	int error;
1487478ca4b0SKonstantin Belousov 
1488478ca4b0SKonstantin Belousov 	/*
1489c6d31b83SKonstantin Belousov 	 * The check for TDA_SUSPEND is racy, but it is enough to
1490478ca4b0SKonstantin Belousov 	 * eventually break the lockstep loop.
1491478ca4b0SKonstantin Belousov 	 */
1492c6d31b83SKonstantin Belousov 	if (!td_ast_pending(td, TDA_SUSPEND))
1493478ca4b0SKonstantin Belousov 		return (0);
1494478ca4b0SKonstantin Belousov 	error = 0;
1495478ca4b0SKonstantin Belousov 	p = td->td_proc;
1496478ca4b0SKonstantin Belousov 	PROC_LOCK(p);
1497478ca4b0SKonstantin Belousov 	if (p->p_flag & P_SINGLE_EXIT)
1498478ca4b0SKonstantin Belousov 		error = EINTR;
1499478ca4b0SKonstantin Belousov 	else if (P_SHOULDSTOP(p) ||
1500478ca4b0SKonstantin Belousov 	    ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND)))
1501478ca4b0SKonstantin Belousov 		error = sleep ? thread_suspend_check(0) : ERESTART;
1502478ca4b0SKonstantin Belousov 	PROC_UNLOCK(p);
1503478ca4b0SKonstantin Belousov 	return (error);
1504478ca4b0SKonstantin Belousov }
1505478ca4b0SKonstantin Belousov 
150635c32a76SDavid Xu void
15076ddcc233SKonstantin Belousov thread_suspend_switch(struct thread *td, struct proc *p)
1508a54e85fdSJeff Roberson {
1509a54e85fdSJeff Roberson 
1510a54e85fdSJeff Roberson 	KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
1511a54e85fdSJeff Roberson 	PROC_LOCK_ASSERT(p, MA_OWNED);
15127b4a950aSDavid Xu 	PROC_SLOCK_ASSERT(p, MA_OWNED);
1513a54e85fdSJeff Roberson 	/*
1514a54e85fdSJeff Roberson 	 * We implement thread_suspend_one in stages here to avoid
1515a54e85fdSJeff Roberson 	 * dropping the proc lock while the thread lock is owned.
1516a54e85fdSJeff Roberson 	 */
15176ddcc233SKonstantin Belousov 	if (p == td->td_proc) {
1518a54e85fdSJeff Roberson 		thread_stopped(p);
1519a54e85fdSJeff Roberson 		p->p_suspcount++;
15206ddcc233SKonstantin Belousov 	}
15213f9be10eSDavid Xu 	PROC_UNLOCK(p);
15227b4a950aSDavid Xu 	thread_lock(td);
1523c6d31b83SKonstantin Belousov 	ast_unsched_locked(td, TDA_SUSPEND);
1524a54e85fdSJeff Roberson 	TD_SET_SUSPENDED(td);
1525c5aa6b58SJeff Roberson 	sched_sleep(td, 0);
15267b4a950aSDavid Xu 	PROC_SUNLOCK(p);
1527a54e85fdSJeff Roberson 	DROP_GIANT();
1528686bcb5cSJeff Roberson 	mi_switch(SW_VOL | SWT_SUSPEND);
1529a54e85fdSJeff Roberson 	PICKUP_GIANT();
1530a54e85fdSJeff Roberson 	PROC_LOCK(p);
15317b4a950aSDavid Xu 	PROC_SLOCK(p);
1532a54e85fdSJeff Roberson }
1533a54e85fdSJeff Roberson 
1534a54e85fdSJeff Roberson void
153535c32a76SDavid Xu thread_suspend_one(struct thread *td)
153635c32a76SDavid Xu {
15376ddcc233SKonstantin Belousov 	struct proc *p;
153835c32a76SDavid Xu 
15396ddcc233SKonstantin Belousov 	p = td->td_proc;
15407b4a950aSDavid Xu 	PROC_SLOCK_ASSERT(p, MA_OWNED);
1541a54e85fdSJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
1542e574e444SDavid Xu 	KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
154335c32a76SDavid Xu 	p->p_suspcount++;
1544c6d31b83SKonstantin Belousov 	ast_unsched_locked(td, TDA_SUSPEND);
154571fad9fdSJulian Elischer 	TD_SET_SUSPENDED(td);
1546c5aa6b58SJeff Roberson 	sched_sleep(td, 0);
154735c32a76SDavid Xu }
154835c32a76SDavid Xu 
154984cdea97SKonstantin Belousov static int
155084cdea97SKonstantin Belousov thread_unsuspend_one(struct thread *td, struct proc *p, bool boundary)
155135c32a76SDavid Xu {
155235c32a76SDavid Xu 
1553a54e85fdSJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
1554ad1e7d28SJulian Elischer 	KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
155571fad9fdSJulian Elischer 	TD_CLR_SUSPENDED(td);
15566ddcc233SKonstantin Belousov 	td->td_flags &= ~TDF_ALLPROCSUSP;
15576ddcc233SKonstantin Belousov 	if (td->td_proc == p) {
15586ddcc233SKonstantin Belousov 		PROC_SLOCK_ASSERT(p, MA_OWNED);
155935c32a76SDavid Xu 		p->p_suspcount--;
156084cdea97SKonstantin Belousov 		if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) {
156184cdea97SKonstantin Belousov 			td->td_flags &= ~TDF_BOUNDARY;
156284cdea97SKonstantin Belousov 			p->p_boundary_count--;
156384cdea97SKonstantin Belousov 		}
15646ddcc233SKonstantin Belousov 	}
156561a74c5cSJeff Roberson 	return (setrunnable(td, 0));
156635c32a76SDavid Xu }
156735c32a76SDavid Xu 
1568af928fdeSKonstantin Belousov void
1569af928fdeSKonstantin Belousov thread_run_flash(struct thread *td)
1570af928fdeSKonstantin Belousov {
1571af928fdeSKonstantin Belousov 	struct proc *p;
1572af928fdeSKonstantin Belousov 
1573af928fdeSKonstantin Belousov 	p = td->td_proc;
1574af928fdeSKonstantin Belousov 	PROC_LOCK_ASSERT(p, MA_OWNED);
1575af928fdeSKonstantin Belousov 
1576af928fdeSKonstantin Belousov 	if (TD_ON_SLEEPQ(td))
1577af928fdeSKonstantin Belousov 		sleepq_remove_nested(td);
1578af928fdeSKonstantin Belousov 	else
1579af928fdeSKonstantin Belousov 		thread_lock(td);
1580af928fdeSKonstantin Belousov 
1581af928fdeSKonstantin Belousov 	THREAD_LOCK_ASSERT(td, MA_OWNED);
1582af928fdeSKonstantin Belousov 	KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
1583af928fdeSKonstantin Belousov 
1584af928fdeSKonstantin Belousov 	TD_CLR_SUSPENDED(td);
1585af928fdeSKonstantin Belousov 	PROC_SLOCK(p);
1586af928fdeSKonstantin Belousov 	MPASS(p->p_suspcount > 0);
1587af928fdeSKonstantin Belousov 	p->p_suspcount--;
1588af928fdeSKonstantin Belousov 	PROC_SUNLOCK(p);
1589af928fdeSKonstantin Belousov 	if (setrunnable(td, 0))
1590af928fdeSKonstantin Belousov 		kick_proc0();
1591af928fdeSKonstantin Belousov }
1592af928fdeSKonstantin Belousov 
159344990b8cSJulian Elischer /*
159444990b8cSJulian Elischer  * Allow all threads blocked by single threading to continue running.
159544990b8cSJulian Elischer  */
159644990b8cSJulian Elischer void
159744990b8cSJulian Elischer thread_unsuspend(struct proc *p)
159844990b8cSJulian Elischer {
159944990b8cSJulian Elischer 	struct thread *td;
16007847a9daSJohn Baldwin 	int wakeup_swapper;
160144990b8cSJulian Elischer 
160244990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
16037b4a950aSDavid Xu 	PROC_SLOCK_ASSERT(p, MA_OWNED);
16047847a9daSJohn Baldwin 	wakeup_swapper = 0;
160544990b8cSJulian Elischer 	if (!P_SHOULDSTOP(p)) {
1606ad1e7d28SJulian Elischer                 FOREACH_THREAD_IN_PROC(p, td) {
1607a54e85fdSJeff Roberson 			thread_lock(td);
1608f829268bSKonstantin Belousov 			if (TD_IS_SUSPENDED(td))
160984cdea97SKonstantin Belousov 				wakeup_swapper |= thread_unsuspend_one(td, p,
161084cdea97SKonstantin Belousov 				    true);
1611f829268bSKonstantin Belousov 			else
1612a54e85fdSJeff Roberson 				thread_unlock(td);
1613ad1e7d28SJulian Elischer 		}
161484cdea97SKonstantin Belousov 	} else if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
161584cdea97SKonstantin Belousov 	    p->p_numthreads == p->p_suspcount) {
161644990b8cSJulian Elischer 		/*
161744990b8cSJulian Elischer 		 * Stopping everything also did the job for the single
161844990b8cSJulian Elischer 		 * threading request. Now we've downgraded to single-threaded,
161944990b8cSJulian Elischer 		 * let it continue.
162044990b8cSJulian Elischer 		 */
16216ddcc233SKonstantin Belousov 		if (p->p_singlethread->td_proc == p) {
1622a54e85fdSJeff Roberson 			thread_lock(p->p_singlethread);
16236ddcc233SKonstantin Belousov 			wakeup_swapper = thread_unsuspend_one(
162484cdea97SKonstantin Belousov 			    p->p_singlethread, p, false);
162544990b8cSJulian Elischer 		}
16266ddcc233SKonstantin Belousov 	}
16277847a9daSJohn Baldwin 	if (wakeup_swapper)
16287847a9daSJohn Baldwin 		kick_proc0();
162944990b8cSJulian Elischer }
163044990b8cSJulian Elischer 
1631ed062c8dSJulian Elischer /*
1632ed062c8dSJulian Elischer  * End the single threading mode..
1633ed062c8dSJulian Elischer  */
163444990b8cSJulian Elischer void
16356ddcc233SKonstantin Belousov thread_single_end(struct proc *p, int mode)
163644990b8cSJulian Elischer {
163744990b8cSJulian Elischer 	struct thread *td;
16387847a9daSJohn Baldwin 	int wakeup_swapper;
163944990b8cSJulian Elischer 
16406ddcc233SKonstantin Belousov 	KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
16416ddcc233SKonstantin Belousov 	    mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
16426ddcc233SKonstantin Belousov 	    ("invalid mode %d", mode));
164344990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
16446ddcc233SKonstantin Belousov 	KASSERT((mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) ||
16456ddcc233SKonstantin Belousov 	    (mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0),
16466ddcc233SKonstantin Belousov 	    ("mode %d does not match P_TOTAL_STOP", mode));
164784cdea97SKonstantin Belousov 	KASSERT(mode == SINGLE_ALLPROC || p->p_singlethread == curthread,
164884cdea97SKonstantin Belousov 	    ("thread_single_end from other thread %p %p",
164984cdea97SKonstantin Belousov 	    curthread, p->p_singlethread));
165084cdea97SKonstantin Belousov 	KASSERT(mode != SINGLE_BOUNDARY ||
165184cdea97SKonstantin Belousov 	    (p->p_flag & P_SINGLE_BOUNDARY) != 0,
165284cdea97SKonstantin Belousov 	    ("mis-matched SINGLE_BOUNDARY flags %x", p->p_flag));
16536ddcc233SKonstantin Belousov 	p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY |
16546ddcc233SKonstantin Belousov 	    P_TOTAL_STOP);
16557b4a950aSDavid Xu 	PROC_SLOCK(p);
165644990b8cSJulian Elischer 	p->p_singlethread = NULL;
16577847a9daSJohn Baldwin 	wakeup_swapper = 0;
165849539972SJulian Elischer 	/*
16597847a9daSJohn Baldwin 	 * If there are other threads they may now run,
166049539972SJulian Elischer 	 * unless of course there is a blanket 'stop order'
166149539972SJulian Elischer 	 * on the process. The single threader must be allowed
166249539972SJulian Elischer 	 * to continue however as this is a bad place to stop.
166349539972SJulian Elischer 	 */
16646ddcc233SKonstantin Belousov 	if (p->p_numthreads != remain_for_mode(mode) && !P_SHOULDSTOP(p)) {
1665ad1e7d28SJulian Elischer                 FOREACH_THREAD_IN_PROC(p, td) {
1666a54e85fdSJeff Roberson 			thread_lock(td);
1667ad1e7d28SJulian Elischer 			if (TD_IS_SUSPENDED(td)) {
166884cdea97SKonstantin Belousov 				wakeup_swapper |= thread_unsuspend_one(td, p,
16692d5ef216SMark Johnston 				    true);
167061a74c5cSJeff Roberson 			} else
1671a54e85fdSJeff Roberson 				thread_unlock(td);
167249539972SJulian Elischer 		}
1673ad1e7d28SJulian Elischer 	}
167484cdea97SKonstantin Belousov 	KASSERT(mode != SINGLE_BOUNDARY || p->p_boundary_count == 0,
167584cdea97SKonstantin Belousov 	    ("inconsistent boundary count %d", p->p_boundary_count));
16767b4a950aSDavid Xu 	PROC_SUNLOCK(p);
16777847a9daSJohn Baldwin 	if (wakeup_swapper)
16787847a9daSJohn Baldwin 		kick_proc0();
1679d7a9e6e7SKonstantin Belousov 	wakeup(&p->p_flag);
168049539972SJulian Elischer }
16814fc21c09SDaniel Eischen 
1682aae3547bSMateusz Guzik /*
1683aae3547bSMateusz Guzik  * Locate a thread by number and return with proc lock held.
1684aae3547bSMateusz Guzik  *
1685aae3547bSMateusz Guzik  * thread exit establishes proc -> tidhash lock ordering, but lookup
1686aae3547bSMateusz Guzik  * takes tidhash first and needs to return locked proc.
1687aae3547bSMateusz Guzik  *
1688aae3547bSMateusz Guzik  * The problem is worked around by relying on type-safety of both
1689aae3547bSMateusz Guzik  * structures and doing the work in 2 steps:
1690aae3547bSMateusz Guzik  * - tidhash-locked lookup which saves both thread and proc pointers
1691aae3547bSMateusz Guzik  * - proc-locked verification that the found thread still matches
1692aae3547bSMateusz Guzik  */
1693aae3547bSMateusz Guzik static bool
1694aae3547bSMateusz Guzik tdfind_hash(lwpid_t tid, pid_t pid, struct proc **pp, struct thread **tdp)
1695cf7d9a8cSDavid Xu {
1696cf7d9a8cSDavid Xu #define RUN_THRESH	16
1697aae3547bSMateusz Guzik 	struct proc *p;
1698cf7d9a8cSDavid Xu 	struct thread *td;
1699aae3547bSMateusz Guzik 	int run;
1700aae3547bSMateusz Guzik 	bool locked;
1701cf7d9a8cSDavid Xu 
1702aae3547bSMateusz Guzik 	run = 0;
170326007fe3SMateusz Guzik 	rw_rlock(TIDHASHLOCK(tid));
1704aae3547bSMateusz Guzik 	locked = true;
1705cf7d9a8cSDavid Xu 	LIST_FOREACH(td, TIDHASH(tid), td_hash) {
1706aae3547bSMateusz Guzik 		if (td->td_tid != tid) {
1707aae3547bSMateusz Guzik 			run++;
1708aae3547bSMateusz Guzik 			continue;
1709cf7d9a8cSDavid Xu 		}
1710aae3547bSMateusz Guzik 		p = td->td_proc;
1711aae3547bSMateusz Guzik 		if (pid != -1 && p->p_pid != pid) {
1712cf7d9a8cSDavid Xu 			td = NULL;
1713cf7d9a8cSDavid Xu 			break;
1714cf7d9a8cSDavid Xu 		}
1715cf7d9a8cSDavid Xu 		if (run > RUN_THRESH) {
171626007fe3SMateusz Guzik 			if (rw_try_upgrade(TIDHASHLOCK(tid))) {
1717cf7d9a8cSDavid Xu 				LIST_REMOVE(td, td_hash);
1718cf7d9a8cSDavid Xu 				LIST_INSERT_HEAD(TIDHASH(td->td_tid),
1719cf7d9a8cSDavid Xu 					td, td_hash);
172026007fe3SMateusz Guzik 				rw_wunlock(TIDHASHLOCK(tid));
1721aae3547bSMateusz Guzik 				locked = false;
1722aae3547bSMateusz Guzik 				break;
1723cf7d9a8cSDavid Xu 			}
1724cf7d9a8cSDavid Xu 		}
1725cf7d9a8cSDavid Xu 		break;
1726cf7d9a8cSDavid Xu 	}
1727aae3547bSMateusz Guzik 	if (locked)
172826007fe3SMateusz Guzik 		rw_runlock(TIDHASHLOCK(tid));
1729aae3547bSMateusz Guzik 	if (td == NULL)
1730aae3547bSMateusz Guzik 		return (false);
1731aae3547bSMateusz Guzik 	*pp = p;
1732aae3547bSMateusz Guzik 	*tdp = td;
1733aae3547bSMateusz Guzik 	return (true);
1734aae3547bSMateusz Guzik }
1735aae3547bSMateusz Guzik 
1736aae3547bSMateusz Guzik struct thread *
1737aae3547bSMateusz Guzik tdfind(lwpid_t tid, pid_t pid)
1738aae3547bSMateusz Guzik {
1739aae3547bSMateusz Guzik 	struct proc *p;
1740aae3547bSMateusz Guzik 	struct thread *td;
1741aae3547bSMateusz Guzik 
1742aae3547bSMateusz Guzik 	td = curthread;
1743aae3547bSMateusz Guzik 	if (td->td_tid == tid) {
1744aae3547bSMateusz Guzik 		if (pid != -1 && td->td_proc->p_pid != pid)
1745aae3547bSMateusz Guzik 			return (NULL);
1746aae3547bSMateusz Guzik 		PROC_LOCK(td->td_proc);
1747cf7d9a8cSDavid Xu 		return (td);
1748cf7d9a8cSDavid Xu 	}
1749cf7d9a8cSDavid Xu 
1750aae3547bSMateusz Guzik 	for (;;) {
1751aae3547bSMateusz Guzik 		if (!tdfind_hash(tid, pid, &p, &td))
1752aae3547bSMateusz Guzik 			return (NULL);
1753aae3547bSMateusz Guzik 		PROC_LOCK(p);
1754aae3547bSMateusz Guzik 		if (td->td_tid != tid) {
1755aae3547bSMateusz Guzik 			PROC_UNLOCK(p);
1756aae3547bSMateusz Guzik 			continue;
1757aae3547bSMateusz Guzik 		}
1758aae3547bSMateusz Guzik 		if (td->td_proc != p) {
1759aae3547bSMateusz Guzik 			PROC_UNLOCK(p);
1760aae3547bSMateusz Guzik 			continue;
1761aae3547bSMateusz Guzik 		}
1762aae3547bSMateusz Guzik 		if (p->p_state == PRS_NEW) {
1763aae3547bSMateusz Guzik 			PROC_UNLOCK(p);
1764aae3547bSMateusz Guzik 			return (NULL);
1765aae3547bSMateusz Guzik 		}
1766aae3547bSMateusz Guzik 		return (td);
1767aae3547bSMateusz Guzik 	}
1768aae3547bSMateusz Guzik }
1769aae3547bSMateusz Guzik 
1770cf7d9a8cSDavid Xu void
1771cf7d9a8cSDavid Xu tidhash_add(struct thread *td)
1772cf7d9a8cSDavid Xu {
177326007fe3SMateusz Guzik 	rw_wlock(TIDHASHLOCK(td->td_tid));
1774cf7d9a8cSDavid Xu 	LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash);
177526007fe3SMateusz Guzik 	rw_wunlock(TIDHASHLOCK(td->td_tid));
1776cf7d9a8cSDavid Xu }
1777cf7d9a8cSDavid Xu 
1778cf7d9a8cSDavid Xu void
1779cf7d9a8cSDavid Xu tidhash_remove(struct thread *td)
1780cf7d9a8cSDavid Xu {
178126007fe3SMateusz Guzik 
178226007fe3SMateusz Guzik 	rw_wlock(TIDHASHLOCK(td->td_tid));
1783cf7d9a8cSDavid Xu 	LIST_REMOVE(td, td_hash);
178426007fe3SMateusz Guzik 	rw_wunlock(TIDHASHLOCK(td->td_tid));
1785cf7d9a8cSDavid Xu }
1786