xref: /freebsd/sys/kern/kern_thread.c (revision 16d95d4f92d26f59208d6ddae1ca337c7a5e44a2)
19454b2d8SWarner Losh /*-
244990b8cSJulian Elischer  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
344990b8cSJulian Elischer  *  All rights reserved.
444990b8cSJulian Elischer  *
544990b8cSJulian Elischer  * Redistribution and use in source and binary forms, with or without
644990b8cSJulian Elischer  * modification, are permitted provided that the following conditions
744990b8cSJulian Elischer  * are met:
844990b8cSJulian Elischer  * 1. Redistributions of source code must retain the above copyright
944990b8cSJulian Elischer  *    notice(s), this list of conditions and the following disclaimer as
1044990b8cSJulian Elischer  *    the first lines of this file unmodified other than the possible
1144990b8cSJulian Elischer  *    addition of one or more copyright notices.
1244990b8cSJulian Elischer  * 2. Redistributions in binary form must reproduce the above copyright
1344990b8cSJulian Elischer  *    notice(s), this list of conditions and the following disclaimer in the
1444990b8cSJulian Elischer  *    documentation and/or other materials provided with the distribution.
1544990b8cSJulian Elischer  *
1644990b8cSJulian Elischer  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
1744990b8cSJulian Elischer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1844990b8cSJulian Elischer  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1944990b8cSJulian Elischer  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
2044990b8cSJulian Elischer  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
2144990b8cSJulian Elischer  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
2244990b8cSJulian Elischer  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
2344990b8cSJulian Elischer  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2444990b8cSJulian Elischer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2544990b8cSJulian Elischer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
2644990b8cSJulian Elischer  * DAMAGE.
2744990b8cSJulian Elischer  */
2844990b8cSJulian Elischer 
293d06b4b3SAttilio Rao #include "opt_witness.h"
3016d95d4fSJoseph Koshy #include "opt_hwpmc_hooks.h"
313d06b4b3SAttilio Rao 
32677b542eSDavid E. O'Brien #include <sys/cdefs.h>
33677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
34677b542eSDavid E. O'Brien 
3544990b8cSJulian Elischer #include <sys/param.h>
3644990b8cSJulian Elischer #include <sys/systm.h>
3744990b8cSJulian Elischer #include <sys/kernel.h>
3844990b8cSJulian Elischer #include <sys/lock.h>
3944990b8cSJulian Elischer #include <sys/mutex.h>
4044990b8cSJulian Elischer #include <sys/proc.h>
41e170bfdaSDavid Xu #include <sys/resourcevar.h>
4294e0a4cdSJulian Elischer #include <sys/smp.h>
4344990b8cSJulian Elischer #include <sys/sysctl.h>
44de028f5aSJeff Roberson #include <sys/sched.h>
4544f3b092SJohn Baldwin #include <sys/sleepqueue.h>
46ace8398dSJeff Roberson #include <sys/selinfo.h>
47961a7b24SJohn Baldwin #include <sys/turnstile.h>
4844990b8cSJulian Elischer #include <sys/ktr.h>
49bc8e6d81SDavid Xu #include <sys/umtx.h>
50d7f687fcSJeff Roberson #include <sys/cpuset.h>
5116d95d4fSJoseph Koshy #ifdef	HWPMC_HOOKS
5216d95d4fSJoseph Koshy #include <sys/pmckern.h>
5316d95d4fSJoseph Koshy #endif
5444990b8cSJulian Elischer 
55911b84b0SRobert Watson #include <security/audit/audit.h>
56911b84b0SRobert Watson 
5744990b8cSJulian Elischer #include <vm/vm.h>
5849a2507bSAlan Cox #include <vm/vm_extern.h>
5944990b8cSJulian Elischer #include <vm/uma.h>
60b209f889SRandall Stewart #include <sys/eventhandler.h>
6102fb42b0SPeter Wemm 
628460a577SJohn Birrell /*
638460a577SJohn Birrell  * thread related storage.
648460a577SJohn Birrell  */
6544990b8cSJulian Elischer static uma_zone_t thread_zone;
6644990b8cSJulian Elischer 
6744990b8cSJulian Elischer SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation");
68fdc5ecd2SDavid Xu 
69345ad866SJulian Elischer int max_threads_per_proc = 1500;
70fdc5ecd2SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
714f0db5e0SJulian Elischer 	&max_threads_per_proc, 0, "Limit on threads per proc");
724f0db5e0SJulian Elischer 
73345ad866SJulian Elischer int max_threads_hits;
740252d203SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD,
750252d203SDavid Xu 	&max_threads_hits, 0, "");
760252d203SDavid Xu 
775215b187SJeff Roberson TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
78c8790f5dSAttilio Rao static struct mtx zombie_lock;
79a54e85fdSJeff Roberson MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
8044990b8cSJulian Elischer 
81ff8fbcffSJeff Roberson static void thread_zombie(struct thread *);
82ff8fbcffSJeff Roberson 
83fdcac928SMarcel Moolenaar struct mtx tid_lock;
841ea7a6f8SPoul-Henning Kamp static struct unrhdr *tid_unrhdr;
85fdcac928SMarcel Moolenaar 
86fdcac928SMarcel Moolenaar /*
87696058c3SJulian Elischer  * Prepare a thread for use.
8844990b8cSJulian Elischer  */
89b23f72e9SBrian Feldman static int
90b23f72e9SBrian Feldman thread_ctor(void *mem, int size, void *arg, int flags)
9144990b8cSJulian Elischer {
9244990b8cSJulian Elischer 	struct thread	*td;
9344990b8cSJulian Elischer 
9444990b8cSJulian Elischer 	td = (struct thread *)mem;
9571fad9fdSJulian Elischer 	td->td_state = TDS_INACTIVE;
96060563ecSJulian Elischer 	td->td_oncpu = NOCPU;
976c27c603SJuli Mallett 
98773eff9dSPoul-Henning Kamp 	td->td_tid = alloc_unr(tid_unrhdr);
99f9bb7538SMohan Srinivasan 	td->td_syscalls = 0;
100773eff9dSPoul-Henning Kamp 
1016c27c603SJuli Mallett 	/*
1026c27c603SJuli Mallett 	 * Note that td_critnest begins life as 1 because the thread is not
1036c27c603SJuli Mallett 	 * running and is thereby implicitly waiting to be on the receiving
104a54e85fdSJeff Roberson 	 * end of a context switch.
1056c27c603SJuli Mallett 	 */
106139b7550SJohn Baldwin 	td->td_critnest = 1;
107b209f889SRandall Stewart 	EVENTHANDLER_INVOKE(thread_ctor, td);
108911b84b0SRobert Watson #ifdef AUDIT
109911b84b0SRobert Watson 	audit_thread_alloc(td);
110911b84b0SRobert Watson #endif
111d10183d9SDavid Xu 	umtx_thread_alloc(td);
112b23f72e9SBrian Feldman 	return (0);
11344990b8cSJulian Elischer }
11444990b8cSJulian Elischer 
11544990b8cSJulian Elischer /*
11644990b8cSJulian Elischer  * Reclaim a thread after use.
11744990b8cSJulian Elischer  */
11844990b8cSJulian Elischer static void
11944990b8cSJulian Elischer thread_dtor(void *mem, int size, void *arg)
12044990b8cSJulian Elischer {
12144990b8cSJulian Elischer 	struct thread *td;
12244990b8cSJulian Elischer 
12344990b8cSJulian Elischer 	td = (struct thread *)mem;
12444990b8cSJulian Elischer 
12544990b8cSJulian Elischer #ifdef INVARIANTS
12644990b8cSJulian Elischer 	/* Verify that this thread is in a safe state to free. */
12744990b8cSJulian Elischer 	switch (td->td_state) {
12871fad9fdSJulian Elischer 	case TDS_INHIBITED:
12971fad9fdSJulian Elischer 	case TDS_RUNNING:
13071fad9fdSJulian Elischer 	case TDS_CAN_RUN:
13144990b8cSJulian Elischer 	case TDS_RUNQ:
13244990b8cSJulian Elischer 		/*
13344990b8cSJulian Elischer 		 * We must never unlink a thread that is in one of
13444990b8cSJulian Elischer 		 * these states, because it is currently active.
13544990b8cSJulian Elischer 		 */
13644990b8cSJulian Elischer 		panic("bad state for thread unlinking");
13744990b8cSJulian Elischer 		/* NOTREACHED */
13871fad9fdSJulian Elischer 	case TDS_INACTIVE:
13944990b8cSJulian Elischer 		break;
14044990b8cSJulian Elischer 	default:
14144990b8cSJulian Elischer 		panic("bad thread state");
14244990b8cSJulian Elischer 		/* NOTREACHED */
14344990b8cSJulian Elischer 	}
14444990b8cSJulian Elischer #endif
1456e8525ceSRobert Watson #ifdef AUDIT
1466e8525ceSRobert Watson 	audit_thread_free(td);
1476e8525ceSRobert Watson #endif
1481ba4a712SPawel Jakub Dawidek 	/* Free all OSD associated to this thread. */
1491ba4a712SPawel Jakub Dawidek 	osd_thread_exit(td);
1501ba4a712SPawel Jakub Dawidek 
151b209f889SRandall Stewart 	EVENTHANDLER_INVOKE(thread_dtor, td);
152773eff9dSPoul-Henning Kamp 	free_unr(tid_unrhdr, td->td_tid);
15344990b8cSJulian Elischer }
15444990b8cSJulian Elischer 
15544990b8cSJulian Elischer /*
15644990b8cSJulian Elischer  * Initialize type-stable parts of a thread (when newly created).
15744990b8cSJulian Elischer  */
158b23f72e9SBrian Feldman static int
159b23f72e9SBrian Feldman thread_init(void *mem, int size, int flags)
16044990b8cSJulian Elischer {
16144990b8cSJulian Elischer 	struct thread *td;
16244990b8cSJulian Elischer 
16344990b8cSJulian Elischer 	td = (struct thread *)mem;
164247aba24SMarcel Moolenaar 
16544f3b092SJohn Baldwin 	td->td_sleepqueue = sleepq_alloc();
166961a7b24SJohn Baldwin 	td->td_turnstile = turnstile_alloc();
167b209f889SRandall Stewart 	EVENTHANDLER_INVOKE(thread_init, td);
168de028f5aSJeff Roberson 	td->td_sched = (struct td_sched *)&td[1];
169d10183d9SDavid Xu 	umtx_thread_init(td);
17089b57fcfSKonstantin Belousov 	td->td_kstack = 0;
171b23f72e9SBrian Feldman 	return (0);
17244990b8cSJulian Elischer }
17344990b8cSJulian Elischer 
17444990b8cSJulian Elischer /*
17544990b8cSJulian Elischer  * Tear down type-stable parts of a thread (just before being discarded).
17644990b8cSJulian Elischer  */
17744990b8cSJulian Elischer static void
17844990b8cSJulian Elischer thread_fini(void *mem, int size)
17944990b8cSJulian Elischer {
18044990b8cSJulian Elischer 	struct thread *td;
18144990b8cSJulian Elischer 
18244990b8cSJulian Elischer 	td = (struct thread *)mem;
183b209f889SRandall Stewart 	EVENTHANDLER_INVOKE(thread_fini, td);
184961a7b24SJohn Baldwin 	turnstile_free(td->td_turnstile);
18544f3b092SJohn Baldwin 	sleepq_free(td->td_sleepqueue);
186d10183d9SDavid Xu 	umtx_thread_fini(td);
187ace8398dSJeff Roberson 	seltdfini(td);
18844990b8cSJulian Elischer }
1895215b187SJeff Roberson 
1905c8329edSJulian Elischer /*
1915215b187SJeff Roberson  * For a newly created process,
1925215b187SJeff Roberson  * link up all the structures and its initial threads etc.
193ed062c8dSJulian Elischer  * called from:
194ed062c8dSJulian Elischer  * {arch}/{arch}/machdep.c   ia64_init(), init386() etc.
195ed062c8dSJulian Elischer  * proc_dtor() (should go away)
196ed062c8dSJulian Elischer  * proc_init()
1975c8329edSJulian Elischer  */
1985c8329edSJulian Elischer void
19989b57fcfSKonstantin Belousov proc_linkup0(struct proc *p, struct thread *td)
20089b57fcfSKonstantin Belousov {
20189b57fcfSKonstantin Belousov 	TAILQ_INIT(&p->p_threads);	     /* all threads in proc */
20289b57fcfSKonstantin Belousov 	proc_linkup(p, td);
20389b57fcfSKonstantin Belousov }
20489b57fcfSKonstantin Belousov 
20589b57fcfSKonstantin Belousov void
2068460a577SJohn Birrell proc_linkup(struct proc *p, struct thread *td)
2075c8329edSJulian Elischer {
208a54e85fdSJeff Roberson 
2099104847fSDavid Xu 	sigqueue_init(&p->p_sigqueue, p);
210ebceaf6dSDavid Xu 	p->p_ksi = ksiginfo_alloc(1);
211ebceaf6dSDavid Xu 	if (p->p_ksi != NULL) {
2125c474517SDavid Xu 		/* XXX p_ksi may be null if ksiginfo zone is not ready */
213ebceaf6dSDavid Xu 		p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
214ebceaf6dSDavid Xu 	}
215b2f92ef9SDavid Xu 	LIST_INIT(&p->p_mqnotifier);
2165c8329edSJulian Elischer 	p->p_numthreads = 0;
2178460a577SJohn Birrell 	thread_link(td, p);
2185c8329edSJulian Elischer }
2195c8329edSJulian Elischer 
2205c8329edSJulian Elischer /*
22144990b8cSJulian Elischer  * Initialize global thread allocation resources.
22244990b8cSJulian Elischer  */
22344990b8cSJulian Elischer void
22444990b8cSJulian Elischer threadinit(void)
22544990b8cSJulian Elischer {
22644990b8cSJulian Elischer 
2271ea7a6f8SPoul-Henning Kamp 	mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
2286829a5c5SJulian Elischer 	/* leave one number for thread0 */
2296829a5c5SJulian Elischer 	tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock);
2301ea7a6f8SPoul-Henning Kamp 
231de028f5aSJeff Roberson 	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
23244990b8cSJulian Elischer 	    thread_ctor, thread_dtor, thread_init, thread_fini,
2334649e92bSJohn Baldwin 	    16 - 1, 0);
23444990b8cSJulian Elischer }
23544990b8cSJulian Elischer 
23644990b8cSJulian Elischer /*
237ff8fbcffSJeff Roberson  * Place an unused thread on the zombie list.
238ad1e7d28SJulian Elischer  * Use the slpq as that must be unused by now.
23944990b8cSJulian Elischer  */
24044990b8cSJulian Elischer void
241ff8fbcffSJeff Roberson thread_zombie(struct thread *td)
24244990b8cSJulian Elischer {
243a54e85fdSJeff Roberson 	mtx_lock_spin(&zombie_lock);
244ad1e7d28SJulian Elischer 	TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq);
245a54e85fdSJeff Roberson 	mtx_unlock_spin(&zombie_lock);
24644990b8cSJulian Elischer }
24744990b8cSJulian Elischer 
2485c8329edSJulian Elischer /*
249ff8fbcffSJeff Roberson  * Release a thread that has exited after cpu_throw().
250ff8fbcffSJeff Roberson  */
251ff8fbcffSJeff Roberson void
252ff8fbcffSJeff Roberson thread_stash(struct thread *td)
253ff8fbcffSJeff Roberson {
254ff8fbcffSJeff Roberson 	atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
255ff8fbcffSJeff Roberson 	thread_zombie(td);
256ff8fbcffSJeff Roberson }
257ff8fbcffSJeff Roberson 
258ff8fbcffSJeff Roberson /*
2596617724cSJeff Roberson  * Reap zombie resources.
26044990b8cSJulian Elischer  */
26144990b8cSJulian Elischer void
26244990b8cSJulian Elischer thread_reap(void)
26344990b8cSJulian Elischer {
2645c8329edSJulian Elischer 	struct thread *td_first, *td_next;
26544990b8cSJulian Elischer 
26644990b8cSJulian Elischer 	/*
2675215b187SJeff Roberson 	 * Don't even bother to lock if none at this instant,
2685215b187SJeff Roberson 	 * we really don't care about the next instant..
26944990b8cSJulian Elischer 	 */
2708460a577SJohn Birrell 	if (!TAILQ_EMPTY(&zombie_threads)) {
271a54e85fdSJeff Roberson 		mtx_lock_spin(&zombie_lock);
2725c8329edSJulian Elischer 		td_first = TAILQ_FIRST(&zombie_threads);
2735c8329edSJulian Elischer 		if (td_first)
2745c8329edSJulian Elischer 			TAILQ_INIT(&zombie_threads);
275a54e85fdSJeff Roberson 		mtx_unlock_spin(&zombie_lock);
2765c8329edSJulian Elischer 		while (td_first) {
277ad1e7d28SJulian Elischer 			td_next = TAILQ_NEXT(td_first, td_slpq);
2785215b187SJeff Roberson 			if (td_first->td_ucred)
2795215b187SJeff Roberson 				crfree(td_first->td_ucred);
2805c8329edSJulian Elischer 			thread_free(td_first);
2815c8329edSJulian Elischer 			td_first = td_next;
28244990b8cSJulian Elischer 		}
28344990b8cSJulian Elischer 	}
284ed062c8dSJulian Elischer }
28544990b8cSJulian Elischer 
2864f0db5e0SJulian Elischer /*
28744990b8cSJulian Elischer  * Allocate a thread.
28844990b8cSJulian Elischer  */
28944990b8cSJulian Elischer struct thread *
2908a945d10SKonstantin Belousov thread_alloc(int pages)
29144990b8cSJulian Elischer {
29289b57fcfSKonstantin Belousov 	struct thread *td;
2938460a577SJohn Birrell 
29444990b8cSJulian Elischer 	thread_reap(); /* check if any zombies to get */
29589b57fcfSKonstantin Belousov 
29689b57fcfSKonstantin Belousov 	td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
29789b57fcfSKonstantin Belousov 	KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
2988a945d10SKonstantin Belousov 	if (!vm_thread_new(td, pages)) {
29989b57fcfSKonstantin Belousov 		uma_zfree(thread_zone, td);
30089b57fcfSKonstantin Belousov 		return (NULL);
30189b57fcfSKonstantin Belousov 	}
3020c3967e7SMarcel Moolenaar 	cpu_thread_alloc(td);
30389b57fcfSKonstantin Belousov 	return (td);
30444990b8cSJulian Elischer }
30544990b8cSJulian Elischer 
3068a945d10SKonstantin Belousov int
3078a945d10SKonstantin Belousov thread_alloc_stack(struct thread *td, int pages)
3088a945d10SKonstantin Belousov {
3098a945d10SKonstantin Belousov 
3108a945d10SKonstantin Belousov 	KASSERT(td->td_kstack == 0,
3118a945d10SKonstantin Belousov 	    ("thread_alloc_stack called on a thread with kstack"));
3128a945d10SKonstantin Belousov 	if (!vm_thread_new(td, pages))
3138a945d10SKonstantin Belousov 		return (0);
3148a945d10SKonstantin Belousov 	cpu_thread_alloc(td);
3158a945d10SKonstantin Belousov 	return (1);
3168a945d10SKonstantin Belousov }
3174f0db5e0SJulian Elischer 
3184f0db5e0SJulian Elischer /*
31944990b8cSJulian Elischer  * Deallocate a thread.
32044990b8cSJulian Elischer  */
32144990b8cSJulian Elischer void
32244990b8cSJulian Elischer thread_free(struct thread *td)
32344990b8cSJulian Elischer {
3242e6b8de4SJeff Roberson 
3252e6b8de4SJeff Roberson 	lock_profile_thread_exit(td);
32645aea8deSJeff Roberson 	if (td->td_cpuset)
327d7f687fcSJeff Roberson 		cpuset_rel(td->td_cpuset);
328d7f687fcSJeff Roberson 	td->td_cpuset = NULL;
3290c3967e7SMarcel Moolenaar 	cpu_thread_free(td);
33089b57fcfSKonstantin Belousov 	if (td->td_kstack != 0)
33189b57fcfSKonstantin Belousov 		vm_thread_dispose(td);
33244990b8cSJulian Elischer 	uma_zfree(thread_zone, td);
33344990b8cSJulian Elischer }
33444990b8cSJulian Elischer 
33544990b8cSJulian Elischer /*
33644990b8cSJulian Elischer  * Discard the current thread and exit from its context.
33794e0a4cdSJulian Elischer  * Always called with scheduler locked.
33844990b8cSJulian Elischer  *
33944990b8cSJulian Elischer  * Because we can't free a thread while we're operating under its context,
340696058c3SJulian Elischer  * push the current thread into our CPU's deadthread holder. This means
341696058c3SJulian Elischer  * we needn't worry about someone else grabbing our context before we
3426617724cSJeff Roberson  * do a cpu_throw().
34344990b8cSJulian Elischer  */
34444990b8cSJulian Elischer void
34544990b8cSJulian Elischer thread_exit(void)
34644990b8cSJulian Elischer {
347e170bfdaSDavid Xu 	uint64_t new_switchtime;
34844990b8cSJulian Elischer 	struct thread *td;
3491c4bcd05SJeff Roberson 	struct thread *td2;
35044990b8cSJulian Elischer 	struct proc *p;
3517847a9daSJohn Baldwin 	int wakeup_swapper;
35244990b8cSJulian Elischer 
35344990b8cSJulian Elischer 	td = curthread;
35444990b8cSJulian Elischer 	p = td->td_proc;
35544990b8cSJulian Elischer 
356a54e85fdSJeff Roberson 	PROC_SLOCK_ASSERT(p, MA_OWNED);
357ed062c8dSJulian Elischer 	mtx_assert(&Giant, MA_NOTOWNED);
358a54e85fdSJeff Roberson 
35944990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
360ed062c8dSJulian Elischer 	KASSERT(p != NULL, ("thread exiting without a process"));
361cc701b73SRobert Watson 	CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
362e01eafefSJulian Elischer 	    (long)p->p_pid, td->td_name);
3639104847fSDavid Xu 	KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
36444990b8cSJulian Elischer 
36589964dd2SRobert Watson #ifdef AUDIT
36689964dd2SRobert Watson 	AUDIT_SYSCALL_EXIT(0, td);
36789964dd2SRobert Watson #endif
368d10183d9SDavid Xu 	umtx_thread_exit(td);
369ed062c8dSJulian Elischer 	/*
370ed062c8dSJulian Elischer 	 * drop FPU & debug register state storage, or any other
371ed062c8dSJulian Elischer 	 * architecture specific resources that
372ed062c8dSJulian Elischer 	 * would not be on a new untouched process.
373ed062c8dSJulian Elischer 	 */
37444990b8cSJulian Elischer 	cpu_thread_exit(td);	/* XXXSMP */
37544990b8cSJulian Elischer 
376e170bfdaSDavid Xu 	/* Do the same timestamp bookkeeping that mi_switch() would do. */
377e170bfdaSDavid Xu 	new_switchtime = cpu_ticks();
378e170bfdaSDavid Xu 	p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
379e170bfdaSDavid Xu 	PCPU_SET(switchtime, new_switchtime);
380e170bfdaSDavid Xu 	PCPU_SET(switchticks, ticks);
381b4b70819SAttilio Rao 	PCPU_INC(cnt.v_swtch);
382a140976eSAttilio Rao 	/* Save our resource usage in our process. */
383a140976eSAttilio Rao 	td->td_ru.ru_nvcsw++;
384a140976eSAttilio Rao 	rucollect(&p->p_ru, &td->td_ru);
385ed062c8dSJulian Elischer 	/*
3861faf202eSJulian Elischer 	 * The last thread is left attached to the process
3871faf202eSJulian Elischer 	 * So that the whole bundle gets recycled. Skip
388ed062c8dSJulian Elischer 	 * all this stuff if we never had threads.
389ed062c8dSJulian Elischer 	 * EXIT clears all sign of other threads when
390ed062c8dSJulian Elischer 	 * it goes to single threading, so the last thread always
391ed062c8dSJulian Elischer 	 * takes the short path.
3921faf202eSJulian Elischer 	 */
393ed062c8dSJulian Elischer 	if (p->p_flag & P_HADTHREADS) {
3941faf202eSJulian Elischer 		if (p->p_numthreads > 1) {
395d3a0bd78SJulian Elischer 			thread_unlink(td);
3961c4bcd05SJeff Roberson 			td2 = FIRST_THREAD_IN_PROC(p);
3971c4bcd05SJeff Roberson 			sched_exit_thread(td2, td);
398ed062c8dSJulian Elischer 
399ed062c8dSJulian Elischer 			/*
40044990b8cSJulian Elischer 			 * The test below is NOT true if we are the
4011faf202eSJulian Elischer 			 * sole exiting thread. P_STOPPED_SNGL is unset
40244990b8cSJulian Elischer 			 * in exit1() after it is the only survivor.
40344990b8cSJulian Elischer 			 */
4041279572aSDavid Xu 			if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
40544990b8cSJulian Elischer 				if (p->p_numthreads == p->p_suspcount) {
406a54e85fdSJeff Roberson 					thread_lock(p->p_singlethread);
4077847a9daSJohn Baldwin 					wakeup_swapper = thread_unsuspend_one(
4087847a9daSJohn Baldwin 						p->p_singlethread);
409a54e85fdSJeff Roberson 					thread_unlock(p->p_singlethread);
4107847a9daSJohn Baldwin 					if (wakeup_swapper)
4117847a9daSJohn Baldwin 						kick_proc0();
41244990b8cSJulian Elischer 				}
41344990b8cSJulian Elischer 			}
41448bfcdddSJulian Elischer 
415ff8fbcffSJeff Roberson 			atomic_add_int(&td->td_proc->p_exitthreads, 1);
416696058c3SJulian Elischer 			PCPU_SET(deadthread, td);
4171faf202eSJulian Elischer 		} else {
418ed062c8dSJulian Elischer 			/*
419ed062c8dSJulian Elischer 			 * The last thread is exiting.. but not through exit()
420ed062c8dSJulian Elischer 			 */
421ed062c8dSJulian Elischer 			panic ("thread_exit: Last thread exiting on its own");
422ed062c8dSJulian Elischer 		}
4231faf202eSJulian Elischer 	}
42416d95d4fSJoseph Koshy #ifdef	HWPMC_HOOKS
42516d95d4fSJoseph Koshy 	/*
42616d95d4fSJoseph Koshy 	 * If this thread is part of a process that is being tracked by hwpmc(4),
42716d95d4fSJoseph Koshy 	 * inform the module of the thread's impending exit.
42816d95d4fSJoseph Koshy 	 */
42916d95d4fSJoseph Koshy 	if (PMC_PROC_IS_USING_PMCS(td->td_proc))
43016d95d4fSJoseph Koshy 		PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
43116d95d4fSJoseph Koshy #endif
432a54e85fdSJeff Roberson 	PROC_UNLOCK(p);
433a54e85fdSJeff Roberson 	thread_lock(td);
434a140976eSAttilio Rao 	/* Save our tick information with both the thread and proc locked */
435a54e85fdSJeff Roberson 	ruxagg(&p->p_rux, td);
436a54e85fdSJeff Roberson 	PROC_SUNLOCK(p);
437dcc9954eSJulian Elischer 	td->td_state = TDS_INACTIVE;
4383d06b4b3SAttilio Rao #ifdef WITNESS
4393d06b4b3SAttilio Rao 	witness_thread_exit(td);
4403d06b4b3SAttilio Rao #endif
441732d9528SJulian Elischer 	CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
442a54e85fdSJeff Roberson 	sched_throw(td);
443cc66ebe2SPeter Wemm 	panic("I'm a teapot!");
44444990b8cSJulian Elischer 	/* NOTREACHED */
44544990b8cSJulian Elischer }
44644990b8cSJulian Elischer 
44744990b8cSJulian Elischer /*
448696058c3SJulian Elischer  * Do any thread specific cleanups that may be needed in wait()
44937814395SPeter Wemm  * called with Giant, proc and schedlock not held.
450696058c3SJulian Elischer  */
451696058c3SJulian Elischer void
452696058c3SJulian Elischer thread_wait(struct proc *p)
453696058c3SJulian Elischer {
454696058c3SJulian Elischer 	struct thread *td;
455696058c3SJulian Elischer 
45637814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
45785495c72SJens Schweikhardt 	KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()"));
458ff8fbcffSJeff Roberson 	td = FIRST_THREAD_IN_PROC(p);
459ff8fbcffSJeff Roberson 	/* Lock the last thread so we spin until it exits cpu_throw(). */
460ff8fbcffSJeff Roberson 	thread_lock(td);
461ff8fbcffSJeff Roberson 	thread_unlock(td);
462ff8fbcffSJeff Roberson 	/* Wait for any remaining threads to exit cpu_throw(). */
463ff8fbcffSJeff Roberson 	while (p->p_exitthreads)
464ff8fbcffSJeff Roberson 		sched_relinquish(curthread);
4652e6b8de4SJeff Roberson 	lock_profile_thread_exit(td);
466d7f687fcSJeff Roberson 	cpuset_rel(td->td_cpuset);
467d7f687fcSJeff Roberson 	td->td_cpuset = NULL;
468696058c3SJulian Elischer 	cpu_thread_clean(td);
469ed062c8dSJulian Elischer 	crfree(td->td_ucred);
470696058c3SJulian Elischer 	thread_reap();	/* check for zombie threads etc. */
471696058c3SJulian Elischer }
472696058c3SJulian Elischer 
473696058c3SJulian Elischer /*
47444990b8cSJulian Elischer  * Link a thread to a process.
4751faf202eSJulian Elischer  * set up anything that needs to be initialized for it to
4761faf202eSJulian Elischer  * be used by the process.
47744990b8cSJulian Elischer  */
47844990b8cSJulian Elischer void
4798460a577SJohn Birrell thread_link(struct thread *td, struct proc *p)
48044990b8cSJulian Elischer {
48144990b8cSJulian Elischer 
482a54e85fdSJeff Roberson 	/*
483a54e85fdSJeff Roberson 	 * XXX This can't be enabled because it's called for proc0 before
484374ae2a3SJeff Roberson 	 * its lock has been created.
485374ae2a3SJeff Roberson 	 * PROC_LOCK_ASSERT(p, MA_OWNED);
486a54e85fdSJeff Roberson 	 */
48771fad9fdSJulian Elischer 	td->td_state    = TDS_INACTIVE;
48844990b8cSJulian Elischer 	td->td_proc     = p;
489b61ce5b0SJeff Roberson 	td->td_flags    = TDF_INMEM;
49044990b8cSJulian Elischer 
4911faf202eSJulian Elischer 	LIST_INIT(&td->td_contested);
492eea4f254SJeff Roberson 	LIST_INIT(&td->td_lprof[0]);
493eea4f254SJeff Roberson 	LIST_INIT(&td->td_lprof[1]);
4949104847fSDavid Xu 	sigqueue_init(&td->td_sigqueue, p);
495c06eb4e2SSam Leffler 	callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
49644990b8cSJulian Elischer 	TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist);
49744990b8cSJulian Elischer 	p->p_numthreads++;
49844990b8cSJulian Elischer }
49944990b8cSJulian Elischer 
500ed062c8dSJulian Elischer /*
501e5bedcefSJulian Elischer  * Convert a process with one thread to an unthreaded process.
502e5bedcefSJulian Elischer  */
503e5bedcefSJulian Elischer void
504e5bedcefSJulian Elischer thread_unthread(struct thread *td)
505e5bedcefSJulian Elischer {
506e5bedcefSJulian Elischer 	struct proc *p = td->td_proc;
507e5bedcefSJulian Elischer 
508e5bedcefSJulian Elischer 	KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads"));
5098460a577SJohn Birrell 	p->p_flag &= ~P_HADTHREADS;
510e5bedcefSJulian Elischer }
511e5bedcefSJulian Elischer 
512e5bedcefSJulian Elischer /*
513ed062c8dSJulian Elischer  * Called from:
514ed062c8dSJulian Elischer  *  thread_exit()
515ed062c8dSJulian Elischer  */
516d3a0bd78SJulian Elischer void
517d3a0bd78SJulian Elischer thread_unlink(struct thread *td)
518d3a0bd78SJulian Elischer {
519d3a0bd78SJulian Elischer 	struct proc *p = td->td_proc;
520d3a0bd78SJulian Elischer 
521374ae2a3SJeff Roberson 	PROC_LOCK_ASSERT(p, MA_OWNED);
522d3a0bd78SJulian Elischer 	TAILQ_REMOVE(&p->p_threads, td, td_plist);
523d3a0bd78SJulian Elischer 	p->p_numthreads--;
524d3a0bd78SJulian Elischer 	/* could clear a few other things here */
5258460a577SJohn Birrell 	/* Must  NOT clear links to proc! */
5265c8329edSJulian Elischer }
5275c8329edSJulian Elischer 
52879799053SKonstantin Belousov static int
52979799053SKonstantin Belousov calc_remaining(struct proc *p, int mode)
53079799053SKonstantin Belousov {
53179799053SKonstantin Belousov 	int remaining;
53279799053SKonstantin Belousov 
53379799053SKonstantin Belousov 	if (mode == SINGLE_EXIT)
53479799053SKonstantin Belousov 		remaining = p->p_numthreads;
53579799053SKonstantin Belousov 	else if (mode == SINGLE_BOUNDARY)
53679799053SKonstantin Belousov 		remaining = p->p_numthreads - p->p_boundary_count;
53779799053SKonstantin Belousov 	else if (mode == SINGLE_NO_EXIT)
53879799053SKonstantin Belousov 		remaining = p->p_numthreads - p->p_suspcount;
53979799053SKonstantin Belousov 	else
54079799053SKonstantin Belousov 		panic("calc_remaining: wrong mode %d", mode);
54179799053SKonstantin Belousov 	return (remaining);
54279799053SKonstantin Belousov }
54379799053SKonstantin Belousov 
5445215b187SJeff Roberson /*
54544990b8cSJulian Elischer  * Enforce single-threading.
54644990b8cSJulian Elischer  *
54744990b8cSJulian Elischer  * Returns 1 if the caller must abort (another thread is waiting to
54844990b8cSJulian Elischer  * exit the process or similar). Process is locked!
54944990b8cSJulian Elischer  * Returns 0 when you are successfully the only thread running.
55044990b8cSJulian Elischer  * A process has successfully single threaded in the suspend mode when
55144990b8cSJulian Elischer  * There are no threads in user mode. Threads in the kernel must be
55244990b8cSJulian Elischer  * allowed to continue until they get to the user boundary. They may even
55344990b8cSJulian Elischer  * copy out their return values and data before suspending. They may however be
554e2668f55SMaxim Konovalov  * accelerated in reaching the user boundary as we will wake up
55544990b8cSJulian Elischer  * any sleeping threads that are interruptable. (PCATCH).
55644990b8cSJulian Elischer  */
55744990b8cSJulian Elischer int
558906ac69dSDavid Xu thread_single(int mode)
55944990b8cSJulian Elischer {
56044990b8cSJulian Elischer 	struct thread *td;
56144990b8cSJulian Elischer 	struct thread *td2;
56244990b8cSJulian Elischer 	struct proc *p;
563da7bbd2cSJohn Baldwin 	int remaining, wakeup_swapper;
56444990b8cSJulian Elischer 
56544990b8cSJulian Elischer 	td = curthread;
56644990b8cSJulian Elischer 	p = td->td_proc;
56737814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
56844990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
56944990b8cSJulian Elischer 	KASSERT((td != NULL), ("curthread is NULL"));
57044990b8cSJulian Elischer 
571ed062c8dSJulian Elischer 	if ((p->p_flag & P_HADTHREADS) == 0)
57244990b8cSJulian Elischer 		return (0);
57344990b8cSJulian Elischer 
574e3b9bf71SJulian Elischer 	/* Is someone already single threading? */
575906ac69dSDavid Xu 	if (p->p_singlethread != NULL && p->p_singlethread != td)
57644990b8cSJulian Elischer 		return (1);
57744990b8cSJulian Elischer 
578906ac69dSDavid Xu 	if (mode == SINGLE_EXIT) {
579906ac69dSDavid Xu 		p->p_flag |= P_SINGLE_EXIT;
580906ac69dSDavid Xu 		p->p_flag &= ~P_SINGLE_BOUNDARY;
581906ac69dSDavid Xu 	} else {
582906ac69dSDavid Xu 		p->p_flag &= ~P_SINGLE_EXIT;
583906ac69dSDavid Xu 		if (mode == SINGLE_BOUNDARY)
584906ac69dSDavid Xu 			p->p_flag |= P_SINGLE_BOUNDARY;
585906ac69dSDavid Xu 		else
586906ac69dSDavid Xu 			p->p_flag &= ~P_SINGLE_BOUNDARY;
587906ac69dSDavid Xu 	}
5881279572aSDavid Xu 	p->p_flag |= P_STOPPED_SINGLE;
5897b4a950aSDavid Xu 	PROC_SLOCK(p);
590112afcb2SJohn Baldwin 	p->p_singlethread = td;
59179799053SKonstantin Belousov 	remaining = calc_remaining(p, mode);
592ec008e96SDavid Xu 	while (remaining != 1) {
593bf1a3220SDavid Xu 		if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
594bf1a3220SDavid Xu 			goto stopme;
595da7bbd2cSJohn Baldwin 		wakeup_swapper = 0;
59644990b8cSJulian Elischer 		FOREACH_THREAD_IN_PROC(p, td2) {
59744990b8cSJulian Elischer 			if (td2 == td)
59844990b8cSJulian Elischer 				continue;
599a54e85fdSJeff Roberson 			thread_lock(td2);
600b7edba77SJeff Roberson 			td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
60171fad9fdSJulian Elischer 			if (TD_IS_INHIBITED(td2)) {
602906ac69dSDavid Xu 				switch (mode) {
603906ac69dSDavid Xu 				case SINGLE_EXIT:
604906ac69dSDavid Xu 					if (TD_IS_SUSPENDED(td2))
6057847a9daSJohn Baldwin 						wakeup_swapper |=
60671fad9fdSJulian Elischer 						    thread_unsuspend_one(td2);
60733862f40SDavid Xu 					if (TD_ON_SLEEPQ(td2) &&
608906ac69dSDavid Xu 					    (td2->td_flags & TDF_SINTR))
6097847a9daSJohn Baldwin 						wakeup_swapper |=
61094f0972bSDavid Xu 						    sleepq_abort(td2, EINTR);
611906ac69dSDavid Xu 					break;
612906ac69dSDavid Xu 				case SINGLE_BOUNDARY:
613ffdc5a34SDavid Xu 					if (TD_IS_SUSPENDED(td2) &&
614ffdc5a34SDavid Xu 					    !(td2->td_flags & TDF_BOUNDARY))
615ffdc5a34SDavid Xu 						wakeup_swapper |=
616ffdc5a34SDavid Xu 						    thread_unsuspend_one(td2);
617ffdc5a34SDavid Xu 					if (TD_ON_SLEEPQ(td2) &&
618ffdc5a34SDavid Xu 					    (td2->td_flags & TDF_SINTR))
619ffdc5a34SDavid Xu 						wakeup_swapper |=
620ffdc5a34SDavid Xu 						    sleepq_abort(td2, ERESTART);
621906ac69dSDavid Xu 					break;
622f33a947bSKonstantin Belousov 				case SINGLE_NO_EXIT:
623f33a947bSKonstantin Belousov 					if (TD_IS_SUSPENDED(td2) &&
624f33a947bSKonstantin Belousov 					    !(td2->td_flags & TDF_BOUNDARY))
625f33a947bSKonstantin Belousov 						wakeup_swapper |=
626f33a947bSKonstantin Belousov 						    thread_unsuspend_one(td2);
627f33a947bSKonstantin Belousov 					if (TD_ON_SLEEPQ(td2) &&
628f33a947bSKonstantin Belousov 					    (td2->td_flags & TDF_SINTR))
629f33a947bSKonstantin Belousov 						wakeup_swapper |=
630f33a947bSKonstantin Belousov 						    sleepq_abort(td2, ERESTART);
631f33a947bSKonstantin Belousov 					break;
632906ac69dSDavid Xu 				default:
633906ac69dSDavid Xu 					break;
63444990b8cSJulian Elischer 				}
63544990b8cSJulian Elischer 			}
636d8267df7SDavid Xu #ifdef SMP
637d8267df7SDavid Xu 			else if (TD_IS_RUNNING(td2) && td != td2) {
638d8267df7SDavid Xu 				forward_signal(td2);
639d8267df7SDavid Xu 			}
640d8267df7SDavid Xu #endif
641a54e85fdSJeff Roberson 			thread_unlock(td2);
6429d102777SJulian Elischer 		}
643da7bbd2cSJohn Baldwin 		if (wakeup_swapper)
644da7bbd2cSJohn Baldwin 			kick_proc0();
64579799053SKonstantin Belousov 		remaining = calc_remaining(p, mode);
646ec008e96SDavid Xu 
6479d102777SJulian Elischer 		/*
6489d102777SJulian Elischer 		 * Maybe we suspended some threads.. was it enough?
6499d102777SJulian Elischer 		 */
650ec008e96SDavid Xu 		if (remaining == 1)
6519d102777SJulian Elischer 			break;
6529d102777SJulian Elischer 
653bf1a3220SDavid Xu stopme:
65444990b8cSJulian Elischer 		/*
65544990b8cSJulian Elischer 		 * Wake us up when everyone else has suspended.
656e3b9bf71SJulian Elischer 		 * In the mean time we suspend as well.
65744990b8cSJulian Elischer 		 */
658a54e85fdSJeff Roberson 		thread_suspend_switch(td);
65979799053SKonstantin Belousov 		remaining = calc_remaining(p, mode);
66044990b8cSJulian Elischer 	}
661906ac69dSDavid Xu 	if (mode == SINGLE_EXIT) {
66291599697SJulian Elischer 		/*
66391599697SJulian Elischer 		 * We have gotten rid of all the other threads and we
66491599697SJulian Elischer 		 * are about to either exit or exec. In either case,
66591599697SJulian Elischer 		 * we try our utmost  to revert to being a non-threaded
66691599697SJulian Elischer 		 * process.
66791599697SJulian Elischer 		 */
668ed062c8dSJulian Elischer 		p->p_singlethread = NULL;
66964895117SDavid Xu 		p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
670e5bedcefSJulian Elischer 		thread_unthread(td);
67191599697SJulian Elischer 	}
6727b4a950aSDavid Xu 	PROC_SUNLOCK(p);
67344990b8cSJulian Elischer 	return (0);
67444990b8cSJulian Elischer }
67544990b8cSJulian Elischer 
67644990b8cSJulian Elischer /*
67744990b8cSJulian Elischer  * Called in from locations that can safely check to see
67844990b8cSJulian Elischer  * whether we have to suspend or at least throttle for a
67944990b8cSJulian Elischer  * single-thread event (e.g. fork).
68044990b8cSJulian Elischer  *
68144990b8cSJulian Elischer  * Such locations include userret().
68244990b8cSJulian Elischer  * If the "return_instead" argument is non zero, the thread must be able to
68344990b8cSJulian Elischer  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
68444990b8cSJulian Elischer  *
68544990b8cSJulian Elischer  * The 'return_instead' argument tells the function if it may do a
68644990b8cSJulian Elischer  * thread_exit() or suspend, or whether the caller must abort and back
68744990b8cSJulian Elischer  * out instead.
68844990b8cSJulian Elischer  *
68944990b8cSJulian Elischer  * If the thread that set the single_threading request has set the
69044990b8cSJulian Elischer  * P_SINGLE_EXIT bit in the process flags then this call will never return
69144990b8cSJulian Elischer  * if 'return_instead' is false, but will exit.
69244990b8cSJulian Elischer  *
69344990b8cSJulian Elischer  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
69444990b8cSJulian Elischer  *---------------+--------------------+---------------------
69544990b8cSJulian Elischer  *       0       | returns 0          |   returns 0 or 1
69644990b8cSJulian Elischer  *               | when ST ends       |   immediatly
69744990b8cSJulian Elischer  *---------------+--------------------+---------------------
69844990b8cSJulian Elischer  *       1       | thread exits       |   returns 1
69944990b8cSJulian Elischer  *               |                    |  immediatly
70044990b8cSJulian Elischer  * 0 = thread_exit() or suspension ok,
70144990b8cSJulian Elischer  * other = return error instead of stopping the thread.
70244990b8cSJulian Elischer  *
70344990b8cSJulian Elischer  * While a full suspension is under effect, even a single threading
70444990b8cSJulian Elischer  * thread would be suspended if it made this call (but it shouldn't).
70544990b8cSJulian Elischer  * This call should only be made from places where
70644990b8cSJulian Elischer  * thread_exit() would be safe as that may be the outcome unless
70744990b8cSJulian Elischer  * return_instead is set.
70844990b8cSJulian Elischer  */
70944990b8cSJulian Elischer int
71044990b8cSJulian Elischer thread_suspend_check(int return_instead)
71144990b8cSJulian Elischer {
712ecafb24bSJuli Mallett 	struct thread *td;
713ecafb24bSJuli Mallett 	struct proc *p;
7147847a9daSJohn Baldwin 	int wakeup_swapper;
71544990b8cSJulian Elischer 
71644990b8cSJulian Elischer 	td = curthread;
71744990b8cSJulian Elischer 	p = td->td_proc;
71837814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
71944990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
720cbf4e354SDavid Xu 	while (P_SHOULDSTOP(p) ||
721904c5ec4SDavid Xu 	      ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) {
7221279572aSDavid Xu 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
72344990b8cSJulian Elischer 			KASSERT(p->p_singlethread != NULL,
72444990b8cSJulian Elischer 			    ("singlethread not set"));
72544990b8cSJulian Elischer 			/*
726e3b9bf71SJulian Elischer 			 * The only suspension in action is a
727e3b9bf71SJulian Elischer 			 * single-threading. Single threader need not stop.
728b6d5995eSJulian Elischer 			 * XXX Should be safe to access unlocked
729b6d5995eSJulian Elischer 			 * as it can only be set to be true by us.
73044990b8cSJulian Elischer 			 */
731e3b9bf71SJulian Elischer 			if (p->p_singlethread == td)
73244990b8cSJulian Elischer 				return (0);	/* Exempt from stopping. */
73344990b8cSJulian Elischer 		}
73445a4bfa1SDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
73594f0972bSDavid Xu 			return (EINTR);
73644990b8cSJulian Elischer 
737906ac69dSDavid Xu 		/* Should we goto user boundary if we didn't come from there? */
738906ac69dSDavid Xu 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
739906ac69dSDavid Xu 		    (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
74094f0972bSDavid Xu 			return (ERESTART);
741906ac69dSDavid Xu 
7429104847fSDavid Xu 		/* If thread will exit, flush its pending signals */
7439104847fSDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
7449104847fSDavid Xu 			sigqueue_flush(&td->td_sigqueue);
7459104847fSDavid Xu 
7467b4a950aSDavid Xu 		PROC_SLOCK(p);
747e574e444SDavid Xu 		thread_stopped(p);
74844990b8cSJulian Elischer 		/*
74944990b8cSJulian Elischer 		 * If the process is waiting for us to exit,
75044990b8cSJulian Elischer 		 * this thread should just suicide.
7511279572aSDavid Xu 		 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
75244990b8cSJulian Elischer 		 */
7537b4a950aSDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
75444990b8cSJulian Elischer 			thread_exit();
755a54e85fdSJeff Roberson 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
756a54e85fdSJeff Roberson 			if (p->p_numthreads == p->p_suspcount + 1) {
757a54e85fdSJeff Roberson 				thread_lock(p->p_singlethread);
7587847a9daSJohn Baldwin 				wakeup_swapper =
759a54e85fdSJeff Roberson 				    thread_unsuspend_one(p->p_singlethread);
760a54e85fdSJeff Roberson 				thread_unlock(p->p_singlethread);
7617847a9daSJohn Baldwin 				if (wakeup_swapper)
7627847a9daSJohn Baldwin 					kick_proc0();
763a54e85fdSJeff Roberson 			}
764a54e85fdSJeff Roberson 		}
7653f9be10eSDavid Xu 		PROC_UNLOCK(p);
7667b4a950aSDavid Xu 		thread_lock(td);
76744990b8cSJulian Elischer 		/*
76844990b8cSJulian Elischer 		 * When a thread suspends, it just
769ad1e7d28SJulian Elischer 		 * gets taken off all queues.
77044990b8cSJulian Elischer 		 */
77171fad9fdSJulian Elischer 		thread_suspend_one(td);
772906ac69dSDavid Xu 		if (return_instead == 0) {
773906ac69dSDavid Xu 			p->p_boundary_count++;
774906ac69dSDavid Xu 			td->td_flags |= TDF_BOUNDARY;
775cf19bf91SJulian Elischer 		}
7767b4a950aSDavid Xu 		PROC_SUNLOCK(p);
7778df78c41SJeff Roberson 		mi_switch(SW_INVOL | SWT_SUSPEND, NULL);
778a54e85fdSJeff Roberson 		if (return_instead == 0)
779906ac69dSDavid Xu 			td->td_flags &= ~TDF_BOUNDARY;
780a54e85fdSJeff Roberson 		thread_unlock(td);
78144990b8cSJulian Elischer 		PROC_LOCK(p);
782a54e85fdSJeff Roberson 		if (return_instead == 0)
783a54e85fdSJeff Roberson 			p->p_boundary_count--;
78444990b8cSJulian Elischer 	}
78544990b8cSJulian Elischer 	return (0);
78644990b8cSJulian Elischer }
78744990b8cSJulian Elischer 
78835c32a76SDavid Xu void
789a54e85fdSJeff Roberson thread_suspend_switch(struct thread *td)
790a54e85fdSJeff Roberson {
791a54e85fdSJeff Roberson 	struct proc *p;
792a54e85fdSJeff Roberson 
793a54e85fdSJeff Roberson 	p = td->td_proc;
794a54e85fdSJeff Roberson 	KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
795a54e85fdSJeff Roberson 	PROC_LOCK_ASSERT(p, MA_OWNED);
7967b4a950aSDavid Xu 	PROC_SLOCK_ASSERT(p, MA_OWNED);
797a54e85fdSJeff Roberson 	/*
798a54e85fdSJeff Roberson 	 * We implement thread_suspend_one in stages here to avoid
799a54e85fdSJeff Roberson 	 * dropping the proc lock while the thread lock is owned.
800a54e85fdSJeff Roberson 	 */
801a54e85fdSJeff Roberson 	thread_stopped(p);
802a54e85fdSJeff Roberson 	p->p_suspcount++;
8033f9be10eSDavid Xu 	PROC_UNLOCK(p);
8047b4a950aSDavid Xu 	thread_lock(td);
805b7edba77SJeff Roberson 	td->td_flags &= ~TDF_NEEDSUSPCHK;
806a54e85fdSJeff Roberson 	TD_SET_SUSPENDED(td);
807c5aa6b58SJeff Roberson 	sched_sleep(td, 0);
8087b4a950aSDavid Xu 	PROC_SUNLOCK(p);
809a54e85fdSJeff Roberson 	DROP_GIANT();
8108df78c41SJeff Roberson 	mi_switch(SW_VOL | SWT_SUSPEND, NULL);
811a54e85fdSJeff Roberson 	thread_unlock(td);
812a54e85fdSJeff Roberson 	PICKUP_GIANT();
813a54e85fdSJeff Roberson 	PROC_LOCK(p);
8147b4a950aSDavid Xu 	PROC_SLOCK(p);
815a54e85fdSJeff Roberson }
816a54e85fdSJeff Roberson 
817a54e85fdSJeff Roberson void
81835c32a76SDavid Xu thread_suspend_one(struct thread *td)
81935c32a76SDavid Xu {
82035c32a76SDavid Xu 	struct proc *p = td->td_proc;
82135c32a76SDavid Xu 
8227b4a950aSDavid Xu 	PROC_SLOCK_ASSERT(p, MA_OWNED);
823a54e85fdSJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
824e574e444SDavid Xu 	KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
82535c32a76SDavid Xu 	p->p_suspcount++;
826b7edba77SJeff Roberson 	td->td_flags &= ~TDF_NEEDSUSPCHK;
82771fad9fdSJulian Elischer 	TD_SET_SUSPENDED(td);
828c5aa6b58SJeff Roberson 	sched_sleep(td, 0);
82935c32a76SDavid Xu }
83035c32a76SDavid Xu 
8317847a9daSJohn Baldwin int
83235c32a76SDavid Xu thread_unsuspend_one(struct thread *td)
83335c32a76SDavid Xu {
83435c32a76SDavid Xu 	struct proc *p = td->td_proc;
83535c32a76SDavid Xu 
8367b4a950aSDavid Xu 	PROC_SLOCK_ASSERT(p, MA_OWNED);
837a54e85fdSJeff Roberson 	THREAD_LOCK_ASSERT(td, MA_OWNED);
838ad1e7d28SJulian Elischer 	KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
83971fad9fdSJulian Elischer 	TD_CLR_SUSPENDED(td);
84035c32a76SDavid Xu 	p->p_suspcount--;
8417847a9daSJohn Baldwin 	return (setrunnable(td));
84235c32a76SDavid Xu }
84335c32a76SDavid Xu 
84444990b8cSJulian Elischer /*
84544990b8cSJulian Elischer  * Allow all threads blocked by single threading to continue running.
84644990b8cSJulian Elischer  */
84744990b8cSJulian Elischer void
84844990b8cSJulian Elischer thread_unsuspend(struct proc *p)
84944990b8cSJulian Elischer {
85044990b8cSJulian Elischer 	struct thread *td;
8517847a9daSJohn Baldwin 	int wakeup_swapper;
85244990b8cSJulian Elischer 
85344990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
8547b4a950aSDavid Xu 	PROC_SLOCK_ASSERT(p, MA_OWNED);
8557847a9daSJohn Baldwin 	wakeup_swapper = 0;
85644990b8cSJulian Elischer 	if (!P_SHOULDSTOP(p)) {
857ad1e7d28SJulian Elischer                 FOREACH_THREAD_IN_PROC(p, td) {
858a54e85fdSJeff Roberson 			thread_lock(td);
859ad1e7d28SJulian Elischer 			if (TD_IS_SUSPENDED(td)) {
8607847a9daSJohn Baldwin 				wakeup_swapper |= thread_unsuspend_one(td);
86144990b8cSJulian Elischer 			}
862a54e85fdSJeff Roberson 			thread_unlock(td);
863ad1e7d28SJulian Elischer 		}
8641279572aSDavid Xu 	} else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) &&
86544990b8cSJulian Elischer 	    (p->p_numthreads == p->p_suspcount)) {
86644990b8cSJulian Elischer 		/*
86744990b8cSJulian Elischer 		 * Stopping everything also did the job for the single
86844990b8cSJulian Elischer 		 * threading request. Now we've downgraded to single-threaded,
86944990b8cSJulian Elischer 		 * let it continue.
87044990b8cSJulian Elischer 		 */
871a54e85fdSJeff Roberson 		thread_lock(p->p_singlethread);
8727847a9daSJohn Baldwin 		wakeup_swapper = thread_unsuspend_one(p->p_singlethread);
873a54e85fdSJeff Roberson 		thread_unlock(p->p_singlethread);
87444990b8cSJulian Elischer 	}
8757847a9daSJohn Baldwin 	if (wakeup_swapper)
8767847a9daSJohn Baldwin 		kick_proc0();
87744990b8cSJulian Elischer }
87844990b8cSJulian Elischer 
879ed062c8dSJulian Elischer /*
880ed062c8dSJulian Elischer  * End the single threading mode..
881ed062c8dSJulian Elischer  */
88244990b8cSJulian Elischer void
88344990b8cSJulian Elischer thread_single_end(void)
88444990b8cSJulian Elischer {
88544990b8cSJulian Elischer 	struct thread *td;
88644990b8cSJulian Elischer 	struct proc *p;
8877847a9daSJohn Baldwin 	int wakeup_swapper;
88844990b8cSJulian Elischer 
88944990b8cSJulian Elischer 	td = curthread;
89044990b8cSJulian Elischer 	p = td->td_proc;
89144990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
892906ac69dSDavid Xu 	p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
8937b4a950aSDavid Xu 	PROC_SLOCK(p);
89444990b8cSJulian Elischer 	p->p_singlethread = NULL;
8957847a9daSJohn Baldwin 	wakeup_swapper = 0;
89649539972SJulian Elischer 	/*
8977847a9daSJohn Baldwin 	 * If there are other threads they may now run,
89849539972SJulian Elischer 	 * unless of course there is a blanket 'stop order'
89949539972SJulian Elischer 	 * on the process. The single threader must be allowed
90049539972SJulian Elischer 	 * to continue however as this is a bad place to stop.
90149539972SJulian Elischer 	 */
90249539972SJulian Elischer 	if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) {
903ad1e7d28SJulian Elischer                 FOREACH_THREAD_IN_PROC(p, td) {
904a54e85fdSJeff Roberson 			thread_lock(td);
905ad1e7d28SJulian Elischer 			if (TD_IS_SUSPENDED(td)) {
9067847a9daSJohn Baldwin 				wakeup_swapper |= thread_unsuspend_one(td);
90744990b8cSJulian Elischer 			}
908a54e85fdSJeff Roberson 			thread_unlock(td);
90949539972SJulian Elischer 		}
910ad1e7d28SJulian Elischer 	}
9117b4a950aSDavid Xu 	PROC_SUNLOCK(p);
9127847a9daSJohn Baldwin 	if (wakeup_swapper)
9137847a9daSJohn Baldwin 		kick_proc0();
91449539972SJulian Elischer }
9154fc21c09SDaniel Eischen 
91644355392SDavid Xu struct thread *
91744355392SDavid Xu thread_find(struct proc *p, lwpid_t tid)
91844355392SDavid Xu {
91944355392SDavid Xu 	struct thread *td;
92044355392SDavid Xu 
92144355392SDavid Xu 	PROC_LOCK_ASSERT(p, MA_OWNED);
92244355392SDavid Xu 	FOREACH_THREAD_IN_PROC(p, td) {
92344355392SDavid Xu 		if (td->td_tid == tid)
92444355392SDavid Xu 			break;
92544355392SDavid Xu 	}
92644355392SDavid Xu 	return (td);
92744355392SDavid Xu }
928