xref: /freebsd/sys/kern/kern_thread.c (revision 94f0972beced4d0e4f4578931d6613ebb42527bb)
19454b2d8SWarner Losh /*-
244990b8cSJulian Elischer  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
344990b8cSJulian Elischer  *  All rights reserved.
444990b8cSJulian Elischer  *
544990b8cSJulian Elischer  * Redistribution and use in source and binary forms, with or without
644990b8cSJulian Elischer  * modification, are permitted provided that the following conditions
744990b8cSJulian Elischer  * are met:
844990b8cSJulian Elischer  * 1. Redistributions of source code must retain the above copyright
944990b8cSJulian Elischer  *    notice(s), this list of conditions and the following disclaimer as
1044990b8cSJulian Elischer  *    the first lines of this file unmodified other than the possible
1144990b8cSJulian Elischer  *    addition of one or more copyright notices.
1244990b8cSJulian Elischer  * 2. Redistributions in binary form must reproduce the above copyright
1344990b8cSJulian Elischer  *    notice(s), this list of conditions and the following disclaimer in the
1444990b8cSJulian Elischer  *    documentation and/or other materials provided with the distribution.
1544990b8cSJulian Elischer  *
1644990b8cSJulian Elischer  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
1744990b8cSJulian Elischer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1844990b8cSJulian Elischer  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1944990b8cSJulian Elischer  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
2044990b8cSJulian Elischer  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
2144990b8cSJulian Elischer  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
2244990b8cSJulian Elischer  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
2344990b8cSJulian Elischer  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2444990b8cSJulian Elischer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2544990b8cSJulian Elischer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
2644990b8cSJulian Elischer  * DAMAGE.
2744990b8cSJulian Elischer  */
2844990b8cSJulian Elischer 
29677b542eSDavid E. O'Brien #include <sys/cdefs.h>
30677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
31677b542eSDavid E. O'Brien 
3244990b8cSJulian Elischer #include <sys/param.h>
3344990b8cSJulian Elischer #include <sys/systm.h>
3444990b8cSJulian Elischer #include <sys/kernel.h>
3544990b8cSJulian Elischer #include <sys/lock.h>
3644990b8cSJulian Elischer #include <sys/mutex.h>
3744990b8cSJulian Elischer #include <sys/proc.h>
3894e0a4cdSJulian Elischer #include <sys/smp.h>
3944990b8cSJulian Elischer #include <sys/sysctl.h>
40de028f5aSJeff Roberson #include <sys/sched.h>
4144f3b092SJohn Baldwin #include <sys/sleepqueue.h>
42961a7b24SJohn Baldwin #include <sys/turnstile.h>
4344990b8cSJulian Elischer #include <sys/ktr.h>
44bc8e6d81SDavid Xu #include <sys/umtx.h>
4544990b8cSJulian Elischer 
46911b84b0SRobert Watson #include <security/audit/audit.h>
47911b84b0SRobert Watson 
4844990b8cSJulian Elischer #include <vm/vm.h>
4949a2507bSAlan Cox #include <vm/vm_extern.h>
5044990b8cSJulian Elischer #include <vm/uma.h>
5102fb42b0SPeter Wemm 
5244990b8cSJulian Elischer /*
534f0db5e0SJulian Elischer  * KSEGRP related storage.
5444990b8cSJulian Elischer  */
554f0db5e0SJulian Elischer static uma_zone_t ksegrp_zone;
5644990b8cSJulian Elischer static uma_zone_t thread_zone;
5744990b8cSJulian Elischer 
584f0db5e0SJulian Elischer /* DEBUG ONLY */
5944990b8cSJulian Elischer SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation");
60696058c3SJulian Elischer static int thread_debug = 0;
61696058c3SJulian Elischer SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW,
62696058c3SJulian Elischer 	&thread_debug, 0, "thread debug");
63fdc5ecd2SDavid Xu 
64345ad866SJulian Elischer int max_threads_per_proc = 1500;
65fdc5ecd2SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
664f0db5e0SJulian Elischer 	&max_threads_per_proc, 0, "Limit on threads per proc");
674f0db5e0SJulian Elischer 
68ed062c8dSJulian Elischer int max_groups_per_proc = 1500;
69fdc5ecd2SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW,
70fdc5ecd2SDavid Xu 	&max_groups_per_proc, 0, "Limit on thread groups per proc");
71fdc5ecd2SDavid Xu 
72345ad866SJulian Elischer int max_threads_hits;
730252d203SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD,
740252d203SDavid Xu 	&max_threads_hits, 0, "");
750252d203SDavid Xu 
7694e0a4cdSJulian Elischer int virtual_cpu;
7794e0a4cdSJulian Elischer 
785215b187SJeff Roberson TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
795c8329edSJulian Elischer TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps);
805215b187SJeff Roberson struct mtx kse_zombie_lock;
815215b187SJeff Roberson MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN);
8244990b8cSJulian Elischer 
8394e0a4cdSJulian Elischer static int
8494e0a4cdSJulian Elischer sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS)
8594e0a4cdSJulian Elischer {
8694e0a4cdSJulian Elischer 	int error, new_val;
8794e0a4cdSJulian Elischer 	int def_val;
8894e0a4cdSJulian Elischer 
8994e0a4cdSJulian Elischer 	def_val = mp_ncpus;
9094e0a4cdSJulian Elischer 	if (virtual_cpu == 0)
9194e0a4cdSJulian Elischer 		new_val = def_val;
9294e0a4cdSJulian Elischer 	else
9394e0a4cdSJulian Elischer 		new_val = virtual_cpu;
9494e0a4cdSJulian Elischer 	error = sysctl_handle_int(oidp, &new_val, 0, req);
9594e0a4cdSJulian Elischer 	if (error != 0 || req->newptr == NULL)
9694e0a4cdSJulian Elischer 		return (error);
9794e0a4cdSJulian Elischer 	if (new_val < 0)
9894e0a4cdSJulian Elischer 		return (EINVAL);
9994e0a4cdSJulian Elischer 	virtual_cpu = new_val;
10094e0a4cdSJulian Elischer 	return (0);
10194e0a4cdSJulian Elischer }
10294e0a4cdSJulian Elischer 
10394e0a4cdSJulian Elischer /* DEBUG ONLY */
10494e0a4cdSJulian Elischer SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW,
10594e0a4cdSJulian Elischer 	0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I",
10694e0a4cdSJulian Elischer 	"debug virtual cpus");
1075c8329edSJulian Elischer 
108fdcac928SMarcel Moolenaar struct mtx tid_lock;
1091ea7a6f8SPoul-Henning Kamp static struct unrhdr *tid_unrhdr;
110fdcac928SMarcel Moolenaar 
111fdcac928SMarcel Moolenaar /*
112696058c3SJulian Elischer  * Prepare a thread for use.
11344990b8cSJulian Elischer  */
114b23f72e9SBrian Feldman static int
115b23f72e9SBrian Feldman thread_ctor(void *mem, int size, void *arg, int flags)
11644990b8cSJulian Elischer {
11744990b8cSJulian Elischer 	struct thread	*td;
11844990b8cSJulian Elischer 
11944990b8cSJulian Elischer 	td = (struct thread *)mem;
12071fad9fdSJulian Elischer 	td->td_state = TDS_INACTIVE;
121060563ecSJulian Elischer 	td->td_oncpu = NOCPU;
1226c27c603SJuli Mallett 
123773eff9dSPoul-Henning Kamp 	td->td_tid = alloc_unr(tid_unrhdr);
124773eff9dSPoul-Henning Kamp 
1256c27c603SJuli Mallett 	/*
1266c27c603SJuli Mallett 	 * Note that td_critnest begins life as 1 because the thread is not
1276c27c603SJuli Mallett 	 * running and is thereby implicitly waiting to be on the receiving
1286c27c603SJuli Mallett 	 * end of a context switch.  A context switch must occur inside a
1296c27c603SJuli Mallett 	 * critical section, and in fact, includes hand-off of the sched_lock.
1306c27c603SJuli Mallett 	 * After a context switch to a newly created thread, it will release
1316c27c603SJuli Mallett 	 * sched_lock for the first time, and its td_critnest will hit 0 for
1326c27c603SJuli Mallett 	 * the first time.  This happens on the far end of a context switch,
1336c27c603SJuli Mallett 	 * and when it context switches away from itself, it will in fact go
1346c27c603SJuli Mallett 	 * back into a critical section, and hand off the sched lock to the
1356c27c603SJuli Mallett 	 * next thread.
1366c27c603SJuli Mallett 	 */
137139b7550SJohn Baldwin 	td->td_critnest = 1;
138911b84b0SRobert Watson 
139911b84b0SRobert Watson #ifdef AUDIT
140911b84b0SRobert Watson 	audit_thread_alloc(td);
141911b84b0SRobert Watson #endif
142b23f72e9SBrian Feldman 	return (0);
14344990b8cSJulian Elischer }
14444990b8cSJulian Elischer 
14544990b8cSJulian Elischer /*
14644990b8cSJulian Elischer  * Reclaim a thread after use.
14744990b8cSJulian Elischer  */
14844990b8cSJulian Elischer static void
14944990b8cSJulian Elischer thread_dtor(void *mem, int size, void *arg)
15044990b8cSJulian Elischer {
15144990b8cSJulian Elischer 	struct thread *td;
15244990b8cSJulian Elischer 
15344990b8cSJulian Elischer 	td = (struct thread *)mem;
15444990b8cSJulian Elischer 
15544990b8cSJulian Elischer #ifdef INVARIANTS
15644990b8cSJulian Elischer 	/* Verify that this thread is in a safe state to free. */
15744990b8cSJulian Elischer 	switch (td->td_state) {
15871fad9fdSJulian Elischer 	case TDS_INHIBITED:
15971fad9fdSJulian Elischer 	case TDS_RUNNING:
16071fad9fdSJulian Elischer 	case TDS_CAN_RUN:
16144990b8cSJulian Elischer 	case TDS_RUNQ:
16244990b8cSJulian Elischer 		/*
16344990b8cSJulian Elischer 		 * We must never unlink a thread that is in one of
16444990b8cSJulian Elischer 		 * these states, because it is currently active.
16544990b8cSJulian Elischer 		 */
16644990b8cSJulian Elischer 		panic("bad state for thread unlinking");
16744990b8cSJulian Elischer 		/* NOTREACHED */
16871fad9fdSJulian Elischer 	case TDS_INACTIVE:
16944990b8cSJulian Elischer 		break;
17044990b8cSJulian Elischer 	default:
17144990b8cSJulian Elischer 		panic("bad thread state");
17244990b8cSJulian Elischer 		/* NOTREACHED */
17344990b8cSJulian Elischer 	}
17444990b8cSJulian Elischer #endif
1756e8525ceSRobert Watson #ifdef AUDIT
1766e8525ceSRobert Watson 	audit_thread_free(td);
1776e8525ceSRobert Watson #endif
178773eff9dSPoul-Henning Kamp 	free_unr(tid_unrhdr, td->td_tid);
179ed062c8dSJulian Elischer 	sched_newthread(td);
18044990b8cSJulian Elischer }
18144990b8cSJulian Elischer 
18244990b8cSJulian Elischer /*
18344990b8cSJulian Elischer  * Initialize type-stable parts of a thread (when newly created).
18444990b8cSJulian Elischer  */
185b23f72e9SBrian Feldman static int
186b23f72e9SBrian Feldman thread_init(void *mem, int size, int flags)
18744990b8cSJulian Elischer {
18844990b8cSJulian Elischer 	struct thread *td;
18944990b8cSJulian Elischer 
19044990b8cSJulian Elischer 	td = (struct thread *)mem;
191247aba24SMarcel Moolenaar 
19249a2507bSAlan Cox 	vm_thread_new(td, 0);
19344990b8cSJulian Elischer 	cpu_thread_setup(td);
19444f3b092SJohn Baldwin 	td->td_sleepqueue = sleepq_alloc();
195961a7b24SJohn Baldwin 	td->td_turnstile = turnstile_alloc();
196bc8e6d81SDavid Xu 	td->td_umtxq = umtxq_alloc();
197de028f5aSJeff Roberson 	td->td_sched = (struct td_sched *)&td[1];
198ed062c8dSJulian Elischer 	sched_newthread(td);
199b23f72e9SBrian Feldman 	return (0);
20044990b8cSJulian Elischer }
20144990b8cSJulian Elischer 
20244990b8cSJulian Elischer /*
20344990b8cSJulian Elischer  * Tear down type-stable parts of a thread (just before being discarded).
20444990b8cSJulian Elischer  */
20544990b8cSJulian Elischer static void
20644990b8cSJulian Elischer thread_fini(void *mem, int size)
20744990b8cSJulian Elischer {
20844990b8cSJulian Elischer 	struct thread *td;
20944990b8cSJulian Elischer 
21044990b8cSJulian Elischer 	td = (struct thread *)mem;
211961a7b24SJohn Baldwin 	turnstile_free(td->td_turnstile);
21244f3b092SJohn Baldwin 	sleepq_free(td->td_sleepqueue);
213bc8e6d81SDavid Xu 	umtxq_free(td->td_umtxq);
21449a2507bSAlan Cox 	vm_thread_dispose(td);
21544990b8cSJulian Elischer }
2165215b187SJeff Roberson 
217de028f5aSJeff Roberson /*
218de028f5aSJeff Roberson  * Initialize type-stable parts of a ksegrp (when newly created).
219de028f5aSJeff Roberson  */
220b23f72e9SBrian Feldman static int
221a9b5dc7dSJulian Elischer ksegrp_ctor(void *mem, int size, void *arg, int flags)
222de028f5aSJeff Roberson {
223de028f5aSJeff Roberson 	struct ksegrp	*kg;
224de028f5aSJeff Roberson 
225de028f5aSJeff Roberson 	kg = (struct ksegrp *)mem;
226a9b5dc7dSJulian Elischer 	bzero(mem, size);
227de028f5aSJeff Roberson 	kg->kg_sched = (struct kg_sched *)&kg[1];
228b23f72e9SBrian Feldman 	return (0);
229de028f5aSJeff Roberson }
23044990b8cSJulian Elischer 
2315c8329edSJulian Elischer void
2325c8329edSJulian Elischer ksegrp_link(struct ksegrp *kg, struct proc *p)
2335c8329edSJulian Elischer {
2345c8329edSJulian Elischer 
2355c8329edSJulian Elischer 	TAILQ_INIT(&kg->kg_threads);
2365c8329edSJulian Elischer 	TAILQ_INIT(&kg->kg_runq);	/* links with td_runq */
2375215b187SJeff Roberson 	TAILQ_INIT(&kg->kg_upcalls);	/* all upcall structure in ksegrp */
2385c8329edSJulian Elischer 	kg->kg_proc = p;
2395215b187SJeff Roberson 	/*
2405215b187SJeff Roberson 	 * the following counters are in the -zero- section
2415215b187SJeff Roberson 	 * and may not need clearing
2425215b187SJeff Roberson 	 */
2435c8329edSJulian Elischer 	kg->kg_numthreads = 0;
2445215b187SJeff Roberson 	kg->kg_numupcalls = 0;
2455c8329edSJulian Elischer 	/* link it in now that it's consistent */
2465c8329edSJulian Elischer 	p->p_numksegrps++;
2475c8329edSJulian Elischer 	TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp);
2485c8329edSJulian Elischer }
2495c8329edSJulian Elischer 
250ed062c8dSJulian Elischer /*
251ed062c8dSJulian Elischer  * Called from:
252ed062c8dSJulian Elischer  *   thread-exit()
253ed062c8dSJulian Elischer  */
2545c8329edSJulian Elischer void
2555c8329edSJulian Elischer ksegrp_unlink(struct ksegrp *kg)
2565c8329edSJulian Elischer {
2575c8329edSJulian Elischer 	struct proc *p;
2585c8329edSJulian Elischer 
2595c8329edSJulian Elischer 	mtx_assert(&sched_lock, MA_OWNED);
2605215b187SJeff Roberson 	KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads"));
2615215b187SJeff Roberson 	KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls"));
2625215b187SJeff Roberson 
2635c8329edSJulian Elischer 	p = kg->kg_proc;
2645c8329edSJulian Elischer 	TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp);
2655c8329edSJulian Elischer 	p->p_numksegrps--;
2665c8329edSJulian Elischer 	/*
2675c8329edSJulian Elischer 	 * Aggregate stats from the KSE
2685c8329edSJulian Elischer 	 */
26921fc3164SDavid Xu 	if (p->p_procscopegrp == kg)
27021fc3164SDavid Xu 		p->p_procscopegrp = NULL;
2715c8329edSJulian Elischer }
2725c8329edSJulian Elischer 
2735c8329edSJulian Elischer /*
2745215b187SJeff Roberson  * For a newly created process,
2755215b187SJeff Roberson  * link up all the structures and its initial threads etc.
276ed062c8dSJulian Elischer  * called from:
277ed062c8dSJulian Elischer  * {arch}/{arch}/machdep.c   ia64_init(), init386() etc.
278ed062c8dSJulian Elischer  * proc_dtor() (should go away)
279ed062c8dSJulian Elischer  * proc_init()
2805c8329edSJulian Elischer  */
2815c8329edSJulian Elischer void
282ed062c8dSJulian Elischer proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td)
2835c8329edSJulian Elischer {
2845c8329edSJulian Elischer 
2855c8329edSJulian Elischer 	TAILQ_INIT(&p->p_ksegrps);	     /* all ksegrps in proc */
2865c8329edSJulian Elischer 	TAILQ_INIT(&p->p_threads);	     /* all threads in proc */
2875c8329edSJulian Elischer 	TAILQ_INIT(&p->p_suspended);	     /* Threads suspended */
2889104847fSDavid Xu 	sigqueue_init(&p->p_sigqueue, p);
289ebceaf6dSDavid Xu 	p->p_ksi = ksiginfo_alloc(1);
290ebceaf6dSDavid Xu 	if (p->p_ksi != NULL) {
2915c474517SDavid Xu 		/* XXX p_ksi may be null if ksiginfo zone is not ready */
292ebceaf6dSDavid Xu 		p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
293ebceaf6dSDavid Xu 	}
294b2f92ef9SDavid Xu 	LIST_INIT(&p->p_mqnotifier);
2955c8329edSJulian Elischer 	p->p_numksegrps = 0;
2965c8329edSJulian Elischer 	p->p_numthreads = 0;
2975c8329edSJulian Elischer 
2985c8329edSJulian Elischer 	ksegrp_link(kg, p);
2995c8329edSJulian Elischer 	thread_link(td, kg);
3005c8329edSJulian Elischer }
3015c8329edSJulian Elischer 
3025c8329edSJulian Elischer /*
30344990b8cSJulian Elischer  * Initialize global thread allocation resources.
30444990b8cSJulian Elischer  */
30544990b8cSJulian Elischer void
30644990b8cSJulian Elischer threadinit(void)
30744990b8cSJulian Elischer {
30844990b8cSJulian Elischer 
3091ea7a6f8SPoul-Henning Kamp 	mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
3101ea7a6f8SPoul-Henning Kamp 	tid_unrhdr = new_unrhdr(PID_MAX + 1, INT_MAX, &tid_lock);
3111ea7a6f8SPoul-Henning Kamp 
312de028f5aSJeff Roberson 	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
31344990b8cSJulian Elischer 	    thread_ctor, thread_dtor, thread_init, thread_fini,
31444990b8cSJulian Elischer 	    UMA_ALIGN_CACHE, 0);
315de028f5aSJeff Roberson 	ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(),
316a9b5dc7dSJulian Elischer 	    ksegrp_ctor, NULL, NULL, NULL,
3174f0db5e0SJulian Elischer 	    UMA_ALIGN_CACHE, 0);
318ed062c8dSJulian Elischer 	kseinit();	/* set up kse specific stuff  e.g. upcall zone*/
31944990b8cSJulian Elischer }
32044990b8cSJulian Elischer 
32144990b8cSJulian Elischer /*
3221faf202eSJulian Elischer  * Stash an embarasingly extra thread into the zombie thread queue.
32344990b8cSJulian Elischer  */
32444990b8cSJulian Elischer void
32544990b8cSJulian Elischer thread_stash(struct thread *td)
32644990b8cSJulian Elischer {
3275215b187SJeff Roberson 	mtx_lock_spin(&kse_zombie_lock);
32844990b8cSJulian Elischer 	TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq);
3295215b187SJeff Roberson 	mtx_unlock_spin(&kse_zombie_lock);
33044990b8cSJulian Elischer }
33144990b8cSJulian Elischer 
33244990b8cSJulian Elischer /*
3335c8329edSJulian Elischer  * Stash an embarasingly extra ksegrp into the zombie ksegrp queue.
3345c8329edSJulian Elischer  */
3355c8329edSJulian Elischer void
3365c8329edSJulian Elischer ksegrp_stash(struct ksegrp *kg)
3375c8329edSJulian Elischer {
3385215b187SJeff Roberson 	mtx_lock_spin(&kse_zombie_lock);
3395c8329edSJulian Elischer 	TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp);
3405215b187SJeff Roberson 	mtx_unlock_spin(&kse_zombie_lock);
3415c8329edSJulian Elischer }
3425c8329edSJulian Elischer 
3435c8329edSJulian Elischer /*
3445215b187SJeff Roberson  * Reap zombie kse resource.
34544990b8cSJulian Elischer  */
34644990b8cSJulian Elischer void
34744990b8cSJulian Elischer thread_reap(void)
34844990b8cSJulian Elischer {
3495c8329edSJulian Elischer 	struct thread *td_first, *td_next;
3505c8329edSJulian Elischer 	struct ksegrp *kg_first, * kg_next;
35144990b8cSJulian Elischer 
35244990b8cSJulian Elischer 	/*
3535215b187SJeff Roberson 	 * Don't even bother to lock if none at this instant,
3545215b187SJeff Roberson 	 * we really don't care about the next instant..
35544990b8cSJulian Elischer 	 */
3565c8329edSJulian Elischer 	if ((!TAILQ_EMPTY(&zombie_threads))
357345ad866SJulian Elischer 	    || (!TAILQ_EMPTY(&zombie_ksegrps))) {
3585215b187SJeff Roberson 		mtx_lock_spin(&kse_zombie_lock);
3595c8329edSJulian Elischer 		td_first = TAILQ_FIRST(&zombie_threads);
3605c8329edSJulian Elischer 		kg_first = TAILQ_FIRST(&zombie_ksegrps);
3615c8329edSJulian Elischer 		if (td_first)
3625c8329edSJulian Elischer 			TAILQ_INIT(&zombie_threads);
3635c8329edSJulian Elischer 		if (kg_first)
3645c8329edSJulian Elischer 			TAILQ_INIT(&zombie_ksegrps);
3655215b187SJeff Roberson 		mtx_unlock_spin(&kse_zombie_lock);
3665c8329edSJulian Elischer 		while (td_first) {
3675c8329edSJulian Elischer 			td_next = TAILQ_NEXT(td_first, td_runq);
3685215b187SJeff Roberson 			if (td_first->td_ucred)
3695215b187SJeff Roberson 				crfree(td_first->td_ucred);
3705c8329edSJulian Elischer 			thread_free(td_first);
3715c8329edSJulian Elischer 			td_first = td_next;
37244990b8cSJulian Elischer 		}
3735c8329edSJulian Elischer 		while (kg_first) {
3745c8329edSJulian Elischer 			kg_next = TAILQ_NEXT(kg_first, kg_ksegrp);
3755c8329edSJulian Elischer 			ksegrp_free(kg_first);
3765c8329edSJulian Elischer 			kg_first = kg_next;
3775c8329edSJulian Elischer 		}
378ed062c8dSJulian Elischer 		/*
379ed062c8dSJulian Elischer 		 * there will always be a thread on the list if one of these
380ed062c8dSJulian Elischer 		 * is there.
381ed062c8dSJulian Elischer 		 */
382345ad866SJulian Elischer 		kse_GC();
38344990b8cSJulian Elischer 	}
384ed062c8dSJulian Elischer }
38544990b8cSJulian Elischer 
38644990b8cSJulian Elischer /*
3874f0db5e0SJulian Elischer  * Allocate a ksegrp.
3884f0db5e0SJulian Elischer  */
3894f0db5e0SJulian Elischer struct ksegrp *
3904f0db5e0SJulian Elischer ksegrp_alloc(void)
3914f0db5e0SJulian Elischer {
392a163d034SWarner Losh 	return (uma_zalloc(ksegrp_zone, M_WAITOK));
3934f0db5e0SJulian Elischer }
3944f0db5e0SJulian Elischer 
3954f0db5e0SJulian Elischer /*
39644990b8cSJulian Elischer  * Allocate a thread.
39744990b8cSJulian Elischer  */
39844990b8cSJulian Elischer struct thread *
39944990b8cSJulian Elischer thread_alloc(void)
40044990b8cSJulian Elischer {
40144990b8cSJulian Elischer 	thread_reap(); /* check if any zombies to get */
402a163d034SWarner Losh 	return (uma_zalloc(thread_zone, M_WAITOK));
40344990b8cSJulian Elischer }
40444990b8cSJulian Elischer 
40544990b8cSJulian Elischer /*
4064f0db5e0SJulian Elischer  * Deallocate a ksegrp.
4074f0db5e0SJulian Elischer  */
4084f0db5e0SJulian Elischer void
4094f0db5e0SJulian Elischer ksegrp_free(struct ksegrp *td)
4104f0db5e0SJulian Elischer {
4114f0db5e0SJulian Elischer 	uma_zfree(ksegrp_zone, td);
4124f0db5e0SJulian Elischer }
4134f0db5e0SJulian Elischer 
4144f0db5e0SJulian Elischer /*
41544990b8cSJulian Elischer  * Deallocate a thread.
41644990b8cSJulian Elischer  */
41744990b8cSJulian Elischer void
41844990b8cSJulian Elischer thread_free(struct thread *td)
41944990b8cSJulian Elischer {
420696058c3SJulian Elischer 
421696058c3SJulian Elischer 	cpu_thread_clean(td);
42244990b8cSJulian Elischer 	uma_zfree(thread_zone, td);
42344990b8cSJulian Elischer }
42444990b8cSJulian Elischer 
42544990b8cSJulian Elischer /*
42644990b8cSJulian Elischer  * Discard the current thread and exit from its context.
42794e0a4cdSJulian Elischer  * Always called with scheduler locked.
42844990b8cSJulian Elischer  *
42944990b8cSJulian Elischer  * Because we can't free a thread while we're operating under its context,
430696058c3SJulian Elischer  * push the current thread into our CPU's deadthread holder. This means
431696058c3SJulian Elischer  * we needn't worry about someone else grabbing our context before we
43294e0a4cdSJulian Elischer  * do a cpu_throw().  This may not be needed now as we are under schedlock.
43394e0a4cdSJulian Elischer  * Maybe we can just do a thread_stash() as thr_exit1 does.
43494e0a4cdSJulian Elischer  */
43594e0a4cdSJulian Elischer /*  XXX
43694e0a4cdSJulian Elischer  * libthr expects its thread exit to return for the last
43794e0a4cdSJulian Elischer  * thread, meaning that the program is back to non-threaded
43894e0a4cdSJulian Elischer  * mode I guess. Because we do this (cpu_throw) unconditionally
43994e0a4cdSJulian Elischer  * here, they have their own version of it. (thr_exit1())
44094e0a4cdSJulian Elischer  * that doesn't do it all if this was the last thread.
44194e0a4cdSJulian Elischer  * It is also called from thread_suspend_check().
44294e0a4cdSJulian Elischer  * Of course in the end, they end up coming here through exit1
44394e0a4cdSJulian Elischer  * anyhow..  After fixing 'thr' to play by the rules we should be able
44494e0a4cdSJulian Elischer  * to merge these two functions together.
445ed062c8dSJulian Elischer  *
446ed062c8dSJulian Elischer  * called from:
447ed062c8dSJulian Elischer  * exit1()
448ed062c8dSJulian Elischer  * kse_exit()
449ed062c8dSJulian Elischer  * thr_exit()
450ed062c8dSJulian Elischer  * thread_user_enter()
451ed062c8dSJulian Elischer  * thread_userret()
452ed062c8dSJulian Elischer  * thread_suspend_check()
45344990b8cSJulian Elischer  */
45444990b8cSJulian Elischer void
45544990b8cSJulian Elischer thread_exit(void)
45644990b8cSJulian Elischer {
45744990b8cSJulian Elischer 	struct thread *td;
45844990b8cSJulian Elischer 	struct proc *p;
45944990b8cSJulian Elischer 	struct ksegrp	*kg;
46044990b8cSJulian Elischer 
46144990b8cSJulian Elischer 	td = curthread;
46244990b8cSJulian Elischer 	kg = td->td_ksegrp;
46344990b8cSJulian Elischer 	p = td->td_proc;
46444990b8cSJulian Elischer 
46544990b8cSJulian Elischer 	mtx_assert(&sched_lock, MA_OWNED);
466ed062c8dSJulian Elischer 	mtx_assert(&Giant, MA_NOTOWNED);
46744990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
468ed062c8dSJulian Elischer 	KASSERT(p != NULL, ("thread exiting without a process"));
469ed062c8dSJulian Elischer 	KASSERT(kg != NULL, ("thread exiting without a kse group"));
470cc701b73SRobert Watson 	CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
471cc701b73SRobert Watson 	    (long)p->p_pid, p->p_comm);
4729104847fSDavid Xu 	KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
47344990b8cSJulian Elischer 
47489964dd2SRobert Watson #ifdef AUDIT
47589964dd2SRobert Watson 	AUDIT_SYSCALL_EXIT(0, td);
47689964dd2SRobert Watson #endif
47789964dd2SRobert Watson 
47848bfcdddSJulian Elischer 	if (td->td_standin != NULL) {
479ed062c8dSJulian Elischer 		/*
480ed062c8dSJulian Elischer 		 * Note that we don't need to free the cred here as it
481ed062c8dSJulian Elischer 		 * is done in thread_reap().
482ed062c8dSJulian Elischer 		 */
48348bfcdddSJulian Elischer 		thread_stash(td->td_standin);
48448bfcdddSJulian Elischer 		td->td_standin = NULL;
48548bfcdddSJulian Elischer 	}
48648bfcdddSJulian Elischer 
487ed062c8dSJulian Elischer 	/*
488ed062c8dSJulian Elischer 	 * drop FPU & debug register state storage, or any other
489ed062c8dSJulian Elischer 	 * architecture specific resources that
490ed062c8dSJulian Elischer 	 * would not be on a new untouched process.
491ed062c8dSJulian Elischer 	 */
49244990b8cSJulian Elischer 	cpu_thread_exit(td);	/* XXXSMP */
49344990b8cSJulian Elischer 
4941faf202eSJulian Elischer 	/*
495ed062c8dSJulian Elischer 	 * The thread is exiting. scheduler can release its stuff
496ed062c8dSJulian Elischer 	 * and collect stats etc.
497ed062c8dSJulian Elischer 	 */
498ed062c8dSJulian Elischer 	sched_thread_exit(td);
499ed062c8dSJulian Elischer 
500ed062c8dSJulian Elischer 	/*
5011faf202eSJulian Elischer 	 * The last thread is left attached to the process
5021faf202eSJulian Elischer 	 * So that the whole bundle gets recycled. Skip
503ed062c8dSJulian Elischer 	 * all this stuff if we never had threads.
504ed062c8dSJulian Elischer 	 * EXIT clears all sign of other threads when
505ed062c8dSJulian Elischer 	 * it goes to single threading, so the last thread always
506ed062c8dSJulian Elischer 	 * takes the short path.
5071faf202eSJulian Elischer 	 */
508ed062c8dSJulian Elischer 	if (p->p_flag & P_HADTHREADS) {
5091faf202eSJulian Elischer 		if (p->p_numthreads > 1) {
510d3a0bd78SJulian Elischer 			thread_unlink(td);
511ed062c8dSJulian Elischer 
512ed062c8dSJulian Elischer 			/* XXX first arg not used in 4BSD or ULE */
513ed062c8dSJulian Elischer 			sched_exit_thread(FIRST_THREAD_IN_PROC(p), td);
514ed062c8dSJulian Elischer 
515ed062c8dSJulian Elischer 			/*
51644990b8cSJulian Elischer 			 * The test below is NOT true if we are the
5171faf202eSJulian Elischer 			 * sole exiting thread. P_STOPPED_SNGL is unset
51844990b8cSJulian Elischer 			 * in exit1() after it is the only survivor.
51944990b8cSJulian Elischer 			 */
5201279572aSDavid Xu 			if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
52144990b8cSJulian Elischer 				if (p->p_numthreads == p->p_suspcount) {
52271fad9fdSJulian Elischer 					thread_unsuspend_one(p->p_singlethread);
52344990b8cSJulian Elischer 				}
52444990b8cSJulian Elischer 			}
52548bfcdddSJulian Elischer 
5265215b187SJeff Roberson 			/*
5275215b187SJeff Roberson 			 * Because each upcall structure has an owner thread,
5285215b187SJeff Roberson 			 * owner thread exits only when process is in exiting
5295215b187SJeff Roberson 			 * state, so upcall to userland is no longer needed,
5305215b187SJeff Roberson 			 * deleting upcall structure is safe here.
5315215b187SJeff Roberson 			 * So when all threads in a group is exited, all upcalls
5325215b187SJeff Roberson 			 * in the group should be automatically freed.
533ed062c8dSJulian Elischer 			 *  XXXKSE This is a KSE thing and should be exported
534ed062c8dSJulian Elischer 			 * there somehow.
5355215b187SJeff Roberson 			 */
5365215b187SJeff Roberson 			upcall_remove(td);
5376f8132a8SJulian Elischer 
53848bfcdddSJulian Elischer 			/*
539ed062c8dSJulian Elischer 			 * If the thread we unlinked above was the last one,
540ed062c8dSJulian Elischer 			 * then this ksegrp should go away too.
54148bfcdddSJulian Elischer 			 */
542ed062c8dSJulian Elischer 			if (kg->kg_numthreads == 0) {
543ed062c8dSJulian Elischer 				/*
544ed062c8dSJulian Elischer 				 * let the scheduler know about this in case
545ed062c8dSJulian Elischer 				 * it needs to recover stats or resources.
546ed062c8dSJulian Elischer 				 * Theoretically we could let
547ed062c8dSJulian Elischer 				 * sched_exit_ksegrp()  do the equivalent of
548ed062c8dSJulian Elischer 				 * setting the concurrency to 0
549ed062c8dSJulian Elischer 				 * but don't do it yet to avoid changing
550ed062c8dSJulian Elischer 				 * the existing scheduler code until we
551ed062c8dSJulian Elischer 				 * are ready.
552ed062c8dSJulian Elischer 				 * We supply a random other ksegrp
553ed062c8dSJulian Elischer 				 * as the recipient of any built up
554ed062c8dSJulian Elischer 				 * cpu usage etc. (If the scheduler wants it).
555ed062c8dSJulian Elischer 				 * XXXKSE
556ed062c8dSJulian Elischer 				 * This is probably not fair so think of
557ed062c8dSJulian Elischer  				 * a better answer.
558ed062c8dSJulian Elischer 				 */
55955d44f79SJulian Elischer 				sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td);
560ed062c8dSJulian Elischer 				sched_set_concurrency(kg, 0); /* XXX TEMP */
561ab2baa72SDavid Xu 				ksegrp_unlink(kg);
562ed062c8dSJulian Elischer 				ksegrp_stash(kg);
563ab2baa72SDavid Xu 			}
5646f8132a8SJulian Elischer 			PROC_UNLOCK(p);
5655c8329edSJulian Elischer 			td->td_ksegrp	= NULL;
566696058c3SJulian Elischer 			PCPU_SET(deadthread, td);
5671faf202eSJulian Elischer 		} else {
568ed062c8dSJulian Elischer 			/*
569ed062c8dSJulian Elischer 			 * The last thread is exiting.. but not through exit()
570ed062c8dSJulian Elischer 			 * what should we do?
571ed062c8dSJulian Elischer 			 * Theoretically this can't happen
572ed062c8dSJulian Elischer  			 * exit1() - clears threading flags before coming here
573ed062c8dSJulian Elischer  			 * kse_exit() - treats last thread specially
574ed062c8dSJulian Elischer  			 * thr_exit() - treats last thread specially
575ed062c8dSJulian Elischer  			 * thread_user_enter() - only if more exist
576ed062c8dSJulian Elischer  			 * thread_userret() - only if more exist
577ed062c8dSJulian Elischer  			 * thread_suspend_check() - only if more exist
578ed062c8dSJulian Elischer 			 */
579ed062c8dSJulian Elischer 			panic ("thread_exit: Last thread exiting on its own");
580ed062c8dSJulian Elischer 		}
581ed062c8dSJulian Elischer 	} else {
582ed062c8dSJulian Elischer 		/*
583ed062c8dSJulian Elischer 		 * non threaded process comes here.
584ed062c8dSJulian Elischer 		 * This includes an EX threaded process that is coming
585ed062c8dSJulian Elischer 		 * here via exit1(). (exit1 dethreads the proc first).
586ed062c8dSJulian Elischer 		 */
5871faf202eSJulian Elischer 		PROC_UNLOCK(p);
5881faf202eSJulian Elischer 	}
589dcc9954eSJulian Elischer 	td->td_state = TDS_INACTIVE;
590732d9528SJulian Elischer 	CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
591cc66ebe2SPeter Wemm 	cpu_throw(td, choosethread());
592cc66ebe2SPeter Wemm 	panic("I'm a teapot!");
59344990b8cSJulian Elischer 	/* NOTREACHED */
59444990b8cSJulian Elischer }
59544990b8cSJulian Elischer 
59644990b8cSJulian Elischer /*
597696058c3SJulian Elischer  * Do any thread specific cleanups that may be needed in wait()
59837814395SPeter Wemm  * called with Giant, proc and schedlock not held.
599696058c3SJulian Elischer  */
600696058c3SJulian Elischer void
601696058c3SJulian Elischer thread_wait(struct proc *p)
602696058c3SJulian Elischer {
603696058c3SJulian Elischer 	struct thread *td;
604696058c3SJulian Elischer 
60537814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
60685495c72SJens Schweikhardt 	KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()"));
60785495c72SJens Schweikhardt 	KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()"));
608696058c3SJulian Elischer 	FOREACH_THREAD_IN_PROC(p, td) {
609696058c3SJulian Elischer 		if (td->td_standin != NULL) {
610b75b0311SJulian Elischer 			if (td->td_standin->td_ucred != NULL) {
611b75b0311SJulian Elischer 				crfree(td->td_standin->td_ucred);
612b75b0311SJulian Elischer 				td->td_standin->td_ucred = NULL;
613b75b0311SJulian Elischer 			}
614696058c3SJulian Elischer 			thread_free(td->td_standin);
615696058c3SJulian Elischer 			td->td_standin = NULL;
616696058c3SJulian Elischer 		}
617696058c3SJulian Elischer 		cpu_thread_clean(td);
618ed062c8dSJulian Elischer 		crfree(td->td_ucred);
619696058c3SJulian Elischer 	}
620696058c3SJulian Elischer 	thread_reap();	/* check for zombie threads etc. */
621696058c3SJulian Elischer }
622696058c3SJulian Elischer 
623696058c3SJulian Elischer /*
62444990b8cSJulian Elischer  * Link a thread to a process.
6251faf202eSJulian Elischer  * set up anything that needs to be initialized for it to
6261faf202eSJulian Elischer  * be used by the process.
62744990b8cSJulian Elischer  *
62844990b8cSJulian Elischer  * Note that we do not link to the proc's ucred here.
62944990b8cSJulian Elischer  * The thread is linked as if running but no KSE assigned.
630ed062c8dSJulian Elischer  * Called from:
631ed062c8dSJulian Elischer  *  proc_linkup()
632ed062c8dSJulian Elischer  *  thread_schedule_upcall()
633ed062c8dSJulian Elischer  *  thr_create()
63444990b8cSJulian Elischer  */
63544990b8cSJulian Elischer void
63644990b8cSJulian Elischer thread_link(struct thread *td, struct ksegrp *kg)
63744990b8cSJulian Elischer {
63844990b8cSJulian Elischer 	struct proc *p;
63944990b8cSJulian Elischer 
64044990b8cSJulian Elischer 	p = kg->kg_proc;
64171fad9fdSJulian Elischer 	td->td_state    = TDS_INACTIVE;
64244990b8cSJulian Elischer 	td->td_proc     = p;
64344990b8cSJulian Elischer 	td->td_ksegrp   = kg;
6445215b187SJeff Roberson 	td->td_flags    = 0;
6454fc21c09SDaniel Eischen 	td->td_kflags	= 0;
64644990b8cSJulian Elischer 
6471faf202eSJulian Elischer 	LIST_INIT(&td->td_contested);
6489104847fSDavid Xu 	sigqueue_init(&td->td_sigqueue, p);
649c06eb4e2SSam Leffler 	callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
65044990b8cSJulian Elischer 	TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist);
65144990b8cSJulian Elischer 	TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist);
65244990b8cSJulian Elischer 	p->p_numthreads++;
65344990b8cSJulian Elischer 	kg->kg_numthreads++;
65444990b8cSJulian Elischer }
65544990b8cSJulian Elischer 
656ed062c8dSJulian Elischer /*
657e5bedcefSJulian Elischer  * Convert a process with one thread to an unthreaded process.
658e5bedcefSJulian Elischer  * Called from:
659e5bedcefSJulian Elischer  *  thread_single(exit)  (called from execve and exit)
660e5bedcefSJulian Elischer  *  kse_exit()		XXX may need cleaning up wrt KSE stuff
661e5bedcefSJulian Elischer  */
662e5bedcefSJulian Elischer void
663e5bedcefSJulian Elischer thread_unthread(struct thread *td)
664e5bedcefSJulian Elischer {
665e5bedcefSJulian Elischer 	struct proc *p = td->td_proc;
666e5bedcefSJulian Elischer 
667e5bedcefSJulian Elischer 	KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads"));
668e5bedcefSJulian Elischer 	upcall_remove(td);
669e5bedcefSJulian Elischer 	p->p_flag &= ~(P_SA|P_HADTHREADS);
670e5bedcefSJulian Elischer 	td->td_mailbox = NULL;
671e5bedcefSJulian Elischer 	td->td_pflags &= ~(TDP_SA | TDP_CAN_UNBIND);
672e5bedcefSJulian Elischer 	if (td->td_standin != NULL) {
673e5bedcefSJulian Elischer 		thread_stash(td->td_standin);
674e5bedcefSJulian Elischer 		td->td_standin = NULL;
675e5bedcefSJulian Elischer 	}
676e5bedcefSJulian Elischer 	sched_set_concurrency(td->td_ksegrp, 1);
677e5bedcefSJulian Elischer }
678e5bedcefSJulian Elischer 
679e5bedcefSJulian Elischer /*
680ed062c8dSJulian Elischer  * Called from:
681ed062c8dSJulian Elischer  *  thread_exit()
682ed062c8dSJulian Elischer  */
683d3a0bd78SJulian Elischer void
684d3a0bd78SJulian Elischer thread_unlink(struct thread *td)
685d3a0bd78SJulian Elischer {
686d3a0bd78SJulian Elischer 	struct proc *p = td->td_proc;
687d3a0bd78SJulian Elischer 	struct ksegrp *kg = td->td_ksegrp;
688d3a0bd78SJulian Elischer 
689112afcb2SJohn Baldwin 	mtx_assert(&sched_lock, MA_OWNED);
690d3a0bd78SJulian Elischer 	TAILQ_REMOVE(&p->p_threads, td, td_plist);
691d3a0bd78SJulian Elischer 	p->p_numthreads--;
692d3a0bd78SJulian Elischer 	TAILQ_REMOVE(&kg->kg_threads, td, td_kglist);
693d3a0bd78SJulian Elischer 	kg->kg_numthreads--;
694d3a0bd78SJulian Elischer 	/* could clear a few other things here */
695ed062c8dSJulian Elischer 	/* Must  NOT clear links to proc and ksegrp! */
6965c8329edSJulian Elischer }
6975c8329edSJulian Elischer 
6985215b187SJeff Roberson /*
69944990b8cSJulian Elischer  * Enforce single-threading.
70044990b8cSJulian Elischer  *
70144990b8cSJulian Elischer  * Returns 1 if the caller must abort (another thread is waiting to
70244990b8cSJulian Elischer  * exit the process or similar). Process is locked!
70344990b8cSJulian Elischer  * Returns 0 when you are successfully the only thread running.
70444990b8cSJulian Elischer  * A process has successfully single threaded in the suspend mode when
70544990b8cSJulian Elischer  * There are no threads in user mode. Threads in the kernel must be
70644990b8cSJulian Elischer  * allowed to continue until they get to the user boundary. They may even
70744990b8cSJulian Elischer  * copy out their return values and data before suspending. They may however be
70844990b8cSJulian Elischer  * accellerated in reaching the user boundary as we will wake up
70944990b8cSJulian Elischer  * any sleeping threads that are interruptable. (PCATCH).
71044990b8cSJulian Elischer  */
71144990b8cSJulian Elischer int
712906ac69dSDavid Xu thread_single(int mode)
71344990b8cSJulian Elischer {
71444990b8cSJulian Elischer 	struct thread *td;
71544990b8cSJulian Elischer 	struct thread *td2;
71644990b8cSJulian Elischer 	struct proc *p;
717ec008e96SDavid Xu 	int remaining;
71844990b8cSJulian Elischer 
71944990b8cSJulian Elischer 	td = curthread;
72044990b8cSJulian Elischer 	p = td->td_proc;
72137814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
72244990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
72344990b8cSJulian Elischer 	KASSERT((td != NULL), ("curthread is NULL"));
72444990b8cSJulian Elischer 
725ed062c8dSJulian Elischer 	if ((p->p_flag & P_HADTHREADS) == 0)
72644990b8cSJulian Elischer 		return (0);
72744990b8cSJulian Elischer 
728e3b9bf71SJulian Elischer 	/* Is someone already single threading? */
729906ac69dSDavid Xu 	if (p->p_singlethread != NULL && p->p_singlethread != td)
73044990b8cSJulian Elischer 		return (1);
73144990b8cSJulian Elischer 
732906ac69dSDavid Xu 	if (mode == SINGLE_EXIT) {
733906ac69dSDavid Xu 		p->p_flag |= P_SINGLE_EXIT;
734906ac69dSDavid Xu 		p->p_flag &= ~P_SINGLE_BOUNDARY;
735906ac69dSDavid Xu 	} else {
736906ac69dSDavid Xu 		p->p_flag &= ~P_SINGLE_EXIT;
737906ac69dSDavid Xu 		if (mode == SINGLE_BOUNDARY)
738906ac69dSDavid Xu 			p->p_flag |= P_SINGLE_BOUNDARY;
739906ac69dSDavid Xu 		else
740906ac69dSDavid Xu 			p->p_flag &= ~P_SINGLE_BOUNDARY;
741906ac69dSDavid Xu 	}
7421279572aSDavid Xu 	p->p_flag |= P_STOPPED_SINGLE;
74371fad9fdSJulian Elischer 	mtx_lock_spin(&sched_lock);
744112afcb2SJohn Baldwin 	p->p_singlethread = td;
745906ac69dSDavid Xu 	if (mode == SINGLE_EXIT)
746ec008e96SDavid Xu 		remaining = p->p_numthreads;
747906ac69dSDavid Xu 	else if (mode == SINGLE_BOUNDARY)
748906ac69dSDavid Xu 		remaining = p->p_numthreads - p->p_boundary_count;
749906ac69dSDavid Xu 	else
750ec008e96SDavid Xu 		remaining = p->p_numthreads - p->p_suspcount;
751ec008e96SDavid Xu 	while (remaining != 1) {
75244990b8cSJulian Elischer 		FOREACH_THREAD_IN_PROC(p, td2) {
75344990b8cSJulian Elischer 			if (td2 == td)
75444990b8cSJulian Elischer 				continue;
755588257e8SDavid Xu 			td2->td_flags |= TDF_ASTPENDING;
75671fad9fdSJulian Elischer 			if (TD_IS_INHIBITED(td2)) {
757906ac69dSDavid Xu 				switch (mode) {
758906ac69dSDavid Xu 				case SINGLE_EXIT:
759cbf4e354SDavid Xu 					if (td->td_flags & TDF_DBSUSPEND)
760cbf4e354SDavid Xu 						td->td_flags &= ~TDF_DBSUSPEND;
761906ac69dSDavid Xu 					if (TD_IS_SUSPENDED(td2))
76271fad9fdSJulian Elischer 						thread_unsuspend_one(td2);
76333862f40SDavid Xu 					if (TD_ON_SLEEPQ(td2) &&
764906ac69dSDavid Xu 					    (td2->td_flags & TDF_SINTR))
76594f0972bSDavid Xu 						sleepq_abort(td2, EINTR);
766906ac69dSDavid Xu 					break;
767906ac69dSDavid Xu 				case SINGLE_BOUNDARY:
768906ac69dSDavid Xu 					if (TD_IS_SUSPENDED(td2) &&
769906ac69dSDavid Xu 					    !(td2->td_flags & TDF_BOUNDARY))
770906ac69dSDavid Xu 						thread_unsuspend_one(td2);
771906ac69dSDavid Xu 					if (TD_ON_SLEEPQ(td2) &&
772906ac69dSDavid Xu 					    (td2->td_flags & TDF_SINTR))
77394f0972bSDavid Xu 						sleepq_abort(td2, ERESTART);
774906ac69dSDavid Xu 					break;
775906ac69dSDavid Xu 				default:
7769d102777SJulian Elischer 					if (TD_IS_SUSPENDED(td2))
7779d102777SJulian Elischer 						continue;
7785215b187SJeff Roberson 					/*
7795215b187SJeff Roberson 					 * maybe other inhibitted states too?
7805215b187SJeff Roberson 					 */
7818acf6057SDavid Xu 					if ((td2->td_flags & TDF_SINTR) &&
7828acf6057SDavid Xu 					    (td2->td_inhibitors &
7838acf6057SDavid Xu 					    (TDI_SLEEPING | TDI_SWAPPED)))
7849d102777SJulian Elischer 						thread_suspend_one(td2);
785906ac69dSDavid Xu 					break;
78644990b8cSJulian Elischer 				}
78744990b8cSJulian Elischer 			}
788d8267df7SDavid Xu #ifdef SMP
789d8267df7SDavid Xu 			else if (TD_IS_RUNNING(td2) && td != td2) {
790d8267df7SDavid Xu 				forward_signal(td2);
791d8267df7SDavid Xu 			}
792d8267df7SDavid Xu #endif
7939d102777SJulian Elischer 		}
794906ac69dSDavid Xu 		if (mode == SINGLE_EXIT)
795ec008e96SDavid Xu 			remaining = p->p_numthreads;
796906ac69dSDavid Xu 		else if (mode == SINGLE_BOUNDARY)
797906ac69dSDavid Xu 			remaining = p->p_numthreads - p->p_boundary_count;
798ec008e96SDavid Xu 		else
799ec008e96SDavid Xu 			remaining = p->p_numthreads - p->p_suspcount;
800ec008e96SDavid Xu 
8019d102777SJulian Elischer 		/*
8029d102777SJulian Elischer 		 * Maybe we suspended some threads.. was it enough?
8039d102777SJulian Elischer 		 */
804ec008e96SDavid Xu 		if (remaining == 1)
8059d102777SJulian Elischer 			break;
8069d102777SJulian Elischer 
80744990b8cSJulian Elischer 		/*
80844990b8cSJulian Elischer 		 * Wake us up when everyone else has suspended.
809e3b9bf71SJulian Elischer 		 * In the mean time we suspend as well.
81044990b8cSJulian Elischer 		 */
81171fad9fdSJulian Elischer 		thread_suspend_one(td);
81244990b8cSJulian Elischer 		PROC_UNLOCK(p);
813bf0acc27SJohn Baldwin 		mi_switch(SW_VOL, NULL);
81444990b8cSJulian Elischer 		mtx_unlock_spin(&sched_lock);
81544990b8cSJulian Elischer 		PROC_LOCK(p);
816112afcb2SJohn Baldwin 		mtx_lock_spin(&sched_lock);
817906ac69dSDavid Xu 		if (mode == SINGLE_EXIT)
818ec008e96SDavid Xu 			remaining = p->p_numthreads;
819906ac69dSDavid Xu 		else if (mode == SINGLE_BOUNDARY)
820906ac69dSDavid Xu 			remaining = p->p_numthreads - p->p_boundary_count;
821ec008e96SDavid Xu 		else
822ec008e96SDavid Xu 			remaining = p->p_numthreads - p->p_suspcount;
82344990b8cSJulian Elischer 	}
824906ac69dSDavid Xu 	if (mode == SINGLE_EXIT) {
82591599697SJulian Elischer 		/*
82691599697SJulian Elischer 		 * We have gotten rid of all the other threads and we
82791599697SJulian Elischer 		 * are about to either exit or exec. In either case,
82891599697SJulian Elischer 		 * we try our utmost  to revert to being a non-threaded
82991599697SJulian Elischer 		 * process.
83091599697SJulian Elischer 		 */
831ed062c8dSJulian Elischer 		p->p_singlethread = NULL;
83264895117SDavid Xu 		p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
833e5bedcefSJulian Elischer 		thread_unthread(td);
83491599697SJulian Elischer 	}
835112afcb2SJohn Baldwin 	mtx_unlock_spin(&sched_lock);
83644990b8cSJulian Elischer 	return (0);
83744990b8cSJulian Elischer }
83844990b8cSJulian Elischer 
83944990b8cSJulian Elischer /*
84044990b8cSJulian Elischer  * Called in from locations that can safely check to see
84144990b8cSJulian Elischer  * whether we have to suspend or at least throttle for a
84244990b8cSJulian Elischer  * single-thread event (e.g. fork).
84344990b8cSJulian Elischer  *
84444990b8cSJulian Elischer  * Such locations include userret().
84544990b8cSJulian Elischer  * If the "return_instead" argument is non zero, the thread must be able to
84644990b8cSJulian Elischer  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
84744990b8cSJulian Elischer  *
84844990b8cSJulian Elischer  * The 'return_instead' argument tells the function if it may do a
84944990b8cSJulian Elischer  * thread_exit() or suspend, or whether the caller must abort and back
85044990b8cSJulian Elischer  * out instead.
85144990b8cSJulian Elischer  *
85244990b8cSJulian Elischer  * If the thread that set the single_threading request has set the
85344990b8cSJulian Elischer  * P_SINGLE_EXIT bit in the process flags then this call will never return
85444990b8cSJulian Elischer  * if 'return_instead' is false, but will exit.
85544990b8cSJulian Elischer  *
85644990b8cSJulian Elischer  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
85744990b8cSJulian Elischer  *---------------+--------------------+---------------------
85844990b8cSJulian Elischer  *       0       | returns 0          |   returns 0 or 1
85944990b8cSJulian Elischer  *               | when ST ends       |   immediatly
86044990b8cSJulian Elischer  *---------------+--------------------+---------------------
86144990b8cSJulian Elischer  *       1       | thread exits       |   returns 1
86244990b8cSJulian Elischer  *               |                    |  immediatly
86344990b8cSJulian Elischer  * 0 = thread_exit() or suspension ok,
86444990b8cSJulian Elischer  * other = return error instead of stopping the thread.
86544990b8cSJulian Elischer  *
86644990b8cSJulian Elischer  * While a full suspension is under effect, even a single threading
86744990b8cSJulian Elischer  * thread would be suspended if it made this call (but it shouldn't).
86844990b8cSJulian Elischer  * This call should only be made from places where
86944990b8cSJulian Elischer  * thread_exit() would be safe as that may be the outcome unless
87044990b8cSJulian Elischer  * return_instead is set.
87144990b8cSJulian Elischer  */
87244990b8cSJulian Elischer int
87344990b8cSJulian Elischer thread_suspend_check(int return_instead)
87444990b8cSJulian Elischer {
875ecafb24bSJuli Mallett 	struct thread *td;
876ecafb24bSJuli Mallett 	struct proc *p;
87744990b8cSJulian Elischer 
87844990b8cSJulian Elischer 	td = curthread;
87944990b8cSJulian Elischer 	p = td->td_proc;
88037814395SPeter Wemm 	mtx_assert(&Giant, MA_NOTOWNED);
88144990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
882cbf4e354SDavid Xu 	while (P_SHOULDSTOP(p) ||
883cbf4e354SDavid Xu 	      ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) {
8841279572aSDavid Xu 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
88544990b8cSJulian Elischer 			KASSERT(p->p_singlethread != NULL,
88644990b8cSJulian Elischer 			    ("singlethread not set"));
88744990b8cSJulian Elischer 			/*
888e3b9bf71SJulian Elischer 			 * The only suspension in action is a
889e3b9bf71SJulian Elischer 			 * single-threading. Single threader need not stop.
890b6d5995eSJulian Elischer 			 * XXX Should be safe to access unlocked
891b6d5995eSJulian Elischer 			 * as it can only be set to be true by us.
89244990b8cSJulian Elischer 			 */
893e3b9bf71SJulian Elischer 			if (p->p_singlethread == td)
89444990b8cSJulian Elischer 				return (0);	/* Exempt from stopping. */
89544990b8cSJulian Elischer 		}
89645a4bfa1SDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
89794f0972bSDavid Xu 			return (EINTR);
89844990b8cSJulian Elischer 
899906ac69dSDavid Xu 		/* Should we goto user boundary if we didn't come from there? */
900906ac69dSDavid Xu 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
901906ac69dSDavid Xu 		    (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
90294f0972bSDavid Xu 			return (ERESTART);
903906ac69dSDavid Xu 
9049104847fSDavid Xu 		/* If thread will exit, flush its pending signals */
9059104847fSDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
9069104847fSDavid Xu 			sigqueue_flush(&td->td_sigqueue);
9079104847fSDavid Xu 
908e574e444SDavid Xu 		mtx_lock_spin(&sched_lock);
909e574e444SDavid Xu 		thread_stopped(p);
91044990b8cSJulian Elischer 		/*
91144990b8cSJulian Elischer 		 * If the process is waiting for us to exit,
91244990b8cSJulian Elischer 		 * this thread should just suicide.
9131279572aSDavid Xu 		 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
91444990b8cSJulian Elischer 		 */
915906ac69dSDavid Xu 		if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
91644990b8cSJulian Elischer 			thread_exit();
91744990b8cSJulian Elischer 
91844990b8cSJulian Elischer 		/*
91944990b8cSJulian Elischer 		 * When a thread suspends, it just
92044990b8cSJulian Elischer 		 * moves to the processes's suspend queue
92144990b8cSJulian Elischer 		 * and stays there.
92244990b8cSJulian Elischer 		 */
92371fad9fdSJulian Elischer 		thread_suspend_one(td);
924906ac69dSDavid Xu 		if (return_instead == 0) {
925906ac69dSDavid Xu 			p->p_boundary_count++;
926906ac69dSDavid Xu 			td->td_flags |= TDF_BOUNDARY;
927cf19bf91SJulian Elischer 		}
928906ac69dSDavid Xu 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
929906ac69dSDavid Xu 			if (p->p_numthreads == p->p_suspcount)
930906ac69dSDavid Xu 				thread_unsuspend_one(p->p_singlethread);
931cf19bf91SJulian Elischer 		}
932a6f37ac9SJohn Baldwin 		PROC_UNLOCK(p);
933bf0acc27SJohn Baldwin 		mi_switch(SW_INVOL, NULL);
934906ac69dSDavid Xu 		if (return_instead == 0) {
935906ac69dSDavid Xu 			p->p_boundary_count--;
936906ac69dSDavid Xu 			td->td_flags &= ~TDF_BOUNDARY;
937906ac69dSDavid Xu 		}
93844990b8cSJulian Elischer 		mtx_unlock_spin(&sched_lock);
93944990b8cSJulian Elischer 		PROC_LOCK(p);
94044990b8cSJulian Elischer 	}
94144990b8cSJulian Elischer 	return (0);
94244990b8cSJulian Elischer }
94344990b8cSJulian Elischer 
94435c32a76SDavid Xu void
94535c32a76SDavid Xu thread_suspend_one(struct thread *td)
94635c32a76SDavid Xu {
94735c32a76SDavid Xu 	struct proc *p = td->td_proc;
94835c32a76SDavid Xu 
94935c32a76SDavid Xu 	mtx_assert(&sched_lock, MA_OWNED);
950112afcb2SJohn Baldwin 	PROC_LOCK_ASSERT(p, MA_OWNED);
951e574e444SDavid Xu 	KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
95235c32a76SDavid Xu 	p->p_suspcount++;
95371fad9fdSJulian Elischer 	TD_SET_SUSPENDED(td);
95435c32a76SDavid Xu 	TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq);
95535c32a76SDavid Xu }
95635c32a76SDavid Xu 
95735c32a76SDavid Xu void
95835c32a76SDavid Xu thread_unsuspend_one(struct thread *td)
95935c32a76SDavid Xu {
96035c32a76SDavid Xu 	struct proc *p = td->td_proc;
96135c32a76SDavid Xu 
96235c32a76SDavid Xu 	mtx_assert(&sched_lock, MA_OWNED);
963112afcb2SJohn Baldwin 	PROC_LOCK_ASSERT(p, MA_OWNED);
96435c32a76SDavid Xu 	TAILQ_REMOVE(&p->p_suspended, td, td_runq);
96571fad9fdSJulian Elischer 	TD_CLR_SUSPENDED(td);
96635c32a76SDavid Xu 	p->p_suspcount--;
96771fad9fdSJulian Elischer 	setrunnable(td);
96835c32a76SDavid Xu }
96935c32a76SDavid Xu 
97044990b8cSJulian Elischer /*
97144990b8cSJulian Elischer  * Allow all threads blocked by single threading to continue running.
97244990b8cSJulian Elischer  */
97344990b8cSJulian Elischer void
97444990b8cSJulian Elischer thread_unsuspend(struct proc *p)
97544990b8cSJulian Elischer {
97644990b8cSJulian Elischer 	struct thread *td;
97744990b8cSJulian Elischer 
978b6d5995eSJulian Elischer 	mtx_assert(&sched_lock, MA_OWNED);
97944990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
98044990b8cSJulian Elischer 	if (!P_SHOULDSTOP(p)) {
98144990b8cSJulian Elischer 		while ((td = TAILQ_FIRST(&p->p_suspended))) {
98235c32a76SDavid Xu 			thread_unsuspend_one(td);
98344990b8cSJulian Elischer 		}
9841279572aSDavid Xu 	} else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) &&
98544990b8cSJulian Elischer 	    (p->p_numthreads == p->p_suspcount)) {
98644990b8cSJulian Elischer 		/*
98744990b8cSJulian Elischer 		 * Stopping everything also did the job for the single
98844990b8cSJulian Elischer 		 * threading request. Now we've downgraded to single-threaded,
98944990b8cSJulian Elischer 		 * let it continue.
99044990b8cSJulian Elischer 		 */
99135c32a76SDavid Xu 		thread_unsuspend_one(p->p_singlethread);
99244990b8cSJulian Elischer 	}
99344990b8cSJulian Elischer }
99444990b8cSJulian Elischer 
995ed062c8dSJulian Elischer /*
996ed062c8dSJulian Elischer  * End the single threading mode..
997ed062c8dSJulian Elischer  */
99844990b8cSJulian Elischer void
99944990b8cSJulian Elischer thread_single_end(void)
100044990b8cSJulian Elischer {
100144990b8cSJulian Elischer 	struct thread *td;
100244990b8cSJulian Elischer 	struct proc *p;
100344990b8cSJulian Elischer 
100444990b8cSJulian Elischer 	td = curthread;
100544990b8cSJulian Elischer 	p = td->td_proc;
100644990b8cSJulian Elischer 	PROC_LOCK_ASSERT(p, MA_OWNED);
1007906ac69dSDavid Xu 	p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
1008112afcb2SJohn Baldwin 	mtx_lock_spin(&sched_lock);
100944990b8cSJulian Elischer 	p->p_singlethread = NULL;
101021fc3164SDavid Xu 	p->p_procscopegrp = NULL;
101149539972SJulian Elischer 	/*
101249539972SJulian Elischer 	 * If there are other threads they mey now run,
101349539972SJulian Elischer 	 * unless of course there is a blanket 'stop order'
101449539972SJulian Elischer 	 * on the process. The single threader must be allowed
101549539972SJulian Elischer 	 * to continue however as this is a bad place to stop.
101649539972SJulian Elischer 	 */
101749539972SJulian Elischer 	if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) {
101849539972SJulian Elischer 		while ((td = TAILQ_FIRST(&p->p_suspended))) {
101971fad9fdSJulian Elischer 			thread_unsuspend_one(td);
102044990b8cSJulian Elischer 		}
102149539972SJulian Elischer 	}
1022112afcb2SJohn Baldwin 	mtx_unlock_spin(&sched_lock);
102349539972SJulian Elischer }
10244fc21c09SDaniel Eischen 
1025007ddf7eSJohn Baldwin /*
1026007ddf7eSJohn Baldwin  * Called before going into an interruptible sleep to see if we have been
1027007ddf7eSJohn Baldwin  * interrupted or requested to exit.
1028007ddf7eSJohn Baldwin  */
1029007ddf7eSJohn Baldwin int
1030007ddf7eSJohn Baldwin thread_sleep_check(struct thread *td)
1031007ddf7eSJohn Baldwin {
1032007ddf7eSJohn Baldwin 	struct proc *p;
1033007ddf7eSJohn Baldwin 
1034007ddf7eSJohn Baldwin 	p = td->td_proc;
1035007ddf7eSJohn Baldwin 	mtx_assert(&sched_lock, MA_OWNED);
10362179a22cSJulian Elischer 	if (p->p_flag & P_HADTHREADS) {
1037906ac69dSDavid Xu 		if (p->p_singlethread != td) {
1038906ac69dSDavid Xu 			if (p->p_flag & P_SINGLE_EXIT)
1039007ddf7eSJohn Baldwin 				return (EINTR);
1040906ac69dSDavid Xu 			if (p->p_flag & P_SINGLE_BOUNDARY)
1041906ac69dSDavid Xu 				return (ERESTART);
1042906ac69dSDavid Xu 		}
1043007ddf7eSJohn Baldwin 		if (td->td_flags & TDF_INTERRUPT)
1044007ddf7eSJohn Baldwin 			return (td->td_intrval);
1045007ddf7eSJohn Baldwin 	}
1046007ddf7eSJohn Baldwin 	return (0);
1047007ddf7eSJohn Baldwin }
104844355392SDavid Xu 
104944355392SDavid Xu struct thread *
105044355392SDavid Xu thread_find(struct proc *p, lwpid_t tid)
105144355392SDavid Xu {
105244355392SDavid Xu 	struct thread *td;
105344355392SDavid Xu 
105444355392SDavid Xu 	PROC_LOCK_ASSERT(p, MA_OWNED);
105544355392SDavid Xu 	mtx_lock_spin(&sched_lock);
105644355392SDavid Xu 	FOREACH_THREAD_IN_PROC(p, td) {
105744355392SDavid Xu 		if (td->td_tid == tid)
105844355392SDavid Xu 			break;
105944355392SDavid Xu 	}
106044355392SDavid Xu 	mtx_unlock_spin(&sched_lock);
106144355392SDavid Xu 	return (td);
106244355392SDavid Xu }
1063