19454b2d8SWarner Losh /*- 244990b8cSJulian Elischer * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 344990b8cSJulian Elischer * All rights reserved. 444990b8cSJulian Elischer * 544990b8cSJulian Elischer * Redistribution and use in source and binary forms, with or without 644990b8cSJulian Elischer * modification, are permitted provided that the following conditions 744990b8cSJulian Elischer * are met: 844990b8cSJulian Elischer * 1. Redistributions of source code must retain the above copyright 944990b8cSJulian Elischer * notice(s), this list of conditions and the following disclaimer as 1044990b8cSJulian Elischer * the first lines of this file unmodified other than the possible 1144990b8cSJulian Elischer * addition of one or more copyright notices. 1244990b8cSJulian Elischer * 2. Redistributions in binary form must reproduce the above copyright 1344990b8cSJulian Elischer * notice(s), this list of conditions and the following disclaimer in the 1444990b8cSJulian Elischer * documentation and/or other materials provided with the distribution. 1544990b8cSJulian Elischer * 1644990b8cSJulian Elischer * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1744990b8cSJulian Elischer * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1844990b8cSJulian Elischer * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1944990b8cSJulian Elischer * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2044990b8cSJulian Elischer * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2144990b8cSJulian Elischer * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2244990b8cSJulian Elischer * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2344990b8cSJulian Elischer * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2444990b8cSJulian Elischer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2544990b8cSJulian Elischer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2644990b8cSJulian Elischer * DAMAGE. 2744990b8cSJulian Elischer */ 2844990b8cSJulian Elischer 29677b542eSDavid E. O'Brien #include <sys/cdefs.h> 30677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 31677b542eSDavid E. O'Brien 3244990b8cSJulian Elischer #include <sys/param.h> 3344990b8cSJulian Elischer #include <sys/systm.h> 3444990b8cSJulian Elischer #include <sys/kernel.h> 3544990b8cSJulian Elischer #include <sys/lock.h> 3644990b8cSJulian Elischer #include <sys/mutex.h> 3744990b8cSJulian Elischer #include <sys/proc.h> 3894e0a4cdSJulian Elischer #include <sys/smp.h> 3944990b8cSJulian Elischer #include <sys/sysctl.h> 40de028f5aSJeff Roberson #include <sys/sched.h> 4144f3b092SJohn Baldwin #include <sys/sleepqueue.h> 42961a7b24SJohn Baldwin #include <sys/turnstile.h> 4344990b8cSJulian Elischer #include <sys/ktr.h> 44bc8e6d81SDavid Xu #include <sys/umtx.h> 4544990b8cSJulian Elischer 46911b84b0SRobert Watson #include <security/audit/audit.h> 47911b84b0SRobert Watson 4844990b8cSJulian Elischer #include <vm/vm.h> 4949a2507bSAlan Cox #include <vm/vm_extern.h> 5044990b8cSJulian Elischer #include <vm/uma.h> 5102fb42b0SPeter Wemm 5244990b8cSJulian Elischer /* 534f0db5e0SJulian Elischer * KSEGRP related storage. 5444990b8cSJulian Elischer */ 554f0db5e0SJulian Elischer static uma_zone_t ksegrp_zone; 5644990b8cSJulian Elischer static uma_zone_t thread_zone; 5744990b8cSJulian Elischer 584f0db5e0SJulian Elischer /* DEBUG ONLY */ 5944990b8cSJulian Elischer SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 60696058c3SJulian Elischer static int thread_debug = 0; 61696058c3SJulian Elischer SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 62696058c3SJulian Elischer &thread_debug, 0, "thread debug"); 63fdc5ecd2SDavid Xu 64345ad866SJulian Elischer int max_threads_per_proc = 1500; 65fdc5ecd2SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 664f0db5e0SJulian Elischer &max_threads_per_proc, 0, "Limit on threads per proc"); 674f0db5e0SJulian Elischer 68ed062c8dSJulian Elischer int max_groups_per_proc = 1500; 69fdc5ecd2SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 70fdc5ecd2SDavid Xu &max_groups_per_proc, 0, "Limit on thread groups per proc"); 71fdc5ecd2SDavid Xu 72345ad866SJulian Elischer int max_threads_hits; 730252d203SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 740252d203SDavid Xu &max_threads_hits, 0, ""); 750252d203SDavid Xu 7694e0a4cdSJulian Elischer int virtual_cpu; 7794e0a4cdSJulian Elischer 785215b187SJeff Roberson TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 795c8329edSJulian Elischer TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 805215b187SJeff Roberson struct mtx kse_zombie_lock; 815215b187SJeff Roberson MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 8244990b8cSJulian Elischer 8394e0a4cdSJulian Elischer static int 8494e0a4cdSJulian Elischer sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 8594e0a4cdSJulian Elischer { 8694e0a4cdSJulian Elischer int error, new_val; 8794e0a4cdSJulian Elischer int def_val; 8894e0a4cdSJulian Elischer 8994e0a4cdSJulian Elischer def_val = mp_ncpus; 9094e0a4cdSJulian Elischer if (virtual_cpu == 0) 9194e0a4cdSJulian Elischer new_val = def_val; 9294e0a4cdSJulian Elischer else 9394e0a4cdSJulian Elischer new_val = virtual_cpu; 9494e0a4cdSJulian Elischer error = sysctl_handle_int(oidp, &new_val, 0, req); 9594e0a4cdSJulian Elischer if (error != 0 || req->newptr == NULL) 9694e0a4cdSJulian Elischer return (error); 9794e0a4cdSJulian Elischer if (new_val < 0) 9894e0a4cdSJulian Elischer return (EINVAL); 9994e0a4cdSJulian Elischer virtual_cpu = new_val; 10094e0a4cdSJulian Elischer return (0); 10194e0a4cdSJulian Elischer } 10294e0a4cdSJulian Elischer 10394e0a4cdSJulian Elischer /* DEBUG ONLY */ 10494e0a4cdSJulian Elischer SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 10594e0a4cdSJulian Elischer 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 10694e0a4cdSJulian Elischer "debug virtual cpus"); 1075c8329edSJulian Elischer 108fdcac928SMarcel Moolenaar struct mtx tid_lock; 1091ea7a6f8SPoul-Henning Kamp static struct unrhdr *tid_unrhdr; 110fdcac928SMarcel Moolenaar 111fdcac928SMarcel Moolenaar /* 112696058c3SJulian Elischer * Prepare a thread for use. 11344990b8cSJulian Elischer */ 114b23f72e9SBrian Feldman static int 115b23f72e9SBrian Feldman thread_ctor(void *mem, int size, void *arg, int flags) 11644990b8cSJulian Elischer { 11744990b8cSJulian Elischer struct thread *td; 11844990b8cSJulian Elischer 11944990b8cSJulian Elischer td = (struct thread *)mem; 12071fad9fdSJulian Elischer td->td_state = TDS_INACTIVE; 121060563ecSJulian Elischer td->td_oncpu = NOCPU; 1226c27c603SJuli Mallett 123773eff9dSPoul-Henning Kamp td->td_tid = alloc_unr(tid_unrhdr); 124773eff9dSPoul-Henning Kamp 1256c27c603SJuli Mallett /* 1266c27c603SJuli Mallett * Note that td_critnest begins life as 1 because the thread is not 1276c27c603SJuli Mallett * running and is thereby implicitly waiting to be on the receiving 1286c27c603SJuli Mallett * end of a context switch. A context switch must occur inside a 1296c27c603SJuli Mallett * critical section, and in fact, includes hand-off of the sched_lock. 1306c27c603SJuli Mallett * After a context switch to a newly created thread, it will release 1316c27c603SJuli Mallett * sched_lock for the first time, and its td_critnest will hit 0 for 1326c27c603SJuli Mallett * the first time. This happens on the far end of a context switch, 1336c27c603SJuli Mallett * and when it context switches away from itself, it will in fact go 1346c27c603SJuli Mallett * back into a critical section, and hand off the sched lock to the 1356c27c603SJuli Mallett * next thread. 1366c27c603SJuli Mallett */ 137139b7550SJohn Baldwin td->td_critnest = 1; 138911b84b0SRobert Watson 139911b84b0SRobert Watson #ifdef AUDIT 140911b84b0SRobert Watson audit_thread_alloc(td); 141911b84b0SRobert Watson #endif 142b23f72e9SBrian Feldman return (0); 14344990b8cSJulian Elischer } 14444990b8cSJulian Elischer 14544990b8cSJulian Elischer /* 14644990b8cSJulian Elischer * Reclaim a thread after use. 14744990b8cSJulian Elischer */ 14844990b8cSJulian Elischer static void 14944990b8cSJulian Elischer thread_dtor(void *mem, int size, void *arg) 15044990b8cSJulian Elischer { 15144990b8cSJulian Elischer struct thread *td; 15244990b8cSJulian Elischer 15344990b8cSJulian Elischer td = (struct thread *)mem; 15444990b8cSJulian Elischer 15544990b8cSJulian Elischer #ifdef INVARIANTS 15644990b8cSJulian Elischer /* Verify that this thread is in a safe state to free. */ 15744990b8cSJulian Elischer switch (td->td_state) { 15871fad9fdSJulian Elischer case TDS_INHIBITED: 15971fad9fdSJulian Elischer case TDS_RUNNING: 16071fad9fdSJulian Elischer case TDS_CAN_RUN: 16144990b8cSJulian Elischer case TDS_RUNQ: 16244990b8cSJulian Elischer /* 16344990b8cSJulian Elischer * We must never unlink a thread that is in one of 16444990b8cSJulian Elischer * these states, because it is currently active. 16544990b8cSJulian Elischer */ 16644990b8cSJulian Elischer panic("bad state for thread unlinking"); 16744990b8cSJulian Elischer /* NOTREACHED */ 16871fad9fdSJulian Elischer case TDS_INACTIVE: 16944990b8cSJulian Elischer break; 17044990b8cSJulian Elischer default: 17144990b8cSJulian Elischer panic("bad thread state"); 17244990b8cSJulian Elischer /* NOTREACHED */ 17344990b8cSJulian Elischer } 17444990b8cSJulian Elischer #endif 1756e8525ceSRobert Watson #ifdef AUDIT 1766e8525ceSRobert Watson audit_thread_free(td); 1776e8525ceSRobert Watson #endif 178773eff9dSPoul-Henning Kamp free_unr(tid_unrhdr, td->td_tid); 179ed062c8dSJulian Elischer sched_newthread(td); 18044990b8cSJulian Elischer } 18144990b8cSJulian Elischer 18244990b8cSJulian Elischer /* 18344990b8cSJulian Elischer * Initialize type-stable parts of a thread (when newly created). 18444990b8cSJulian Elischer */ 185b23f72e9SBrian Feldman static int 186b23f72e9SBrian Feldman thread_init(void *mem, int size, int flags) 18744990b8cSJulian Elischer { 18844990b8cSJulian Elischer struct thread *td; 18944990b8cSJulian Elischer 19044990b8cSJulian Elischer td = (struct thread *)mem; 191247aba24SMarcel Moolenaar 19249a2507bSAlan Cox vm_thread_new(td, 0); 19344990b8cSJulian Elischer cpu_thread_setup(td); 19444f3b092SJohn Baldwin td->td_sleepqueue = sleepq_alloc(); 195961a7b24SJohn Baldwin td->td_turnstile = turnstile_alloc(); 196bc8e6d81SDavid Xu td->td_umtxq = umtxq_alloc(); 197de028f5aSJeff Roberson td->td_sched = (struct td_sched *)&td[1]; 198ed062c8dSJulian Elischer sched_newthread(td); 199b23f72e9SBrian Feldman return (0); 20044990b8cSJulian Elischer } 20144990b8cSJulian Elischer 20244990b8cSJulian Elischer /* 20344990b8cSJulian Elischer * Tear down type-stable parts of a thread (just before being discarded). 20444990b8cSJulian Elischer */ 20544990b8cSJulian Elischer static void 20644990b8cSJulian Elischer thread_fini(void *mem, int size) 20744990b8cSJulian Elischer { 20844990b8cSJulian Elischer struct thread *td; 20944990b8cSJulian Elischer 21044990b8cSJulian Elischer td = (struct thread *)mem; 211961a7b24SJohn Baldwin turnstile_free(td->td_turnstile); 21244f3b092SJohn Baldwin sleepq_free(td->td_sleepqueue); 213bc8e6d81SDavid Xu umtxq_free(td->td_umtxq); 21449a2507bSAlan Cox vm_thread_dispose(td); 21544990b8cSJulian Elischer } 2165215b187SJeff Roberson 217de028f5aSJeff Roberson /* 218de028f5aSJeff Roberson * Initialize type-stable parts of a ksegrp (when newly created). 219de028f5aSJeff Roberson */ 220b23f72e9SBrian Feldman static int 221a9b5dc7dSJulian Elischer ksegrp_ctor(void *mem, int size, void *arg, int flags) 222de028f5aSJeff Roberson { 223de028f5aSJeff Roberson struct ksegrp *kg; 224de028f5aSJeff Roberson 225de028f5aSJeff Roberson kg = (struct ksegrp *)mem; 226a9b5dc7dSJulian Elischer bzero(mem, size); 227de028f5aSJeff Roberson kg->kg_sched = (struct kg_sched *)&kg[1]; 228b23f72e9SBrian Feldman return (0); 229de028f5aSJeff Roberson } 23044990b8cSJulian Elischer 2315c8329edSJulian Elischer void 2325c8329edSJulian Elischer ksegrp_link(struct ksegrp *kg, struct proc *p) 2335c8329edSJulian Elischer { 2345c8329edSJulian Elischer 2355c8329edSJulian Elischer TAILQ_INIT(&kg->kg_threads); 2365c8329edSJulian Elischer TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 2375215b187SJeff Roberson TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 2385c8329edSJulian Elischer kg->kg_proc = p; 2395215b187SJeff Roberson /* 2405215b187SJeff Roberson * the following counters are in the -zero- section 2415215b187SJeff Roberson * and may not need clearing 2425215b187SJeff Roberson */ 2435c8329edSJulian Elischer kg->kg_numthreads = 0; 2445215b187SJeff Roberson kg->kg_numupcalls = 0; 2455c8329edSJulian Elischer /* link it in now that it's consistent */ 2465c8329edSJulian Elischer p->p_numksegrps++; 2475c8329edSJulian Elischer TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 2485c8329edSJulian Elischer } 2495c8329edSJulian Elischer 250ed062c8dSJulian Elischer /* 251ed062c8dSJulian Elischer * Called from: 252ed062c8dSJulian Elischer * thread-exit() 253ed062c8dSJulian Elischer */ 2545c8329edSJulian Elischer void 2555c8329edSJulian Elischer ksegrp_unlink(struct ksegrp *kg) 2565c8329edSJulian Elischer { 2575c8329edSJulian Elischer struct proc *p; 2585c8329edSJulian Elischer 2595c8329edSJulian Elischer mtx_assert(&sched_lock, MA_OWNED); 2605215b187SJeff Roberson KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 2615215b187SJeff Roberson KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 2625215b187SJeff Roberson 2635c8329edSJulian Elischer p = kg->kg_proc; 2645c8329edSJulian Elischer TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 2655c8329edSJulian Elischer p->p_numksegrps--; 2665c8329edSJulian Elischer /* 2675c8329edSJulian Elischer * Aggregate stats from the KSE 2685c8329edSJulian Elischer */ 26921fc3164SDavid Xu if (p->p_procscopegrp == kg) 27021fc3164SDavid Xu p->p_procscopegrp = NULL; 2715c8329edSJulian Elischer } 2725c8329edSJulian Elischer 2735c8329edSJulian Elischer /* 2745215b187SJeff Roberson * For a newly created process, 2755215b187SJeff Roberson * link up all the structures and its initial threads etc. 276ed062c8dSJulian Elischer * called from: 277ed062c8dSJulian Elischer * {arch}/{arch}/machdep.c ia64_init(), init386() etc. 278ed062c8dSJulian Elischer * proc_dtor() (should go away) 279ed062c8dSJulian Elischer * proc_init() 2805c8329edSJulian Elischer */ 2815c8329edSJulian Elischer void 282ed062c8dSJulian Elischer proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td) 2835c8329edSJulian Elischer { 2845c8329edSJulian Elischer 2855c8329edSJulian Elischer TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 2865c8329edSJulian Elischer TAILQ_INIT(&p->p_threads); /* all threads in proc */ 2875c8329edSJulian Elischer TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 2889104847fSDavid Xu sigqueue_init(&p->p_sigqueue, p); 289ebceaf6dSDavid Xu p->p_ksi = ksiginfo_alloc(1); 290ebceaf6dSDavid Xu if (p->p_ksi != NULL) { 2915c474517SDavid Xu /* XXX p_ksi may be null if ksiginfo zone is not ready */ 292ebceaf6dSDavid Xu p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 293ebceaf6dSDavid Xu } 294b2f92ef9SDavid Xu LIST_INIT(&p->p_mqnotifier); 2955c8329edSJulian Elischer p->p_numksegrps = 0; 2965c8329edSJulian Elischer p->p_numthreads = 0; 2975c8329edSJulian Elischer 2985c8329edSJulian Elischer ksegrp_link(kg, p); 2995c8329edSJulian Elischer thread_link(td, kg); 3005c8329edSJulian Elischer } 3015c8329edSJulian Elischer 3025c8329edSJulian Elischer /* 30344990b8cSJulian Elischer * Initialize global thread allocation resources. 30444990b8cSJulian Elischer */ 30544990b8cSJulian Elischer void 30644990b8cSJulian Elischer threadinit(void) 30744990b8cSJulian Elischer { 30844990b8cSJulian Elischer 3091ea7a6f8SPoul-Henning Kamp mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 3101ea7a6f8SPoul-Henning Kamp tid_unrhdr = new_unrhdr(PID_MAX + 1, INT_MAX, &tid_lock); 3111ea7a6f8SPoul-Henning Kamp 312de028f5aSJeff Roberson thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 31344990b8cSJulian Elischer thread_ctor, thread_dtor, thread_init, thread_fini, 31444990b8cSJulian Elischer UMA_ALIGN_CACHE, 0); 315de028f5aSJeff Roberson ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 316a9b5dc7dSJulian Elischer ksegrp_ctor, NULL, NULL, NULL, 3174f0db5e0SJulian Elischer UMA_ALIGN_CACHE, 0); 318ed062c8dSJulian Elischer kseinit(); /* set up kse specific stuff e.g. upcall zone*/ 31944990b8cSJulian Elischer } 32044990b8cSJulian Elischer 32144990b8cSJulian Elischer /* 3221faf202eSJulian Elischer * Stash an embarasingly extra thread into the zombie thread queue. 32344990b8cSJulian Elischer */ 32444990b8cSJulian Elischer void 32544990b8cSJulian Elischer thread_stash(struct thread *td) 32644990b8cSJulian Elischer { 3275215b187SJeff Roberson mtx_lock_spin(&kse_zombie_lock); 32844990b8cSJulian Elischer TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 3295215b187SJeff Roberson mtx_unlock_spin(&kse_zombie_lock); 33044990b8cSJulian Elischer } 33144990b8cSJulian Elischer 33244990b8cSJulian Elischer /* 3335c8329edSJulian Elischer * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 3345c8329edSJulian Elischer */ 3355c8329edSJulian Elischer void 3365c8329edSJulian Elischer ksegrp_stash(struct ksegrp *kg) 3375c8329edSJulian Elischer { 3385215b187SJeff Roberson mtx_lock_spin(&kse_zombie_lock); 3395c8329edSJulian Elischer TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 3405215b187SJeff Roberson mtx_unlock_spin(&kse_zombie_lock); 3415c8329edSJulian Elischer } 3425c8329edSJulian Elischer 3435c8329edSJulian Elischer /* 3445215b187SJeff Roberson * Reap zombie kse resource. 34544990b8cSJulian Elischer */ 34644990b8cSJulian Elischer void 34744990b8cSJulian Elischer thread_reap(void) 34844990b8cSJulian Elischer { 3495c8329edSJulian Elischer struct thread *td_first, *td_next; 3505c8329edSJulian Elischer struct ksegrp *kg_first, * kg_next; 35144990b8cSJulian Elischer 35244990b8cSJulian Elischer /* 3535215b187SJeff Roberson * Don't even bother to lock if none at this instant, 3545215b187SJeff Roberson * we really don't care about the next instant.. 35544990b8cSJulian Elischer */ 3565c8329edSJulian Elischer if ((!TAILQ_EMPTY(&zombie_threads)) 357345ad866SJulian Elischer || (!TAILQ_EMPTY(&zombie_ksegrps))) { 3585215b187SJeff Roberson mtx_lock_spin(&kse_zombie_lock); 3595c8329edSJulian Elischer td_first = TAILQ_FIRST(&zombie_threads); 3605c8329edSJulian Elischer kg_first = TAILQ_FIRST(&zombie_ksegrps); 3615c8329edSJulian Elischer if (td_first) 3625c8329edSJulian Elischer TAILQ_INIT(&zombie_threads); 3635c8329edSJulian Elischer if (kg_first) 3645c8329edSJulian Elischer TAILQ_INIT(&zombie_ksegrps); 3655215b187SJeff Roberson mtx_unlock_spin(&kse_zombie_lock); 3665c8329edSJulian Elischer while (td_first) { 3675c8329edSJulian Elischer td_next = TAILQ_NEXT(td_first, td_runq); 3685215b187SJeff Roberson if (td_first->td_ucred) 3695215b187SJeff Roberson crfree(td_first->td_ucred); 3705c8329edSJulian Elischer thread_free(td_first); 3715c8329edSJulian Elischer td_first = td_next; 37244990b8cSJulian Elischer } 3735c8329edSJulian Elischer while (kg_first) { 3745c8329edSJulian Elischer kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 3755c8329edSJulian Elischer ksegrp_free(kg_first); 3765c8329edSJulian Elischer kg_first = kg_next; 3775c8329edSJulian Elischer } 378ed062c8dSJulian Elischer /* 379ed062c8dSJulian Elischer * there will always be a thread on the list if one of these 380ed062c8dSJulian Elischer * is there. 381ed062c8dSJulian Elischer */ 382345ad866SJulian Elischer kse_GC(); 38344990b8cSJulian Elischer } 384ed062c8dSJulian Elischer } 38544990b8cSJulian Elischer 38644990b8cSJulian Elischer /* 3874f0db5e0SJulian Elischer * Allocate a ksegrp. 3884f0db5e0SJulian Elischer */ 3894f0db5e0SJulian Elischer struct ksegrp * 3904f0db5e0SJulian Elischer ksegrp_alloc(void) 3914f0db5e0SJulian Elischer { 392a163d034SWarner Losh return (uma_zalloc(ksegrp_zone, M_WAITOK)); 3934f0db5e0SJulian Elischer } 3944f0db5e0SJulian Elischer 3954f0db5e0SJulian Elischer /* 39644990b8cSJulian Elischer * Allocate a thread. 39744990b8cSJulian Elischer */ 39844990b8cSJulian Elischer struct thread * 39944990b8cSJulian Elischer thread_alloc(void) 40044990b8cSJulian Elischer { 40144990b8cSJulian Elischer thread_reap(); /* check if any zombies to get */ 402a163d034SWarner Losh return (uma_zalloc(thread_zone, M_WAITOK)); 40344990b8cSJulian Elischer } 40444990b8cSJulian Elischer 40544990b8cSJulian Elischer /* 4064f0db5e0SJulian Elischer * Deallocate a ksegrp. 4074f0db5e0SJulian Elischer */ 4084f0db5e0SJulian Elischer void 4094f0db5e0SJulian Elischer ksegrp_free(struct ksegrp *td) 4104f0db5e0SJulian Elischer { 4114f0db5e0SJulian Elischer uma_zfree(ksegrp_zone, td); 4124f0db5e0SJulian Elischer } 4134f0db5e0SJulian Elischer 4144f0db5e0SJulian Elischer /* 41544990b8cSJulian Elischer * Deallocate a thread. 41644990b8cSJulian Elischer */ 41744990b8cSJulian Elischer void 41844990b8cSJulian Elischer thread_free(struct thread *td) 41944990b8cSJulian Elischer { 420696058c3SJulian Elischer 421696058c3SJulian Elischer cpu_thread_clean(td); 42244990b8cSJulian Elischer uma_zfree(thread_zone, td); 42344990b8cSJulian Elischer } 42444990b8cSJulian Elischer 42544990b8cSJulian Elischer /* 42644990b8cSJulian Elischer * Discard the current thread and exit from its context. 42794e0a4cdSJulian Elischer * Always called with scheduler locked. 42844990b8cSJulian Elischer * 42944990b8cSJulian Elischer * Because we can't free a thread while we're operating under its context, 430696058c3SJulian Elischer * push the current thread into our CPU's deadthread holder. This means 431696058c3SJulian Elischer * we needn't worry about someone else grabbing our context before we 43294e0a4cdSJulian Elischer * do a cpu_throw(). This may not be needed now as we are under schedlock. 43394e0a4cdSJulian Elischer * Maybe we can just do a thread_stash() as thr_exit1 does. 43494e0a4cdSJulian Elischer */ 43594e0a4cdSJulian Elischer /* XXX 43694e0a4cdSJulian Elischer * libthr expects its thread exit to return for the last 43794e0a4cdSJulian Elischer * thread, meaning that the program is back to non-threaded 43894e0a4cdSJulian Elischer * mode I guess. Because we do this (cpu_throw) unconditionally 43994e0a4cdSJulian Elischer * here, they have their own version of it. (thr_exit1()) 44094e0a4cdSJulian Elischer * that doesn't do it all if this was the last thread. 44194e0a4cdSJulian Elischer * It is also called from thread_suspend_check(). 44294e0a4cdSJulian Elischer * Of course in the end, they end up coming here through exit1 44394e0a4cdSJulian Elischer * anyhow.. After fixing 'thr' to play by the rules we should be able 44494e0a4cdSJulian Elischer * to merge these two functions together. 445ed062c8dSJulian Elischer * 446ed062c8dSJulian Elischer * called from: 447ed062c8dSJulian Elischer * exit1() 448ed062c8dSJulian Elischer * kse_exit() 449ed062c8dSJulian Elischer * thr_exit() 450ed062c8dSJulian Elischer * thread_user_enter() 451ed062c8dSJulian Elischer * thread_userret() 452ed062c8dSJulian Elischer * thread_suspend_check() 45344990b8cSJulian Elischer */ 45444990b8cSJulian Elischer void 45544990b8cSJulian Elischer thread_exit(void) 45644990b8cSJulian Elischer { 45744990b8cSJulian Elischer struct thread *td; 45844990b8cSJulian Elischer struct proc *p; 45944990b8cSJulian Elischer struct ksegrp *kg; 46044990b8cSJulian Elischer 46144990b8cSJulian Elischer td = curthread; 46244990b8cSJulian Elischer kg = td->td_ksegrp; 46344990b8cSJulian Elischer p = td->td_proc; 46444990b8cSJulian Elischer 46544990b8cSJulian Elischer mtx_assert(&sched_lock, MA_OWNED); 466ed062c8dSJulian Elischer mtx_assert(&Giant, MA_NOTOWNED); 46744990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 468ed062c8dSJulian Elischer KASSERT(p != NULL, ("thread exiting without a process")); 469ed062c8dSJulian Elischer KASSERT(kg != NULL, ("thread exiting without a kse group")); 470cc701b73SRobert Watson CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 471cc701b73SRobert Watson (long)p->p_pid, p->p_comm); 4729104847fSDavid Xu KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 47344990b8cSJulian Elischer 47489964dd2SRobert Watson #ifdef AUDIT 47589964dd2SRobert Watson AUDIT_SYSCALL_EXIT(0, td); 47689964dd2SRobert Watson #endif 47789964dd2SRobert Watson 47848bfcdddSJulian Elischer if (td->td_standin != NULL) { 479ed062c8dSJulian Elischer /* 480ed062c8dSJulian Elischer * Note that we don't need to free the cred here as it 481ed062c8dSJulian Elischer * is done in thread_reap(). 482ed062c8dSJulian Elischer */ 48348bfcdddSJulian Elischer thread_stash(td->td_standin); 48448bfcdddSJulian Elischer td->td_standin = NULL; 48548bfcdddSJulian Elischer } 48648bfcdddSJulian Elischer 487ed062c8dSJulian Elischer /* 488ed062c8dSJulian Elischer * drop FPU & debug register state storage, or any other 489ed062c8dSJulian Elischer * architecture specific resources that 490ed062c8dSJulian Elischer * would not be on a new untouched process. 491ed062c8dSJulian Elischer */ 49244990b8cSJulian Elischer cpu_thread_exit(td); /* XXXSMP */ 49344990b8cSJulian Elischer 4941faf202eSJulian Elischer /* 495ed062c8dSJulian Elischer * The thread is exiting. scheduler can release its stuff 496ed062c8dSJulian Elischer * and collect stats etc. 497ed062c8dSJulian Elischer */ 498ed062c8dSJulian Elischer sched_thread_exit(td); 499ed062c8dSJulian Elischer 500ed062c8dSJulian Elischer /* 5011faf202eSJulian Elischer * The last thread is left attached to the process 5021faf202eSJulian Elischer * So that the whole bundle gets recycled. Skip 503ed062c8dSJulian Elischer * all this stuff if we never had threads. 504ed062c8dSJulian Elischer * EXIT clears all sign of other threads when 505ed062c8dSJulian Elischer * it goes to single threading, so the last thread always 506ed062c8dSJulian Elischer * takes the short path. 5071faf202eSJulian Elischer */ 508ed062c8dSJulian Elischer if (p->p_flag & P_HADTHREADS) { 5091faf202eSJulian Elischer if (p->p_numthreads > 1) { 510d3a0bd78SJulian Elischer thread_unlink(td); 511ed062c8dSJulian Elischer 512ed062c8dSJulian Elischer /* XXX first arg not used in 4BSD or ULE */ 513ed062c8dSJulian Elischer sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); 514ed062c8dSJulian Elischer 515ed062c8dSJulian Elischer /* 51644990b8cSJulian Elischer * The test below is NOT true if we are the 5171faf202eSJulian Elischer * sole exiting thread. P_STOPPED_SNGL is unset 51844990b8cSJulian Elischer * in exit1() after it is the only survivor. 51944990b8cSJulian Elischer */ 5201279572aSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 52144990b8cSJulian Elischer if (p->p_numthreads == p->p_suspcount) { 52271fad9fdSJulian Elischer thread_unsuspend_one(p->p_singlethread); 52344990b8cSJulian Elischer } 52444990b8cSJulian Elischer } 52548bfcdddSJulian Elischer 5265215b187SJeff Roberson /* 5275215b187SJeff Roberson * Because each upcall structure has an owner thread, 5285215b187SJeff Roberson * owner thread exits only when process is in exiting 5295215b187SJeff Roberson * state, so upcall to userland is no longer needed, 5305215b187SJeff Roberson * deleting upcall structure is safe here. 5315215b187SJeff Roberson * So when all threads in a group is exited, all upcalls 5325215b187SJeff Roberson * in the group should be automatically freed. 533ed062c8dSJulian Elischer * XXXKSE This is a KSE thing and should be exported 534ed062c8dSJulian Elischer * there somehow. 5355215b187SJeff Roberson */ 5365215b187SJeff Roberson upcall_remove(td); 5376f8132a8SJulian Elischer 53848bfcdddSJulian Elischer /* 539ed062c8dSJulian Elischer * If the thread we unlinked above was the last one, 540ed062c8dSJulian Elischer * then this ksegrp should go away too. 54148bfcdddSJulian Elischer */ 542ed062c8dSJulian Elischer if (kg->kg_numthreads == 0) { 543ed062c8dSJulian Elischer /* 544ed062c8dSJulian Elischer * let the scheduler know about this in case 545ed062c8dSJulian Elischer * it needs to recover stats or resources. 546ed062c8dSJulian Elischer * Theoretically we could let 547ed062c8dSJulian Elischer * sched_exit_ksegrp() do the equivalent of 548ed062c8dSJulian Elischer * setting the concurrency to 0 549ed062c8dSJulian Elischer * but don't do it yet to avoid changing 550ed062c8dSJulian Elischer * the existing scheduler code until we 551ed062c8dSJulian Elischer * are ready. 552ed062c8dSJulian Elischer * We supply a random other ksegrp 553ed062c8dSJulian Elischer * as the recipient of any built up 554ed062c8dSJulian Elischer * cpu usage etc. (If the scheduler wants it). 555ed062c8dSJulian Elischer * XXXKSE 556ed062c8dSJulian Elischer * This is probably not fair so think of 557ed062c8dSJulian Elischer * a better answer. 558ed062c8dSJulian Elischer */ 55955d44f79SJulian Elischer sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); 560ed062c8dSJulian Elischer sched_set_concurrency(kg, 0); /* XXX TEMP */ 561ab2baa72SDavid Xu ksegrp_unlink(kg); 562ed062c8dSJulian Elischer ksegrp_stash(kg); 563ab2baa72SDavid Xu } 5646f8132a8SJulian Elischer PROC_UNLOCK(p); 5655c8329edSJulian Elischer td->td_ksegrp = NULL; 566696058c3SJulian Elischer PCPU_SET(deadthread, td); 5671faf202eSJulian Elischer } else { 568ed062c8dSJulian Elischer /* 569ed062c8dSJulian Elischer * The last thread is exiting.. but not through exit() 570ed062c8dSJulian Elischer * what should we do? 571ed062c8dSJulian Elischer * Theoretically this can't happen 572ed062c8dSJulian Elischer * exit1() - clears threading flags before coming here 573ed062c8dSJulian Elischer * kse_exit() - treats last thread specially 574ed062c8dSJulian Elischer * thr_exit() - treats last thread specially 575ed062c8dSJulian Elischer * thread_user_enter() - only if more exist 576ed062c8dSJulian Elischer * thread_userret() - only if more exist 577ed062c8dSJulian Elischer * thread_suspend_check() - only if more exist 578ed062c8dSJulian Elischer */ 579ed062c8dSJulian Elischer panic ("thread_exit: Last thread exiting on its own"); 580ed062c8dSJulian Elischer } 581ed062c8dSJulian Elischer } else { 582ed062c8dSJulian Elischer /* 583ed062c8dSJulian Elischer * non threaded process comes here. 584ed062c8dSJulian Elischer * This includes an EX threaded process that is coming 585ed062c8dSJulian Elischer * here via exit1(). (exit1 dethreads the proc first). 586ed062c8dSJulian Elischer */ 5871faf202eSJulian Elischer PROC_UNLOCK(p); 5881faf202eSJulian Elischer } 589dcc9954eSJulian Elischer td->td_state = TDS_INACTIVE; 590732d9528SJulian Elischer CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 591cc66ebe2SPeter Wemm cpu_throw(td, choosethread()); 592cc66ebe2SPeter Wemm panic("I'm a teapot!"); 59344990b8cSJulian Elischer /* NOTREACHED */ 59444990b8cSJulian Elischer } 59544990b8cSJulian Elischer 59644990b8cSJulian Elischer /* 597696058c3SJulian Elischer * Do any thread specific cleanups that may be needed in wait() 59837814395SPeter Wemm * called with Giant, proc and schedlock not held. 599696058c3SJulian Elischer */ 600696058c3SJulian Elischer void 601696058c3SJulian Elischer thread_wait(struct proc *p) 602696058c3SJulian Elischer { 603696058c3SJulian Elischer struct thread *td; 604696058c3SJulian Elischer 60537814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 60685495c72SJens Schweikhardt KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 60785495c72SJens Schweikhardt KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); 608696058c3SJulian Elischer FOREACH_THREAD_IN_PROC(p, td) { 609696058c3SJulian Elischer if (td->td_standin != NULL) { 610b75b0311SJulian Elischer if (td->td_standin->td_ucred != NULL) { 611b75b0311SJulian Elischer crfree(td->td_standin->td_ucred); 612b75b0311SJulian Elischer td->td_standin->td_ucred = NULL; 613b75b0311SJulian Elischer } 614696058c3SJulian Elischer thread_free(td->td_standin); 615696058c3SJulian Elischer td->td_standin = NULL; 616696058c3SJulian Elischer } 617696058c3SJulian Elischer cpu_thread_clean(td); 618ed062c8dSJulian Elischer crfree(td->td_ucred); 619696058c3SJulian Elischer } 620696058c3SJulian Elischer thread_reap(); /* check for zombie threads etc. */ 621696058c3SJulian Elischer } 622696058c3SJulian Elischer 623696058c3SJulian Elischer /* 62444990b8cSJulian Elischer * Link a thread to a process. 6251faf202eSJulian Elischer * set up anything that needs to be initialized for it to 6261faf202eSJulian Elischer * be used by the process. 62744990b8cSJulian Elischer * 62844990b8cSJulian Elischer * Note that we do not link to the proc's ucred here. 62944990b8cSJulian Elischer * The thread is linked as if running but no KSE assigned. 630ed062c8dSJulian Elischer * Called from: 631ed062c8dSJulian Elischer * proc_linkup() 632ed062c8dSJulian Elischer * thread_schedule_upcall() 633ed062c8dSJulian Elischer * thr_create() 63444990b8cSJulian Elischer */ 63544990b8cSJulian Elischer void 63644990b8cSJulian Elischer thread_link(struct thread *td, struct ksegrp *kg) 63744990b8cSJulian Elischer { 63844990b8cSJulian Elischer struct proc *p; 63944990b8cSJulian Elischer 64044990b8cSJulian Elischer p = kg->kg_proc; 64171fad9fdSJulian Elischer td->td_state = TDS_INACTIVE; 64244990b8cSJulian Elischer td->td_proc = p; 64344990b8cSJulian Elischer td->td_ksegrp = kg; 6445215b187SJeff Roberson td->td_flags = 0; 6454fc21c09SDaniel Eischen td->td_kflags = 0; 64644990b8cSJulian Elischer 6471faf202eSJulian Elischer LIST_INIT(&td->td_contested); 6489104847fSDavid Xu sigqueue_init(&td->td_sigqueue, p); 649c06eb4e2SSam Leffler callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 65044990b8cSJulian Elischer TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 65144990b8cSJulian Elischer TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 65244990b8cSJulian Elischer p->p_numthreads++; 65344990b8cSJulian Elischer kg->kg_numthreads++; 65444990b8cSJulian Elischer } 65544990b8cSJulian Elischer 656ed062c8dSJulian Elischer /* 657e5bedcefSJulian Elischer * Convert a process with one thread to an unthreaded process. 658e5bedcefSJulian Elischer * Called from: 659e5bedcefSJulian Elischer * thread_single(exit) (called from execve and exit) 660e5bedcefSJulian Elischer * kse_exit() XXX may need cleaning up wrt KSE stuff 661e5bedcefSJulian Elischer */ 662e5bedcefSJulian Elischer void 663e5bedcefSJulian Elischer thread_unthread(struct thread *td) 664e5bedcefSJulian Elischer { 665e5bedcefSJulian Elischer struct proc *p = td->td_proc; 666e5bedcefSJulian Elischer 667e5bedcefSJulian Elischer KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); 668e5bedcefSJulian Elischer upcall_remove(td); 669e5bedcefSJulian Elischer p->p_flag &= ~(P_SA|P_HADTHREADS); 670e5bedcefSJulian Elischer td->td_mailbox = NULL; 671e5bedcefSJulian Elischer td->td_pflags &= ~(TDP_SA | TDP_CAN_UNBIND); 672e5bedcefSJulian Elischer if (td->td_standin != NULL) { 673e5bedcefSJulian Elischer thread_stash(td->td_standin); 674e5bedcefSJulian Elischer td->td_standin = NULL; 675e5bedcefSJulian Elischer } 676e5bedcefSJulian Elischer sched_set_concurrency(td->td_ksegrp, 1); 677e5bedcefSJulian Elischer } 678e5bedcefSJulian Elischer 679e5bedcefSJulian Elischer /* 680ed062c8dSJulian Elischer * Called from: 681ed062c8dSJulian Elischer * thread_exit() 682ed062c8dSJulian Elischer */ 683d3a0bd78SJulian Elischer void 684d3a0bd78SJulian Elischer thread_unlink(struct thread *td) 685d3a0bd78SJulian Elischer { 686d3a0bd78SJulian Elischer struct proc *p = td->td_proc; 687d3a0bd78SJulian Elischer struct ksegrp *kg = td->td_ksegrp; 688d3a0bd78SJulian Elischer 689112afcb2SJohn Baldwin mtx_assert(&sched_lock, MA_OWNED); 690d3a0bd78SJulian Elischer TAILQ_REMOVE(&p->p_threads, td, td_plist); 691d3a0bd78SJulian Elischer p->p_numthreads--; 692d3a0bd78SJulian Elischer TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 693d3a0bd78SJulian Elischer kg->kg_numthreads--; 694d3a0bd78SJulian Elischer /* could clear a few other things here */ 695ed062c8dSJulian Elischer /* Must NOT clear links to proc and ksegrp! */ 6965c8329edSJulian Elischer } 6975c8329edSJulian Elischer 6985215b187SJeff Roberson /* 69944990b8cSJulian Elischer * Enforce single-threading. 70044990b8cSJulian Elischer * 70144990b8cSJulian Elischer * Returns 1 if the caller must abort (another thread is waiting to 70244990b8cSJulian Elischer * exit the process or similar). Process is locked! 70344990b8cSJulian Elischer * Returns 0 when you are successfully the only thread running. 70444990b8cSJulian Elischer * A process has successfully single threaded in the suspend mode when 70544990b8cSJulian Elischer * There are no threads in user mode. Threads in the kernel must be 70644990b8cSJulian Elischer * allowed to continue until they get to the user boundary. They may even 70744990b8cSJulian Elischer * copy out their return values and data before suspending. They may however be 70844990b8cSJulian Elischer * accellerated in reaching the user boundary as we will wake up 70944990b8cSJulian Elischer * any sleeping threads that are interruptable. (PCATCH). 71044990b8cSJulian Elischer */ 71144990b8cSJulian Elischer int 712906ac69dSDavid Xu thread_single(int mode) 71344990b8cSJulian Elischer { 71444990b8cSJulian Elischer struct thread *td; 71544990b8cSJulian Elischer struct thread *td2; 71644990b8cSJulian Elischer struct proc *p; 717ec008e96SDavid Xu int remaining; 71844990b8cSJulian Elischer 71944990b8cSJulian Elischer td = curthread; 72044990b8cSJulian Elischer p = td->td_proc; 72137814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 72244990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 72344990b8cSJulian Elischer KASSERT((td != NULL), ("curthread is NULL")); 72444990b8cSJulian Elischer 725ed062c8dSJulian Elischer if ((p->p_flag & P_HADTHREADS) == 0) 72644990b8cSJulian Elischer return (0); 72744990b8cSJulian Elischer 728e3b9bf71SJulian Elischer /* Is someone already single threading? */ 729906ac69dSDavid Xu if (p->p_singlethread != NULL && p->p_singlethread != td) 73044990b8cSJulian Elischer return (1); 73144990b8cSJulian Elischer 732906ac69dSDavid Xu if (mode == SINGLE_EXIT) { 733906ac69dSDavid Xu p->p_flag |= P_SINGLE_EXIT; 734906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_BOUNDARY; 735906ac69dSDavid Xu } else { 736906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_EXIT; 737906ac69dSDavid Xu if (mode == SINGLE_BOUNDARY) 738906ac69dSDavid Xu p->p_flag |= P_SINGLE_BOUNDARY; 739906ac69dSDavid Xu else 740906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_BOUNDARY; 741906ac69dSDavid Xu } 7421279572aSDavid Xu p->p_flag |= P_STOPPED_SINGLE; 74371fad9fdSJulian Elischer mtx_lock_spin(&sched_lock); 744112afcb2SJohn Baldwin p->p_singlethread = td; 745906ac69dSDavid Xu if (mode == SINGLE_EXIT) 746ec008e96SDavid Xu remaining = p->p_numthreads; 747906ac69dSDavid Xu else if (mode == SINGLE_BOUNDARY) 748906ac69dSDavid Xu remaining = p->p_numthreads - p->p_boundary_count; 749906ac69dSDavid Xu else 750ec008e96SDavid Xu remaining = p->p_numthreads - p->p_suspcount; 751ec008e96SDavid Xu while (remaining != 1) { 75244990b8cSJulian Elischer FOREACH_THREAD_IN_PROC(p, td2) { 75344990b8cSJulian Elischer if (td2 == td) 75444990b8cSJulian Elischer continue; 755588257e8SDavid Xu td2->td_flags |= TDF_ASTPENDING; 75671fad9fdSJulian Elischer if (TD_IS_INHIBITED(td2)) { 757906ac69dSDavid Xu switch (mode) { 758906ac69dSDavid Xu case SINGLE_EXIT: 759cbf4e354SDavid Xu if (td->td_flags & TDF_DBSUSPEND) 760cbf4e354SDavid Xu td->td_flags &= ~TDF_DBSUSPEND; 761906ac69dSDavid Xu if (TD_IS_SUSPENDED(td2)) 76271fad9fdSJulian Elischer thread_unsuspend_one(td2); 76333862f40SDavid Xu if (TD_ON_SLEEPQ(td2) && 764906ac69dSDavid Xu (td2->td_flags & TDF_SINTR)) 76594f0972bSDavid Xu sleepq_abort(td2, EINTR); 766906ac69dSDavid Xu break; 767906ac69dSDavid Xu case SINGLE_BOUNDARY: 768906ac69dSDavid Xu if (TD_IS_SUSPENDED(td2) && 769906ac69dSDavid Xu !(td2->td_flags & TDF_BOUNDARY)) 770906ac69dSDavid Xu thread_unsuspend_one(td2); 771906ac69dSDavid Xu if (TD_ON_SLEEPQ(td2) && 772906ac69dSDavid Xu (td2->td_flags & TDF_SINTR)) 77394f0972bSDavid Xu sleepq_abort(td2, ERESTART); 774906ac69dSDavid Xu break; 775906ac69dSDavid Xu default: 7769d102777SJulian Elischer if (TD_IS_SUSPENDED(td2)) 7779d102777SJulian Elischer continue; 7785215b187SJeff Roberson /* 7795215b187SJeff Roberson * maybe other inhibitted states too? 7805215b187SJeff Roberson */ 7818acf6057SDavid Xu if ((td2->td_flags & TDF_SINTR) && 7828acf6057SDavid Xu (td2->td_inhibitors & 7838acf6057SDavid Xu (TDI_SLEEPING | TDI_SWAPPED))) 7849d102777SJulian Elischer thread_suspend_one(td2); 785906ac69dSDavid Xu break; 78644990b8cSJulian Elischer } 78744990b8cSJulian Elischer } 788d8267df7SDavid Xu #ifdef SMP 789d8267df7SDavid Xu else if (TD_IS_RUNNING(td2) && td != td2) { 790d8267df7SDavid Xu forward_signal(td2); 791d8267df7SDavid Xu } 792d8267df7SDavid Xu #endif 7939d102777SJulian Elischer } 794906ac69dSDavid Xu if (mode == SINGLE_EXIT) 795ec008e96SDavid Xu remaining = p->p_numthreads; 796906ac69dSDavid Xu else if (mode == SINGLE_BOUNDARY) 797906ac69dSDavid Xu remaining = p->p_numthreads - p->p_boundary_count; 798ec008e96SDavid Xu else 799ec008e96SDavid Xu remaining = p->p_numthreads - p->p_suspcount; 800ec008e96SDavid Xu 8019d102777SJulian Elischer /* 8029d102777SJulian Elischer * Maybe we suspended some threads.. was it enough? 8039d102777SJulian Elischer */ 804ec008e96SDavid Xu if (remaining == 1) 8059d102777SJulian Elischer break; 8069d102777SJulian Elischer 80744990b8cSJulian Elischer /* 80844990b8cSJulian Elischer * Wake us up when everyone else has suspended. 809e3b9bf71SJulian Elischer * In the mean time we suspend as well. 81044990b8cSJulian Elischer */ 81171fad9fdSJulian Elischer thread_suspend_one(td); 81244990b8cSJulian Elischer PROC_UNLOCK(p); 813bf0acc27SJohn Baldwin mi_switch(SW_VOL, NULL); 81444990b8cSJulian Elischer mtx_unlock_spin(&sched_lock); 81544990b8cSJulian Elischer PROC_LOCK(p); 816112afcb2SJohn Baldwin mtx_lock_spin(&sched_lock); 817906ac69dSDavid Xu if (mode == SINGLE_EXIT) 818ec008e96SDavid Xu remaining = p->p_numthreads; 819906ac69dSDavid Xu else if (mode == SINGLE_BOUNDARY) 820906ac69dSDavid Xu remaining = p->p_numthreads - p->p_boundary_count; 821ec008e96SDavid Xu else 822ec008e96SDavid Xu remaining = p->p_numthreads - p->p_suspcount; 82344990b8cSJulian Elischer } 824906ac69dSDavid Xu if (mode == SINGLE_EXIT) { 82591599697SJulian Elischer /* 82691599697SJulian Elischer * We have gotten rid of all the other threads and we 82791599697SJulian Elischer * are about to either exit or exec. In either case, 82891599697SJulian Elischer * we try our utmost to revert to being a non-threaded 82991599697SJulian Elischer * process. 83091599697SJulian Elischer */ 831ed062c8dSJulian Elischer p->p_singlethread = NULL; 83264895117SDavid Xu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 833e5bedcefSJulian Elischer thread_unthread(td); 83491599697SJulian Elischer } 835112afcb2SJohn Baldwin mtx_unlock_spin(&sched_lock); 83644990b8cSJulian Elischer return (0); 83744990b8cSJulian Elischer } 83844990b8cSJulian Elischer 83944990b8cSJulian Elischer /* 84044990b8cSJulian Elischer * Called in from locations that can safely check to see 84144990b8cSJulian Elischer * whether we have to suspend or at least throttle for a 84244990b8cSJulian Elischer * single-thread event (e.g. fork). 84344990b8cSJulian Elischer * 84444990b8cSJulian Elischer * Such locations include userret(). 84544990b8cSJulian Elischer * If the "return_instead" argument is non zero, the thread must be able to 84644990b8cSJulian Elischer * accept 0 (caller may continue), or 1 (caller must abort) as a result. 84744990b8cSJulian Elischer * 84844990b8cSJulian Elischer * The 'return_instead' argument tells the function if it may do a 84944990b8cSJulian Elischer * thread_exit() or suspend, or whether the caller must abort and back 85044990b8cSJulian Elischer * out instead. 85144990b8cSJulian Elischer * 85244990b8cSJulian Elischer * If the thread that set the single_threading request has set the 85344990b8cSJulian Elischer * P_SINGLE_EXIT bit in the process flags then this call will never return 85444990b8cSJulian Elischer * if 'return_instead' is false, but will exit. 85544990b8cSJulian Elischer * 85644990b8cSJulian Elischer * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 85744990b8cSJulian Elischer *---------------+--------------------+--------------------- 85844990b8cSJulian Elischer * 0 | returns 0 | returns 0 or 1 85944990b8cSJulian Elischer * | when ST ends | immediatly 86044990b8cSJulian Elischer *---------------+--------------------+--------------------- 86144990b8cSJulian Elischer * 1 | thread exits | returns 1 86244990b8cSJulian Elischer * | | immediatly 86344990b8cSJulian Elischer * 0 = thread_exit() or suspension ok, 86444990b8cSJulian Elischer * other = return error instead of stopping the thread. 86544990b8cSJulian Elischer * 86644990b8cSJulian Elischer * While a full suspension is under effect, even a single threading 86744990b8cSJulian Elischer * thread would be suspended if it made this call (but it shouldn't). 86844990b8cSJulian Elischer * This call should only be made from places where 86944990b8cSJulian Elischer * thread_exit() would be safe as that may be the outcome unless 87044990b8cSJulian Elischer * return_instead is set. 87144990b8cSJulian Elischer */ 87244990b8cSJulian Elischer int 87344990b8cSJulian Elischer thread_suspend_check(int return_instead) 87444990b8cSJulian Elischer { 875ecafb24bSJuli Mallett struct thread *td; 876ecafb24bSJuli Mallett struct proc *p; 87744990b8cSJulian Elischer 87844990b8cSJulian Elischer td = curthread; 87944990b8cSJulian Elischer p = td->td_proc; 88037814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 88144990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 882cbf4e354SDavid Xu while (P_SHOULDSTOP(p) || 883cbf4e354SDavid Xu ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) { 8841279572aSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 88544990b8cSJulian Elischer KASSERT(p->p_singlethread != NULL, 88644990b8cSJulian Elischer ("singlethread not set")); 88744990b8cSJulian Elischer /* 888e3b9bf71SJulian Elischer * The only suspension in action is a 889e3b9bf71SJulian Elischer * single-threading. Single threader need not stop. 890b6d5995eSJulian Elischer * XXX Should be safe to access unlocked 891b6d5995eSJulian Elischer * as it can only be set to be true by us. 89244990b8cSJulian Elischer */ 893e3b9bf71SJulian Elischer if (p->p_singlethread == td) 89444990b8cSJulian Elischer return (0); /* Exempt from stopping. */ 89544990b8cSJulian Elischer } 89645a4bfa1SDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 89794f0972bSDavid Xu return (EINTR); 89844990b8cSJulian Elischer 899906ac69dSDavid Xu /* Should we goto user boundary if we didn't come from there? */ 900906ac69dSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 901906ac69dSDavid Xu (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 90294f0972bSDavid Xu return (ERESTART); 903906ac69dSDavid Xu 9049104847fSDavid Xu /* If thread will exit, flush its pending signals */ 9059104847fSDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 9069104847fSDavid Xu sigqueue_flush(&td->td_sigqueue); 9079104847fSDavid Xu 908e574e444SDavid Xu mtx_lock_spin(&sched_lock); 909e574e444SDavid Xu thread_stopped(p); 91044990b8cSJulian Elischer /* 91144990b8cSJulian Elischer * If the process is waiting for us to exit, 91244990b8cSJulian Elischer * this thread should just suicide. 9131279572aSDavid Xu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 91444990b8cSJulian Elischer */ 915906ac69dSDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 91644990b8cSJulian Elischer thread_exit(); 91744990b8cSJulian Elischer 91844990b8cSJulian Elischer /* 91944990b8cSJulian Elischer * When a thread suspends, it just 92044990b8cSJulian Elischer * moves to the processes's suspend queue 92144990b8cSJulian Elischer * and stays there. 92244990b8cSJulian Elischer */ 92371fad9fdSJulian Elischer thread_suspend_one(td); 924906ac69dSDavid Xu if (return_instead == 0) { 925906ac69dSDavid Xu p->p_boundary_count++; 926906ac69dSDavid Xu td->td_flags |= TDF_BOUNDARY; 927cf19bf91SJulian Elischer } 928906ac69dSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 929906ac69dSDavid Xu if (p->p_numthreads == p->p_suspcount) 930906ac69dSDavid Xu thread_unsuspend_one(p->p_singlethread); 931cf19bf91SJulian Elischer } 932a6f37ac9SJohn Baldwin PROC_UNLOCK(p); 933bf0acc27SJohn Baldwin mi_switch(SW_INVOL, NULL); 934906ac69dSDavid Xu if (return_instead == 0) { 935906ac69dSDavid Xu p->p_boundary_count--; 936906ac69dSDavid Xu td->td_flags &= ~TDF_BOUNDARY; 937906ac69dSDavid Xu } 93844990b8cSJulian Elischer mtx_unlock_spin(&sched_lock); 93944990b8cSJulian Elischer PROC_LOCK(p); 94044990b8cSJulian Elischer } 94144990b8cSJulian Elischer return (0); 94244990b8cSJulian Elischer } 94344990b8cSJulian Elischer 94435c32a76SDavid Xu void 94535c32a76SDavid Xu thread_suspend_one(struct thread *td) 94635c32a76SDavid Xu { 94735c32a76SDavid Xu struct proc *p = td->td_proc; 94835c32a76SDavid Xu 94935c32a76SDavid Xu mtx_assert(&sched_lock, MA_OWNED); 950112afcb2SJohn Baldwin PROC_LOCK_ASSERT(p, MA_OWNED); 951e574e444SDavid Xu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 95235c32a76SDavid Xu p->p_suspcount++; 95371fad9fdSJulian Elischer TD_SET_SUSPENDED(td); 95435c32a76SDavid Xu TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 95535c32a76SDavid Xu } 95635c32a76SDavid Xu 95735c32a76SDavid Xu void 95835c32a76SDavid Xu thread_unsuspend_one(struct thread *td) 95935c32a76SDavid Xu { 96035c32a76SDavid Xu struct proc *p = td->td_proc; 96135c32a76SDavid Xu 96235c32a76SDavid Xu mtx_assert(&sched_lock, MA_OWNED); 963112afcb2SJohn Baldwin PROC_LOCK_ASSERT(p, MA_OWNED); 96435c32a76SDavid Xu TAILQ_REMOVE(&p->p_suspended, td, td_runq); 96571fad9fdSJulian Elischer TD_CLR_SUSPENDED(td); 96635c32a76SDavid Xu p->p_suspcount--; 96771fad9fdSJulian Elischer setrunnable(td); 96835c32a76SDavid Xu } 96935c32a76SDavid Xu 97044990b8cSJulian Elischer /* 97144990b8cSJulian Elischer * Allow all threads blocked by single threading to continue running. 97244990b8cSJulian Elischer */ 97344990b8cSJulian Elischer void 97444990b8cSJulian Elischer thread_unsuspend(struct proc *p) 97544990b8cSJulian Elischer { 97644990b8cSJulian Elischer struct thread *td; 97744990b8cSJulian Elischer 978b6d5995eSJulian Elischer mtx_assert(&sched_lock, MA_OWNED); 97944990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 98044990b8cSJulian Elischer if (!P_SHOULDSTOP(p)) { 98144990b8cSJulian Elischer while ((td = TAILQ_FIRST(&p->p_suspended))) { 98235c32a76SDavid Xu thread_unsuspend_one(td); 98344990b8cSJulian Elischer } 9841279572aSDavid Xu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 98544990b8cSJulian Elischer (p->p_numthreads == p->p_suspcount)) { 98644990b8cSJulian Elischer /* 98744990b8cSJulian Elischer * Stopping everything also did the job for the single 98844990b8cSJulian Elischer * threading request. Now we've downgraded to single-threaded, 98944990b8cSJulian Elischer * let it continue. 99044990b8cSJulian Elischer */ 99135c32a76SDavid Xu thread_unsuspend_one(p->p_singlethread); 99244990b8cSJulian Elischer } 99344990b8cSJulian Elischer } 99444990b8cSJulian Elischer 995ed062c8dSJulian Elischer /* 996ed062c8dSJulian Elischer * End the single threading mode.. 997ed062c8dSJulian Elischer */ 99844990b8cSJulian Elischer void 99944990b8cSJulian Elischer thread_single_end(void) 100044990b8cSJulian Elischer { 100144990b8cSJulian Elischer struct thread *td; 100244990b8cSJulian Elischer struct proc *p; 100344990b8cSJulian Elischer 100444990b8cSJulian Elischer td = curthread; 100544990b8cSJulian Elischer p = td->td_proc; 100644990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 1007906ac69dSDavid Xu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); 1008112afcb2SJohn Baldwin mtx_lock_spin(&sched_lock); 100944990b8cSJulian Elischer p->p_singlethread = NULL; 101021fc3164SDavid Xu p->p_procscopegrp = NULL; 101149539972SJulian Elischer /* 101249539972SJulian Elischer * If there are other threads they mey now run, 101349539972SJulian Elischer * unless of course there is a blanket 'stop order' 101449539972SJulian Elischer * on the process. The single threader must be allowed 101549539972SJulian Elischer * to continue however as this is a bad place to stop. 101649539972SJulian Elischer */ 101749539972SJulian Elischer if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 101849539972SJulian Elischer while ((td = TAILQ_FIRST(&p->p_suspended))) { 101971fad9fdSJulian Elischer thread_unsuspend_one(td); 102044990b8cSJulian Elischer } 102149539972SJulian Elischer } 1022112afcb2SJohn Baldwin mtx_unlock_spin(&sched_lock); 102349539972SJulian Elischer } 10244fc21c09SDaniel Eischen 1025007ddf7eSJohn Baldwin /* 1026007ddf7eSJohn Baldwin * Called before going into an interruptible sleep to see if we have been 1027007ddf7eSJohn Baldwin * interrupted or requested to exit. 1028007ddf7eSJohn Baldwin */ 1029007ddf7eSJohn Baldwin int 1030007ddf7eSJohn Baldwin thread_sleep_check(struct thread *td) 1031007ddf7eSJohn Baldwin { 1032007ddf7eSJohn Baldwin struct proc *p; 1033007ddf7eSJohn Baldwin 1034007ddf7eSJohn Baldwin p = td->td_proc; 1035007ddf7eSJohn Baldwin mtx_assert(&sched_lock, MA_OWNED); 10362179a22cSJulian Elischer if (p->p_flag & P_HADTHREADS) { 1037906ac69dSDavid Xu if (p->p_singlethread != td) { 1038906ac69dSDavid Xu if (p->p_flag & P_SINGLE_EXIT) 1039007ddf7eSJohn Baldwin return (EINTR); 1040906ac69dSDavid Xu if (p->p_flag & P_SINGLE_BOUNDARY) 1041906ac69dSDavid Xu return (ERESTART); 1042906ac69dSDavid Xu } 1043007ddf7eSJohn Baldwin if (td->td_flags & TDF_INTERRUPT) 1044007ddf7eSJohn Baldwin return (td->td_intrval); 1045007ddf7eSJohn Baldwin } 1046007ddf7eSJohn Baldwin return (0); 1047007ddf7eSJohn Baldwin } 104844355392SDavid Xu 104944355392SDavid Xu struct thread * 105044355392SDavid Xu thread_find(struct proc *p, lwpid_t tid) 105144355392SDavid Xu { 105244355392SDavid Xu struct thread *td; 105344355392SDavid Xu 105444355392SDavid Xu PROC_LOCK_ASSERT(p, MA_OWNED); 105544355392SDavid Xu mtx_lock_spin(&sched_lock); 105644355392SDavid Xu FOREACH_THREAD_IN_PROC(p, td) { 105744355392SDavid Xu if (td->td_tid == tid) 105844355392SDavid Xu break; 105944355392SDavid Xu } 106044355392SDavid Xu mtx_unlock_spin(&sched_lock); 106144355392SDavid Xu return (td); 106244355392SDavid Xu } 1063