19454b2d8SWarner Losh /*- 244990b8cSJulian Elischer * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 344990b8cSJulian Elischer * All rights reserved. 444990b8cSJulian Elischer * 544990b8cSJulian Elischer * Redistribution and use in source and binary forms, with or without 644990b8cSJulian Elischer * modification, are permitted provided that the following conditions 744990b8cSJulian Elischer * are met: 844990b8cSJulian Elischer * 1. Redistributions of source code must retain the above copyright 944990b8cSJulian Elischer * notice(s), this list of conditions and the following disclaimer as 1044990b8cSJulian Elischer * the first lines of this file unmodified other than the possible 1144990b8cSJulian Elischer * addition of one or more copyright notices. 1244990b8cSJulian Elischer * 2. Redistributions in binary form must reproduce the above copyright 1344990b8cSJulian Elischer * notice(s), this list of conditions and the following disclaimer in the 1444990b8cSJulian Elischer * documentation and/or other materials provided with the distribution. 1544990b8cSJulian Elischer * 1644990b8cSJulian Elischer * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1744990b8cSJulian Elischer * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1844990b8cSJulian Elischer * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1944990b8cSJulian Elischer * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2044990b8cSJulian Elischer * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2144990b8cSJulian Elischer * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2244990b8cSJulian Elischer * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2344990b8cSJulian Elischer * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2444990b8cSJulian Elischer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2544990b8cSJulian Elischer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2644990b8cSJulian Elischer * DAMAGE. 2744990b8cSJulian Elischer */ 2844990b8cSJulian Elischer 293d06b4b3SAttilio Rao #include "opt_witness.h" 3016d95d4fSJoseph Koshy #include "opt_hwpmc_hooks.h" 313d06b4b3SAttilio Rao 32677b542eSDavid E. O'Brien #include <sys/cdefs.h> 33677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 34677b542eSDavid E. O'Brien 3544990b8cSJulian Elischer #include <sys/param.h> 3644990b8cSJulian Elischer #include <sys/systm.h> 3744990b8cSJulian Elischer #include <sys/kernel.h> 3844990b8cSJulian Elischer #include <sys/lock.h> 3944990b8cSJulian Elischer #include <sys/mutex.h> 4044990b8cSJulian Elischer #include <sys/proc.h> 41e170bfdaSDavid Xu #include <sys/resourcevar.h> 4294e0a4cdSJulian Elischer #include <sys/smp.h> 4344990b8cSJulian Elischer #include <sys/sysctl.h> 44de028f5aSJeff Roberson #include <sys/sched.h> 4544f3b092SJohn Baldwin #include <sys/sleepqueue.h> 46ace8398dSJeff Roberson #include <sys/selinfo.h> 47961a7b24SJohn Baldwin #include <sys/turnstile.h> 4844990b8cSJulian Elischer #include <sys/ktr.h> 49bc8e6d81SDavid Xu #include <sys/umtx.h> 50d7f687fcSJeff Roberson #include <sys/cpuset.h> 5116d95d4fSJoseph Koshy #ifdef HWPMC_HOOKS 5216d95d4fSJoseph Koshy #include <sys/pmckern.h> 5316d95d4fSJoseph Koshy #endif 5444990b8cSJulian Elischer 55911b84b0SRobert Watson #include <security/audit/audit.h> 56911b84b0SRobert Watson 5744990b8cSJulian Elischer #include <vm/vm.h> 5849a2507bSAlan Cox #include <vm/vm_extern.h> 5944990b8cSJulian Elischer #include <vm/uma.h> 60b209f889SRandall Stewart #include <sys/eventhandler.h> 6102fb42b0SPeter Wemm 628460a577SJohn Birrell /* 638460a577SJohn Birrell * thread related storage. 648460a577SJohn Birrell */ 6544990b8cSJulian Elischer static uma_zone_t thread_zone; 6644990b8cSJulian Elischer 6744990b8cSJulian Elischer SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 68fdc5ecd2SDavid Xu 69345ad866SJulian Elischer int max_threads_per_proc = 1500; 70fdc5ecd2SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 714f0db5e0SJulian Elischer &max_threads_per_proc, 0, "Limit on threads per proc"); 724f0db5e0SJulian Elischer 73345ad866SJulian Elischer int max_threads_hits; 740252d203SDavid Xu SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 750252d203SDavid Xu &max_threads_hits, 0, ""); 760252d203SDavid Xu 775215b187SJeff Roberson TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 78c8790f5dSAttilio Rao static struct mtx zombie_lock; 79a54e85fdSJeff Roberson MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); 8044990b8cSJulian Elischer 81ff8fbcffSJeff Roberson static void thread_zombie(struct thread *); 82ff8fbcffSJeff Roberson 83fdcac928SMarcel Moolenaar struct mtx tid_lock; 841ea7a6f8SPoul-Henning Kamp static struct unrhdr *tid_unrhdr; 85fdcac928SMarcel Moolenaar 86fdcac928SMarcel Moolenaar /* 87696058c3SJulian Elischer * Prepare a thread for use. 8844990b8cSJulian Elischer */ 89b23f72e9SBrian Feldman static int 90b23f72e9SBrian Feldman thread_ctor(void *mem, int size, void *arg, int flags) 9144990b8cSJulian Elischer { 9244990b8cSJulian Elischer struct thread *td; 9344990b8cSJulian Elischer 9444990b8cSJulian Elischer td = (struct thread *)mem; 9571fad9fdSJulian Elischer td->td_state = TDS_INACTIVE; 96060563ecSJulian Elischer td->td_oncpu = NOCPU; 976c27c603SJuli Mallett 98773eff9dSPoul-Henning Kamp td->td_tid = alloc_unr(tid_unrhdr); 99f9bb7538SMohan Srinivasan td->td_syscalls = 0; 100773eff9dSPoul-Henning Kamp 1016c27c603SJuli Mallett /* 1026c27c603SJuli Mallett * Note that td_critnest begins life as 1 because the thread is not 1036c27c603SJuli Mallett * running and is thereby implicitly waiting to be on the receiving 104a54e85fdSJeff Roberson * end of a context switch. 1056c27c603SJuli Mallett */ 106139b7550SJohn Baldwin td->td_critnest = 1; 107b209f889SRandall Stewart EVENTHANDLER_INVOKE(thread_ctor, td); 108911b84b0SRobert Watson #ifdef AUDIT 109911b84b0SRobert Watson audit_thread_alloc(td); 110911b84b0SRobert Watson #endif 111d10183d9SDavid Xu umtx_thread_alloc(td); 112b23f72e9SBrian Feldman return (0); 11344990b8cSJulian Elischer } 11444990b8cSJulian Elischer 11544990b8cSJulian Elischer /* 11644990b8cSJulian Elischer * Reclaim a thread after use. 11744990b8cSJulian Elischer */ 11844990b8cSJulian Elischer static void 11944990b8cSJulian Elischer thread_dtor(void *mem, int size, void *arg) 12044990b8cSJulian Elischer { 12144990b8cSJulian Elischer struct thread *td; 12244990b8cSJulian Elischer 12344990b8cSJulian Elischer td = (struct thread *)mem; 12444990b8cSJulian Elischer 12544990b8cSJulian Elischer #ifdef INVARIANTS 12644990b8cSJulian Elischer /* Verify that this thread is in a safe state to free. */ 12744990b8cSJulian Elischer switch (td->td_state) { 12871fad9fdSJulian Elischer case TDS_INHIBITED: 12971fad9fdSJulian Elischer case TDS_RUNNING: 13071fad9fdSJulian Elischer case TDS_CAN_RUN: 13144990b8cSJulian Elischer case TDS_RUNQ: 13244990b8cSJulian Elischer /* 13344990b8cSJulian Elischer * We must never unlink a thread that is in one of 13444990b8cSJulian Elischer * these states, because it is currently active. 13544990b8cSJulian Elischer */ 13644990b8cSJulian Elischer panic("bad state for thread unlinking"); 13744990b8cSJulian Elischer /* NOTREACHED */ 13871fad9fdSJulian Elischer case TDS_INACTIVE: 13944990b8cSJulian Elischer break; 14044990b8cSJulian Elischer default: 14144990b8cSJulian Elischer panic("bad thread state"); 14244990b8cSJulian Elischer /* NOTREACHED */ 14344990b8cSJulian Elischer } 14444990b8cSJulian Elischer #endif 1456e8525ceSRobert Watson #ifdef AUDIT 1466e8525ceSRobert Watson audit_thread_free(td); 1476e8525ceSRobert Watson #endif 1481ba4a712SPawel Jakub Dawidek /* Free all OSD associated to this thread. */ 1491ba4a712SPawel Jakub Dawidek osd_thread_exit(td); 1501ba4a712SPawel Jakub Dawidek 151b209f889SRandall Stewart EVENTHANDLER_INVOKE(thread_dtor, td); 152773eff9dSPoul-Henning Kamp free_unr(tid_unrhdr, td->td_tid); 15344990b8cSJulian Elischer } 15444990b8cSJulian Elischer 15544990b8cSJulian Elischer /* 15644990b8cSJulian Elischer * Initialize type-stable parts of a thread (when newly created). 15744990b8cSJulian Elischer */ 158b23f72e9SBrian Feldman static int 159b23f72e9SBrian Feldman thread_init(void *mem, int size, int flags) 16044990b8cSJulian Elischer { 16144990b8cSJulian Elischer struct thread *td; 16244990b8cSJulian Elischer 16344990b8cSJulian Elischer td = (struct thread *)mem; 164247aba24SMarcel Moolenaar 16544f3b092SJohn Baldwin td->td_sleepqueue = sleepq_alloc(); 166961a7b24SJohn Baldwin td->td_turnstile = turnstile_alloc(); 167b209f889SRandall Stewart EVENTHANDLER_INVOKE(thread_init, td); 168de028f5aSJeff Roberson td->td_sched = (struct td_sched *)&td[1]; 169d10183d9SDavid Xu umtx_thread_init(td); 17089b57fcfSKonstantin Belousov td->td_kstack = 0; 171b23f72e9SBrian Feldman return (0); 17244990b8cSJulian Elischer } 17344990b8cSJulian Elischer 17444990b8cSJulian Elischer /* 17544990b8cSJulian Elischer * Tear down type-stable parts of a thread (just before being discarded). 17644990b8cSJulian Elischer */ 17744990b8cSJulian Elischer static void 17844990b8cSJulian Elischer thread_fini(void *mem, int size) 17944990b8cSJulian Elischer { 18044990b8cSJulian Elischer struct thread *td; 18144990b8cSJulian Elischer 18244990b8cSJulian Elischer td = (struct thread *)mem; 183b209f889SRandall Stewart EVENTHANDLER_INVOKE(thread_fini, td); 184961a7b24SJohn Baldwin turnstile_free(td->td_turnstile); 18544f3b092SJohn Baldwin sleepq_free(td->td_sleepqueue); 186d10183d9SDavid Xu umtx_thread_fini(td); 187ace8398dSJeff Roberson seltdfini(td); 18844990b8cSJulian Elischer } 1895215b187SJeff Roberson 1905c8329edSJulian Elischer /* 1915215b187SJeff Roberson * For a newly created process, 1925215b187SJeff Roberson * link up all the structures and its initial threads etc. 193ed062c8dSJulian Elischer * called from: 194ed062c8dSJulian Elischer * {arch}/{arch}/machdep.c ia64_init(), init386() etc. 195ed062c8dSJulian Elischer * proc_dtor() (should go away) 196ed062c8dSJulian Elischer * proc_init() 1975c8329edSJulian Elischer */ 1985c8329edSJulian Elischer void 19989b57fcfSKonstantin Belousov proc_linkup0(struct proc *p, struct thread *td) 20089b57fcfSKonstantin Belousov { 20189b57fcfSKonstantin Belousov TAILQ_INIT(&p->p_threads); /* all threads in proc */ 20289b57fcfSKonstantin Belousov proc_linkup(p, td); 20389b57fcfSKonstantin Belousov } 20489b57fcfSKonstantin Belousov 20589b57fcfSKonstantin Belousov void 2068460a577SJohn Birrell proc_linkup(struct proc *p, struct thread *td) 2075c8329edSJulian Elischer { 208a54e85fdSJeff Roberson 2099104847fSDavid Xu sigqueue_init(&p->p_sigqueue, p); 210ebceaf6dSDavid Xu p->p_ksi = ksiginfo_alloc(1); 211ebceaf6dSDavid Xu if (p->p_ksi != NULL) { 2125c474517SDavid Xu /* XXX p_ksi may be null if ksiginfo zone is not ready */ 213ebceaf6dSDavid Xu p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 214ebceaf6dSDavid Xu } 215b2f92ef9SDavid Xu LIST_INIT(&p->p_mqnotifier); 2165c8329edSJulian Elischer p->p_numthreads = 0; 2178460a577SJohn Birrell thread_link(td, p); 2185c8329edSJulian Elischer } 2195c8329edSJulian Elischer 2205c8329edSJulian Elischer /* 22144990b8cSJulian Elischer * Initialize global thread allocation resources. 22244990b8cSJulian Elischer */ 22344990b8cSJulian Elischer void 22444990b8cSJulian Elischer threadinit(void) 22544990b8cSJulian Elischer { 22644990b8cSJulian Elischer 2271ea7a6f8SPoul-Henning Kamp mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 2286829a5c5SJulian Elischer /* leave one number for thread0 */ 2296829a5c5SJulian Elischer tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock); 2301ea7a6f8SPoul-Henning Kamp 231de028f5aSJeff Roberson thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 23244990b8cSJulian Elischer thread_ctor, thread_dtor, thread_init, thread_fini, 2334649e92bSJohn Baldwin 16 - 1, 0); 23444990b8cSJulian Elischer } 23544990b8cSJulian Elischer 23644990b8cSJulian Elischer /* 237ff8fbcffSJeff Roberson * Place an unused thread on the zombie list. 238ad1e7d28SJulian Elischer * Use the slpq as that must be unused by now. 23944990b8cSJulian Elischer */ 24044990b8cSJulian Elischer void 241ff8fbcffSJeff Roberson thread_zombie(struct thread *td) 24244990b8cSJulian Elischer { 243a54e85fdSJeff Roberson mtx_lock_spin(&zombie_lock); 244ad1e7d28SJulian Elischer TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); 245a54e85fdSJeff Roberson mtx_unlock_spin(&zombie_lock); 24644990b8cSJulian Elischer } 24744990b8cSJulian Elischer 2485c8329edSJulian Elischer /* 249ff8fbcffSJeff Roberson * Release a thread that has exited after cpu_throw(). 250ff8fbcffSJeff Roberson */ 251ff8fbcffSJeff Roberson void 252ff8fbcffSJeff Roberson thread_stash(struct thread *td) 253ff8fbcffSJeff Roberson { 254ff8fbcffSJeff Roberson atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); 255ff8fbcffSJeff Roberson thread_zombie(td); 256ff8fbcffSJeff Roberson } 257ff8fbcffSJeff Roberson 258ff8fbcffSJeff Roberson /* 2596617724cSJeff Roberson * Reap zombie resources. 26044990b8cSJulian Elischer */ 26144990b8cSJulian Elischer void 26244990b8cSJulian Elischer thread_reap(void) 26344990b8cSJulian Elischer { 2645c8329edSJulian Elischer struct thread *td_first, *td_next; 26544990b8cSJulian Elischer 26644990b8cSJulian Elischer /* 2675215b187SJeff Roberson * Don't even bother to lock if none at this instant, 2685215b187SJeff Roberson * we really don't care about the next instant.. 26944990b8cSJulian Elischer */ 2708460a577SJohn Birrell if (!TAILQ_EMPTY(&zombie_threads)) { 271a54e85fdSJeff Roberson mtx_lock_spin(&zombie_lock); 2725c8329edSJulian Elischer td_first = TAILQ_FIRST(&zombie_threads); 2735c8329edSJulian Elischer if (td_first) 2745c8329edSJulian Elischer TAILQ_INIT(&zombie_threads); 275a54e85fdSJeff Roberson mtx_unlock_spin(&zombie_lock); 2765c8329edSJulian Elischer while (td_first) { 277ad1e7d28SJulian Elischer td_next = TAILQ_NEXT(td_first, td_slpq); 2785215b187SJeff Roberson if (td_first->td_ucred) 2795215b187SJeff Roberson crfree(td_first->td_ucred); 2805c8329edSJulian Elischer thread_free(td_first); 2815c8329edSJulian Elischer td_first = td_next; 28244990b8cSJulian Elischer } 28344990b8cSJulian Elischer } 284ed062c8dSJulian Elischer } 28544990b8cSJulian Elischer 2864f0db5e0SJulian Elischer /* 28744990b8cSJulian Elischer * Allocate a thread. 28844990b8cSJulian Elischer */ 28944990b8cSJulian Elischer struct thread * 2908a945d10SKonstantin Belousov thread_alloc(int pages) 29144990b8cSJulian Elischer { 29289b57fcfSKonstantin Belousov struct thread *td; 2938460a577SJohn Birrell 29444990b8cSJulian Elischer thread_reap(); /* check if any zombies to get */ 29589b57fcfSKonstantin Belousov 29689b57fcfSKonstantin Belousov td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK); 29789b57fcfSKonstantin Belousov KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); 2988a945d10SKonstantin Belousov if (!vm_thread_new(td, pages)) { 29989b57fcfSKonstantin Belousov uma_zfree(thread_zone, td); 30089b57fcfSKonstantin Belousov return (NULL); 30189b57fcfSKonstantin Belousov } 3020c3967e7SMarcel Moolenaar cpu_thread_alloc(td); 30389b57fcfSKonstantin Belousov return (td); 30444990b8cSJulian Elischer } 30544990b8cSJulian Elischer 3068a945d10SKonstantin Belousov int 3078a945d10SKonstantin Belousov thread_alloc_stack(struct thread *td, int pages) 3088a945d10SKonstantin Belousov { 3098a945d10SKonstantin Belousov 3108a945d10SKonstantin Belousov KASSERT(td->td_kstack == 0, 3118a945d10SKonstantin Belousov ("thread_alloc_stack called on a thread with kstack")); 3128a945d10SKonstantin Belousov if (!vm_thread_new(td, pages)) 3138a945d10SKonstantin Belousov return (0); 3148a945d10SKonstantin Belousov cpu_thread_alloc(td); 3158a945d10SKonstantin Belousov return (1); 3168a945d10SKonstantin Belousov } 3174f0db5e0SJulian Elischer 3184f0db5e0SJulian Elischer /* 31944990b8cSJulian Elischer * Deallocate a thread. 32044990b8cSJulian Elischer */ 32144990b8cSJulian Elischer void 32244990b8cSJulian Elischer thread_free(struct thread *td) 32344990b8cSJulian Elischer { 3242e6b8de4SJeff Roberson 3252e6b8de4SJeff Roberson lock_profile_thread_exit(td); 32645aea8deSJeff Roberson if (td->td_cpuset) 327d7f687fcSJeff Roberson cpuset_rel(td->td_cpuset); 328d7f687fcSJeff Roberson td->td_cpuset = NULL; 3290c3967e7SMarcel Moolenaar cpu_thread_free(td); 33089b57fcfSKonstantin Belousov if (td->td_kstack != 0) 33189b57fcfSKonstantin Belousov vm_thread_dispose(td); 33244990b8cSJulian Elischer uma_zfree(thread_zone, td); 33344990b8cSJulian Elischer } 33444990b8cSJulian Elischer 33544990b8cSJulian Elischer /* 33644990b8cSJulian Elischer * Discard the current thread and exit from its context. 33794e0a4cdSJulian Elischer * Always called with scheduler locked. 33844990b8cSJulian Elischer * 33944990b8cSJulian Elischer * Because we can't free a thread while we're operating under its context, 340696058c3SJulian Elischer * push the current thread into our CPU's deadthread holder. This means 341696058c3SJulian Elischer * we needn't worry about someone else grabbing our context before we 3426617724cSJeff Roberson * do a cpu_throw(). 34344990b8cSJulian Elischer */ 34444990b8cSJulian Elischer void 34544990b8cSJulian Elischer thread_exit(void) 34644990b8cSJulian Elischer { 347e170bfdaSDavid Xu uint64_t new_switchtime; 34844990b8cSJulian Elischer struct thread *td; 3491c4bcd05SJeff Roberson struct thread *td2; 35044990b8cSJulian Elischer struct proc *p; 3517847a9daSJohn Baldwin int wakeup_swapper; 35244990b8cSJulian Elischer 35344990b8cSJulian Elischer td = curthread; 35444990b8cSJulian Elischer p = td->td_proc; 35544990b8cSJulian Elischer 356a54e85fdSJeff Roberson PROC_SLOCK_ASSERT(p, MA_OWNED); 357ed062c8dSJulian Elischer mtx_assert(&Giant, MA_NOTOWNED); 358a54e85fdSJeff Roberson 35944990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 360ed062c8dSJulian Elischer KASSERT(p != NULL, ("thread exiting without a process")); 361cc701b73SRobert Watson CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 362e01eafefSJulian Elischer (long)p->p_pid, td->td_name); 3639104847fSDavid Xu KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 36444990b8cSJulian Elischer 36589964dd2SRobert Watson #ifdef AUDIT 36689964dd2SRobert Watson AUDIT_SYSCALL_EXIT(0, td); 36789964dd2SRobert Watson #endif 368d10183d9SDavid Xu umtx_thread_exit(td); 369ed062c8dSJulian Elischer /* 370ed062c8dSJulian Elischer * drop FPU & debug register state storage, or any other 371ed062c8dSJulian Elischer * architecture specific resources that 372ed062c8dSJulian Elischer * would not be on a new untouched process. 373ed062c8dSJulian Elischer */ 37444990b8cSJulian Elischer cpu_thread_exit(td); /* XXXSMP */ 37544990b8cSJulian Elischer 376e170bfdaSDavid Xu /* Do the same timestamp bookkeeping that mi_switch() would do. */ 377e170bfdaSDavid Xu new_switchtime = cpu_ticks(); 378e170bfdaSDavid Xu p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime)); 379e170bfdaSDavid Xu PCPU_SET(switchtime, new_switchtime); 380e170bfdaSDavid Xu PCPU_SET(switchticks, ticks); 381b4b70819SAttilio Rao PCPU_INC(cnt.v_swtch); 382a140976eSAttilio Rao /* Save our resource usage in our process. */ 383a140976eSAttilio Rao td->td_ru.ru_nvcsw++; 384a140976eSAttilio Rao rucollect(&p->p_ru, &td->td_ru); 385ed062c8dSJulian Elischer /* 3861faf202eSJulian Elischer * The last thread is left attached to the process 3871faf202eSJulian Elischer * So that the whole bundle gets recycled. Skip 388ed062c8dSJulian Elischer * all this stuff if we never had threads. 389ed062c8dSJulian Elischer * EXIT clears all sign of other threads when 390ed062c8dSJulian Elischer * it goes to single threading, so the last thread always 391ed062c8dSJulian Elischer * takes the short path. 3921faf202eSJulian Elischer */ 393ed062c8dSJulian Elischer if (p->p_flag & P_HADTHREADS) { 3941faf202eSJulian Elischer if (p->p_numthreads > 1) { 395d3a0bd78SJulian Elischer thread_unlink(td); 3961c4bcd05SJeff Roberson td2 = FIRST_THREAD_IN_PROC(p); 3971c4bcd05SJeff Roberson sched_exit_thread(td2, td); 398ed062c8dSJulian Elischer 399ed062c8dSJulian Elischer /* 40044990b8cSJulian Elischer * The test below is NOT true if we are the 4011faf202eSJulian Elischer * sole exiting thread. P_STOPPED_SNGL is unset 40244990b8cSJulian Elischer * in exit1() after it is the only survivor. 40344990b8cSJulian Elischer */ 4041279572aSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 40544990b8cSJulian Elischer if (p->p_numthreads == p->p_suspcount) { 406a54e85fdSJeff Roberson thread_lock(p->p_singlethread); 4077847a9daSJohn Baldwin wakeup_swapper = thread_unsuspend_one( 4087847a9daSJohn Baldwin p->p_singlethread); 409a54e85fdSJeff Roberson thread_unlock(p->p_singlethread); 4107847a9daSJohn Baldwin if (wakeup_swapper) 4117847a9daSJohn Baldwin kick_proc0(); 41244990b8cSJulian Elischer } 41344990b8cSJulian Elischer } 41448bfcdddSJulian Elischer 415ff8fbcffSJeff Roberson atomic_add_int(&td->td_proc->p_exitthreads, 1); 416696058c3SJulian Elischer PCPU_SET(deadthread, td); 4171faf202eSJulian Elischer } else { 418ed062c8dSJulian Elischer /* 419ed062c8dSJulian Elischer * The last thread is exiting.. but not through exit() 420ed062c8dSJulian Elischer */ 421ed062c8dSJulian Elischer panic ("thread_exit: Last thread exiting on its own"); 422ed062c8dSJulian Elischer } 4231faf202eSJulian Elischer } 42416d95d4fSJoseph Koshy #ifdef HWPMC_HOOKS 42516d95d4fSJoseph Koshy /* 42616d95d4fSJoseph Koshy * If this thread is part of a process that is being tracked by hwpmc(4), 42716d95d4fSJoseph Koshy * inform the module of the thread's impending exit. 42816d95d4fSJoseph Koshy */ 42916d95d4fSJoseph Koshy if (PMC_PROC_IS_USING_PMCS(td->td_proc)) 43016d95d4fSJoseph Koshy PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); 43116d95d4fSJoseph Koshy #endif 432a54e85fdSJeff Roberson PROC_UNLOCK(p); 433a54e85fdSJeff Roberson thread_lock(td); 434a140976eSAttilio Rao /* Save our tick information with both the thread and proc locked */ 435a54e85fdSJeff Roberson ruxagg(&p->p_rux, td); 436a54e85fdSJeff Roberson PROC_SUNLOCK(p); 437dcc9954eSJulian Elischer td->td_state = TDS_INACTIVE; 4383d06b4b3SAttilio Rao #ifdef WITNESS 4393d06b4b3SAttilio Rao witness_thread_exit(td); 4403d06b4b3SAttilio Rao #endif 441732d9528SJulian Elischer CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 442a54e85fdSJeff Roberson sched_throw(td); 443cc66ebe2SPeter Wemm panic("I'm a teapot!"); 44444990b8cSJulian Elischer /* NOTREACHED */ 44544990b8cSJulian Elischer } 44644990b8cSJulian Elischer 44744990b8cSJulian Elischer /* 448696058c3SJulian Elischer * Do any thread specific cleanups that may be needed in wait() 44937814395SPeter Wemm * called with Giant, proc and schedlock not held. 450696058c3SJulian Elischer */ 451696058c3SJulian Elischer void 452696058c3SJulian Elischer thread_wait(struct proc *p) 453696058c3SJulian Elischer { 454696058c3SJulian Elischer struct thread *td; 455696058c3SJulian Elischer 45637814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 45785495c72SJens Schweikhardt KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 458ff8fbcffSJeff Roberson td = FIRST_THREAD_IN_PROC(p); 459ff8fbcffSJeff Roberson /* Lock the last thread so we spin until it exits cpu_throw(). */ 460ff8fbcffSJeff Roberson thread_lock(td); 461ff8fbcffSJeff Roberson thread_unlock(td); 462ff8fbcffSJeff Roberson /* Wait for any remaining threads to exit cpu_throw(). */ 463ff8fbcffSJeff Roberson while (p->p_exitthreads) 464ff8fbcffSJeff Roberson sched_relinquish(curthread); 4652e6b8de4SJeff Roberson lock_profile_thread_exit(td); 466d7f687fcSJeff Roberson cpuset_rel(td->td_cpuset); 467d7f687fcSJeff Roberson td->td_cpuset = NULL; 468696058c3SJulian Elischer cpu_thread_clean(td); 469ed062c8dSJulian Elischer crfree(td->td_ucred); 470696058c3SJulian Elischer thread_reap(); /* check for zombie threads etc. */ 471696058c3SJulian Elischer } 472696058c3SJulian Elischer 473696058c3SJulian Elischer /* 47444990b8cSJulian Elischer * Link a thread to a process. 4751faf202eSJulian Elischer * set up anything that needs to be initialized for it to 4761faf202eSJulian Elischer * be used by the process. 47744990b8cSJulian Elischer */ 47844990b8cSJulian Elischer void 4798460a577SJohn Birrell thread_link(struct thread *td, struct proc *p) 48044990b8cSJulian Elischer { 48144990b8cSJulian Elischer 482a54e85fdSJeff Roberson /* 483a54e85fdSJeff Roberson * XXX This can't be enabled because it's called for proc0 before 484374ae2a3SJeff Roberson * its lock has been created. 485374ae2a3SJeff Roberson * PROC_LOCK_ASSERT(p, MA_OWNED); 486a54e85fdSJeff Roberson */ 48771fad9fdSJulian Elischer td->td_state = TDS_INACTIVE; 48844990b8cSJulian Elischer td->td_proc = p; 489b61ce5b0SJeff Roberson td->td_flags = TDF_INMEM; 49044990b8cSJulian Elischer 4911faf202eSJulian Elischer LIST_INIT(&td->td_contested); 492eea4f254SJeff Roberson LIST_INIT(&td->td_lprof[0]); 493eea4f254SJeff Roberson LIST_INIT(&td->td_lprof[1]); 4949104847fSDavid Xu sigqueue_init(&td->td_sigqueue, p); 495c06eb4e2SSam Leffler callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 49644990b8cSJulian Elischer TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 49744990b8cSJulian Elischer p->p_numthreads++; 49844990b8cSJulian Elischer } 49944990b8cSJulian Elischer 500ed062c8dSJulian Elischer /* 501e5bedcefSJulian Elischer * Convert a process with one thread to an unthreaded process. 502e5bedcefSJulian Elischer */ 503e5bedcefSJulian Elischer void 504e5bedcefSJulian Elischer thread_unthread(struct thread *td) 505e5bedcefSJulian Elischer { 506e5bedcefSJulian Elischer struct proc *p = td->td_proc; 507e5bedcefSJulian Elischer 508e5bedcefSJulian Elischer KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); 5098460a577SJohn Birrell p->p_flag &= ~P_HADTHREADS; 510e5bedcefSJulian Elischer } 511e5bedcefSJulian Elischer 512e5bedcefSJulian Elischer /* 513ed062c8dSJulian Elischer * Called from: 514ed062c8dSJulian Elischer * thread_exit() 515ed062c8dSJulian Elischer */ 516d3a0bd78SJulian Elischer void 517d3a0bd78SJulian Elischer thread_unlink(struct thread *td) 518d3a0bd78SJulian Elischer { 519d3a0bd78SJulian Elischer struct proc *p = td->td_proc; 520d3a0bd78SJulian Elischer 521374ae2a3SJeff Roberson PROC_LOCK_ASSERT(p, MA_OWNED); 522d3a0bd78SJulian Elischer TAILQ_REMOVE(&p->p_threads, td, td_plist); 523d3a0bd78SJulian Elischer p->p_numthreads--; 524d3a0bd78SJulian Elischer /* could clear a few other things here */ 5258460a577SJohn Birrell /* Must NOT clear links to proc! */ 5265c8329edSJulian Elischer } 5275c8329edSJulian Elischer 52879799053SKonstantin Belousov static int 52979799053SKonstantin Belousov calc_remaining(struct proc *p, int mode) 53079799053SKonstantin Belousov { 53179799053SKonstantin Belousov int remaining; 53279799053SKonstantin Belousov 53379799053SKonstantin Belousov if (mode == SINGLE_EXIT) 53479799053SKonstantin Belousov remaining = p->p_numthreads; 53579799053SKonstantin Belousov else if (mode == SINGLE_BOUNDARY) 53679799053SKonstantin Belousov remaining = p->p_numthreads - p->p_boundary_count; 53779799053SKonstantin Belousov else if (mode == SINGLE_NO_EXIT) 53879799053SKonstantin Belousov remaining = p->p_numthreads - p->p_suspcount; 53979799053SKonstantin Belousov else 54079799053SKonstantin Belousov panic("calc_remaining: wrong mode %d", mode); 54179799053SKonstantin Belousov return (remaining); 54279799053SKonstantin Belousov } 54379799053SKonstantin Belousov 5445215b187SJeff Roberson /* 54544990b8cSJulian Elischer * Enforce single-threading. 54644990b8cSJulian Elischer * 54744990b8cSJulian Elischer * Returns 1 if the caller must abort (another thread is waiting to 54844990b8cSJulian Elischer * exit the process or similar). Process is locked! 54944990b8cSJulian Elischer * Returns 0 when you are successfully the only thread running. 55044990b8cSJulian Elischer * A process has successfully single threaded in the suspend mode when 55144990b8cSJulian Elischer * There are no threads in user mode. Threads in the kernel must be 55244990b8cSJulian Elischer * allowed to continue until they get to the user boundary. They may even 55344990b8cSJulian Elischer * copy out their return values and data before suspending. They may however be 554e2668f55SMaxim Konovalov * accelerated in reaching the user boundary as we will wake up 55544990b8cSJulian Elischer * any sleeping threads that are interruptable. (PCATCH). 55644990b8cSJulian Elischer */ 55744990b8cSJulian Elischer int 558906ac69dSDavid Xu thread_single(int mode) 55944990b8cSJulian Elischer { 56044990b8cSJulian Elischer struct thread *td; 56144990b8cSJulian Elischer struct thread *td2; 56244990b8cSJulian Elischer struct proc *p; 563da7bbd2cSJohn Baldwin int remaining, wakeup_swapper; 56444990b8cSJulian Elischer 56544990b8cSJulian Elischer td = curthread; 56644990b8cSJulian Elischer p = td->td_proc; 56737814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 56844990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 56944990b8cSJulian Elischer KASSERT((td != NULL), ("curthread is NULL")); 57044990b8cSJulian Elischer 571ed062c8dSJulian Elischer if ((p->p_flag & P_HADTHREADS) == 0) 57244990b8cSJulian Elischer return (0); 57344990b8cSJulian Elischer 574e3b9bf71SJulian Elischer /* Is someone already single threading? */ 575906ac69dSDavid Xu if (p->p_singlethread != NULL && p->p_singlethread != td) 57644990b8cSJulian Elischer return (1); 57744990b8cSJulian Elischer 578906ac69dSDavid Xu if (mode == SINGLE_EXIT) { 579906ac69dSDavid Xu p->p_flag |= P_SINGLE_EXIT; 580906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_BOUNDARY; 581906ac69dSDavid Xu } else { 582906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_EXIT; 583906ac69dSDavid Xu if (mode == SINGLE_BOUNDARY) 584906ac69dSDavid Xu p->p_flag |= P_SINGLE_BOUNDARY; 585906ac69dSDavid Xu else 586906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_BOUNDARY; 587906ac69dSDavid Xu } 5881279572aSDavid Xu p->p_flag |= P_STOPPED_SINGLE; 5897b4a950aSDavid Xu PROC_SLOCK(p); 590112afcb2SJohn Baldwin p->p_singlethread = td; 59179799053SKonstantin Belousov remaining = calc_remaining(p, mode); 592ec008e96SDavid Xu while (remaining != 1) { 593bf1a3220SDavid Xu if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 594bf1a3220SDavid Xu goto stopme; 595da7bbd2cSJohn Baldwin wakeup_swapper = 0; 59644990b8cSJulian Elischer FOREACH_THREAD_IN_PROC(p, td2) { 59744990b8cSJulian Elischer if (td2 == td) 59844990b8cSJulian Elischer continue; 599a54e85fdSJeff Roberson thread_lock(td2); 600b7edba77SJeff Roberson td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK; 60171fad9fdSJulian Elischer if (TD_IS_INHIBITED(td2)) { 602906ac69dSDavid Xu switch (mode) { 603906ac69dSDavid Xu case SINGLE_EXIT: 604906ac69dSDavid Xu if (TD_IS_SUSPENDED(td2)) 6057847a9daSJohn Baldwin wakeup_swapper |= 60671fad9fdSJulian Elischer thread_unsuspend_one(td2); 60733862f40SDavid Xu if (TD_ON_SLEEPQ(td2) && 608906ac69dSDavid Xu (td2->td_flags & TDF_SINTR)) 6097847a9daSJohn Baldwin wakeup_swapper |= 61094f0972bSDavid Xu sleepq_abort(td2, EINTR); 611906ac69dSDavid Xu break; 612906ac69dSDavid Xu case SINGLE_BOUNDARY: 613ffdc5a34SDavid Xu if (TD_IS_SUSPENDED(td2) && 614ffdc5a34SDavid Xu !(td2->td_flags & TDF_BOUNDARY)) 615ffdc5a34SDavid Xu wakeup_swapper |= 616ffdc5a34SDavid Xu thread_unsuspend_one(td2); 617ffdc5a34SDavid Xu if (TD_ON_SLEEPQ(td2) && 618ffdc5a34SDavid Xu (td2->td_flags & TDF_SINTR)) 619ffdc5a34SDavid Xu wakeup_swapper |= 620ffdc5a34SDavid Xu sleepq_abort(td2, ERESTART); 621906ac69dSDavid Xu break; 622f33a947bSKonstantin Belousov case SINGLE_NO_EXIT: 623f33a947bSKonstantin Belousov if (TD_IS_SUSPENDED(td2) && 624f33a947bSKonstantin Belousov !(td2->td_flags & TDF_BOUNDARY)) 625f33a947bSKonstantin Belousov wakeup_swapper |= 626f33a947bSKonstantin Belousov thread_unsuspend_one(td2); 627f33a947bSKonstantin Belousov if (TD_ON_SLEEPQ(td2) && 628f33a947bSKonstantin Belousov (td2->td_flags & TDF_SINTR)) 629f33a947bSKonstantin Belousov wakeup_swapper |= 630f33a947bSKonstantin Belousov sleepq_abort(td2, ERESTART); 631f33a947bSKonstantin Belousov break; 632906ac69dSDavid Xu default: 633906ac69dSDavid Xu break; 63444990b8cSJulian Elischer } 63544990b8cSJulian Elischer } 636d8267df7SDavid Xu #ifdef SMP 637d8267df7SDavid Xu else if (TD_IS_RUNNING(td2) && td != td2) { 638d8267df7SDavid Xu forward_signal(td2); 639d8267df7SDavid Xu } 640d8267df7SDavid Xu #endif 641a54e85fdSJeff Roberson thread_unlock(td2); 6429d102777SJulian Elischer } 643da7bbd2cSJohn Baldwin if (wakeup_swapper) 644da7bbd2cSJohn Baldwin kick_proc0(); 64579799053SKonstantin Belousov remaining = calc_remaining(p, mode); 646ec008e96SDavid Xu 6479d102777SJulian Elischer /* 6489d102777SJulian Elischer * Maybe we suspended some threads.. was it enough? 6499d102777SJulian Elischer */ 650ec008e96SDavid Xu if (remaining == 1) 6519d102777SJulian Elischer break; 6529d102777SJulian Elischer 653bf1a3220SDavid Xu stopme: 65444990b8cSJulian Elischer /* 65544990b8cSJulian Elischer * Wake us up when everyone else has suspended. 656e3b9bf71SJulian Elischer * In the mean time we suspend as well. 65744990b8cSJulian Elischer */ 658a54e85fdSJeff Roberson thread_suspend_switch(td); 65979799053SKonstantin Belousov remaining = calc_remaining(p, mode); 66044990b8cSJulian Elischer } 661906ac69dSDavid Xu if (mode == SINGLE_EXIT) { 66291599697SJulian Elischer /* 66391599697SJulian Elischer * We have gotten rid of all the other threads and we 66491599697SJulian Elischer * are about to either exit or exec. In either case, 66591599697SJulian Elischer * we try our utmost to revert to being a non-threaded 66691599697SJulian Elischer * process. 66791599697SJulian Elischer */ 668ed062c8dSJulian Elischer p->p_singlethread = NULL; 66964895117SDavid Xu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 670e5bedcefSJulian Elischer thread_unthread(td); 67191599697SJulian Elischer } 6727b4a950aSDavid Xu PROC_SUNLOCK(p); 67344990b8cSJulian Elischer return (0); 67444990b8cSJulian Elischer } 67544990b8cSJulian Elischer 67644990b8cSJulian Elischer /* 67744990b8cSJulian Elischer * Called in from locations that can safely check to see 67844990b8cSJulian Elischer * whether we have to suspend or at least throttle for a 67944990b8cSJulian Elischer * single-thread event (e.g. fork). 68044990b8cSJulian Elischer * 68144990b8cSJulian Elischer * Such locations include userret(). 68244990b8cSJulian Elischer * If the "return_instead" argument is non zero, the thread must be able to 68344990b8cSJulian Elischer * accept 0 (caller may continue), or 1 (caller must abort) as a result. 68444990b8cSJulian Elischer * 68544990b8cSJulian Elischer * The 'return_instead' argument tells the function if it may do a 68644990b8cSJulian Elischer * thread_exit() or suspend, or whether the caller must abort and back 68744990b8cSJulian Elischer * out instead. 68844990b8cSJulian Elischer * 68944990b8cSJulian Elischer * If the thread that set the single_threading request has set the 69044990b8cSJulian Elischer * P_SINGLE_EXIT bit in the process flags then this call will never return 69144990b8cSJulian Elischer * if 'return_instead' is false, but will exit. 69244990b8cSJulian Elischer * 69344990b8cSJulian Elischer * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 69444990b8cSJulian Elischer *---------------+--------------------+--------------------- 69544990b8cSJulian Elischer * 0 | returns 0 | returns 0 or 1 69644990b8cSJulian Elischer * | when ST ends | immediatly 69744990b8cSJulian Elischer *---------------+--------------------+--------------------- 69844990b8cSJulian Elischer * 1 | thread exits | returns 1 69944990b8cSJulian Elischer * | | immediatly 70044990b8cSJulian Elischer * 0 = thread_exit() or suspension ok, 70144990b8cSJulian Elischer * other = return error instead of stopping the thread. 70244990b8cSJulian Elischer * 70344990b8cSJulian Elischer * While a full suspension is under effect, even a single threading 70444990b8cSJulian Elischer * thread would be suspended if it made this call (but it shouldn't). 70544990b8cSJulian Elischer * This call should only be made from places where 70644990b8cSJulian Elischer * thread_exit() would be safe as that may be the outcome unless 70744990b8cSJulian Elischer * return_instead is set. 70844990b8cSJulian Elischer */ 70944990b8cSJulian Elischer int 71044990b8cSJulian Elischer thread_suspend_check(int return_instead) 71144990b8cSJulian Elischer { 712ecafb24bSJuli Mallett struct thread *td; 713ecafb24bSJuli Mallett struct proc *p; 7147847a9daSJohn Baldwin int wakeup_swapper; 71544990b8cSJulian Elischer 71644990b8cSJulian Elischer td = curthread; 71744990b8cSJulian Elischer p = td->td_proc; 71837814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 71944990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 720cbf4e354SDavid Xu while (P_SHOULDSTOP(p) || 721904c5ec4SDavid Xu ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) { 7221279572aSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 72344990b8cSJulian Elischer KASSERT(p->p_singlethread != NULL, 72444990b8cSJulian Elischer ("singlethread not set")); 72544990b8cSJulian Elischer /* 726e3b9bf71SJulian Elischer * The only suspension in action is a 727e3b9bf71SJulian Elischer * single-threading. Single threader need not stop. 728b6d5995eSJulian Elischer * XXX Should be safe to access unlocked 729b6d5995eSJulian Elischer * as it can only be set to be true by us. 73044990b8cSJulian Elischer */ 731e3b9bf71SJulian Elischer if (p->p_singlethread == td) 73244990b8cSJulian Elischer return (0); /* Exempt from stopping. */ 73344990b8cSJulian Elischer } 73445a4bfa1SDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 73594f0972bSDavid Xu return (EINTR); 73644990b8cSJulian Elischer 737906ac69dSDavid Xu /* Should we goto user boundary if we didn't come from there? */ 738906ac69dSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 739906ac69dSDavid Xu (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 74094f0972bSDavid Xu return (ERESTART); 741906ac69dSDavid Xu 7429104847fSDavid Xu /* If thread will exit, flush its pending signals */ 7439104847fSDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 7449104847fSDavid Xu sigqueue_flush(&td->td_sigqueue); 7459104847fSDavid Xu 7467b4a950aSDavid Xu PROC_SLOCK(p); 747e574e444SDavid Xu thread_stopped(p); 74844990b8cSJulian Elischer /* 74944990b8cSJulian Elischer * If the process is waiting for us to exit, 75044990b8cSJulian Elischer * this thread should just suicide. 7511279572aSDavid Xu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 75244990b8cSJulian Elischer */ 7537b4a950aSDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 75444990b8cSJulian Elischer thread_exit(); 755a54e85fdSJeff Roberson if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 756a54e85fdSJeff Roberson if (p->p_numthreads == p->p_suspcount + 1) { 757a54e85fdSJeff Roberson thread_lock(p->p_singlethread); 7587847a9daSJohn Baldwin wakeup_swapper = 759a54e85fdSJeff Roberson thread_unsuspend_one(p->p_singlethread); 760a54e85fdSJeff Roberson thread_unlock(p->p_singlethread); 7617847a9daSJohn Baldwin if (wakeup_swapper) 7627847a9daSJohn Baldwin kick_proc0(); 763a54e85fdSJeff Roberson } 764a54e85fdSJeff Roberson } 7653f9be10eSDavid Xu PROC_UNLOCK(p); 7667b4a950aSDavid Xu thread_lock(td); 76744990b8cSJulian Elischer /* 76844990b8cSJulian Elischer * When a thread suspends, it just 769ad1e7d28SJulian Elischer * gets taken off all queues. 77044990b8cSJulian Elischer */ 77171fad9fdSJulian Elischer thread_suspend_one(td); 772906ac69dSDavid Xu if (return_instead == 0) { 773906ac69dSDavid Xu p->p_boundary_count++; 774906ac69dSDavid Xu td->td_flags |= TDF_BOUNDARY; 775cf19bf91SJulian Elischer } 7767b4a950aSDavid Xu PROC_SUNLOCK(p); 7778df78c41SJeff Roberson mi_switch(SW_INVOL | SWT_SUSPEND, NULL); 778a54e85fdSJeff Roberson if (return_instead == 0) 779906ac69dSDavid Xu td->td_flags &= ~TDF_BOUNDARY; 780a54e85fdSJeff Roberson thread_unlock(td); 78144990b8cSJulian Elischer PROC_LOCK(p); 782a54e85fdSJeff Roberson if (return_instead == 0) 783a54e85fdSJeff Roberson p->p_boundary_count--; 78444990b8cSJulian Elischer } 78544990b8cSJulian Elischer return (0); 78644990b8cSJulian Elischer } 78744990b8cSJulian Elischer 78835c32a76SDavid Xu void 789a54e85fdSJeff Roberson thread_suspend_switch(struct thread *td) 790a54e85fdSJeff Roberson { 791a54e85fdSJeff Roberson struct proc *p; 792a54e85fdSJeff Roberson 793a54e85fdSJeff Roberson p = td->td_proc; 794a54e85fdSJeff Roberson KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 795a54e85fdSJeff Roberson PROC_LOCK_ASSERT(p, MA_OWNED); 7967b4a950aSDavid Xu PROC_SLOCK_ASSERT(p, MA_OWNED); 797a54e85fdSJeff Roberson /* 798a54e85fdSJeff Roberson * We implement thread_suspend_one in stages here to avoid 799a54e85fdSJeff Roberson * dropping the proc lock while the thread lock is owned. 800a54e85fdSJeff Roberson */ 801a54e85fdSJeff Roberson thread_stopped(p); 802a54e85fdSJeff Roberson p->p_suspcount++; 8033f9be10eSDavid Xu PROC_UNLOCK(p); 8047b4a950aSDavid Xu thread_lock(td); 805b7edba77SJeff Roberson td->td_flags &= ~TDF_NEEDSUSPCHK; 806a54e85fdSJeff Roberson TD_SET_SUSPENDED(td); 807c5aa6b58SJeff Roberson sched_sleep(td, 0); 8087b4a950aSDavid Xu PROC_SUNLOCK(p); 809a54e85fdSJeff Roberson DROP_GIANT(); 8108df78c41SJeff Roberson mi_switch(SW_VOL | SWT_SUSPEND, NULL); 811a54e85fdSJeff Roberson thread_unlock(td); 812a54e85fdSJeff Roberson PICKUP_GIANT(); 813a54e85fdSJeff Roberson PROC_LOCK(p); 8147b4a950aSDavid Xu PROC_SLOCK(p); 815a54e85fdSJeff Roberson } 816a54e85fdSJeff Roberson 817a54e85fdSJeff Roberson void 81835c32a76SDavid Xu thread_suspend_one(struct thread *td) 81935c32a76SDavid Xu { 82035c32a76SDavid Xu struct proc *p = td->td_proc; 82135c32a76SDavid Xu 8227b4a950aSDavid Xu PROC_SLOCK_ASSERT(p, MA_OWNED); 823a54e85fdSJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 824e574e444SDavid Xu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 82535c32a76SDavid Xu p->p_suspcount++; 826b7edba77SJeff Roberson td->td_flags &= ~TDF_NEEDSUSPCHK; 82771fad9fdSJulian Elischer TD_SET_SUSPENDED(td); 828c5aa6b58SJeff Roberson sched_sleep(td, 0); 82935c32a76SDavid Xu } 83035c32a76SDavid Xu 8317847a9daSJohn Baldwin int 83235c32a76SDavid Xu thread_unsuspend_one(struct thread *td) 83335c32a76SDavid Xu { 83435c32a76SDavid Xu struct proc *p = td->td_proc; 83535c32a76SDavid Xu 8367b4a950aSDavid Xu PROC_SLOCK_ASSERT(p, MA_OWNED); 837a54e85fdSJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 838ad1e7d28SJulian Elischer KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 83971fad9fdSJulian Elischer TD_CLR_SUSPENDED(td); 84035c32a76SDavid Xu p->p_suspcount--; 8417847a9daSJohn Baldwin return (setrunnable(td)); 84235c32a76SDavid Xu } 84335c32a76SDavid Xu 84444990b8cSJulian Elischer /* 84544990b8cSJulian Elischer * Allow all threads blocked by single threading to continue running. 84644990b8cSJulian Elischer */ 84744990b8cSJulian Elischer void 84844990b8cSJulian Elischer thread_unsuspend(struct proc *p) 84944990b8cSJulian Elischer { 85044990b8cSJulian Elischer struct thread *td; 8517847a9daSJohn Baldwin int wakeup_swapper; 85244990b8cSJulian Elischer 85344990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 8547b4a950aSDavid Xu PROC_SLOCK_ASSERT(p, MA_OWNED); 8557847a9daSJohn Baldwin wakeup_swapper = 0; 85644990b8cSJulian Elischer if (!P_SHOULDSTOP(p)) { 857ad1e7d28SJulian Elischer FOREACH_THREAD_IN_PROC(p, td) { 858a54e85fdSJeff Roberson thread_lock(td); 859ad1e7d28SJulian Elischer if (TD_IS_SUSPENDED(td)) { 8607847a9daSJohn Baldwin wakeup_swapper |= thread_unsuspend_one(td); 86144990b8cSJulian Elischer } 862a54e85fdSJeff Roberson thread_unlock(td); 863ad1e7d28SJulian Elischer } 8641279572aSDavid Xu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 86544990b8cSJulian Elischer (p->p_numthreads == p->p_suspcount)) { 86644990b8cSJulian Elischer /* 86744990b8cSJulian Elischer * Stopping everything also did the job for the single 86844990b8cSJulian Elischer * threading request. Now we've downgraded to single-threaded, 86944990b8cSJulian Elischer * let it continue. 87044990b8cSJulian Elischer */ 871a54e85fdSJeff Roberson thread_lock(p->p_singlethread); 8727847a9daSJohn Baldwin wakeup_swapper = thread_unsuspend_one(p->p_singlethread); 873a54e85fdSJeff Roberson thread_unlock(p->p_singlethread); 87444990b8cSJulian Elischer } 8757847a9daSJohn Baldwin if (wakeup_swapper) 8767847a9daSJohn Baldwin kick_proc0(); 87744990b8cSJulian Elischer } 87844990b8cSJulian Elischer 879ed062c8dSJulian Elischer /* 880ed062c8dSJulian Elischer * End the single threading mode.. 881ed062c8dSJulian Elischer */ 88244990b8cSJulian Elischer void 88344990b8cSJulian Elischer thread_single_end(void) 88444990b8cSJulian Elischer { 88544990b8cSJulian Elischer struct thread *td; 88644990b8cSJulian Elischer struct proc *p; 8877847a9daSJohn Baldwin int wakeup_swapper; 88844990b8cSJulian Elischer 88944990b8cSJulian Elischer td = curthread; 89044990b8cSJulian Elischer p = td->td_proc; 89144990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 892906ac69dSDavid Xu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); 8937b4a950aSDavid Xu PROC_SLOCK(p); 89444990b8cSJulian Elischer p->p_singlethread = NULL; 8957847a9daSJohn Baldwin wakeup_swapper = 0; 89649539972SJulian Elischer /* 8977847a9daSJohn Baldwin * If there are other threads they may now run, 89849539972SJulian Elischer * unless of course there is a blanket 'stop order' 89949539972SJulian Elischer * on the process. The single threader must be allowed 90049539972SJulian Elischer * to continue however as this is a bad place to stop. 90149539972SJulian Elischer */ 90249539972SJulian Elischer if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 903ad1e7d28SJulian Elischer FOREACH_THREAD_IN_PROC(p, td) { 904a54e85fdSJeff Roberson thread_lock(td); 905ad1e7d28SJulian Elischer if (TD_IS_SUSPENDED(td)) { 9067847a9daSJohn Baldwin wakeup_swapper |= thread_unsuspend_one(td); 90744990b8cSJulian Elischer } 908a54e85fdSJeff Roberson thread_unlock(td); 90949539972SJulian Elischer } 910ad1e7d28SJulian Elischer } 9117b4a950aSDavid Xu PROC_SUNLOCK(p); 9127847a9daSJohn Baldwin if (wakeup_swapper) 9137847a9daSJohn Baldwin kick_proc0(); 91449539972SJulian Elischer } 9154fc21c09SDaniel Eischen 91644355392SDavid Xu struct thread * 91744355392SDavid Xu thread_find(struct proc *p, lwpid_t tid) 91844355392SDavid Xu { 91944355392SDavid Xu struct thread *td; 92044355392SDavid Xu 92144355392SDavid Xu PROC_LOCK_ASSERT(p, MA_OWNED); 92244355392SDavid Xu FOREACH_THREAD_IN_PROC(p, td) { 92344355392SDavid Xu if (td->td_tid == tid) 92444355392SDavid Xu break; 92544355392SDavid Xu } 92644355392SDavid Xu return (td); 92744355392SDavid Xu } 928