19454b2d8SWarner Losh /*- 28a36da99SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 38a36da99SPedro F. Giffuni * 444990b8cSJulian Elischer * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 544990b8cSJulian Elischer * All rights reserved. 644990b8cSJulian Elischer * 744990b8cSJulian Elischer * Redistribution and use in source and binary forms, with or without 844990b8cSJulian Elischer * modification, are permitted provided that the following conditions 944990b8cSJulian Elischer * are met: 1044990b8cSJulian Elischer * 1. Redistributions of source code must retain the above copyright 1144990b8cSJulian Elischer * notice(s), this list of conditions and the following disclaimer as 1244990b8cSJulian Elischer * the first lines of this file unmodified other than the possible 1344990b8cSJulian Elischer * addition of one or more copyright notices. 1444990b8cSJulian Elischer * 2. Redistributions in binary form must reproduce the above copyright 1544990b8cSJulian Elischer * notice(s), this list of conditions and the following disclaimer in the 1644990b8cSJulian Elischer * documentation and/or other materials provided with the distribution. 1744990b8cSJulian Elischer * 1844990b8cSJulian Elischer * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1944990b8cSJulian Elischer * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 2044990b8cSJulian Elischer * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 2144990b8cSJulian Elischer * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2244990b8cSJulian Elischer * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2344990b8cSJulian Elischer * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2444990b8cSJulian Elischer * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2544990b8cSJulian Elischer * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2644990b8cSJulian Elischer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2744990b8cSJulian Elischer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2844990b8cSJulian Elischer * DAMAGE. 2944990b8cSJulian Elischer */ 3044990b8cSJulian Elischer 313d06b4b3SAttilio Rao #include "opt_witness.h" 3216d95d4fSJoseph Koshy #include "opt_hwpmc_hooks.h" 333d06b4b3SAttilio Rao 34677b542eSDavid E. O'Brien #include <sys/cdefs.h> 35677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 36677b542eSDavid E. O'Brien 3744990b8cSJulian Elischer #include <sys/param.h> 3844990b8cSJulian Elischer #include <sys/systm.h> 3944990b8cSJulian Elischer #include <sys/kernel.h> 4044990b8cSJulian Elischer #include <sys/lock.h> 415dda15adSMark Johnston #include <sys/msan.h> 4244990b8cSJulian Elischer #include <sys/mutex.h> 4344990b8cSJulian Elischer #include <sys/proc.h> 4435bb59edSMateusz Guzik #include <sys/bitstring.h> 456febf180SGleb Smirnoff #include <sys/epoch.h> 468f0e9130SKonstantin Belousov #include <sys/rangelock.h> 47e170bfdaSDavid Xu #include <sys/resourcevar.h> 48b3e9e682SRyan Stone #include <sys/sdt.h> 4994e0a4cdSJulian Elischer #include <sys/smp.h> 50de028f5aSJeff Roberson #include <sys/sched.h> 5144f3b092SJohn Baldwin #include <sys/sleepqueue.h> 52ace8398dSJeff Roberson #include <sys/selinfo.h> 53d1e7a4a5SJohn Baldwin #include <sys/syscallsubr.h> 54598f2b81SMateusz Guzik #include <sys/dtrace_bsd.h> 5591d1786fSDmitry Chagin #include <sys/sysent.h> 56961a7b24SJohn Baldwin #include <sys/turnstile.h> 57d116b9f1SMateusz Guzik #include <sys/taskqueue.h> 5844990b8cSJulian Elischer #include <sys/ktr.h> 59cf7d9a8cSDavid Xu #include <sys/rwlock.h> 60af29f399SDmitry Chagin #include <sys/umtxvar.h> 619ed01c32SGleb Smirnoff #include <sys/vmmeter.h> 62d7f687fcSJeff Roberson #include <sys/cpuset.h> 6316d95d4fSJoseph Koshy #ifdef HWPMC_HOOKS 6416d95d4fSJoseph Koshy #include <sys/pmckern.h> 6516d95d4fSJoseph Koshy #endif 661bd3cf5dSMateusz Guzik #include <sys/priv.h> 6744990b8cSJulian Elischer 68911b84b0SRobert Watson #include <security/audit/audit.h> 69911b84b0SRobert Watson 70d116b9f1SMateusz Guzik #include <vm/pmap.h> 7144990b8cSJulian Elischer #include <vm/vm.h> 7249a2507bSAlan Cox #include <vm/vm_extern.h> 7344990b8cSJulian Elischer #include <vm/uma.h> 74d116b9f1SMateusz Guzik #include <vm/vm_phys.h> 75b209f889SRandall Stewart #include <sys/eventhandler.h> 7602fb42b0SPeter Wemm 77acd9f517SKonstantin Belousov /* 78acd9f517SKonstantin Belousov * Asserts below verify the stability of struct thread and struct proc 79acd9f517SKonstantin Belousov * layout, as exposed by KBI to modules. On head, the KBI is allowed 80acd9f517SKonstantin Belousov * to drift, change to the structures must be accompanied by the 81acd9f517SKonstantin Belousov * assert update. 82acd9f517SKonstantin Belousov * 83acd9f517SKonstantin Belousov * On the stable branches after KBI freeze, conditions must not be 84acd9f517SKonstantin Belousov * violated. Typically new fields are moved to the end of the 85acd9f517SKonstantin Belousov * structures. 86acd9f517SKonstantin Belousov */ 87acd9f517SKonstantin Belousov #ifdef __amd64__ 88a422084aSMark Johnston _Static_assert(offsetof(struct thread, td_flags) == 0x108, 89acd9f517SKonstantin Belousov "struct thread KBI td_flags"); 90c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_pflags) == 0x114, 91acd9f517SKonstantin Belousov "struct thread KBI td_pflags"); 92c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_frame) == 0x4b0, 93acd9f517SKonstantin Belousov "struct thread KBI td_frame"); 94c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_emuldata) == 0x6c0, 95acd9f517SKonstantin Belousov "struct thread KBI td_emuldata"); 9685078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_flag) == 0xb8, 97acd9f517SKonstantin Belousov "struct proc KBI p_flag"); 9885078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_pid) == 0xc4, 99acd9f517SKonstantin Belousov "struct proc KBI p_pid"); 100351d5f7fSKonstantin Belousov _Static_assert(offsetof(struct proc, p_filemon) == 0x3c8, 101acd9f517SKonstantin Belousov "struct proc KBI p_filemon"); 1025e5675cbSKonstantin Belousov _Static_assert(offsetof(struct proc, p_comm) == 0x3e0, 103acd9f517SKonstantin Belousov "struct proc KBI p_comm"); 104351d5f7fSKonstantin Belousov _Static_assert(offsetof(struct proc, p_emuldata) == 0x4c8, 105acd9f517SKonstantin Belousov "struct proc KBI p_emuldata"); 106acd9f517SKonstantin Belousov #endif 107acd9f517SKonstantin Belousov #ifdef __i386__ 108a422084aSMark Johnston _Static_assert(offsetof(struct thread, td_flags) == 0x9c, 109acd9f517SKonstantin Belousov "struct thread KBI td_flags"); 110c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_pflags) == 0xa8, 111acd9f517SKonstantin Belousov "struct thread KBI td_pflags"); 112c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_frame) == 0x30c, 113acd9f517SKonstantin Belousov "struct thread KBI td_frame"); 114c6d31b83SKonstantin Belousov _Static_assert(offsetof(struct thread, td_emuldata) == 0x350, 115acd9f517SKonstantin Belousov "struct thread KBI td_emuldata"); 11685078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_flag) == 0x6c, 117acd9f517SKonstantin Belousov "struct proc KBI p_flag"); 11885078b85SConrad Meyer _Static_assert(offsetof(struct proc, p_pid) == 0x78, 119acd9f517SKonstantin Belousov "struct proc KBI p_pid"); 1204d675b80SKonstantin Belousov _Static_assert(offsetof(struct proc, p_filemon) == 0x270, 121acd9f517SKonstantin Belousov "struct proc KBI p_filemon"); 1225e5675cbSKonstantin Belousov _Static_assert(offsetof(struct proc, p_comm) == 0x284, 123acd9f517SKonstantin Belousov "struct proc KBI p_comm"); 1245e5675cbSKonstantin Belousov _Static_assert(offsetof(struct proc, p_emuldata) == 0x310, 125acd9f517SKonstantin Belousov "struct proc KBI p_emuldata"); 126acd9f517SKonstantin Belousov #endif 127acd9f517SKonstantin Belousov 128b3e9e682SRyan Stone SDT_PROVIDER_DECLARE(proc); 129d9fae5abSAndriy Gapon SDT_PROBE_DEFINE(proc, , , lwp__exit); 130b3e9e682SRyan Stone 1318460a577SJohn Birrell /* 1328460a577SJohn Birrell * thread related storage. 1338460a577SJohn Birrell */ 13444990b8cSJulian Elischer static uma_zone_t thread_zone; 13544990b8cSJulian Elischer 136d116b9f1SMateusz Guzik struct thread_domain_data { 137d116b9f1SMateusz Guzik struct thread *tdd_zombies; 138d116b9f1SMateusz Guzik int tdd_reapticks; 139d116b9f1SMateusz Guzik } __aligned(CACHE_LINE_SIZE); 140d116b9f1SMateusz Guzik 141d116b9f1SMateusz Guzik static struct thread_domain_data thread_domain_data[MAXMEMDOM]; 142d116b9f1SMateusz Guzik 143d116b9f1SMateusz Guzik static struct task thread_reap_task; 144d116b9f1SMateusz Guzik static struct callout thread_reap_callout; 14544990b8cSJulian Elischer 146ff8fbcffSJeff Roberson static void thread_zombie(struct thread *); 147b83e94beSMateusz Guzik static void thread_reap(void); 148d116b9f1SMateusz Guzik static void thread_reap_all(void); 149d116b9f1SMateusz Guzik static void thread_reap_task_cb(void *, int); 150d116b9f1SMateusz Guzik static void thread_reap_callout_cb(void *); 15184cdea97SKonstantin Belousov static int thread_unsuspend_one(struct thread *td, struct proc *p, 15284cdea97SKonstantin Belousov bool boundary); 153755341dfSMateusz Guzik static void thread_free_batched(struct thread *td); 154ff8fbcffSJeff Roberson 155d1ca25beSMateusz Guzik static __exclusive_cache_line struct mtx tid_lock; 156934e7e5eSMateusz Guzik static bitstr_t *tid_bitmap; 15735bb59edSMateusz Guzik 158cf7d9a8cSDavid Xu static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash"); 159cf7d9a8cSDavid Xu 1601bd3cf5dSMateusz Guzik static int maxthread; 1611bd3cf5dSMateusz Guzik SYSCTL_INT(_kern, OID_AUTO, maxthread, CTLFLAG_RDTUN, 1621bd3cf5dSMateusz Guzik &maxthread, 0, "Maximum number of threads"); 1631bd3cf5dSMateusz Guzik 16462dbc992SMateusz Guzik static __exclusive_cache_line int nthreads; 1651bd3cf5dSMateusz Guzik 166aae3547bSMateusz Guzik static LIST_HEAD(tidhashhead, thread) *tidhashtbl; 167aae3547bSMateusz Guzik static u_long tidhash; 16826007fe3SMateusz Guzik static u_long tidhashlock; 16926007fe3SMateusz Guzik static struct rwlock *tidhashtbl_lock; 170aae3547bSMateusz Guzik #define TIDHASH(tid) (&tidhashtbl[(tid) & tidhash]) 17126007fe3SMateusz Guzik #define TIDHASHLOCK(tid) (&tidhashtbl_lock[(tid) & tidhashlock]) 172cf7d9a8cSDavid Xu 1732ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_ctor); 1742ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_dtor); 1752ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_init); 1762ca45184SMatt Joras EVENTHANDLER_LIST_DEFINE(thread_fini); 1772ca45184SMatt Joras 17862dbc992SMateusz Guzik static bool 179d116b9f1SMateusz Guzik thread_count_inc_try(void) 180ec6ea5e8SDavid Xu { 18162dbc992SMateusz Guzik int nthreads_new; 182ec6ea5e8SDavid Xu 18362dbc992SMateusz Guzik nthreads_new = atomic_fetchadd_int(&nthreads, 1) + 1; 18462dbc992SMateusz Guzik if (nthreads_new >= maxthread - 100) { 1851bd3cf5dSMateusz Guzik if (priv_check_cred(curthread->td_ucred, PRIV_MAXPROC) != 0 || 18662dbc992SMateusz Guzik nthreads_new >= maxthread) { 18762dbc992SMateusz Guzik atomic_subtract_int(&nthreads, 1); 188d116b9f1SMateusz Guzik return (false); 189d116b9f1SMateusz Guzik } 190d116b9f1SMateusz Guzik } 191d116b9f1SMateusz Guzik return (true); 192d116b9f1SMateusz Guzik } 193d116b9f1SMateusz Guzik 194d116b9f1SMateusz Guzik static bool 195d116b9f1SMateusz Guzik thread_count_inc(void) 196d116b9f1SMateusz Guzik { 197d116b9f1SMateusz Guzik static struct timeval lastfail; 198d116b9f1SMateusz Guzik static int curfail; 199d116b9f1SMateusz Guzik 200d116b9f1SMateusz Guzik thread_reap(); 201d116b9f1SMateusz Guzik if (thread_count_inc_try()) { 202d116b9f1SMateusz Guzik return (true); 203d116b9f1SMateusz Guzik } 204d116b9f1SMateusz Guzik 205d116b9f1SMateusz Guzik thread_reap_all(); 206d116b9f1SMateusz Guzik if (thread_count_inc_try()) { 207d116b9f1SMateusz Guzik return (true); 208d116b9f1SMateusz Guzik } 209d116b9f1SMateusz Guzik 2101bd3cf5dSMateusz Guzik if (ppsratecheck(&lastfail, &curfail, 1)) { 2111bd3cf5dSMateusz Guzik printf("maxthread limit exceeded by uid %u " 2121bd3cf5dSMateusz Guzik "(pid %d); consider increasing kern.maxthread\n", 2131bd3cf5dSMateusz Guzik curthread->td_ucred->cr_ruid, curproc->p_pid); 2141bd3cf5dSMateusz Guzik } 21562dbc992SMateusz Guzik return (false); 2161bd3cf5dSMateusz Guzik } 2171bd3cf5dSMateusz Guzik 21862dbc992SMateusz Guzik static void 21962dbc992SMateusz Guzik thread_count_sub(int n) 22062dbc992SMateusz Guzik { 22162dbc992SMateusz Guzik 22262dbc992SMateusz Guzik atomic_subtract_int(&nthreads, n); 22362dbc992SMateusz Guzik } 22462dbc992SMateusz Guzik 22562dbc992SMateusz Guzik static void 22662dbc992SMateusz Guzik thread_count_dec(void) 22762dbc992SMateusz Guzik { 22862dbc992SMateusz Guzik 22962dbc992SMateusz Guzik thread_count_sub(1); 23062dbc992SMateusz Guzik } 23162dbc992SMateusz Guzik 23262dbc992SMateusz Guzik static lwpid_t 23362dbc992SMateusz Guzik tid_alloc(void) 23462dbc992SMateusz Guzik { 23562dbc992SMateusz Guzik static lwpid_t trytid; 23662dbc992SMateusz Guzik lwpid_t tid; 23762dbc992SMateusz Guzik 23862dbc992SMateusz Guzik mtx_lock(&tid_lock); 23935bb59edSMateusz Guzik /* 24035bb59edSMateusz Guzik * It is an invariant that the bitmap is big enough to hold maxthread 24135bb59edSMateusz Guzik * IDs. If we got to this point there has to be at least one free. 24235bb59edSMateusz Guzik */ 24335bb59edSMateusz Guzik if (trytid >= maxthread) 24435bb59edSMateusz Guzik trytid = 0; 24535bb59edSMateusz Guzik bit_ffc_at(tid_bitmap, trytid, maxthread, &tid); 24635bb59edSMateusz Guzik if (tid == -1) { 24735bb59edSMateusz Guzik KASSERT(trytid != 0, ("unexpectedly ran out of IDs")); 24835bb59edSMateusz Guzik trytid = 0; 24935bb59edSMateusz Guzik bit_ffc_at(tid_bitmap, trytid, maxthread, &tid); 25035bb59edSMateusz Guzik KASSERT(tid != -1, ("unexpectedly ran out of IDs")); 251ec6ea5e8SDavid Xu } 25235bb59edSMateusz Guzik bit_set(tid_bitmap, tid); 253934e7e5eSMateusz Guzik trytid = tid + 1; 254ec6ea5e8SDavid Xu mtx_unlock(&tid_lock); 25535bb59edSMateusz Guzik return (tid + NO_PID); 256ec6ea5e8SDavid Xu } 257ec6ea5e8SDavid Xu 258ec6ea5e8SDavid Xu static void 259755341dfSMateusz Guzik tid_free_locked(lwpid_t rtid) 260ec6ea5e8SDavid Xu { 26135bb59edSMateusz Guzik lwpid_t tid; 262ec6ea5e8SDavid Xu 263755341dfSMateusz Guzik mtx_assert(&tid_lock, MA_OWNED); 26435bb59edSMateusz Guzik KASSERT(rtid >= NO_PID, 26535bb59edSMateusz Guzik ("%s: invalid tid %d\n", __func__, rtid)); 26635bb59edSMateusz Guzik tid = rtid - NO_PID; 26735bb59edSMateusz Guzik KASSERT(bit_test(tid_bitmap, tid) != 0, 26835bb59edSMateusz Guzik ("thread ID %d not allocated\n", rtid)); 26935bb59edSMateusz Guzik bit_clear(tid_bitmap, tid); 270755341dfSMateusz Guzik } 271755341dfSMateusz Guzik 272755341dfSMateusz Guzik static void 273755341dfSMateusz Guzik tid_free(lwpid_t rtid) 274755341dfSMateusz Guzik { 275755341dfSMateusz Guzik 276755341dfSMateusz Guzik mtx_lock(&tid_lock); 277755341dfSMateusz Guzik tid_free_locked(rtid); 278755341dfSMateusz Guzik mtx_unlock(&tid_lock); 279755341dfSMateusz Guzik } 280755341dfSMateusz Guzik 281755341dfSMateusz Guzik static void 282755341dfSMateusz Guzik tid_free_batch(lwpid_t *batch, int n) 283755341dfSMateusz Guzik { 284755341dfSMateusz Guzik int i; 285755341dfSMateusz Guzik 286755341dfSMateusz Guzik mtx_lock(&tid_lock); 287755341dfSMateusz Guzik for (i = 0; i < n; i++) { 288755341dfSMateusz Guzik tid_free_locked(batch[i]); 289755341dfSMateusz Guzik } 290ec6ea5e8SDavid Xu mtx_unlock(&tid_lock); 291ec6ea5e8SDavid Xu } 292ec6ea5e8SDavid Xu 293fdcac928SMarcel Moolenaar /* 2945ef7b7a0SMateusz Guzik * Batching for thread reapping. 2955ef7b7a0SMateusz Guzik */ 2965ef7b7a0SMateusz Guzik struct tidbatch { 2975ef7b7a0SMateusz Guzik lwpid_t tab[16]; 2985ef7b7a0SMateusz Guzik int n; 2995ef7b7a0SMateusz Guzik }; 3005ef7b7a0SMateusz Guzik 3015ef7b7a0SMateusz Guzik static void 3025ef7b7a0SMateusz Guzik tidbatch_prep(struct tidbatch *tb) 3035ef7b7a0SMateusz Guzik { 3045ef7b7a0SMateusz Guzik 3055ef7b7a0SMateusz Guzik tb->n = 0; 3065ef7b7a0SMateusz Guzik } 3075ef7b7a0SMateusz Guzik 3085ef7b7a0SMateusz Guzik static void 3095ef7b7a0SMateusz Guzik tidbatch_add(struct tidbatch *tb, struct thread *td) 3105ef7b7a0SMateusz Guzik { 3115ef7b7a0SMateusz Guzik 3125ef7b7a0SMateusz Guzik KASSERT(tb->n < nitems(tb->tab), 3135ef7b7a0SMateusz Guzik ("%s: count too high %d", __func__, tb->n)); 3145ef7b7a0SMateusz Guzik tb->tab[tb->n] = td->td_tid; 3155ef7b7a0SMateusz Guzik tb->n++; 3165ef7b7a0SMateusz Guzik } 3175ef7b7a0SMateusz Guzik 3185ef7b7a0SMateusz Guzik static void 3195ef7b7a0SMateusz Guzik tidbatch_process(struct tidbatch *tb) 3205ef7b7a0SMateusz Guzik { 3215ef7b7a0SMateusz Guzik 3225ef7b7a0SMateusz Guzik KASSERT(tb->n <= nitems(tb->tab), 3235ef7b7a0SMateusz Guzik ("%s: count too high %d", __func__, tb->n)); 3245ef7b7a0SMateusz Guzik if (tb->n == nitems(tb->tab)) { 3255ef7b7a0SMateusz Guzik tid_free_batch(tb->tab, tb->n); 3265ef7b7a0SMateusz Guzik tb->n = 0; 3275ef7b7a0SMateusz Guzik } 3285ef7b7a0SMateusz Guzik } 3295ef7b7a0SMateusz Guzik 3305ef7b7a0SMateusz Guzik static void 3315ef7b7a0SMateusz Guzik tidbatch_final(struct tidbatch *tb) 3325ef7b7a0SMateusz Guzik { 3335ef7b7a0SMateusz Guzik 3345ef7b7a0SMateusz Guzik KASSERT(tb->n <= nitems(tb->tab), 3355ef7b7a0SMateusz Guzik ("%s: count too high %d", __func__, tb->n)); 3365ef7b7a0SMateusz Guzik if (tb->n != 0) { 3375ef7b7a0SMateusz Guzik tid_free_batch(tb->tab, tb->n); 3385ef7b7a0SMateusz Guzik } 3395ef7b7a0SMateusz Guzik } 3405ef7b7a0SMateusz Guzik 3415ef7b7a0SMateusz Guzik /* 342696058c3SJulian Elischer * Prepare a thread for use. 34344990b8cSJulian Elischer */ 344b23f72e9SBrian Feldman static int 345b23f72e9SBrian Feldman thread_ctor(void *mem, int size, void *arg, int flags) 34644990b8cSJulian Elischer { 34744990b8cSJulian Elischer struct thread *td; 34844990b8cSJulian Elischer 34944990b8cSJulian Elischer td = (struct thread *)mem; 350fa2528acSAlex Richardson TD_SET_STATE(td, TDS_INACTIVE); 35194dd54b9SKonstantin Belousov td->td_lastcpu = td->td_oncpu = NOCPU; 3526c27c603SJuli Mallett 3536c27c603SJuli Mallett /* 3546c27c603SJuli Mallett * Note that td_critnest begins life as 1 because the thread is not 3556c27c603SJuli Mallett * running and is thereby implicitly waiting to be on the receiving 356a54e85fdSJeff Roberson * end of a context switch. 3576c27c603SJuli Mallett */ 358139b7550SJohn Baldwin td->td_critnest = 1; 359acbe332aSDavid Xu td->td_lend_user_pri = PRI_MAX; 360911b84b0SRobert Watson #ifdef AUDIT 361911b84b0SRobert Watson audit_thread_alloc(td); 362911b84b0SRobert Watson #endif 363598f2b81SMateusz Guzik #ifdef KDTRACE_HOOKS 364598f2b81SMateusz Guzik kdtrace_thread_ctor(td); 365598f2b81SMateusz Guzik #endif 366d10183d9SDavid Xu umtx_thread_alloc(td); 36719d3e47dSMateusz Guzik MPASS(td->td_sel == NULL); 368b23f72e9SBrian Feldman return (0); 36944990b8cSJulian Elischer } 37044990b8cSJulian Elischer 37144990b8cSJulian Elischer /* 37244990b8cSJulian Elischer * Reclaim a thread after use. 37344990b8cSJulian Elischer */ 37444990b8cSJulian Elischer static void 37544990b8cSJulian Elischer thread_dtor(void *mem, int size, void *arg) 37644990b8cSJulian Elischer { 37744990b8cSJulian Elischer struct thread *td; 37844990b8cSJulian Elischer 37944990b8cSJulian Elischer td = (struct thread *)mem; 38044990b8cSJulian Elischer 38144990b8cSJulian Elischer #ifdef INVARIANTS 38244990b8cSJulian Elischer /* Verify that this thread is in a safe state to free. */ 383fa2528acSAlex Richardson switch (TD_GET_STATE(td)) { 38471fad9fdSJulian Elischer case TDS_INHIBITED: 38571fad9fdSJulian Elischer case TDS_RUNNING: 38671fad9fdSJulian Elischer case TDS_CAN_RUN: 38744990b8cSJulian Elischer case TDS_RUNQ: 38844990b8cSJulian Elischer /* 38944990b8cSJulian Elischer * We must never unlink a thread that is in one of 39044990b8cSJulian Elischer * these states, because it is currently active. 39144990b8cSJulian Elischer */ 39244990b8cSJulian Elischer panic("bad state for thread unlinking"); 39344990b8cSJulian Elischer /* NOTREACHED */ 39471fad9fdSJulian Elischer case TDS_INACTIVE: 39544990b8cSJulian Elischer break; 39644990b8cSJulian Elischer default: 39744990b8cSJulian Elischer panic("bad thread state"); 39844990b8cSJulian Elischer /* NOTREACHED */ 39944990b8cSJulian Elischer } 40044990b8cSJulian Elischer #endif 4016e8525ceSRobert Watson #ifdef AUDIT 4026e8525ceSRobert Watson audit_thread_free(td); 4036e8525ceSRobert Watson #endif 404598f2b81SMateusz Guzik #ifdef KDTRACE_HOOKS 405598f2b81SMateusz Guzik kdtrace_thread_dtor(td); 406598f2b81SMateusz Guzik #endif 4071ba4a712SPawel Jakub Dawidek /* Free all OSD associated to this thread. */ 4081ba4a712SPawel Jakub Dawidek osd_thread_exit(td); 409c6d31b83SKonstantin Belousov ast_kclear(td); 41019d3e47dSMateusz Guzik seltdfini(td); 41144990b8cSJulian Elischer } 41244990b8cSJulian Elischer 41344990b8cSJulian Elischer /* 41444990b8cSJulian Elischer * Initialize type-stable parts of a thread (when newly created). 41544990b8cSJulian Elischer */ 416b23f72e9SBrian Feldman static int 417b23f72e9SBrian Feldman thread_init(void *mem, int size, int flags) 41844990b8cSJulian Elischer { 41944990b8cSJulian Elischer struct thread *td; 42044990b8cSJulian Elischer 42144990b8cSJulian Elischer td = (struct thread *)mem; 422247aba24SMarcel Moolenaar 423b83e94beSMateusz Guzik td->td_allocdomain = vm_phys_domain(vtophys(td)); 42444f3b092SJohn Baldwin td->td_sleepqueue = sleepq_alloc(); 425961a7b24SJohn Baldwin td->td_turnstile = turnstile_alloc(); 4268f0e9130SKonstantin Belousov td->td_rlqe = NULL; 4272ca45184SMatt Joras EVENTHANDLER_DIRECT_INVOKE(thread_init, td); 428d10183d9SDavid Xu umtx_thread_init(td); 42989b57fcfSKonstantin Belousov td->td_kstack = 0; 430ad8b1d85SKonstantin Belousov td->td_sel = NULL; 431b23f72e9SBrian Feldman return (0); 43244990b8cSJulian Elischer } 43344990b8cSJulian Elischer 43444990b8cSJulian Elischer /* 43544990b8cSJulian Elischer * Tear down type-stable parts of a thread (just before being discarded). 43644990b8cSJulian Elischer */ 43744990b8cSJulian Elischer static void 43844990b8cSJulian Elischer thread_fini(void *mem, int size) 43944990b8cSJulian Elischer { 44044990b8cSJulian Elischer struct thread *td; 44144990b8cSJulian Elischer 44244990b8cSJulian Elischer td = (struct thread *)mem; 4432ca45184SMatt Joras EVENTHANDLER_DIRECT_INVOKE(thread_fini, td); 4448f0e9130SKonstantin Belousov rlqentry_free(td->td_rlqe); 445961a7b24SJohn Baldwin turnstile_free(td->td_turnstile); 44644f3b092SJohn Baldwin sleepq_free(td->td_sleepqueue); 447d10183d9SDavid Xu umtx_thread_fini(td); 44819d3e47dSMateusz Guzik MPASS(td->td_sel == NULL); 44944990b8cSJulian Elischer } 4505215b187SJeff Roberson 4515c8329edSJulian Elischer /* 4525215b187SJeff Roberson * For a newly created process, 4535215b187SJeff Roberson * link up all the structures and its initial threads etc. 454ed062c8dSJulian Elischer * called from: 455e7d939bdSMarcel Moolenaar * {arch}/{arch}/machdep.c {arch}_init(), init386() etc. 456ed062c8dSJulian Elischer * proc_dtor() (should go away) 457ed062c8dSJulian Elischer * proc_init() 4585c8329edSJulian Elischer */ 4595c8329edSJulian Elischer void 46089b57fcfSKonstantin Belousov proc_linkup0(struct proc *p, struct thread *td) 46189b57fcfSKonstantin Belousov { 46289b57fcfSKonstantin Belousov TAILQ_INIT(&p->p_threads); /* all threads in proc */ 46389b57fcfSKonstantin Belousov proc_linkup(p, td); 46489b57fcfSKonstantin Belousov } 46589b57fcfSKonstantin Belousov 46689b57fcfSKonstantin Belousov void 4678460a577SJohn Birrell proc_linkup(struct proc *p, struct thread *td) 4685c8329edSJulian Elischer { 469a54e85fdSJeff Roberson 4709104847fSDavid Xu sigqueue_init(&p->p_sigqueue, p); 471cc29f221SKonstantin Belousov p->p_ksi = ksiginfo_alloc(M_WAITOK); 472ebceaf6dSDavid Xu if (p->p_ksi != NULL) { 4735c474517SDavid Xu /* XXX p_ksi may be null if ksiginfo zone is not ready */ 474ebceaf6dSDavid Xu p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 475ebceaf6dSDavid Xu } 476b2f92ef9SDavid Xu LIST_INIT(&p->p_mqnotifier); 4775c8329edSJulian Elischer p->p_numthreads = 0; 4788460a577SJohn Birrell thread_link(td, p); 4795c8329edSJulian Elischer } 4805c8329edSJulian Elischer 481c6d31b83SKonstantin Belousov static void 482c6d31b83SKonstantin Belousov ast_suspend(struct thread *td, int tda __unused) 483c6d31b83SKonstantin Belousov { 484c6d31b83SKonstantin Belousov struct proc *p; 485c6d31b83SKonstantin Belousov 486c6d31b83SKonstantin Belousov p = td->td_proc; 487c6d31b83SKonstantin Belousov /* 488c6d31b83SKonstantin Belousov * We need to check to see if we have to exit or wait due to a 489c6d31b83SKonstantin Belousov * single threading requirement or some other STOP condition. 490c6d31b83SKonstantin Belousov */ 491c6d31b83SKonstantin Belousov PROC_LOCK(p); 492c6d31b83SKonstantin Belousov thread_suspend_check(0); 493c6d31b83SKonstantin Belousov PROC_UNLOCK(p); 494c6d31b83SKonstantin Belousov } 495c6d31b83SKonstantin Belousov 4961bd3cf5dSMateusz Guzik extern int max_threads_per_proc; 4971bd3cf5dSMateusz Guzik 4985c8329edSJulian Elischer /* 49944990b8cSJulian Elischer * Initialize global thread allocation resources. 50044990b8cSJulian Elischer */ 50144990b8cSJulian Elischer void 50244990b8cSJulian Elischer threadinit(void) 50344990b8cSJulian Elischer { 50426007fe3SMateusz Guzik u_long i; 505cf31cadeSMateusz Guzik lwpid_t tid0; 50644990b8cSJulian Elischer 5071bd3cf5dSMateusz Guzik /* 5081bd3cf5dSMateusz Guzik * Place an upper limit on threads which can be allocated. 5091bd3cf5dSMateusz Guzik * 5101bd3cf5dSMateusz Guzik * Note that other factors may make the de facto limit much lower. 5111bd3cf5dSMateusz Guzik * 5121bd3cf5dSMateusz Guzik * Platform limits are somewhat arbitrary but deemed "more than good 5131bd3cf5dSMateusz Guzik * enough" for the foreseable future. 5141bd3cf5dSMateusz Guzik */ 5151bd3cf5dSMateusz Guzik if (maxthread == 0) { 5161bd3cf5dSMateusz Guzik #ifdef _LP64 5171bd3cf5dSMateusz Guzik maxthread = MIN(maxproc * max_threads_per_proc, 1000000); 5181bd3cf5dSMateusz Guzik #else 5191bd3cf5dSMateusz Guzik maxthread = MIN(maxproc * max_threads_per_proc, 100000); 5201bd3cf5dSMateusz Guzik #endif 5211bd3cf5dSMateusz Guzik } 5221bd3cf5dSMateusz Guzik 5231ea7a6f8SPoul-Henning Kamp mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 52435bb59edSMateusz Guzik tid_bitmap = bit_alloc(maxthread, M_TIDHASH, M_WAITOK); 52562dbc992SMateusz Guzik /* 52662dbc992SMateusz Guzik * Handle thread0. 52762dbc992SMateusz Guzik */ 52862dbc992SMateusz Guzik thread_count_inc(); 529cf31cadeSMateusz Guzik tid0 = tid_alloc(); 530cf31cadeSMateusz Guzik if (tid0 != THREAD0_TID) 531cf31cadeSMateusz Guzik panic("tid0 %d != %d\n", tid0, THREAD0_TID); 5321ea7a6f8SPoul-Henning Kamp 533de028f5aSJeff Roberson thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 53444990b8cSJulian Elischer thread_ctor, thread_dtor, thread_init, thread_fini, 5352c10be9eSMark Johnston 32 - 1, UMA_ZONE_NOFREE); 536cf7d9a8cSDavid Xu tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash); 53726007fe3SMateusz Guzik tidhashlock = (tidhash + 1) / 64; 53826007fe3SMateusz Guzik if (tidhashlock > 0) 53926007fe3SMateusz Guzik tidhashlock--; 54026007fe3SMateusz Guzik tidhashtbl_lock = malloc(sizeof(*tidhashtbl_lock) * (tidhashlock + 1), 54126007fe3SMateusz Guzik M_TIDHASH, M_WAITOK | M_ZERO); 54226007fe3SMateusz Guzik for (i = 0; i < tidhashlock + 1; i++) 54326007fe3SMateusz Guzik rw_init(&tidhashtbl_lock[i], "tidhash"); 544d116b9f1SMateusz Guzik 545d116b9f1SMateusz Guzik TASK_INIT(&thread_reap_task, 0, thread_reap_task_cb, NULL); 546d116b9f1SMateusz Guzik callout_init(&thread_reap_callout, 1); 547845d7797SKonstantin Belousov callout_reset(&thread_reap_callout, 5 * hz, 548845d7797SKonstantin Belousov thread_reap_callout_cb, NULL); 549c6d31b83SKonstantin Belousov ast_register(TDA_SUSPEND, ASTR_ASTF_REQUIRED, 0, ast_suspend); 55044990b8cSJulian Elischer } 55144990b8cSJulian Elischer 55244990b8cSJulian Elischer /* 553ff8fbcffSJeff Roberson * Place an unused thread on the zombie list. 55444990b8cSJulian Elischer */ 55544990b8cSJulian Elischer void 556ff8fbcffSJeff Roberson thread_zombie(struct thread *td) 55744990b8cSJulian Elischer { 558d116b9f1SMateusz Guzik struct thread_domain_data *tdd; 559c5315f51SMateusz Guzik struct thread *ztd; 560c5315f51SMateusz Guzik 561a9568cd2SMateusz Guzik tdd = &thread_domain_data[td->td_allocdomain]; 562d116b9f1SMateusz Guzik ztd = atomic_load_ptr(&tdd->tdd_zombies); 563c5315f51SMateusz Guzik for (;;) { 564c5315f51SMateusz Guzik td->td_zombie = ztd; 565d116b9f1SMateusz Guzik if (atomic_fcmpset_rel_ptr((uintptr_t *)&tdd->tdd_zombies, 566c5315f51SMateusz Guzik (uintptr_t *)&ztd, (uintptr_t)td)) 567c5315f51SMateusz Guzik break; 568c5315f51SMateusz Guzik continue; 569c5315f51SMateusz Guzik } 57044990b8cSJulian Elischer } 57144990b8cSJulian Elischer 5725c8329edSJulian Elischer /* 573ff8fbcffSJeff Roberson * Release a thread that has exited after cpu_throw(). 574ff8fbcffSJeff Roberson */ 575ff8fbcffSJeff Roberson void 576ff8fbcffSJeff Roberson thread_stash(struct thread *td) 577ff8fbcffSJeff Roberson { 578ff8fbcffSJeff Roberson atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); 579ff8fbcffSJeff Roberson thread_zombie(td); 580ff8fbcffSJeff Roberson } 581ff8fbcffSJeff Roberson 582ff8fbcffSJeff Roberson /* 583d116b9f1SMateusz Guzik * Reap zombies from passed domain. 58444990b8cSJulian Elischer */ 585d116b9f1SMateusz Guzik static void 586d116b9f1SMateusz Guzik thread_reap_domain(struct thread_domain_data *tdd) 58744990b8cSJulian Elischer { 588c5315f51SMateusz Guzik struct thread *itd, *ntd; 5895ef7b7a0SMateusz Guzik struct tidbatch tidbatch; 590f34a2f56SMateusz Guzik struct credbatch credbatch; 5915ef7b7a0SMateusz Guzik int tdcount; 592fb8ab680SMateusz Guzik struct plimit *lim; 593fb8ab680SMateusz Guzik int limcount; 59444990b8cSJulian Elischer 59544990b8cSJulian Elischer /* 596c5315f51SMateusz Guzik * Reading upfront is pessimal if followed by concurrent atomic_swap, 597c5315f51SMateusz Guzik * but most of the time the list is empty. 59844990b8cSJulian Elischer */ 599d116b9f1SMateusz Guzik if (tdd->tdd_zombies == NULL) 600c5315f51SMateusz Guzik return; 601c5315f51SMateusz Guzik 602d116b9f1SMateusz Guzik itd = (struct thread *)atomic_swap_ptr((uintptr_t *)&tdd->tdd_zombies, 603c5315f51SMateusz Guzik (uintptr_t)NULL); 6045ef7b7a0SMateusz Guzik if (itd == NULL) 6055ef7b7a0SMateusz Guzik return; 6065ef7b7a0SMateusz Guzik 607d116b9f1SMateusz Guzik /* 608d116b9f1SMateusz Guzik * Multiple CPUs can get here, the race is fine as ticks is only 609d116b9f1SMateusz Guzik * advisory. 610d116b9f1SMateusz Guzik */ 611d116b9f1SMateusz Guzik tdd->tdd_reapticks = ticks; 612d116b9f1SMateusz Guzik 6135ef7b7a0SMateusz Guzik tidbatch_prep(&tidbatch); 614f34a2f56SMateusz Guzik credbatch_prep(&credbatch); 6155ef7b7a0SMateusz Guzik tdcount = 0; 616fb8ab680SMateusz Guzik lim = NULL; 617fb8ab680SMateusz Guzik limcount = 0; 618d116b9f1SMateusz Guzik 619c5315f51SMateusz Guzik while (itd != NULL) { 620c5315f51SMateusz Guzik ntd = itd->td_zombie; 6215ef7b7a0SMateusz Guzik EVENTHANDLER_DIRECT_INVOKE(thread_dtor, itd); 6225ef7b7a0SMateusz Guzik tidbatch_add(&tidbatch, itd); 623f34a2f56SMateusz Guzik credbatch_add(&credbatch, itd); 624fb8ab680SMateusz Guzik MPASS(itd->td_limit != NULL); 625fb8ab680SMateusz Guzik if (lim != itd->td_limit) { 626fb8ab680SMateusz Guzik if (limcount != 0) { 627fb8ab680SMateusz Guzik lim_freen(lim, limcount); 628fb8ab680SMateusz Guzik limcount = 0; 629fb8ab680SMateusz Guzik } 630fb8ab680SMateusz Guzik } 631fb8ab680SMateusz Guzik lim = itd->td_limit; 632fb8ab680SMateusz Guzik limcount++; 633755341dfSMateusz Guzik thread_free_batched(itd); 6345ef7b7a0SMateusz Guzik tidbatch_process(&tidbatch); 635f34a2f56SMateusz Guzik credbatch_process(&credbatch); 6365ef7b7a0SMateusz Guzik tdcount++; 6375ef7b7a0SMateusz Guzik if (tdcount == 32) { 6385ef7b7a0SMateusz Guzik thread_count_sub(tdcount); 6395ef7b7a0SMateusz Guzik tdcount = 0; 640755341dfSMateusz Guzik } 641c5315f51SMateusz Guzik itd = ntd; 64244990b8cSJulian Elischer } 643755341dfSMateusz Guzik 6445ef7b7a0SMateusz Guzik tidbatch_final(&tidbatch); 645f34a2f56SMateusz Guzik credbatch_final(&credbatch); 6465ef7b7a0SMateusz Guzik if (tdcount != 0) { 6475ef7b7a0SMateusz Guzik thread_count_sub(tdcount); 648755341dfSMateusz Guzik } 649fb8ab680SMateusz Guzik MPASS(limcount != 0); 650fb8ab680SMateusz Guzik lim_freen(lim, limcount); 651ed062c8dSJulian Elischer } 65244990b8cSJulian Elischer 6534f0db5e0SJulian Elischer /* 654d116b9f1SMateusz Guzik * Reap zombies from all domains. 655d116b9f1SMateusz Guzik */ 656d116b9f1SMateusz Guzik static void 657d116b9f1SMateusz Guzik thread_reap_all(void) 658d116b9f1SMateusz Guzik { 659d116b9f1SMateusz Guzik struct thread_domain_data *tdd; 660d116b9f1SMateusz Guzik int i, domain; 661d116b9f1SMateusz Guzik 662d116b9f1SMateusz Guzik domain = PCPU_GET(domain); 663d116b9f1SMateusz Guzik for (i = 0; i < vm_ndomains; i++) { 664d116b9f1SMateusz Guzik tdd = &thread_domain_data[(i + domain) % vm_ndomains]; 665d116b9f1SMateusz Guzik thread_reap_domain(tdd); 666d116b9f1SMateusz Guzik } 667d116b9f1SMateusz Guzik } 668d116b9f1SMateusz Guzik 669d116b9f1SMateusz Guzik /* 670d116b9f1SMateusz Guzik * Reap zombies from local domain. 671d116b9f1SMateusz Guzik */ 672b83e94beSMateusz Guzik static void 673d116b9f1SMateusz Guzik thread_reap(void) 674d116b9f1SMateusz Guzik { 675d116b9f1SMateusz Guzik struct thread_domain_data *tdd; 676d116b9f1SMateusz Guzik int domain; 677d116b9f1SMateusz Guzik 678d116b9f1SMateusz Guzik domain = PCPU_GET(domain); 679d116b9f1SMateusz Guzik tdd = &thread_domain_data[domain]; 680d116b9f1SMateusz Guzik 681d116b9f1SMateusz Guzik thread_reap_domain(tdd); 682d116b9f1SMateusz Guzik } 683d116b9f1SMateusz Guzik 684d116b9f1SMateusz Guzik static void 685d116b9f1SMateusz Guzik thread_reap_task_cb(void *arg __unused, int pending __unused) 686d116b9f1SMateusz Guzik { 687d116b9f1SMateusz Guzik 688d116b9f1SMateusz Guzik thread_reap_all(); 689d116b9f1SMateusz Guzik } 690d116b9f1SMateusz Guzik 691d116b9f1SMateusz Guzik static void 692d116b9f1SMateusz Guzik thread_reap_callout_cb(void *arg __unused) 693d116b9f1SMateusz Guzik { 694d116b9f1SMateusz Guzik struct thread_domain_data *tdd; 695d116b9f1SMateusz Guzik int i, cticks, lticks; 696d116b9f1SMateusz Guzik bool wantreap; 697d116b9f1SMateusz Guzik 698d116b9f1SMateusz Guzik wantreap = false; 699d116b9f1SMateusz Guzik cticks = atomic_load_int(&ticks); 700d116b9f1SMateusz Guzik for (i = 0; i < vm_ndomains; i++) { 701d116b9f1SMateusz Guzik tdd = &thread_domain_data[i]; 702d116b9f1SMateusz Guzik lticks = tdd->tdd_reapticks; 703d116b9f1SMateusz Guzik if (tdd->tdd_zombies != NULL && 704d116b9f1SMateusz Guzik (u_int)(cticks - lticks) > 5 * hz) { 705d116b9f1SMateusz Guzik wantreap = true; 706d116b9f1SMateusz Guzik break; 707d116b9f1SMateusz Guzik } 708d116b9f1SMateusz Guzik } 709d116b9f1SMateusz Guzik 710d116b9f1SMateusz Guzik if (wantreap) 711d116b9f1SMateusz Guzik taskqueue_enqueue(taskqueue_thread, &thread_reap_task); 712845d7797SKonstantin Belousov callout_reset(&thread_reap_callout, 5 * hz, 713845d7797SKonstantin Belousov thread_reap_callout_cb, NULL); 714d116b9f1SMateusz Guzik } 715d116b9f1SMateusz Guzik 716d116b9f1SMateusz Guzik /* 717f62c7e54SKonstantin Belousov * Calling this function guarantees that any thread that exited before 718f62c7e54SKonstantin Belousov * the call is reaped when the function returns. By 'exited' we mean 719f62c7e54SKonstantin Belousov * a thread removed from the process linkage with thread_unlink(). 720f62c7e54SKonstantin Belousov * Practically this means that caller must lock/unlock corresponding 721f62c7e54SKonstantin Belousov * process lock before the call, to synchronize with thread_exit(). 722f62c7e54SKonstantin Belousov */ 723f62c7e54SKonstantin Belousov void 724f62c7e54SKonstantin Belousov thread_reap_barrier(void) 725f62c7e54SKonstantin Belousov { 726f62c7e54SKonstantin Belousov struct task *t; 727f62c7e54SKonstantin Belousov 728f62c7e54SKonstantin Belousov /* 729f62c7e54SKonstantin Belousov * First do context switches to each CPU to ensure that all 730f62c7e54SKonstantin Belousov * PCPU pc_deadthreads are moved to zombie list. 731f62c7e54SKonstantin Belousov */ 732f62c7e54SKonstantin Belousov quiesce_all_cpus("", PDROP); 733f62c7e54SKonstantin Belousov 734f62c7e54SKonstantin Belousov /* 735f62c7e54SKonstantin Belousov * Second, fire the task in the same thread as normal 736f62c7e54SKonstantin Belousov * thread_reap() is done, to serialize reaping. 737f62c7e54SKonstantin Belousov */ 738f62c7e54SKonstantin Belousov t = malloc(sizeof(*t), M_TEMP, M_WAITOK); 739f62c7e54SKonstantin Belousov TASK_INIT(t, 0, thread_reap_task_cb, t); 740f62c7e54SKonstantin Belousov taskqueue_enqueue(taskqueue_thread, t); 741f62c7e54SKonstantin Belousov taskqueue_drain(taskqueue_thread, t); 742f62c7e54SKonstantin Belousov free(t, M_TEMP); 743f62c7e54SKonstantin Belousov } 744f62c7e54SKonstantin Belousov 745f62c7e54SKonstantin Belousov /* 74644990b8cSJulian Elischer * Allocate a thread. 74744990b8cSJulian Elischer */ 74844990b8cSJulian Elischer struct thread * 7498a945d10SKonstantin Belousov thread_alloc(int pages) 75044990b8cSJulian Elischer { 75189b57fcfSKonstantin Belousov struct thread *td; 7521bd3cf5dSMateusz Guzik lwpid_t tid; 7538460a577SJohn Birrell 75462dbc992SMateusz Guzik if (!thread_count_inc()) { 7551bd3cf5dSMateusz Guzik return (NULL); 7561bd3cf5dSMateusz Guzik } 7571bd3cf5dSMateusz Guzik 75862dbc992SMateusz Guzik tid = tid_alloc(); 7591bd3cf5dSMateusz Guzik td = uma_zalloc(thread_zone, M_WAITOK); 76089b57fcfSKonstantin Belousov KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); 7618a945d10SKonstantin Belousov if (!vm_thread_new(td, pages)) { 76289b57fcfSKonstantin Belousov uma_zfree(thread_zone, td); 7631bd3cf5dSMateusz Guzik tid_free(tid); 76462dbc992SMateusz Guzik thread_count_dec(); 76589b57fcfSKonstantin Belousov return (NULL); 76689b57fcfSKonstantin Belousov } 7671bd3cf5dSMateusz Guzik td->td_tid = tid; 768f575573cSKonstantin Belousov bzero(&td->td_sa.args, sizeof(td->td_sa.args)); 7695dda15adSMark Johnston kmsan_thread_alloc(td); 7700c3967e7SMarcel Moolenaar cpu_thread_alloc(td); 7711bd3cf5dSMateusz Guzik EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td); 77289b57fcfSKonstantin Belousov return (td); 77344990b8cSJulian Elischer } 77444990b8cSJulian Elischer 7758a945d10SKonstantin Belousov int 7768a945d10SKonstantin Belousov thread_alloc_stack(struct thread *td, int pages) 7778a945d10SKonstantin Belousov { 7788a945d10SKonstantin Belousov 7798a945d10SKonstantin Belousov KASSERT(td->td_kstack == 0, 7808a945d10SKonstantin Belousov ("thread_alloc_stack called on a thread with kstack")); 7818a945d10SKonstantin Belousov if (!vm_thread_new(td, pages)) 7828a945d10SKonstantin Belousov return (0); 7838a945d10SKonstantin Belousov cpu_thread_alloc(td); 7848a945d10SKonstantin Belousov return (1); 7858a945d10SKonstantin Belousov } 7864f0db5e0SJulian Elischer 7874f0db5e0SJulian Elischer /* 78844990b8cSJulian Elischer * Deallocate a thread. 78944990b8cSJulian Elischer */ 790755341dfSMateusz Guzik static void 791755341dfSMateusz Guzik thread_free_batched(struct thread *td) 79244990b8cSJulian Elischer { 7932e6b8de4SJeff Roberson 7942e6b8de4SJeff Roberson lock_profile_thread_exit(td); 79545aea8deSJeff Roberson if (td->td_cpuset) 796d7f687fcSJeff Roberson cpuset_rel(td->td_cpuset); 797d7f687fcSJeff Roberson td->td_cpuset = NULL; 7980c3967e7SMarcel Moolenaar cpu_thread_free(td); 79989b57fcfSKonstantin Belousov if (td->td_kstack != 0) 80089b57fcfSKonstantin Belousov vm_thread_dispose(td); 8012d19b736SKonstantin Belousov callout_drain(&td->td_slpcallout); 802755341dfSMateusz Guzik /* 803755341dfSMateusz Guzik * Freeing handled by the caller. 804755341dfSMateusz Guzik */ 8051bd3cf5dSMateusz Guzik td->td_tid = -1; 8065dda15adSMark Johnston kmsan_thread_free(td); 80744990b8cSJulian Elischer uma_zfree(thread_zone, td); 80844990b8cSJulian Elischer } 80944990b8cSJulian Elischer 8104ea6a9a2SMateusz Guzik void 811755341dfSMateusz Guzik thread_free(struct thread *td) 812755341dfSMateusz Guzik { 813755341dfSMateusz Guzik lwpid_t tid; 814755341dfSMateusz Guzik 8155ef7b7a0SMateusz Guzik EVENTHANDLER_DIRECT_INVOKE(thread_dtor, td); 816755341dfSMateusz Guzik tid = td->td_tid; 817755341dfSMateusz Guzik thread_free_batched(td); 818755341dfSMateusz Guzik tid_free(tid); 81962dbc992SMateusz Guzik thread_count_dec(); 820755341dfSMateusz Guzik } 821755341dfSMateusz Guzik 822755341dfSMateusz Guzik void 8234ea6a9a2SMateusz Guzik thread_cow_get_proc(struct thread *newtd, struct proc *p) 8244ea6a9a2SMateusz Guzik { 8254ea6a9a2SMateusz Guzik 8264ea6a9a2SMateusz Guzik PROC_LOCK_ASSERT(p, MA_OWNED); 8271724c563SMateusz Guzik newtd->td_realucred = crcowget(p->p_ucred); 8281724c563SMateusz Guzik newtd->td_ucred = newtd->td_realucred; 829f6f6d240SMateusz Guzik newtd->td_limit = lim_hold(p->p_limit); 8304ea6a9a2SMateusz Guzik newtd->td_cowgen = p->p_cowgen; 8314ea6a9a2SMateusz Guzik } 8324ea6a9a2SMateusz Guzik 8334ea6a9a2SMateusz Guzik void 8344ea6a9a2SMateusz Guzik thread_cow_get(struct thread *newtd, struct thread *td) 8354ea6a9a2SMateusz Guzik { 8364ea6a9a2SMateusz Guzik 8371724c563SMateusz Guzik MPASS(td->td_realucred == td->td_ucred); 8381724c563SMateusz Guzik newtd->td_realucred = crcowget(td->td_realucred); 8391724c563SMateusz Guzik newtd->td_ucred = newtd->td_realucred; 840f6f6d240SMateusz Guzik newtd->td_limit = lim_hold(td->td_limit); 8414ea6a9a2SMateusz Guzik newtd->td_cowgen = td->td_cowgen; 8424ea6a9a2SMateusz Guzik } 8434ea6a9a2SMateusz Guzik 8444ea6a9a2SMateusz Guzik void 8454ea6a9a2SMateusz Guzik thread_cow_free(struct thread *td) 8464ea6a9a2SMateusz Guzik { 8474ea6a9a2SMateusz Guzik 8481724c563SMateusz Guzik if (td->td_realucred != NULL) 8491724c563SMateusz Guzik crcowfree(td); 850cd672ca6SMateusz Guzik if (td->td_limit != NULL) 851f6f6d240SMateusz Guzik lim_free(td->td_limit); 8524ea6a9a2SMateusz Guzik } 8534ea6a9a2SMateusz Guzik 8544ea6a9a2SMateusz Guzik void 8554ea6a9a2SMateusz Guzik thread_cow_update(struct thread *td) 8564ea6a9a2SMateusz Guzik { 8574ea6a9a2SMateusz Guzik struct proc *p; 858cd672ca6SMateusz Guzik struct ucred *oldcred; 859cd672ca6SMateusz Guzik struct plimit *oldlimit; 8604ea6a9a2SMateusz Guzik 8614ea6a9a2SMateusz Guzik p = td->td_proc; 8624ea6a9a2SMateusz Guzik PROC_LOCK(p); 8631724c563SMateusz Guzik oldcred = crcowsync(); 8648a0cb04dSMateusz Guzik oldlimit = lim_cowsync(); 8654ea6a9a2SMateusz Guzik td->td_cowgen = p->p_cowgen; 8664ea6a9a2SMateusz Guzik PROC_UNLOCK(p); 867cd672ca6SMateusz Guzik if (oldcred != NULL) 868cd672ca6SMateusz Guzik crfree(oldcred); 869cd672ca6SMateusz Guzik if (oldlimit != NULL) 870cd672ca6SMateusz Guzik lim_free(oldlimit); 8714ea6a9a2SMateusz Guzik } 8724ea6a9a2SMateusz Guzik 87332114b63SMateusz Guzik void 87432114b63SMateusz Guzik thread_cow_synced(struct thread *td) 87532114b63SMateusz Guzik { 87632114b63SMateusz Guzik struct proc *p; 87732114b63SMateusz Guzik 87832114b63SMateusz Guzik p = td->td_proc; 87932114b63SMateusz Guzik PROC_LOCK_ASSERT(p, MA_OWNED); 88032114b63SMateusz Guzik MPASS(td->td_cowgen != p->p_cowgen); 88132114b63SMateusz Guzik MPASS(td->td_ucred == p->p_ucred); 88232114b63SMateusz Guzik MPASS(td->td_limit == p->p_limit); 88332114b63SMateusz Guzik td->td_cowgen = p->p_cowgen; 88432114b63SMateusz Guzik } 88532114b63SMateusz Guzik 88644990b8cSJulian Elischer /* 88744990b8cSJulian Elischer * Discard the current thread and exit from its context. 88894e0a4cdSJulian Elischer * Always called with scheduler locked. 88944990b8cSJulian Elischer * 89044990b8cSJulian Elischer * Because we can't free a thread while we're operating under its context, 891696058c3SJulian Elischer * push the current thread into our CPU's deadthread holder. This means 892696058c3SJulian Elischer * we needn't worry about someone else grabbing our context before we 8936617724cSJeff Roberson * do a cpu_throw(). 89444990b8cSJulian Elischer */ 89544990b8cSJulian Elischer void 89644990b8cSJulian Elischer thread_exit(void) 89744990b8cSJulian Elischer { 8987e3a96eaSJohn Baldwin uint64_t runtime, new_switchtime; 89944990b8cSJulian Elischer struct thread *td; 9001c4bcd05SJeff Roberson struct thread *td2; 90144990b8cSJulian Elischer struct proc *p; 9027847a9daSJohn Baldwin int wakeup_swapper; 90344990b8cSJulian Elischer 90444990b8cSJulian Elischer td = curthread; 90544990b8cSJulian Elischer p = td->td_proc; 90644990b8cSJulian Elischer 907a54e85fdSJeff Roberson PROC_SLOCK_ASSERT(p, MA_OWNED); 908ed062c8dSJulian Elischer mtx_assert(&Giant, MA_NOTOWNED); 909a54e85fdSJeff Roberson 91044990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 911ed062c8dSJulian Elischer KASSERT(p != NULL, ("thread exiting without a process")); 912cc701b73SRobert Watson CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 913e01eafefSJulian Elischer (long)p->p_pid, td->td_name); 9146c9271a9SAndriy Gapon SDT_PROBE0(proc, , , lwp__exit); 9159104847fSDavid Xu KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 916936c24faSMateusz Guzik MPASS(td->td_realucred == td->td_ucred); 91744990b8cSJulian Elischer 918ed062c8dSJulian Elischer /* 919ed062c8dSJulian Elischer * drop FPU & debug register state storage, or any other 920ed062c8dSJulian Elischer * architecture specific resources that 921ed062c8dSJulian Elischer * would not be on a new untouched process. 922ed062c8dSJulian Elischer */ 923bd07998eSKonstantin Belousov cpu_thread_exit(td); 92444990b8cSJulian Elischer 925ed062c8dSJulian Elischer /* 9261faf202eSJulian Elischer * The last thread is left attached to the process 9271faf202eSJulian Elischer * So that the whole bundle gets recycled. Skip 928ed062c8dSJulian Elischer * all this stuff if we never had threads. 929ed062c8dSJulian Elischer * EXIT clears all sign of other threads when 930ed062c8dSJulian Elischer * it goes to single threading, so the last thread always 931ed062c8dSJulian Elischer * takes the short path. 9321faf202eSJulian Elischer */ 933ed062c8dSJulian Elischer if (p->p_flag & P_HADTHREADS) { 9341faf202eSJulian Elischer if (p->p_numthreads > 1) { 935fd229b5bSKonstantin Belousov atomic_add_int(&td->td_proc->p_exitthreads, 1); 936d3a0bd78SJulian Elischer thread_unlink(td); 9371c4bcd05SJeff Roberson td2 = FIRST_THREAD_IN_PROC(p); 9381c4bcd05SJeff Roberson sched_exit_thread(td2, td); 939ed062c8dSJulian Elischer 940ed062c8dSJulian Elischer /* 94144990b8cSJulian Elischer * The test below is NOT true if we are the 9429182554aSKonstantin Belousov * sole exiting thread. P_STOPPED_SINGLE is unset 94344990b8cSJulian Elischer * in exit1() after it is the only survivor. 94444990b8cSJulian Elischer */ 9451279572aSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 94644990b8cSJulian Elischer if (p->p_numthreads == p->p_suspcount) { 947a54e85fdSJeff Roberson thread_lock(p->p_singlethread); 9487847a9daSJohn Baldwin wakeup_swapper = thread_unsuspend_one( 94984cdea97SKonstantin Belousov p->p_singlethread, p, false); 9507847a9daSJohn Baldwin if (wakeup_swapper) 9517847a9daSJohn Baldwin kick_proc0(); 95244990b8cSJulian Elischer } 95344990b8cSJulian Elischer } 95448bfcdddSJulian Elischer 955696058c3SJulian Elischer PCPU_SET(deadthread, td); 9561faf202eSJulian Elischer } else { 957ed062c8dSJulian Elischer /* 958ed062c8dSJulian Elischer * The last thread is exiting.. but not through exit() 959ed062c8dSJulian Elischer */ 960ed062c8dSJulian Elischer panic ("thread_exit: Last thread exiting on its own"); 961ed062c8dSJulian Elischer } 9621faf202eSJulian Elischer } 96316d95d4fSJoseph Koshy #ifdef HWPMC_HOOKS 96416d95d4fSJoseph Koshy /* 96516d95d4fSJoseph Koshy * If this thread is part of a process that is being tracked by hwpmc(4), 96616d95d4fSJoseph Koshy * inform the module of the thread's impending exit. 96716d95d4fSJoseph Koshy */ 9686161b98cSMatt Macy if (PMC_PROC_IS_USING_PMCS(td->td_proc)) { 96916d95d4fSJoseph Koshy PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); 9706161b98cSMatt Macy PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT, NULL); 971ebfaf69cSMatt Macy } else if (PMC_SYSTEM_SAMPLING_ACTIVE()) 972ebfaf69cSMatt Macy PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT_LOG, NULL); 97316d95d4fSJoseph Koshy #endif 974a54e85fdSJeff Roberson PROC_UNLOCK(p); 9755c7bebf9SKonstantin Belousov PROC_STATLOCK(p); 9765c7bebf9SKonstantin Belousov thread_lock(td); 9775c7bebf9SKonstantin Belousov PROC_SUNLOCK(p); 9787e3a96eaSJohn Baldwin 9797e3a96eaSJohn Baldwin /* Do the same timestamp bookkeeping that mi_switch() would do. */ 9807e3a96eaSJohn Baldwin new_switchtime = cpu_ticks(); 9817e3a96eaSJohn Baldwin runtime = new_switchtime - PCPU_GET(switchtime); 9827e3a96eaSJohn Baldwin td->td_runtime += runtime; 9837e3a96eaSJohn Baldwin td->td_incruntime += runtime; 9847e3a96eaSJohn Baldwin PCPU_SET(switchtime, new_switchtime); 9857e3a96eaSJohn Baldwin PCPU_SET(switchticks, ticks); 98683c9dea1SGleb Smirnoff VM_CNT_INC(v_swtch); 9877e3a96eaSJohn Baldwin 9887e3a96eaSJohn Baldwin /* Save our resource usage in our process. */ 9897e3a96eaSJohn Baldwin td->td_ru.ru_nvcsw++; 99061a74c5cSJeff Roberson ruxagg_locked(p, td); 9917e3a96eaSJohn Baldwin rucollect(&p->p_ru, &td->td_ru); 9925c7bebf9SKonstantin Belousov PROC_STATUNLOCK(p); 9937e3a96eaSJohn Baldwin 994fa2528acSAlex Richardson TD_SET_STATE(td, TDS_INACTIVE); 9953d06b4b3SAttilio Rao #ifdef WITNESS 9963d06b4b3SAttilio Rao witness_thread_exit(td); 9973d06b4b3SAttilio Rao #endif 998732d9528SJulian Elischer CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 999a54e85fdSJeff Roberson sched_throw(td); 1000cc66ebe2SPeter Wemm panic("I'm a teapot!"); 100144990b8cSJulian Elischer /* NOTREACHED */ 100244990b8cSJulian Elischer } 100344990b8cSJulian Elischer 100444990b8cSJulian Elischer /* 1005696058c3SJulian Elischer * Do any thread specific cleanups that may be needed in wait() 100637814395SPeter Wemm * called with Giant, proc and schedlock not held. 1007696058c3SJulian Elischer */ 1008696058c3SJulian Elischer void 1009696058c3SJulian Elischer thread_wait(struct proc *p) 1010696058c3SJulian Elischer { 1011696058c3SJulian Elischer struct thread *td; 1012696058c3SJulian Elischer 101337814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 1014624bf9e1SKonstantin Belousov KASSERT(p->p_numthreads == 1, ("multiple threads in thread_wait()")); 1015624bf9e1SKonstantin Belousov KASSERT(p->p_exitthreads == 0, ("p_exitthreads leaking")); 1016ff8fbcffSJeff Roberson td = FIRST_THREAD_IN_PROC(p); 1017ff8fbcffSJeff Roberson /* Lock the last thread so we spin until it exits cpu_throw(). */ 1018ff8fbcffSJeff Roberson thread_lock(td); 1019ff8fbcffSJeff Roberson thread_unlock(td); 10202e6b8de4SJeff Roberson lock_profile_thread_exit(td); 1021d7f687fcSJeff Roberson cpuset_rel(td->td_cpuset); 1022d7f687fcSJeff Roberson td->td_cpuset = NULL; 1023696058c3SJulian Elischer cpu_thread_clean(td); 10244ea6a9a2SMateusz Guzik thread_cow_free(td); 10252d19b736SKonstantin Belousov callout_drain(&td->td_slpcallout); 1026696058c3SJulian Elischer thread_reap(); /* check for zombie threads etc. */ 1027696058c3SJulian Elischer } 1028696058c3SJulian Elischer 1029696058c3SJulian Elischer /* 103044990b8cSJulian Elischer * Link a thread to a process. 10311faf202eSJulian Elischer * set up anything that needs to be initialized for it to 10321faf202eSJulian Elischer * be used by the process. 103344990b8cSJulian Elischer */ 103444990b8cSJulian Elischer void 10358460a577SJohn Birrell thread_link(struct thread *td, struct proc *p) 103644990b8cSJulian Elischer { 103744990b8cSJulian Elischer 1038a54e85fdSJeff Roberson /* 1039a54e85fdSJeff Roberson * XXX This can't be enabled because it's called for proc0 before 1040374ae2a3SJeff Roberson * its lock has been created. 1041374ae2a3SJeff Roberson * PROC_LOCK_ASSERT(p, MA_OWNED); 1042a54e85fdSJeff Roberson */ 1043fa2528acSAlex Richardson TD_SET_STATE(td, TDS_INACTIVE); 104444990b8cSJulian Elischer td->td_proc = p; 1045b61ce5b0SJeff Roberson td->td_flags = TDF_INMEM; 104644990b8cSJulian Elischer 10471faf202eSJulian Elischer LIST_INIT(&td->td_contested); 1048eea4f254SJeff Roberson LIST_INIT(&td->td_lprof[0]); 1049eea4f254SJeff Roberson LIST_INIT(&td->td_lprof[1]); 1050f6eccf96SGleb Smirnoff #ifdef EPOCH_TRACE 1051dd902d01SGleb Smirnoff SLIST_INIT(&td->td_epochs); 1052f6eccf96SGleb Smirnoff #endif 10539104847fSDavid Xu sigqueue_init(&td->td_sigqueue, p); 1054fd90e2edSJung-uk Kim callout_init(&td->td_slpcallout, 1); 105566d8df9dSDaniel Eischen TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist); 105644990b8cSJulian Elischer p->p_numthreads++; 105744990b8cSJulian Elischer } 105844990b8cSJulian Elischer 1059ed062c8dSJulian Elischer /* 1060ed062c8dSJulian Elischer * Called from: 1061ed062c8dSJulian Elischer * thread_exit() 1062ed062c8dSJulian Elischer */ 1063d3a0bd78SJulian Elischer void 1064d3a0bd78SJulian Elischer thread_unlink(struct thread *td) 1065d3a0bd78SJulian Elischer { 1066d3a0bd78SJulian Elischer struct proc *p = td->td_proc; 1067d3a0bd78SJulian Elischer 1068374ae2a3SJeff Roberson PROC_LOCK_ASSERT(p, MA_OWNED); 1069f6eccf96SGleb Smirnoff #ifdef EPOCH_TRACE 1070dd902d01SGleb Smirnoff MPASS(SLIST_EMPTY(&td->td_epochs)); 1071f6eccf96SGleb Smirnoff #endif 1072dd902d01SGleb Smirnoff 1073d3a0bd78SJulian Elischer TAILQ_REMOVE(&p->p_threads, td, td_plist); 1074d3a0bd78SJulian Elischer p->p_numthreads--; 1075d3a0bd78SJulian Elischer /* could clear a few other things here */ 10768460a577SJohn Birrell /* Must NOT clear links to proc! */ 10775c8329edSJulian Elischer } 10785c8329edSJulian Elischer 107979799053SKonstantin Belousov static int 108079799053SKonstantin Belousov calc_remaining(struct proc *p, int mode) 108179799053SKonstantin Belousov { 108279799053SKonstantin Belousov int remaining; 108379799053SKonstantin Belousov 10847b519077SKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED); 10857b519077SKonstantin Belousov PROC_SLOCK_ASSERT(p, MA_OWNED); 108679799053SKonstantin Belousov if (mode == SINGLE_EXIT) 108779799053SKonstantin Belousov remaining = p->p_numthreads; 108879799053SKonstantin Belousov else if (mode == SINGLE_BOUNDARY) 108979799053SKonstantin Belousov remaining = p->p_numthreads - p->p_boundary_count; 10906ddcc233SKonstantin Belousov else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC) 109179799053SKonstantin Belousov remaining = p->p_numthreads - p->p_suspcount; 109279799053SKonstantin Belousov else 109379799053SKonstantin Belousov panic("calc_remaining: wrong mode %d", mode); 109479799053SKonstantin Belousov return (remaining); 109579799053SKonstantin Belousov } 109679799053SKonstantin Belousov 109707a9368aSKonstantin Belousov static int 109807a9368aSKonstantin Belousov remain_for_mode(int mode) 109907a9368aSKonstantin Belousov { 110007a9368aSKonstantin Belousov 11016ddcc233SKonstantin Belousov return (mode == SINGLE_ALLPROC ? 0 : 1); 110207a9368aSKonstantin Belousov } 110307a9368aSKonstantin Belousov 110407a9368aSKonstantin Belousov static int 110507a9368aSKonstantin Belousov weed_inhib(int mode, struct thread *td2, struct proc *p) 110607a9368aSKonstantin Belousov { 110707a9368aSKonstantin Belousov int wakeup_swapper; 110807a9368aSKonstantin Belousov 110907a9368aSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED); 111007a9368aSKonstantin Belousov PROC_SLOCK_ASSERT(p, MA_OWNED); 111107a9368aSKonstantin Belousov THREAD_LOCK_ASSERT(td2, MA_OWNED); 111207a9368aSKonstantin Belousov 111307a9368aSKonstantin Belousov wakeup_swapper = 0; 111461a74c5cSJeff Roberson 111561a74c5cSJeff Roberson /* 111661a74c5cSJeff Roberson * Since the thread lock is dropped by the scheduler we have 111761a74c5cSJeff Roberson * to retry to check for races. 111861a74c5cSJeff Roberson */ 111961a74c5cSJeff Roberson restart: 112007a9368aSKonstantin Belousov switch (mode) { 112107a9368aSKonstantin Belousov case SINGLE_EXIT: 112261a74c5cSJeff Roberson if (TD_IS_SUSPENDED(td2)) { 112384cdea97SKonstantin Belousov wakeup_swapper |= thread_unsuspend_one(td2, p, true); 112461a74c5cSJeff Roberson thread_lock(td2); 112561a74c5cSJeff Roberson goto restart; 112661a74c5cSJeff Roberson } 112761a74c5cSJeff Roberson if (TD_CAN_ABORT(td2)) { 112807a9368aSKonstantin Belousov wakeup_swapper |= sleepq_abort(td2, EINTR); 112961a74c5cSJeff Roberson return (wakeup_swapper); 113061a74c5cSJeff Roberson } 113107a9368aSKonstantin Belousov break; 113207a9368aSKonstantin Belousov case SINGLE_BOUNDARY: 113307a9368aSKonstantin Belousov case SINGLE_NO_EXIT: 113461a74c5cSJeff Roberson if (TD_IS_SUSPENDED(td2) && 113561a74c5cSJeff Roberson (td2->td_flags & TDF_BOUNDARY) == 0) { 113684cdea97SKonstantin Belousov wakeup_swapper |= thread_unsuspend_one(td2, p, false); 113761a74c5cSJeff Roberson thread_lock(td2); 113861a74c5cSJeff Roberson goto restart; 113961a74c5cSJeff Roberson } 114061a74c5cSJeff Roberson if (TD_CAN_ABORT(td2)) { 114107a9368aSKonstantin Belousov wakeup_swapper |= sleepq_abort(td2, ERESTART); 114261a74c5cSJeff Roberson return (wakeup_swapper); 114361a74c5cSJeff Roberson } 1144917dd390SKonstantin Belousov break; 11456ddcc233SKonstantin Belousov case SINGLE_ALLPROC: 11466ddcc233SKonstantin Belousov /* 11476ddcc233SKonstantin Belousov * ALLPROC suspend tries to avoid spurious EINTR for 11486ddcc233SKonstantin Belousov * threads sleeping interruptable, by suspending the 11496ddcc233SKonstantin Belousov * thread directly, similarly to sig_suspend_threads(). 1150dd883e9aSKonstantin Belousov * Since such sleep is not neccessary performed at the user 1151dd883e9aSKonstantin Belousov * boundary, TDF_ALLPROCSUSP is used to avoid immediate 1152dd883e9aSKonstantin Belousov * un-suspend. 11536ddcc233SKonstantin Belousov */ 1154*5c274b36SBrooks Davis if (TD_IS_SUSPENDED(td2) && 1155*5c274b36SBrooks Davis (td2->td_flags & TDF_ALLPROCSUSP) == 0) { 115684cdea97SKonstantin Belousov wakeup_swapper |= thread_unsuspend_one(td2, p, false); 115761a74c5cSJeff Roberson thread_lock(td2); 115861a74c5cSJeff Roberson goto restart; 115961a74c5cSJeff Roberson } 116061a74c5cSJeff Roberson if (TD_CAN_ABORT(td2)) { 11616ddcc233SKonstantin Belousov td2->td_flags |= TDF_ALLPROCSUSP; 11626ddcc233SKonstantin Belousov wakeup_swapper |= sleepq_abort(td2, ERESTART); 116361a74c5cSJeff Roberson return (wakeup_swapper); 11646ddcc233SKonstantin Belousov } 116507a9368aSKonstantin Belousov break; 116661a74c5cSJeff Roberson default: 116761a74c5cSJeff Roberson break; 116807a9368aSKonstantin Belousov } 116961a74c5cSJeff Roberson thread_unlock(td2); 117007a9368aSKonstantin Belousov return (wakeup_swapper); 117107a9368aSKonstantin Belousov } 117207a9368aSKonstantin Belousov 11735215b187SJeff Roberson /* 117444990b8cSJulian Elischer * Enforce single-threading. 117544990b8cSJulian Elischer * 117644990b8cSJulian Elischer * Returns 1 if the caller must abort (another thread is waiting to 117744990b8cSJulian Elischer * exit the process or similar). Process is locked! 117844990b8cSJulian Elischer * Returns 0 when you are successfully the only thread running. 117944990b8cSJulian Elischer * A process has successfully single threaded in the suspend mode when 118044990b8cSJulian Elischer * There are no threads in user mode. Threads in the kernel must be 118144990b8cSJulian Elischer * allowed to continue until they get to the user boundary. They may even 118244990b8cSJulian Elischer * copy out their return values and data before suspending. They may however be 1183e2668f55SMaxim Konovalov * accelerated in reaching the user boundary as we will wake up 118444990b8cSJulian Elischer * any sleeping threads that are interruptable. (PCATCH). 118544990b8cSJulian Elischer */ 118644990b8cSJulian Elischer int 11876ddcc233SKonstantin Belousov thread_single(struct proc *p, int mode) 118844990b8cSJulian Elischer { 118944990b8cSJulian Elischer struct thread *td; 119044990b8cSJulian Elischer struct thread *td2; 1191da7bbd2cSJohn Baldwin int remaining, wakeup_swapper; 119244990b8cSJulian Elischer 119344990b8cSJulian Elischer td = curthread; 11946ddcc233SKonstantin Belousov KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY || 11956ddcc233SKonstantin Belousov mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT, 11966ddcc233SKonstantin Belousov ("invalid mode %d", mode)); 11976ddcc233SKonstantin Belousov /* 11986ddcc233SKonstantin Belousov * If allowing non-ALLPROC singlethreading for non-curproc 11996ddcc233SKonstantin Belousov * callers, calc_remaining() and remain_for_mode() should be 12006ddcc233SKonstantin Belousov * adjusted to also account for td->td_proc != p. For now 12016ddcc233SKonstantin Belousov * this is not implemented because it is not used. 12026ddcc233SKonstantin Belousov */ 12036ddcc233SKonstantin Belousov KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) || 12046ddcc233SKonstantin Belousov (mode != SINGLE_ALLPROC && td->td_proc == p), 12056ddcc233SKonstantin Belousov ("mode %d proc %p curproc %p", mode, p, td->td_proc)); 120637814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 120744990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 120844990b8cSJulian Elischer 1209d7a9e6e7SKonstantin Belousov /* 1210d7a9e6e7SKonstantin Belousov * Is someone already single threading? 1211d7a9e6e7SKonstantin Belousov * Or may be singlethreading is not needed at all. 1212d7a9e6e7SKonstantin Belousov */ 1213d7a9e6e7SKonstantin Belousov if (mode == SINGLE_ALLPROC) { 1214d7a9e6e7SKonstantin Belousov while ((p->p_flag & P_STOPPED_SINGLE) != 0) { 1215d7a9e6e7SKonstantin Belousov if ((p->p_flag2 & P2_WEXIT) != 0) 1216d7a9e6e7SKonstantin Belousov return (1); 1217d7a9e6e7SKonstantin Belousov msleep(&p->p_flag, &p->p_mtx, PCATCH, "thrsgl", 0); 1218d7a9e6e7SKonstantin Belousov } 1219d7a9e6e7SKonstantin Belousov } else if ((p->p_flag & P_HADTHREADS) == 0) 122044990b8cSJulian Elischer return (0); 1221906ac69dSDavid Xu if (p->p_singlethread != NULL && p->p_singlethread != td) 122244990b8cSJulian Elischer return (1); 122344990b8cSJulian Elischer 1224906ac69dSDavid Xu if (mode == SINGLE_EXIT) { 1225906ac69dSDavid Xu p->p_flag |= P_SINGLE_EXIT; 1226906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_BOUNDARY; 1227906ac69dSDavid Xu } else { 1228906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_EXIT; 1229906ac69dSDavid Xu if (mode == SINGLE_BOUNDARY) 1230906ac69dSDavid Xu p->p_flag |= P_SINGLE_BOUNDARY; 1231906ac69dSDavid Xu else 1232906ac69dSDavid Xu p->p_flag &= ~P_SINGLE_BOUNDARY; 1233906ac69dSDavid Xu } 1234f829268bSKonstantin Belousov if (mode == SINGLE_ALLPROC) 12356ddcc233SKonstantin Belousov p->p_flag |= P_TOTAL_STOP; 12361279572aSDavid Xu p->p_flag |= P_STOPPED_SINGLE; 12377b4a950aSDavid Xu PROC_SLOCK(p); 1238112afcb2SJohn Baldwin p->p_singlethread = td; 123979799053SKonstantin Belousov remaining = calc_remaining(p, mode); 124007a9368aSKonstantin Belousov while (remaining != remain_for_mode(mode)) { 1241bf1a3220SDavid Xu if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 1242bf1a3220SDavid Xu goto stopme; 1243da7bbd2cSJohn Baldwin wakeup_swapper = 0; 124444990b8cSJulian Elischer FOREACH_THREAD_IN_PROC(p, td2) { 124544990b8cSJulian Elischer if (td2 == td) 124644990b8cSJulian Elischer continue; 1247a54e85fdSJeff Roberson thread_lock(td2); 1248c6d31b83SKonstantin Belousov ast_sched_locked(td2, TDA_SUSPEND); 12496ddcc233SKonstantin Belousov if (TD_IS_INHIBITED(td2)) { 125007a9368aSKonstantin Belousov wakeup_swapper |= weed_inhib(mode, td2, p); 1251d8267df7SDavid Xu #ifdef SMP 1252b9009b17SKonstantin Belousov } else if (TD_IS_RUNNING(td2)) { 1253d8267df7SDavid Xu forward_signal(td2); 125461a74c5cSJeff Roberson thread_unlock(td2); 1255d8267df7SDavid Xu #endif 125661a74c5cSJeff Roberson } else 1257a54e85fdSJeff Roberson thread_unlock(td2); 12589d102777SJulian Elischer } 1259da7bbd2cSJohn Baldwin if (wakeup_swapper) 1260da7bbd2cSJohn Baldwin kick_proc0(); 126179799053SKonstantin Belousov remaining = calc_remaining(p, mode); 1262ec008e96SDavid Xu 12639d102777SJulian Elischer /* 12649d102777SJulian Elischer * Maybe we suspended some threads.. was it enough? 12659d102777SJulian Elischer */ 126607a9368aSKonstantin Belousov if (remaining == remain_for_mode(mode)) 12679d102777SJulian Elischer break; 12689d102777SJulian Elischer 1269bf1a3220SDavid Xu stopme: 127044990b8cSJulian Elischer /* 127144990b8cSJulian Elischer * Wake us up when everyone else has suspended. 1272e3b9bf71SJulian Elischer * In the mean time we suspend as well. 127344990b8cSJulian Elischer */ 12746ddcc233SKonstantin Belousov thread_suspend_switch(td, p); 127579799053SKonstantin Belousov remaining = calc_remaining(p, mode); 127644990b8cSJulian Elischer } 1277906ac69dSDavid Xu if (mode == SINGLE_EXIT) { 127891599697SJulian Elischer /* 12798626a0ddSKonstantin Belousov * Convert the process to an unthreaded process. The 12808626a0ddSKonstantin Belousov * SINGLE_EXIT is called by exit1() or execve(), in 12818626a0ddSKonstantin Belousov * both cases other threads must be retired. 128291599697SJulian Elischer */ 12838626a0ddSKonstantin Belousov KASSERT(p->p_numthreads == 1, ("Unthreading with >1 threads")); 1284ed062c8dSJulian Elischer p->p_singlethread = NULL; 12858626a0ddSKonstantin Belousov p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS); 1286fd229b5bSKonstantin Belousov 1287fd229b5bSKonstantin Belousov /* 1288fd229b5bSKonstantin Belousov * Wait for any remaining threads to exit cpu_throw(). 1289fd229b5bSKonstantin Belousov */ 1290fd229b5bSKonstantin Belousov while (p->p_exitthreads != 0) { 1291fd229b5bSKonstantin Belousov PROC_SUNLOCK(p); 1292fd229b5bSKonstantin Belousov PROC_UNLOCK(p); 1293fd229b5bSKonstantin Belousov sched_relinquish(td); 1294fd229b5bSKonstantin Belousov PROC_LOCK(p); 1295fd229b5bSKonstantin Belousov PROC_SLOCK(p); 1296fd229b5bSKonstantin Belousov } 1297ac437c07SKonstantin Belousov } else if (mode == SINGLE_BOUNDARY) { 1298ac437c07SKonstantin Belousov /* 1299ac437c07SKonstantin Belousov * Wait until all suspended threads are removed from 1300ac437c07SKonstantin Belousov * the processors. The thread_suspend_check() 1301ac437c07SKonstantin Belousov * increments p_boundary_count while it is still 1302ac437c07SKonstantin Belousov * running, which makes it possible for the execve() 1303ac437c07SKonstantin Belousov * to destroy vmspace while our other threads are 1304ac437c07SKonstantin Belousov * still using the address space. 1305ac437c07SKonstantin Belousov * 1306ac437c07SKonstantin Belousov * We lock the thread, which is only allowed to 1307ac437c07SKonstantin Belousov * succeed after context switch code finished using 1308ac437c07SKonstantin Belousov * the address space. 1309ac437c07SKonstantin Belousov */ 1310ac437c07SKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td2) { 1311ac437c07SKonstantin Belousov if (td2 == td) 1312ac437c07SKonstantin Belousov continue; 1313ac437c07SKonstantin Belousov thread_lock(td2); 1314ac437c07SKonstantin Belousov KASSERT((td2->td_flags & TDF_BOUNDARY) != 0, 1315ac437c07SKonstantin Belousov ("td %p not on boundary", td2)); 1316ac437c07SKonstantin Belousov KASSERT(TD_IS_SUSPENDED(td2), 1317ac437c07SKonstantin Belousov ("td %p is not suspended", td2)); 1318ac437c07SKonstantin Belousov thread_unlock(td2); 1319ac437c07SKonstantin Belousov } 132091599697SJulian Elischer } 13217b4a950aSDavid Xu PROC_SUNLOCK(p); 132244990b8cSJulian Elischer return (0); 132344990b8cSJulian Elischer } 132444990b8cSJulian Elischer 13258638fe7bSKonstantin Belousov bool 13268638fe7bSKonstantin Belousov thread_suspend_check_needed(void) 13278638fe7bSKonstantin Belousov { 13288638fe7bSKonstantin Belousov struct proc *p; 13298638fe7bSKonstantin Belousov struct thread *td; 13308638fe7bSKonstantin Belousov 13318638fe7bSKonstantin Belousov td = curthread; 13328638fe7bSKonstantin Belousov p = td->td_proc; 13338638fe7bSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED); 13348638fe7bSKonstantin Belousov return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 && 13358638fe7bSKonstantin Belousov (td->td_dbgflags & TDB_SUSPEND) != 0)); 13368638fe7bSKonstantin Belousov } 13378638fe7bSKonstantin Belousov 133844990b8cSJulian Elischer /* 133944990b8cSJulian Elischer * Called in from locations that can safely check to see 134044990b8cSJulian Elischer * whether we have to suspend or at least throttle for a 134144990b8cSJulian Elischer * single-thread event (e.g. fork). 134244990b8cSJulian Elischer * 134344990b8cSJulian Elischer * Such locations include userret(). 134444990b8cSJulian Elischer * If the "return_instead" argument is non zero, the thread must be able to 134544990b8cSJulian Elischer * accept 0 (caller may continue), or 1 (caller must abort) as a result. 134644990b8cSJulian Elischer * 134744990b8cSJulian Elischer * The 'return_instead' argument tells the function if it may do a 134844990b8cSJulian Elischer * thread_exit() or suspend, or whether the caller must abort and back 134944990b8cSJulian Elischer * out instead. 135044990b8cSJulian Elischer * 135144990b8cSJulian Elischer * If the thread that set the single_threading request has set the 135244990b8cSJulian Elischer * P_SINGLE_EXIT bit in the process flags then this call will never return 135344990b8cSJulian Elischer * if 'return_instead' is false, but will exit. 135444990b8cSJulian Elischer * 135544990b8cSJulian Elischer * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 135644990b8cSJulian Elischer *---------------+--------------------+--------------------- 135744990b8cSJulian Elischer * 0 | returns 0 | returns 0 or 1 1358353374b5SJohn Baldwin * | when ST ends | immediately 135944990b8cSJulian Elischer *---------------+--------------------+--------------------- 136044990b8cSJulian Elischer * 1 | thread exits | returns 1 1361353374b5SJohn Baldwin * | | immediately 136244990b8cSJulian Elischer * 0 = thread_exit() or suspension ok, 136344990b8cSJulian Elischer * other = return error instead of stopping the thread. 136444990b8cSJulian Elischer * 136544990b8cSJulian Elischer * While a full suspension is under effect, even a single threading 136644990b8cSJulian Elischer * thread would be suspended if it made this call (but it shouldn't). 136744990b8cSJulian Elischer * This call should only be made from places where 136844990b8cSJulian Elischer * thread_exit() would be safe as that may be the outcome unless 136944990b8cSJulian Elischer * return_instead is set. 137044990b8cSJulian Elischer */ 137144990b8cSJulian Elischer int 137244990b8cSJulian Elischer thread_suspend_check(int return_instead) 137344990b8cSJulian Elischer { 1374ecafb24bSJuli Mallett struct thread *td; 1375ecafb24bSJuli Mallett struct proc *p; 137646e47c4fSKonstantin Belousov int wakeup_swapper; 137744990b8cSJulian Elischer 137844990b8cSJulian Elischer td = curthread; 137944990b8cSJulian Elischer p = td->td_proc; 138037814395SPeter Wemm mtx_assert(&Giant, MA_NOTOWNED); 138144990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 13828638fe7bSKonstantin Belousov while (thread_suspend_check_needed()) { 13831279572aSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 138444990b8cSJulian Elischer KASSERT(p->p_singlethread != NULL, 138544990b8cSJulian Elischer ("singlethread not set")); 138644990b8cSJulian Elischer /* 1387e3b9bf71SJulian Elischer * The only suspension in action is a 1388e3b9bf71SJulian Elischer * single-threading. Single threader need not stop. 1389bd07998eSKonstantin Belousov * It is safe to access p->p_singlethread unlocked 1390bd07998eSKonstantin Belousov * because it can only be set to our address by us. 139144990b8cSJulian Elischer */ 1392e3b9bf71SJulian Elischer if (p->p_singlethread == td) 139344990b8cSJulian Elischer return (0); /* Exempt from stopping. */ 139444990b8cSJulian Elischer } 139545a4bfa1SDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 139694f0972bSDavid Xu return (EINTR); 139744990b8cSJulian Elischer 1398906ac69dSDavid Xu /* Should we goto user boundary if we didn't come from there? */ 1399906ac69dSDavid Xu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 1400906ac69dSDavid Xu (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 140194f0972bSDavid Xu return (ERESTART); 1402906ac69dSDavid Xu 140344990b8cSJulian Elischer /* 14043077f938SKonstantin Belousov * Ignore suspend requests if they are deferred. 1405d071a6faSJohn Baldwin */ 14063077f938SKonstantin Belousov if ((td->td_flags & TDF_SBDRY) != 0) { 1407d071a6faSJohn Baldwin KASSERT(return_instead, 1408d071a6faSJohn Baldwin ("TDF_SBDRY set for unsafe thread_suspend_check")); 140946e47c4fSKonstantin Belousov KASSERT((td->td_flags & (TDF_SEINTR | TDF_SERESTART)) != 141046e47c4fSKonstantin Belousov (TDF_SEINTR | TDF_SERESTART), 141146e47c4fSKonstantin Belousov ("both TDF_SEINTR and TDF_SERESTART")); 141246e47c4fSKonstantin Belousov return (TD_SBDRY_INTR(td) ? TD_SBDRY_ERRNO(td) : 0); 1413d071a6faSJohn Baldwin } 1414d071a6faSJohn Baldwin 1415d071a6faSJohn Baldwin /* 141644990b8cSJulian Elischer * If the process is waiting for us to exit, 141744990b8cSJulian Elischer * this thread should just suicide. 14181279572aSDavid Xu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 141944990b8cSJulian Elischer */ 1420cf7d9a8cSDavid Xu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1421cf7d9a8cSDavid Xu PROC_UNLOCK(p); 142291d1786fSDmitry Chagin 142391d1786fSDmitry Chagin /* 142491d1786fSDmitry Chagin * Allow Linux emulation layer to do some work 142591d1786fSDmitry Chagin * before thread suicide. 142691d1786fSDmitry Chagin */ 142791d1786fSDmitry Chagin if (__predict_false(p->p_sysent->sv_thread_detach != NULL)) 142891d1786fSDmitry Chagin (p->p_sysent->sv_thread_detach)(td); 14292a339d9eSKonstantin Belousov umtx_thread_exit(td); 1430d1e7a4a5SJohn Baldwin kern_thr_exit(td); 1431d1e7a4a5SJohn Baldwin panic("stopped thread did not exit"); 1432cf7d9a8cSDavid Xu } 143321ecd1e9SDavid Xu 143421ecd1e9SDavid Xu PROC_SLOCK(p); 143521ecd1e9SDavid Xu thread_stopped(p); 1436a54e85fdSJeff Roberson if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1437a54e85fdSJeff Roberson if (p->p_numthreads == p->p_suspcount + 1) { 1438a54e85fdSJeff Roberson thread_lock(p->p_singlethread); 143984cdea97SKonstantin Belousov wakeup_swapper = thread_unsuspend_one( 144084cdea97SKonstantin Belousov p->p_singlethread, p, false); 14417847a9daSJohn Baldwin if (wakeup_swapper) 14427847a9daSJohn Baldwin kick_proc0(); 1443a54e85fdSJeff Roberson } 1444a54e85fdSJeff Roberson } 14453f9be10eSDavid Xu PROC_UNLOCK(p); 14467b4a950aSDavid Xu thread_lock(td); 144744990b8cSJulian Elischer /* 144844990b8cSJulian Elischer * When a thread suspends, it just 1449ad1e7d28SJulian Elischer * gets taken off all queues. 145044990b8cSJulian Elischer */ 145171fad9fdSJulian Elischer thread_suspend_one(td); 1452906ac69dSDavid Xu if (return_instead == 0) { 1453906ac69dSDavid Xu p->p_boundary_count++; 1454906ac69dSDavid Xu td->td_flags |= TDF_BOUNDARY; 1455cf19bf91SJulian Elischer } 14567b4a950aSDavid Xu PROC_SUNLOCK(p); 1457686bcb5cSJeff Roberson mi_switch(SW_INVOL | SWT_SUSPEND); 145844990b8cSJulian Elischer PROC_LOCK(p); 145944990b8cSJulian Elischer } 146044990b8cSJulian Elischer return (0); 146144990b8cSJulian Elischer } 146244990b8cSJulian Elischer 1463478ca4b0SKonstantin Belousov /* 1464478ca4b0SKonstantin Belousov * Check for possible stops and suspensions while executing a 1465478ca4b0SKonstantin Belousov * casueword or similar transiently failing operation. 1466478ca4b0SKonstantin Belousov * 1467478ca4b0SKonstantin Belousov * The sleep argument controls whether the function can handle a stop 1468478ca4b0SKonstantin Belousov * request itself or it should return ERESTART and the request is 1469478ca4b0SKonstantin Belousov * proceed at the kernel/user boundary in ast. 1470478ca4b0SKonstantin Belousov * 1471478ca4b0SKonstantin Belousov * Typically, when retrying due to casueword(9) failure (rv == 1), we 1472478ca4b0SKonstantin Belousov * should handle the stop requests there, with exception of cases when 1473478ca4b0SKonstantin Belousov * the thread owns a kernel resource, for instance busied the umtx 1474300b525dSKonstantin Belousov * key, or when functions return immediately if thread_check_susp() 1475478ca4b0SKonstantin Belousov * returned non-zero. On the other hand, retrying the whole lock 1476478ca4b0SKonstantin Belousov * operation, we better not stop there but delegate the handling to 1477478ca4b0SKonstantin Belousov * ast. 1478478ca4b0SKonstantin Belousov * 1479478ca4b0SKonstantin Belousov * If the request is for thread termination P_SINGLE_EXIT, we cannot 1480478ca4b0SKonstantin Belousov * handle it at all, and simply return EINTR. 1481478ca4b0SKonstantin Belousov */ 1482478ca4b0SKonstantin Belousov int 1483478ca4b0SKonstantin Belousov thread_check_susp(struct thread *td, bool sleep) 1484478ca4b0SKonstantin Belousov { 1485478ca4b0SKonstantin Belousov struct proc *p; 1486478ca4b0SKonstantin Belousov int error; 1487478ca4b0SKonstantin Belousov 1488478ca4b0SKonstantin Belousov /* 1489c6d31b83SKonstantin Belousov * The check for TDA_SUSPEND is racy, but it is enough to 1490478ca4b0SKonstantin Belousov * eventually break the lockstep loop. 1491478ca4b0SKonstantin Belousov */ 1492c6d31b83SKonstantin Belousov if (!td_ast_pending(td, TDA_SUSPEND)) 1493478ca4b0SKonstantin Belousov return (0); 1494478ca4b0SKonstantin Belousov error = 0; 1495478ca4b0SKonstantin Belousov p = td->td_proc; 1496478ca4b0SKonstantin Belousov PROC_LOCK(p); 1497478ca4b0SKonstantin Belousov if (p->p_flag & P_SINGLE_EXIT) 1498478ca4b0SKonstantin Belousov error = EINTR; 1499478ca4b0SKonstantin Belousov else if (P_SHOULDSTOP(p) || 1500478ca4b0SKonstantin Belousov ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) 1501478ca4b0SKonstantin Belousov error = sleep ? thread_suspend_check(0) : ERESTART; 1502478ca4b0SKonstantin Belousov PROC_UNLOCK(p); 1503478ca4b0SKonstantin Belousov return (error); 1504478ca4b0SKonstantin Belousov } 1505478ca4b0SKonstantin Belousov 150635c32a76SDavid Xu void 15076ddcc233SKonstantin Belousov thread_suspend_switch(struct thread *td, struct proc *p) 1508a54e85fdSJeff Roberson { 1509a54e85fdSJeff Roberson 1510a54e85fdSJeff Roberson KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 1511a54e85fdSJeff Roberson PROC_LOCK_ASSERT(p, MA_OWNED); 15127b4a950aSDavid Xu PROC_SLOCK_ASSERT(p, MA_OWNED); 1513a54e85fdSJeff Roberson /* 1514a54e85fdSJeff Roberson * We implement thread_suspend_one in stages here to avoid 1515a54e85fdSJeff Roberson * dropping the proc lock while the thread lock is owned. 1516a54e85fdSJeff Roberson */ 15176ddcc233SKonstantin Belousov if (p == td->td_proc) { 1518a54e85fdSJeff Roberson thread_stopped(p); 1519a54e85fdSJeff Roberson p->p_suspcount++; 15206ddcc233SKonstantin Belousov } 15213f9be10eSDavid Xu PROC_UNLOCK(p); 15227b4a950aSDavid Xu thread_lock(td); 1523c6d31b83SKonstantin Belousov ast_unsched_locked(td, TDA_SUSPEND); 1524a54e85fdSJeff Roberson TD_SET_SUSPENDED(td); 1525c5aa6b58SJeff Roberson sched_sleep(td, 0); 15267b4a950aSDavid Xu PROC_SUNLOCK(p); 1527a54e85fdSJeff Roberson DROP_GIANT(); 1528686bcb5cSJeff Roberson mi_switch(SW_VOL | SWT_SUSPEND); 1529a54e85fdSJeff Roberson PICKUP_GIANT(); 1530a54e85fdSJeff Roberson PROC_LOCK(p); 15317b4a950aSDavid Xu PROC_SLOCK(p); 1532a54e85fdSJeff Roberson } 1533a54e85fdSJeff Roberson 1534a54e85fdSJeff Roberson void 153535c32a76SDavid Xu thread_suspend_one(struct thread *td) 153635c32a76SDavid Xu { 15376ddcc233SKonstantin Belousov struct proc *p; 153835c32a76SDavid Xu 15396ddcc233SKonstantin Belousov p = td->td_proc; 15407b4a950aSDavid Xu PROC_SLOCK_ASSERT(p, MA_OWNED); 1541a54e85fdSJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 1542e574e444SDavid Xu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 154335c32a76SDavid Xu p->p_suspcount++; 1544c6d31b83SKonstantin Belousov ast_unsched_locked(td, TDA_SUSPEND); 154571fad9fdSJulian Elischer TD_SET_SUSPENDED(td); 1546c5aa6b58SJeff Roberson sched_sleep(td, 0); 154735c32a76SDavid Xu } 154835c32a76SDavid Xu 154984cdea97SKonstantin Belousov static int 155084cdea97SKonstantin Belousov thread_unsuspend_one(struct thread *td, struct proc *p, bool boundary) 155135c32a76SDavid Xu { 155235c32a76SDavid Xu 1553a54e85fdSJeff Roberson THREAD_LOCK_ASSERT(td, MA_OWNED); 1554ad1e7d28SJulian Elischer KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 155571fad9fdSJulian Elischer TD_CLR_SUSPENDED(td); 15566ddcc233SKonstantin Belousov td->td_flags &= ~TDF_ALLPROCSUSP; 15576ddcc233SKonstantin Belousov if (td->td_proc == p) { 15586ddcc233SKonstantin Belousov PROC_SLOCK_ASSERT(p, MA_OWNED); 155935c32a76SDavid Xu p->p_suspcount--; 156084cdea97SKonstantin Belousov if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) { 156184cdea97SKonstantin Belousov td->td_flags &= ~TDF_BOUNDARY; 156284cdea97SKonstantin Belousov p->p_boundary_count--; 156384cdea97SKonstantin Belousov } 15646ddcc233SKonstantin Belousov } 156561a74c5cSJeff Roberson return (setrunnable(td, 0)); 156635c32a76SDavid Xu } 156735c32a76SDavid Xu 1568af928fdeSKonstantin Belousov void 1569af928fdeSKonstantin Belousov thread_run_flash(struct thread *td) 1570af928fdeSKonstantin Belousov { 1571af928fdeSKonstantin Belousov struct proc *p; 1572af928fdeSKonstantin Belousov 1573af928fdeSKonstantin Belousov p = td->td_proc; 1574af928fdeSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED); 1575af928fdeSKonstantin Belousov 1576af928fdeSKonstantin Belousov if (TD_ON_SLEEPQ(td)) 1577af928fdeSKonstantin Belousov sleepq_remove_nested(td); 1578af928fdeSKonstantin Belousov else 1579af928fdeSKonstantin Belousov thread_lock(td); 1580af928fdeSKonstantin Belousov 1581af928fdeSKonstantin Belousov THREAD_LOCK_ASSERT(td, MA_OWNED); 1582af928fdeSKonstantin Belousov KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 1583af928fdeSKonstantin Belousov 1584af928fdeSKonstantin Belousov TD_CLR_SUSPENDED(td); 1585af928fdeSKonstantin Belousov PROC_SLOCK(p); 1586af928fdeSKonstantin Belousov MPASS(p->p_suspcount > 0); 1587af928fdeSKonstantin Belousov p->p_suspcount--; 1588af928fdeSKonstantin Belousov PROC_SUNLOCK(p); 1589af928fdeSKonstantin Belousov if (setrunnable(td, 0)) 1590af928fdeSKonstantin Belousov kick_proc0(); 1591af928fdeSKonstantin Belousov } 1592af928fdeSKonstantin Belousov 159344990b8cSJulian Elischer /* 159444990b8cSJulian Elischer * Allow all threads blocked by single threading to continue running. 159544990b8cSJulian Elischer */ 159644990b8cSJulian Elischer void 159744990b8cSJulian Elischer thread_unsuspend(struct proc *p) 159844990b8cSJulian Elischer { 159944990b8cSJulian Elischer struct thread *td; 16007847a9daSJohn Baldwin int wakeup_swapper; 160144990b8cSJulian Elischer 160244990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 16037b4a950aSDavid Xu PROC_SLOCK_ASSERT(p, MA_OWNED); 16047847a9daSJohn Baldwin wakeup_swapper = 0; 160544990b8cSJulian Elischer if (!P_SHOULDSTOP(p)) { 1606ad1e7d28SJulian Elischer FOREACH_THREAD_IN_PROC(p, td) { 1607a54e85fdSJeff Roberson thread_lock(td); 1608f829268bSKonstantin Belousov if (TD_IS_SUSPENDED(td)) 160984cdea97SKonstantin Belousov wakeup_swapper |= thread_unsuspend_one(td, p, 161084cdea97SKonstantin Belousov true); 1611f829268bSKonstantin Belousov else 1612a54e85fdSJeff Roberson thread_unlock(td); 1613ad1e7d28SJulian Elischer } 161484cdea97SKonstantin Belousov } else if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 161584cdea97SKonstantin Belousov p->p_numthreads == p->p_suspcount) { 161644990b8cSJulian Elischer /* 161744990b8cSJulian Elischer * Stopping everything also did the job for the single 161844990b8cSJulian Elischer * threading request. Now we've downgraded to single-threaded, 161944990b8cSJulian Elischer * let it continue. 162044990b8cSJulian Elischer */ 16216ddcc233SKonstantin Belousov if (p->p_singlethread->td_proc == p) { 1622a54e85fdSJeff Roberson thread_lock(p->p_singlethread); 16236ddcc233SKonstantin Belousov wakeup_swapper = thread_unsuspend_one( 162484cdea97SKonstantin Belousov p->p_singlethread, p, false); 162544990b8cSJulian Elischer } 16266ddcc233SKonstantin Belousov } 16277847a9daSJohn Baldwin if (wakeup_swapper) 16287847a9daSJohn Baldwin kick_proc0(); 162944990b8cSJulian Elischer } 163044990b8cSJulian Elischer 1631ed062c8dSJulian Elischer /* 1632ed062c8dSJulian Elischer * End the single threading mode.. 1633ed062c8dSJulian Elischer */ 163444990b8cSJulian Elischer void 16356ddcc233SKonstantin Belousov thread_single_end(struct proc *p, int mode) 163644990b8cSJulian Elischer { 163744990b8cSJulian Elischer struct thread *td; 16387847a9daSJohn Baldwin int wakeup_swapper; 163944990b8cSJulian Elischer 16406ddcc233SKonstantin Belousov KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY || 16416ddcc233SKonstantin Belousov mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT, 16426ddcc233SKonstantin Belousov ("invalid mode %d", mode)); 164344990b8cSJulian Elischer PROC_LOCK_ASSERT(p, MA_OWNED); 16446ddcc233SKonstantin Belousov KASSERT((mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) || 16456ddcc233SKonstantin Belousov (mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0), 16466ddcc233SKonstantin Belousov ("mode %d does not match P_TOTAL_STOP", mode)); 164784cdea97SKonstantin Belousov KASSERT(mode == SINGLE_ALLPROC || p->p_singlethread == curthread, 164884cdea97SKonstantin Belousov ("thread_single_end from other thread %p %p", 164984cdea97SKonstantin Belousov curthread, p->p_singlethread)); 165084cdea97SKonstantin Belousov KASSERT(mode != SINGLE_BOUNDARY || 165184cdea97SKonstantin Belousov (p->p_flag & P_SINGLE_BOUNDARY) != 0, 165284cdea97SKonstantin Belousov ("mis-matched SINGLE_BOUNDARY flags %x", p->p_flag)); 16536ddcc233SKonstantin Belousov p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY | 16546ddcc233SKonstantin Belousov P_TOTAL_STOP); 16557b4a950aSDavid Xu PROC_SLOCK(p); 165644990b8cSJulian Elischer p->p_singlethread = NULL; 16577847a9daSJohn Baldwin wakeup_swapper = 0; 165849539972SJulian Elischer /* 16597847a9daSJohn Baldwin * If there are other threads they may now run, 166049539972SJulian Elischer * unless of course there is a blanket 'stop order' 166149539972SJulian Elischer * on the process. The single threader must be allowed 166249539972SJulian Elischer * to continue however as this is a bad place to stop. 166349539972SJulian Elischer */ 16646ddcc233SKonstantin Belousov if (p->p_numthreads != remain_for_mode(mode) && !P_SHOULDSTOP(p)) { 1665ad1e7d28SJulian Elischer FOREACH_THREAD_IN_PROC(p, td) { 1666a54e85fdSJeff Roberson thread_lock(td); 1667ad1e7d28SJulian Elischer if (TD_IS_SUSPENDED(td)) { 166884cdea97SKonstantin Belousov wakeup_swapper |= thread_unsuspend_one(td, p, 16692d5ef216SMark Johnston true); 167061a74c5cSJeff Roberson } else 1671a54e85fdSJeff Roberson thread_unlock(td); 167249539972SJulian Elischer } 1673ad1e7d28SJulian Elischer } 167484cdea97SKonstantin Belousov KASSERT(mode != SINGLE_BOUNDARY || p->p_boundary_count == 0, 167584cdea97SKonstantin Belousov ("inconsistent boundary count %d", p->p_boundary_count)); 16767b4a950aSDavid Xu PROC_SUNLOCK(p); 16777847a9daSJohn Baldwin if (wakeup_swapper) 16787847a9daSJohn Baldwin kick_proc0(); 1679d7a9e6e7SKonstantin Belousov wakeup(&p->p_flag); 168049539972SJulian Elischer } 16814fc21c09SDaniel Eischen 1682aae3547bSMateusz Guzik /* 1683aae3547bSMateusz Guzik * Locate a thread by number and return with proc lock held. 1684aae3547bSMateusz Guzik * 1685aae3547bSMateusz Guzik * thread exit establishes proc -> tidhash lock ordering, but lookup 1686aae3547bSMateusz Guzik * takes tidhash first and needs to return locked proc. 1687aae3547bSMateusz Guzik * 1688aae3547bSMateusz Guzik * The problem is worked around by relying on type-safety of both 1689aae3547bSMateusz Guzik * structures and doing the work in 2 steps: 1690aae3547bSMateusz Guzik * - tidhash-locked lookup which saves both thread and proc pointers 1691aae3547bSMateusz Guzik * - proc-locked verification that the found thread still matches 1692aae3547bSMateusz Guzik */ 1693aae3547bSMateusz Guzik static bool 1694aae3547bSMateusz Guzik tdfind_hash(lwpid_t tid, pid_t pid, struct proc **pp, struct thread **tdp) 1695cf7d9a8cSDavid Xu { 1696cf7d9a8cSDavid Xu #define RUN_THRESH 16 1697aae3547bSMateusz Guzik struct proc *p; 1698cf7d9a8cSDavid Xu struct thread *td; 1699aae3547bSMateusz Guzik int run; 1700aae3547bSMateusz Guzik bool locked; 1701cf7d9a8cSDavid Xu 1702aae3547bSMateusz Guzik run = 0; 170326007fe3SMateusz Guzik rw_rlock(TIDHASHLOCK(tid)); 1704aae3547bSMateusz Guzik locked = true; 1705cf7d9a8cSDavid Xu LIST_FOREACH(td, TIDHASH(tid), td_hash) { 1706aae3547bSMateusz Guzik if (td->td_tid != tid) { 1707aae3547bSMateusz Guzik run++; 1708aae3547bSMateusz Guzik continue; 1709cf7d9a8cSDavid Xu } 1710aae3547bSMateusz Guzik p = td->td_proc; 1711aae3547bSMateusz Guzik if (pid != -1 && p->p_pid != pid) { 1712cf7d9a8cSDavid Xu td = NULL; 1713cf7d9a8cSDavid Xu break; 1714cf7d9a8cSDavid Xu } 1715cf7d9a8cSDavid Xu if (run > RUN_THRESH) { 171626007fe3SMateusz Guzik if (rw_try_upgrade(TIDHASHLOCK(tid))) { 1717cf7d9a8cSDavid Xu LIST_REMOVE(td, td_hash); 1718cf7d9a8cSDavid Xu LIST_INSERT_HEAD(TIDHASH(td->td_tid), 1719cf7d9a8cSDavid Xu td, td_hash); 172026007fe3SMateusz Guzik rw_wunlock(TIDHASHLOCK(tid)); 1721aae3547bSMateusz Guzik locked = false; 1722aae3547bSMateusz Guzik break; 1723cf7d9a8cSDavid Xu } 1724cf7d9a8cSDavid Xu } 1725cf7d9a8cSDavid Xu break; 1726cf7d9a8cSDavid Xu } 1727aae3547bSMateusz Guzik if (locked) 172826007fe3SMateusz Guzik rw_runlock(TIDHASHLOCK(tid)); 1729aae3547bSMateusz Guzik if (td == NULL) 1730aae3547bSMateusz Guzik return (false); 1731aae3547bSMateusz Guzik *pp = p; 1732aae3547bSMateusz Guzik *tdp = td; 1733aae3547bSMateusz Guzik return (true); 1734aae3547bSMateusz Guzik } 1735aae3547bSMateusz Guzik 1736aae3547bSMateusz Guzik struct thread * 1737aae3547bSMateusz Guzik tdfind(lwpid_t tid, pid_t pid) 1738aae3547bSMateusz Guzik { 1739aae3547bSMateusz Guzik struct proc *p; 1740aae3547bSMateusz Guzik struct thread *td; 1741aae3547bSMateusz Guzik 1742aae3547bSMateusz Guzik td = curthread; 1743aae3547bSMateusz Guzik if (td->td_tid == tid) { 1744aae3547bSMateusz Guzik if (pid != -1 && td->td_proc->p_pid != pid) 1745aae3547bSMateusz Guzik return (NULL); 1746aae3547bSMateusz Guzik PROC_LOCK(td->td_proc); 1747cf7d9a8cSDavid Xu return (td); 1748cf7d9a8cSDavid Xu } 1749cf7d9a8cSDavid Xu 1750aae3547bSMateusz Guzik for (;;) { 1751aae3547bSMateusz Guzik if (!tdfind_hash(tid, pid, &p, &td)) 1752aae3547bSMateusz Guzik return (NULL); 1753aae3547bSMateusz Guzik PROC_LOCK(p); 1754aae3547bSMateusz Guzik if (td->td_tid != tid) { 1755aae3547bSMateusz Guzik PROC_UNLOCK(p); 1756aae3547bSMateusz Guzik continue; 1757aae3547bSMateusz Guzik } 1758aae3547bSMateusz Guzik if (td->td_proc != p) { 1759aae3547bSMateusz Guzik PROC_UNLOCK(p); 1760aae3547bSMateusz Guzik continue; 1761aae3547bSMateusz Guzik } 1762aae3547bSMateusz Guzik if (p->p_state == PRS_NEW) { 1763aae3547bSMateusz Guzik PROC_UNLOCK(p); 1764aae3547bSMateusz Guzik return (NULL); 1765aae3547bSMateusz Guzik } 1766aae3547bSMateusz Guzik return (td); 1767aae3547bSMateusz Guzik } 1768aae3547bSMateusz Guzik } 1769aae3547bSMateusz Guzik 1770cf7d9a8cSDavid Xu void 1771cf7d9a8cSDavid Xu tidhash_add(struct thread *td) 1772cf7d9a8cSDavid Xu { 177326007fe3SMateusz Guzik rw_wlock(TIDHASHLOCK(td->td_tid)); 1774cf7d9a8cSDavid Xu LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); 177526007fe3SMateusz Guzik rw_wunlock(TIDHASHLOCK(td->td_tid)); 1776cf7d9a8cSDavid Xu } 1777cf7d9a8cSDavid Xu 1778cf7d9a8cSDavid Xu void 1779cf7d9a8cSDavid Xu tidhash_remove(struct thread *td) 1780cf7d9a8cSDavid Xu { 178126007fe3SMateusz Guzik 178226007fe3SMateusz Guzik rw_wlock(TIDHASHLOCK(td->td_tid)); 1783cf7d9a8cSDavid Xu LIST_REMOVE(td, td_hash); 178426007fe3SMateusz Guzik rw_wunlock(TIDHASHLOCK(td->td_tid)); 1785cf7d9a8cSDavid Xu } 1786