/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include "lint.h" #include "thr_uberdata.h" #include #include #include #undef errno extern int errno; int __threaded = 0; /* zero until first thr_create() */ /* * thr_concurrency and pthread_concurrency are not used by the library. * They exist solely to hold and return the values set by calls to * thr_setconcurrency() and pthread_setconcurrency(). * Because thr_concurrency is affected by the THR_NEW_LWP flag * to thr_create(), thr_concurrency is protected by link_lock. */ static int thr_concurrency = 1; static int pthread_concurrency; size_t _lpagesize; #define HASHTBLSZ 1024 /* must be a power of two */ #define TIDHASH(tid, udp) (tid & (udp)->hash_mask) /* initial allocation, just enough for one lwp */ #pragma align 64(init_hash_table) thr_hash_table_t init_hash_table[1] = { { DEFAULTMUTEX, DEFAULTCV, NULL }, }; extern const Lc_interface rtld_funcs[]; /* * The weak version is known to libc_db and mdb. */ #pragma weak _uberdata = __uberdata uberdata_t __uberdata = { { DEFAULTMUTEX, DEFAULTCV }, /* link_lock */ { DEFAULTMUTEX, DEFAULTCV }, /* fork_lock */ { DEFAULTMUTEX, DEFAULTCV }, /* tdb_hash_lock */ { 0, }, /* tdb_hash_lock_stats */ { { 0 }, }, /* siguaction[NSIG] */ {{ DEFAULTMUTEX, NULL, 0 }, /* bucket[NBUCKETS] */ { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }, { DEFAULTMUTEX, NULL, 0 }}, { RECURSIVEMUTEX, NULL, NULL }, /* atexit_root */ { DEFAULTMUTEX, 0, 0, NULL }, /* tsd_metadata */ { DEFAULTMUTEX, {0, 0}, {0, 0} }, /* tls_metadata */ 0, /* primary_map */ 0, /* bucket_init */ 0, /* pad[0] */ 0, /* pad[1] */ { 0 }, /* uberflags */ NULL, /* queue_head */ init_hash_table, /* thr_hash_table */ 1, /* hash_size: size of the hash table */ 0, /* hash_mask: hash_size - 1 */ NULL, /* ulwp_one */ NULL, /* all_lwps */ NULL, /* all_zombies */ 0, /* nthreads */ 0, /* nzombies */ 0, /* ndaemons */ 0, /* pid */ sigacthandler, /* sigacthandler */ NULL, /* lwp_stacks */ NULL, /* lwp_laststack */ 0, /* nfreestack */ 10, /* thread_stack_cache */ NULL, /* ulwp_freelist */ NULL, /* ulwp_lastfree */ NULL, /* ulwp_replace_free */ NULL, /* ulwp_replace_last */ NULL, /* atforklist */ NULL, /* __tdb_bootstrap */ { /* tdb */ NULL, /* tdb_sync_addr_hash */ 0, /* tdb_register_count */ 0, /* tdb_hash_alloc_failed */ NULL, /* tdb_sync_addr_free */ NULL, /* tdb_sync_addr_last */ 0, /* tdb_sync_alloc */ { 0, 0 }, /* tdb_ev_global_mask */ tdb_events, /* tdb_events array */ }, }; /* * The weak version is known to libc_db and mdb. */ #pragma weak _tdb_bootstrap = __tdb_bootstrap uberdata_t **__tdb_bootstrap = NULL; int thread_queue_fifo = 4; int thread_queue_dump = 0; int thread_cond_wait_defer = 0; int thread_error_detection = 0; int thread_async_safe = 0; int thread_stack_cache = 10; int thread_door_noreserve = 0; static ulwp_t *ulwp_alloc(void); static void ulwp_free(ulwp_t *); /* * Insert the lwp into the hash table. */ void hash_in_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp) { ulwp->ul_hash = udp->thr_hash_table[ix].hash_bucket; udp->thr_hash_table[ix].hash_bucket = ulwp; ulwp->ul_ix = ix; } void hash_in(ulwp_t *ulwp, uberdata_t *udp) { int ix = TIDHASH(ulwp->ul_lwpid, udp); mutex_t *mp = &udp->thr_hash_table[ix].hash_lock; lmutex_lock(mp); hash_in_unlocked(ulwp, ix, udp); lmutex_unlock(mp); } /* * Delete the lwp from the hash table. */ void hash_out_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp) { ulwp_t **ulwpp; for (ulwpp = &udp->thr_hash_table[ix].hash_bucket; ulwp != *ulwpp; ulwpp = &(*ulwpp)->ul_hash) ; *ulwpp = ulwp->ul_hash; ulwp->ul_hash = NULL; ulwp->ul_ix = -1; } void hash_out(ulwp_t *ulwp, uberdata_t *udp) { int ix; if ((ix = ulwp->ul_ix) >= 0) { mutex_t *mp = &udp->thr_hash_table[ix].hash_lock; lmutex_lock(mp); hash_out_unlocked(ulwp, ix, udp); lmutex_unlock(mp); } } static void ulwp_clean(ulwp_t *ulwp) { ulwp->ul_self = NULL; ulwp->ul_rval = NULL; ulwp->ul_lwpid = 0; ulwp->ul_pri = 0; ulwp->ul_mappedpri = 0; ulwp->ul_policy = 0; ulwp->ul_pri_mapped = 0; ulwp->ul_mutator = 0; ulwp->ul_pleasestop = 0; ulwp->ul_stop = 0; ulwp->ul_dead = 0; ulwp->ul_unwind = 0; ulwp->ul_detached = 0; ulwp->ul_stopping = 0; ulwp->ul_sp = 0; ulwp->ul_critical = 0; ulwp->ul_cancelable = 0; ulwp->ul_preempt = 0; ulwp->ul_sigsuspend = 0; ulwp->ul_cancel_pending = 0; ulwp->ul_cancel_disabled = 0; ulwp->ul_cancel_async = 0; ulwp->ul_save_async = 0; ulwp->ul_cursig = 0; ulwp->ul_created = 0; ulwp->ul_replace = 0; ulwp->ul_schedctl_called = NULL; ulwp->ul_errno = 0; ulwp->ul_errnop = NULL; ulwp->ul_clnup_hdr = NULL; ulwp->ul_schedctl = NULL; ulwp->ul_bindflags = 0; (void) _private_memset(&ulwp->ul_td_evbuf, 0, sizeof (ulwp->ul_td_evbuf)); ulwp->ul_td_events_enable = 0; ulwp->ul_qtype = 0; ulwp->ul_usropts = 0; ulwp->ul_startpc = NULL; ulwp->ul_startarg = NULL; ulwp->ul_wchan = NULL; ulwp->ul_link = NULL; ulwp->ul_sleepq = NULL; ulwp->ul_mxchain = NULL; ulwp->ul_epri = 0; ulwp->ul_emappedpri = 0; /* PROBE_SUPPORT begin */ ulwp->ul_tpdp = NULL; /* PROBE_SUPPORT end */ ulwp->ul_siglink = NULL; (void) _private_memset(ulwp->ul_ftsd, 0, sizeof (void *) * TSD_NFAST); ulwp->ul_stsd = NULL; (void) _private_memset(&ulwp->ul_spinlock, 0, sizeof (ulwp->ul_spinlock)); ulwp->ul_spin_lock_spin = 0; ulwp->ul_spin_lock_spin2 = 0; ulwp->ul_spin_lock_sleep = 0; ulwp->ul_spin_lock_wakeup = 0; ulwp->ul_ex_unwind = NULL; } static int stackprot; /* * Answer the question, "Is the lwp in question really dead?" * We must inquire of the operating system to be really sure * because the lwp may have called lwp_exit() but it has not * yet completed the exit. */ static int dead_and_buried(ulwp_t *ulwp) { if (ulwp->ul_lwpid == (lwpid_t)(-1)) return (1); if (ulwp->ul_dead && ulwp->ul_detached && __lwp_kill(ulwp->ul_lwpid, 0) == ESRCH) { ulwp->ul_lwpid = (lwpid_t)(-1); return (1); } return (0); } /* * Attempt to keep the stack cache within the specified cache limit. */ static void trim_stack_cache(int cache_limit) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *prev = NULL; ulwp_t **ulwpp = &udp->lwp_stacks; ulwp_t *ulwp; ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, self)); while (udp->nfreestack > cache_limit && (ulwp = *ulwpp) != NULL) { if (dead_and_buried(ulwp)) { *ulwpp = ulwp->ul_next; if (ulwp == udp->lwp_laststack) udp->lwp_laststack = prev; hash_out(ulwp, udp); udp->nfreestack--; (void) _private_munmap(ulwp->ul_stk, ulwp->ul_mapsiz); /* * Now put the free ulwp on the ulwp freelist. */ ulwp->ul_mapsiz = 0; ulwp->ul_next = NULL; if (udp->ulwp_freelist == NULL) udp->ulwp_freelist = udp->ulwp_lastfree = ulwp; else { udp->ulwp_lastfree->ul_next = ulwp; udp->ulwp_lastfree = ulwp; } } else { prev = ulwp; ulwpp = &ulwp->ul_next; } } } /* * Find an unused stack of the requested size * or create a new stack of the requested size. * Return a pointer to the ulwp_t structure referring to the stack, or NULL. * thr_exit() stores 1 in the ul_dead member. * thr_join() stores -1 in the ul_lwpid member. */ ulwp_t * find_stack(size_t stksize, size_t guardsize) { uberdata_t *udp = curthread->ul_uberdata; size_t mapsize; ulwp_t *prev; ulwp_t *ulwp; ulwp_t **ulwpp; void *stk; /* * The stack is allocated PROT_READ|PROT_WRITE|PROT_EXEC * unless overridden by the system's configuration. */ if (stackprot == 0) { /* do this once */ long lprot = _sysconf(_SC_STACK_PROT); if (lprot <= 0) lprot = (PROT_READ|PROT_WRITE|PROT_EXEC); stackprot = (int)lprot; } if (_lpagesize == 0) _lpagesize = _sysconf(_SC_PAGESIZE); /* * One megabyte stacks by default, but subtract off * two pages for the system-created red zones. * Round up a non-zero stack size to a pagesize multiple. */ if (stksize == 0) stksize = DEFAULTSTACK - 2 * _lpagesize; else stksize = ((stksize + _lpagesize - 1) & -_lpagesize); /* * Round up the mapping size to a multiple of pagesize. * Note: mmap() provides at least one page of red zone * so we deduct that from the value of guardsize. */ if (guardsize != 0) guardsize = ((guardsize + _lpagesize - 1) & -_lpagesize) - _lpagesize; mapsize = stksize + guardsize; lmutex_lock(&udp->link_lock); for (prev = NULL, ulwpp = &udp->lwp_stacks; (ulwp = *ulwpp) != NULL; prev = ulwp, ulwpp = &ulwp->ul_next) { if (ulwp->ul_mapsiz == mapsize && ulwp->ul_guardsize == guardsize && dead_and_buried(ulwp)) { /* * The previous lwp is gone; reuse the stack. * Remove the ulwp from the stack list. */ *ulwpp = ulwp->ul_next; ulwp->ul_next = NULL; if (ulwp == udp->lwp_laststack) udp->lwp_laststack = prev; hash_out(ulwp, udp); udp->nfreestack--; lmutex_unlock(&udp->link_lock); ulwp_clean(ulwp); return (ulwp); } } /* * None of the cached stacks matched our mapping size. * Reduce the stack cache to get rid of possibly * very old stacks that will never be reused. */ if (udp->nfreestack > udp->thread_stack_cache) trim_stack_cache(udp->thread_stack_cache); else if (udp->nfreestack > 0) trim_stack_cache(udp->nfreestack - 1); lmutex_unlock(&udp->link_lock); /* * Create a new stack. */ if ((stk = _private_mmap(NULL, mapsize, stackprot, MAP_PRIVATE|MAP_NORESERVE|MAP_ANON, -1, (off_t)0)) != MAP_FAILED) { /* * We have allocated our stack. Now allocate the ulwp. */ ulwp = ulwp_alloc(); if (ulwp == NULL) (void) _private_munmap(stk, mapsize); else { ulwp->ul_stk = stk; ulwp->ul_mapsiz = mapsize; ulwp->ul_guardsize = guardsize; ulwp->ul_stktop = (uintptr_t)stk + mapsize; ulwp->ul_stksiz = stksize; ulwp->ul_ix = -1; if (guardsize) /* protect the extra red zone */ (void) _private_mprotect(stk, guardsize, PROT_NONE); } } return (ulwp); } /* * Get a ulwp_t structure from the free list or allocate a new one. * Such ulwp_t's do not have a stack allocated by the library. */ static ulwp_t * ulwp_alloc(void) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; size_t tls_size; ulwp_t *prev; ulwp_t *ulwp; ulwp_t **ulwpp; caddr_t data; lmutex_lock(&udp->link_lock); for (prev = NULL, ulwpp = &udp->ulwp_freelist; (ulwp = *ulwpp) != NULL; prev = ulwp, ulwpp = &ulwp->ul_next) { if (dead_and_buried(ulwp)) { *ulwpp = ulwp->ul_next; ulwp->ul_next = NULL; if (ulwp == udp->ulwp_lastfree) udp->ulwp_lastfree = prev; hash_out(ulwp, udp); lmutex_unlock(&udp->link_lock); ulwp_clean(ulwp); return (ulwp); } } lmutex_unlock(&udp->link_lock); tls_size = roundup64(udp->tls_metadata.static_tls.tls_size); data = lmalloc(sizeof (*ulwp) + tls_size); if (data != NULL) { /* LINTED pointer cast may result in improper alignment */ ulwp = (ulwp_t *)(data + tls_size); } return (ulwp); } /* * Free a ulwp structure. * If there is an associated stack, put it on the stack list and * munmap() previously freed stacks up to the residual cache limit. * Else put it on the ulwp free list and never call lfree() on it. */ static void ulwp_free(ulwp_t *ulwp) { uberdata_t *udp = curthread->ul_uberdata; ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, curthread)); ulwp->ul_next = NULL; if (ulwp == udp->ulwp_one) /* don't reuse the primoridal stack */ /*EMPTY*/; else if (ulwp->ul_mapsiz != 0) { if (udp->lwp_stacks == NULL) udp->lwp_stacks = udp->lwp_laststack = ulwp; else { udp->lwp_laststack->ul_next = ulwp; udp->lwp_laststack = ulwp; } if (++udp->nfreestack > udp->thread_stack_cache) trim_stack_cache(udp->thread_stack_cache); } else { if (udp->ulwp_freelist == NULL) udp->ulwp_freelist = udp->ulwp_lastfree = ulwp; else { udp->ulwp_lastfree->ul_next = ulwp; udp->ulwp_lastfree = ulwp; } } } /* * Find a named lwp and return a pointer to its hash list location. * On success, returns with the hash lock held. */ ulwp_t ** find_lwpp(thread_t tid) { uberdata_t *udp = curthread->ul_uberdata; int ix = TIDHASH(tid, udp); mutex_t *mp = &udp->thr_hash_table[ix].hash_lock; ulwp_t *ulwp; ulwp_t **ulwpp; if (tid == 0) return (NULL); lmutex_lock(mp); for (ulwpp = &udp->thr_hash_table[ix].hash_bucket; (ulwp = *ulwpp) != NULL; ulwpp = &ulwp->ul_hash) { if (ulwp->ul_lwpid == tid) return (ulwpp); } lmutex_unlock(mp); return (NULL); } /* * Wake up all lwps waiting on this lwp for some reason. */ void ulwp_broadcast(ulwp_t *ulwp) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self)); (void) cond_broadcast_internal(ulwp_condvar(ulwp, udp)); } /* * Find a named lwp and return a pointer to it. * Returns with the hash lock held. */ ulwp_t * find_lwp(thread_t tid) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *ulwp = NULL; ulwp_t **ulwpp; if (self->ul_lwpid == tid) { ulwp = self; ulwp_lock(ulwp, udp); } else if ((ulwpp = find_lwpp(tid)) != NULL) { ulwp = *ulwpp; } if (ulwp && ulwp->ul_dead) { ulwp_unlock(ulwp, udp); ulwp = NULL; } return (ulwp); } int _thrp_create(void *stk, size_t stksize, void *(*func)(void *), void *arg, long flags, thread_t *new_thread, pri_t priority, int policy, size_t guardsize) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ucontext_t uc; uint_t lwp_flags; thread_t tid; int error = 0; ulwp_t *ulwp; /* * Enforce the restriction of not creating any threads * until the primary link map has been initialized. * Also, disallow thread creation to a child of vfork(). */ if (!self->ul_primarymap || self->ul_vfork) return (ENOTSUP); if (udp->hash_size == 1) finish_init(); if (((stk || stksize) && stksize < MINSTACK) || priority < THREAD_MIN_PRIORITY || priority > THREAD_MAX_PRIORITY) return (EINVAL); if (stk == NULL) { if ((ulwp = find_stack(stksize, guardsize)) == NULL) return (ENOMEM); stksize = ulwp->ul_mapsiz - ulwp->ul_guardsize; } else { /* initialize the private stack */ if ((ulwp = ulwp_alloc()) == NULL) return (ENOMEM); ulwp->ul_stk = stk; ulwp->ul_stktop = (uintptr_t)stk + stksize; ulwp->ul_stksiz = stksize; ulwp->ul_ix = -1; } ulwp->ul_errnop = &ulwp->ul_errno; lwp_flags = LWP_SUSPENDED; if (flags & (THR_DETACHED|THR_DAEMON)) { flags |= THR_DETACHED; lwp_flags |= LWP_DETACHED; } if (flags & THR_DAEMON) lwp_flags |= LWP_DAEMON; /* creating a thread: enforce mt-correctness in _mutex_lock() */ self->ul_async_safe = 1; /* per-thread copies of global variables, for speed */ ulwp->ul_queue_fifo = self->ul_queue_fifo; ulwp->ul_cond_wait_defer = self->ul_cond_wait_defer; ulwp->ul_error_detection = self->ul_error_detection; ulwp->ul_async_safe = self->ul_async_safe; ulwp->ul_max_spinners = self->ul_max_spinners; ulwp->ul_adaptive_spin = self->ul_adaptive_spin; ulwp->ul_release_spin = self->ul_release_spin; ulwp->ul_queue_spin = self->ul_queue_spin; ulwp->ul_door_noreserve = self->ul_door_noreserve; ulwp->ul_primarymap = self->ul_primarymap; ulwp->ul_self = ulwp; ulwp->ul_uberdata = udp; /* debugger support */ ulwp->ul_usropts = flags; #ifdef __sparc /* * We cache several instructions in the thread structure for use * by the fasttrap DTrace provider. When changing this, read the * comment in fasttrap.h for the all the other places that must * be changed. */ ulwp->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */ ulwp->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */ ulwp->ul_dftret = 0x91d0203a; /* ta 0x3a */ ulwp->ul_dreturn = 0x81ca0000; /* return %o0 */ #endif ulwp->ul_startpc = func; ulwp->ul_startarg = arg; _fpinherit(ulwp); /* * Defer signals on the new thread until its TLS constructors * have been called. _thr_setup() will call sigon() after * it has called tls_setup(). */ ulwp->ul_sigdefer = 1; if (setup_context(&uc, _thr_setup, ulwp, (caddr_t)ulwp->ul_stk + ulwp->ul_guardsize, stksize) != 0) error = EAGAIN; /* * Call enter_critical() to avoid being suspended until we * have linked the new thread into the proper lists. * This is necessary because forkall() and fork1() must * suspend all threads and they must see a complete list. */ enter_critical(self); uc.uc_sigmask = ulwp->ul_sigmask = self->ul_sigmask; if (error != 0 || (error = __lwp_create(&uc, lwp_flags, &tid)) != 0) { exit_critical(self); ulwp->ul_lwpid = (lwpid_t)(-1); ulwp->ul_dead = 1; ulwp->ul_detached = 1; lmutex_lock(&udp->link_lock); ulwp_free(ulwp); lmutex_unlock(&udp->link_lock); return (error); } self->ul_nocancel = 0; /* cancellation is now possible */ ulwp->ul_nocancel = 0; udp->uberflags.uf_mt = 1; if (new_thread) *new_thread = tid; if (flags & THR_DETACHED) ulwp->ul_detached = 1; ulwp->ul_lwpid = tid; ulwp->ul_stop = TSTP_REGULAR; ulwp->ul_created = 1; ulwp->ul_policy = policy; ulwp->ul_pri = priority; lmutex_lock(&udp->link_lock); ulwp->ul_forw = udp->all_lwps; ulwp->ul_back = udp->all_lwps->ul_back; ulwp->ul_back->ul_forw = ulwp; ulwp->ul_forw->ul_back = ulwp; hash_in(ulwp, udp); udp->nthreads++; if (flags & THR_DAEMON) udp->ndaemons++; if (flags & THR_NEW_LWP) thr_concurrency++; __threaded = 1; /* inform stdio */ lmutex_unlock(&udp->link_lock); if (__td_event_report(self, TD_CREATE, udp)) { self->ul_td_evbuf.eventnum = TD_CREATE; self->ul_td_evbuf.eventdata = (void *)(uintptr_t)tid; tdb_event(TD_CREATE, udp); } if (!(flags & THR_SUSPENDED)) { ulwp->ul_created = 0; (void) _thrp_continue(tid, TSTP_REGULAR); } exit_critical(self); return (0); } #pragma weak thr_create = _thr_create int _thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg, long flags, thread_t *new_thread) { return (_thrp_create(stk, stksize, func, arg, flags, new_thread, curthread->ul_pri, curthread->ul_policy, 0)); } /* * A special cancellation cleanup hook for DCE. * cleanuphndlr, when it is not NULL, will contain a callback * function to be called before a thread is terminated in * _thr_exit() as a result of being cancelled. */ static void (*cleanuphndlr)(void) = NULL; /* * _pthread_setcleanupinit: sets the cleanup hook. */ int _pthread_setcleanupinit(void (*func)(void)) { cleanuphndlr = func; return (0); } void _thrp_exit() { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *replace = NULL; if (__td_event_report(self, TD_DEATH, udp)) { self->ul_td_evbuf.eventnum = TD_DEATH; tdb_event(TD_DEATH, udp); } ASSERT(self->ul_sigdefer != 0); lmutex_lock(&udp->link_lock); udp->nthreads--; if (self->ul_usropts & THR_NEW_LWP) thr_concurrency--; if (self->ul_usropts & THR_DAEMON) udp->ndaemons--; else if (udp->nthreads == udp->ndaemons) { /* * We are the last non-daemon thread exiting. * Exit the process. We retain our TSD and TLS so * that atexit() application functions can use them. */ lmutex_unlock(&udp->link_lock); exit(0); thr_panic("_thrp_exit(): exit(0) returned"); } lmutex_unlock(&udp->link_lock); tsd_exit(); /* deallocate thread-specific data */ tls_exit(); /* deallocate thread-local storage */ /* block all signals to finish exiting */ block_all_signals(self); /* also prevent ourself from being suspended */ enter_critical(self); rwl_free(self); lmutex_lock(&udp->link_lock); ulwp_free(self); (void) ulwp_lock(self, udp); if (self->ul_mapsiz && !self->ul_detached) { /* * We want to free the stack for reuse but must keep * the ulwp_t struct for the benefit of thr_join(). * For this purpose we allocate a replacement ulwp_t. */ if ((replace = udp->ulwp_replace_free) == NULL) replace = lmalloc(REPLACEMENT_SIZE); else if ((udp->ulwp_replace_free = replace->ul_next) == NULL) udp->ulwp_replace_last = NULL; } if (udp->all_lwps == self) udp->all_lwps = self->ul_forw; if (udp->all_lwps == self) udp->all_lwps = NULL; else { self->ul_forw->ul_back = self->ul_back; self->ul_back->ul_forw = self->ul_forw; } self->ul_forw = self->ul_back = NULL; /* collect queue lock statistics before marking ourself dead */ record_spin_locks(self); self->ul_dead = 1; self->ul_pleasestop = 0; if (replace != NULL) { int ix = self->ul_ix; /* the hash index */ (void) _private_memcpy(replace, self, REPLACEMENT_SIZE); replace->ul_self = replace; replace->ul_gs = 0; /* clone does not carry %gs */ replace->ul_next = NULL; /* clone not on stack list */ replace->ul_mapsiz = 0; /* allows clone to be freed */ replace->ul_replace = 1; /* requires clone to be freed */ hash_out_unlocked(self, ix, udp); hash_in_unlocked(replace, ix, udp); ASSERT(!(self->ul_detached)); self->ul_detached = 1; /* this frees the stack */ self->ul_schedctl = NULL; self->ul_schedctl_called = &udp->uberflags; set_curthread(self = replace); /* * Having just changed the address of curthread, we * must reset the ownership of the locks we hold so * that assertions will not fire when we release them. */ udp->link_lock.mutex_owner = (uintptr_t)self; ulwp_mutex(self, udp)->mutex_owner = (uintptr_t)self; /* * NOTE: * On i386, %gs still references the original, not the * replacement, ulwp structure. Fetching the replacement * curthread pointer via %gs:0 works correctly since the * original ulwp structure will not be reallocated until * this lwp has completed its lwp_exit() system call (see * dead_and_buried()), but from here on out, we must make * no references to %gs: other than %gs:0. */ } /* * Put non-detached terminated threads in the all_zombies list. */ if (!self->ul_detached) { udp->nzombies++; if (udp->all_zombies == NULL) { ASSERT(udp->nzombies == 1); udp->all_zombies = self->ul_forw = self->ul_back = self; } else { self->ul_forw = udp->all_zombies; self->ul_back = udp->all_zombies->ul_back; self->ul_back->ul_forw = self; self->ul_forw->ul_back = self; } } /* * Notify everyone waiting for this thread. */ ulwp_broadcast(self); (void) ulwp_unlock(self, udp); /* * Prevent any more references to the schedctl data. * We are exiting and continue_fork() may not find us. * Do this just before dropping link_lock, since fork * serializes on link_lock. */ self->ul_schedctl = NULL; self->ul_schedctl_called = &udp->uberflags; lmutex_unlock(&udp->link_lock); ASSERT(self->ul_critical == 1); ASSERT(self->ul_preempt == 0); _lwp_terminate(); /* never returns */ thr_panic("_thrp_exit(): _lwp_terminate() returned"); } void collect_queue_statistics() { uberdata_t *udp = curthread->ul_uberdata; ulwp_t *ulwp; if (thread_queue_dump) { lmutex_lock(&udp->link_lock); if ((ulwp = udp->all_lwps) != NULL) { do { record_spin_locks(ulwp); } while ((ulwp = ulwp->ul_forw) != udp->all_lwps); } lmutex_unlock(&udp->link_lock); } } void _thr_exit_common(void *status, int unwind) { ulwp_t *self = curthread; int cancelled = (self->ul_cancel_pending && status == PTHREAD_CANCELED); ASSERT(self->ul_critical == 0 && self->ul_preempt == 0); /* * Disable cancellation and call the special DCE cancellation * cleanup hook if it is enabled. Do nothing else before calling * the DCE cancellation cleanup hook; it may call longjmp() and * never return here. */ self->ul_cancel_disabled = 1; self->ul_cancel_async = 0; self->ul_save_async = 0; self->ul_cancelable = 0; self->ul_cancel_pending = 0; if (cancelled && cleanuphndlr != NULL) (*cleanuphndlr)(); /* * Block application signals while we are exiting. * We call out to C++, TSD, and TLS destructors while exiting * and these are application-defined, so we cannot be assured * that they won't reset the signal mask. We use sigoff() to * defer any signals that may be received as a result of this * bad behavior. Such signals will be lost to the process * when the thread finishes exiting. */ (void) _thr_sigsetmask(SIG_SETMASK, &maskset, NULL); sigoff(self); self->ul_rval = status; /* * If thr_exit is being called from the places where * C++ destructors are to be called such as cancellation * points, then set this flag. It is checked in _t_cancel() * to decide whether _ex_unwind() is to be called or not. */ if (unwind) self->ul_unwind = 1; /* * _thrp_unwind() will eventually call _thrp_exit(). * It never returns. */ _thrp_unwind(NULL); thr_panic("_thr_exit_common(): _thrp_unwind() returned"); } /* * Called when a thread returns from its start function. * We are at the top of the stack; no unwinding is necessary. */ void _thr_terminate(void *status) { _thr_exit_common(status, 0); } #pragma weak thr_exit = _thr_exit #pragma weak pthread_exit = _thr_exit #pragma weak _pthread_exit = _thr_exit void _thr_exit(void *status) { _thr_exit_common(status, 1); } int _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel) { uberdata_t *udp = curthread->ul_uberdata; mutex_t *mp; void *rval; thread_t found; ulwp_t *ulwp; ulwp_t **ulwpp; int replace; int error; if (do_cancel) error = lwp_wait(tid, &found); else { while ((error = __lwp_wait(tid, &found)) == EINTR) ; } if (error) return (error); /* * We must hold link_lock to avoid a race condition with find_stack(). */ lmutex_lock(&udp->link_lock); if ((ulwpp = find_lwpp(found)) == NULL) { /* * lwp_wait() found an lwp that the library doesn't know * about. It must have been created with _lwp_create(). * Just return its lwpid; we can't know its status. */ lmutex_unlock(&udp->link_lock); rval = NULL; } else { /* * Remove ulwp from the hash table. */ ulwp = *ulwpp; *ulwpp = ulwp->ul_hash; ulwp->ul_hash = NULL; /* * Remove ulwp from all_zombies list. */ ASSERT(udp->nzombies >= 1); if (udp->all_zombies == ulwp) udp->all_zombies = ulwp->ul_forw; if (udp->all_zombies == ulwp) udp->all_zombies = NULL; else { ulwp->ul_forw->ul_back = ulwp->ul_back; ulwp->ul_back->ul_forw = ulwp->ul_forw; } ulwp->ul_forw = ulwp->ul_back = NULL; udp->nzombies--; ASSERT(ulwp->ul_dead && !ulwp->ul_detached && !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON))); /* * We can't call ulwp_unlock(ulwp) after we set * ulwp->ul_ix = -1 so we have to get a pointer to the * ulwp's hash table mutex now in order to unlock it below. */ mp = ulwp_mutex(ulwp, udp); ulwp->ul_lwpid = (lwpid_t)(-1); ulwp->ul_ix = -1; rval = ulwp->ul_rval; replace = ulwp->ul_replace; lmutex_unlock(mp); if (replace) { ulwp->ul_next = NULL; if (udp->ulwp_replace_free == NULL) udp->ulwp_replace_free = udp->ulwp_replace_last = ulwp; else { udp->ulwp_replace_last->ul_next = ulwp; udp->ulwp_replace_last = ulwp; } } lmutex_unlock(&udp->link_lock); } if (departed != NULL) *departed = found; if (status != NULL) *status = rval; return (0); } #pragma weak thr_join = _thr_join int _thr_join(thread_t tid, thread_t *departed, void **status) { int error = _thrp_join(tid, departed, status, 1); return ((error == EINVAL)? ESRCH : error); } /* * pthread_join() differs from Solaris thr_join(): * It does not return the departed thread's id * and hence does not have a "departed" argument. * It returns EINVAL if tid refers to a detached thread. */ #pragma weak pthread_join = _pthread_join int _pthread_join(pthread_t tid, void **status) { return ((tid == 0)? ESRCH : _thrp_join(tid, NULL, status, 1)); } #pragma weak pthread_detach = _thr_detach #pragma weak _pthread_detach = _thr_detach int _thr_detach(thread_t tid) { uberdata_t *udp = curthread->ul_uberdata; ulwp_t *ulwp; ulwp_t **ulwpp; int error = 0; if ((ulwpp = find_lwpp(tid)) == NULL) return (ESRCH); ulwp = *ulwpp; if (ulwp->ul_dead) { ulwp_unlock(ulwp, udp); error = _thrp_join(tid, NULL, NULL, 0); } else { error = __lwp_detach(tid); ulwp->ul_detached = 1; ulwp->ul_usropts |= THR_DETACHED; ulwp_unlock(ulwp, udp); } return (error); } /* * Static local string compare function to avoid calling strncmp() * (and hence the dynamic linker) during library initialization. */ static int sncmp(const char *s1, const char *s2, size_t n) { n++; while (--n != 0 && *s1 == *s2++) if (*s1++ == '\0') return (0); return (n == 0 ? 0 : *(uchar_t *)s1 - *(uchar_t *)--s2); } static const char * ematch(const char *ev, const char *match) { int c; while ((c = *match++) != '\0') { if (*ev++ != c) return (NULL); } if (*ev++ != '=') return (NULL); return (ev); } static int envvar(const char *ev, const char *match, int limit) { int val = -1; const char *ename; if ((ename = ematch(ev, match)) != NULL) { int c; for (val = 0; (c = *ename) != '\0'; ename++) { if (!isdigit(c)) { val = -1; break; } val = val * 10 + (c - '0'); if (val > limit) { val = limit; break; } } } return (val); } static void etest(const char *ev) { int value; if ((value = envvar(ev, "QUEUE_SPIN", 1000000)) >= 0) thread_queue_spin = value; if ((value = envvar(ev, "ADAPTIVE_SPIN", 1000000)) >= 0) { thread_adaptive_spin = value; thread_release_spin = (value + 1) / 2; } if ((value = envvar(ev, "RELEASE_SPIN", 1000000)) >= 0) thread_release_spin = value; if ((value = envvar(ev, "MAX_SPINNERS", 100)) >= 0) thread_max_spinners = value; if ((value = envvar(ev, "QUEUE_FIFO", 8)) >= 0) thread_queue_fifo = value; #if defined(THREAD_DEBUG) if ((value = envvar(ev, "QUEUE_VERIFY", 1)) >= 0) thread_queue_verify = value; #endif if ((value = envvar(ev, "QUEUE_DUMP", 1)) >= 0) thread_queue_dump = value; if ((value = envvar(ev, "STACK_CACHE", 10000)) >= 0) thread_stack_cache = value; if ((value = envvar(ev, "COND_WAIT_DEFER", 1)) >= 0) thread_cond_wait_defer = value; if ((value = envvar(ev, "ERROR_DETECTION", 2)) >= 0) thread_error_detection = value; if ((value = envvar(ev, "ASYNC_SAFE", 1)) >= 0) thread_async_safe = value; if ((value = envvar(ev, "DOOR_NORESERVE", 1)) >= 0) thread_door_noreserve = value; } /* * Look for and evaluate environment variables of the form "_THREAD_*". * For compatibility with the past, we also look for environment * names of the form "LIBTHREAD_*". */ static void set_thread_vars() { extern const char **_environ; const char **pev; const char *ev; char c; if ((pev = _environ) == NULL) return; while ((ev = *pev++) != NULL) { c = *ev; if (c == '_' && sncmp(ev, "_THREAD_", 8) == 0) etest(ev + 8); if (c == 'L' && sncmp(ev, "LIBTHREAD_", 10) == 0) etest(ev + 10); } } /* PROBE_SUPPORT begin */ #pragma weak __tnf_probe_notify extern void __tnf_probe_notify(void); /* PROBE_SUPPORT end */ /* same as atexit() but private to the library */ extern int _atexit(void (*)(void)); /* same as _cleanup() but private to the library */ extern void __cleanup(void); extern void atfork_init(void); #ifdef __amd64 extern void __amd64id(void); #endif /* * libc_init() is called by ld.so.1 for library initialization. * We perform minimal initialization; enough to work with the main thread. */ void libc_init(void) { uberdata_t *udp = &__uberdata; ulwp_t *oldself = __curthread(); ucontext_t uc; ulwp_t *self; struct rlimit rl; caddr_t data; size_t tls_size; int setmask; /* * For the initial stage of initialization, we must be careful * not to call any function that could possibly call _cerror(). * For this purpose, we call only the raw system call wrappers. */ #ifdef __amd64 /* * Gather information about cache layouts for optimized * AMD assembler strfoo() and memfoo() functions. */ __amd64id(); #endif /* * Every libc, regardless of which link map, must register __cleanup(). */ (void) _atexit(__cleanup); /* * We keep our uberdata on one of (a) the first alternate link map * or (b) the primary link map. We switch to the primary link map * and stay there once we see it. All intermediate link maps are * subject to being unloaded at any time. */ if (oldself != NULL && (oldself->ul_primarymap || !primary_link_map)) { __tdb_bootstrap = oldself->ul_uberdata->tdb_bootstrap; mutex_setup(); atfork_init(); /* every link map needs atfork() processing */ return; } /* * To establish the main stack information, we have to get our context. * This is also convenient to use for getting our signal mask. */ uc.uc_flags = UC_ALL; (void) __getcontext_syscall(&uc); ASSERT(uc.uc_link == NULL); tls_size = roundup64(udp->tls_metadata.static_tls.tls_size); ASSERT(primary_link_map || tls_size == 0); data = lmalloc(sizeof (ulwp_t) + tls_size); if (data == NULL) thr_panic("cannot allocate thread structure for main thread"); /* LINTED pointer cast may result in improper alignment */ self = (ulwp_t *)(data + tls_size); init_hash_table[0].hash_bucket = self; self->ul_sigmask = uc.uc_sigmask; delete_reserved_signals(&self->ul_sigmask); /* * Are the old and new sets different? * (This can happen if we are currently blocking SIGCANCEL.) * If so, we must explicitly set our signal mask, below. */ setmask = ((self->ul_sigmask.__sigbits[0] ^ uc.uc_sigmask.__sigbits[0]) | (self->ul_sigmask.__sigbits[1] ^ uc.uc_sigmask.__sigbits[1])); #ifdef __sparc /* * We cache several instructions in the thread structure for use * by the fasttrap DTrace provider. When changing this, read the * comment in fasttrap.h for the all the other places that must * be changed. */ self->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */ self->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */ self->ul_dftret = 0x91d0203a; /* ta 0x3a */ self->ul_dreturn = 0x81ca0000; /* return %o0 */ #endif self->ul_stktop = (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size; (void) _private_getrlimit(RLIMIT_STACK, &rl); self->ul_stksiz = rl.rlim_cur; self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz); self->ul_forw = self->ul_back = self; self->ul_hash = NULL; self->ul_ix = 0; self->ul_lwpid = 1; /* __lwp_self() */ self->ul_main = 1; self->ul_self = self; self->ul_uberdata = udp; if (oldself != NULL) { int i; ASSERT(primary_link_map); ASSERT(oldself->ul_main == 1); self->ul_stsd = oldself->ul_stsd; for (i = 0; i < TSD_NFAST; i++) self->ul_ftsd[i] = oldself->ul_ftsd[i]; self->ul_tls = oldself->ul_tls; /* * Retrieve all pointers to uberdata allocated * while running on previous link maps. * This is a giant structure assignment. */ *udp = *oldself->ul_uberdata; /* * These items point to global data on the primary link map. */ udp->thr_hash_table = init_hash_table; udp->sigacthandler = sigacthandler; udp->tdb.tdb_events = tdb_events; ASSERT(udp->nthreads == 1 && !udp->uberflags.uf_mt); ASSERT(udp->lwp_stacks == NULL); ASSERT(udp->ulwp_freelist == NULL); ASSERT(udp->ulwp_replace_free == NULL); ASSERT(udp->hash_size == 1); } udp->all_lwps = self; udp->ulwp_one = self; udp->pid = _private_getpid(); udp->nthreads = 1; /* * In every link map, tdb_bootstrap points to the same piece of * allocated memory. When the primary link map is initialized, * the allocated memory is assigned a pointer to the one true * uberdata. This allows libc_db to initialize itself regardless * of which instance of libc it finds in the address space. */ if (udp->tdb_bootstrap == NULL) udp->tdb_bootstrap = lmalloc(sizeof (uberdata_t *)); __tdb_bootstrap = udp->tdb_bootstrap; if (primary_link_map) { self->ul_primarymap = 1; udp->primary_map = 1; *udp->tdb_bootstrap = udp; } /* * Cancellation can't happen until: * pthread_cancel() is called * or: * another thread is created * For now, as a single-threaded process, set the flag that tells * PROLOGUE/EPILOGUE (in scalls.c) that cancellation can't happen. */ self->ul_nocancel = 1; #if defined(__amd64) self->ul_gs = ___lwp_private(_LWP_SETPRIVATE, _LWP_FSBASE, self); #elif defined(__i386) self->ul_gs = ___lwp_private(_LWP_SETPRIVATE, _LWP_GSBASE, self); #endif /* __i386 || __amd64 */ set_curthread(self); /* redundant on i386 */ /* * Now curthread is established and it is safe to call any * function in libc except one that uses thread-local storage. */ self->ul_errnop = &errno; if (oldself != NULL) { /* tls_size was zero when oldself was allocated */ lfree(oldself, sizeof (ulwp_t)); } mutex_setup(); atfork_init(); signal_init(); /* * If the stack is unlimited, we set the size to zero to disable * stack checking. * XXX: Work harder here. Get the stack size from /proc/self/rmap */ if (self->ul_stksiz == RLIM_INFINITY) { self->ul_ustack.ss_sp = (void *)self->ul_stktop; self->ul_ustack.ss_size = 0; } else { self->ul_ustack.ss_sp = self->ul_stk; self->ul_ustack.ss_size = self->ul_stksiz; } self->ul_ustack.ss_flags = 0; (void) _private_setustack(&self->ul_ustack); /* * Get the variables that affect thread behavior from the environment. */ set_thread_vars(); udp->uberflags.uf_thread_error_detection = (char)thread_error_detection; udp->thread_stack_cache = thread_stack_cache; /* * Make per-thread copies of global variables, for speed. */ self->ul_queue_fifo = (char)thread_queue_fifo; self->ul_cond_wait_defer = (char)thread_cond_wait_defer; self->ul_error_detection = (char)thread_error_detection; self->ul_async_safe = (char)thread_async_safe; self->ul_door_noreserve = (char)thread_door_noreserve; self->ul_max_spinners = (uchar_t)thread_max_spinners; self->ul_adaptive_spin = thread_adaptive_spin; self->ul_release_spin = thread_release_spin; self->ul_queue_spin = thread_queue_spin; /* * When we have initialized the primary link map, inform * the dynamic linker about our interface functions. */ if (self->ul_primarymap) _ld_libc((void *)rtld_funcs); /* * Defer signals until TLS constructors have been called. */ sigoff(self); tls_setup(); sigon(self); if (setmask) (void) restore_signals(self); /* PROBE_SUPPORT begin */ if (self->ul_primarymap && __tnf_probe_notify != NULL) __tnf_probe_notify(); /* PROBE_SUPPORT end */ } #pragma fini(libc_fini) void libc_fini() { /* * If we are doing fini processing for the instance of libc * on the first alternate link map (this happens only when * the dynamic linker rejects a bad audit library), then clear * __curthread(). We abandon whatever memory was allocated by * lmalloc() while running on this alternate link-map but we * don't care (and can't find the memory in any case); we just * want to protect the application from this bad audit library. * No fini processing is done by libc in the normal case. */ uberdata_t *udp = curthread->ul_uberdata; if (udp->primary_map == 0 && udp == &__uberdata) set_curthread(NULL); } /* * finish_init is called when we are about to become multi-threaded, * that is, on the first call to thr_create(). */ void finish_init() { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; thr_hash_table_t *htp; void *data; int i; /* * No locks needed here; we are single-threaded on the first call. * We can be called only after the primary link map has been set up. */ ASSERT(self->ul_primarymap); ASSERT(self == udp->ulwp_one); ASSERT(!udp->uberflags.uf_mt); ASSERT(udp->hash_size == 1); /* * First allocate the queue_head array if not already allocated. */ if (udp->queue_head == NULL) queue_alloc(); /* * Now allocate the thread hash table. */ if ((data = _private_mmap(NULL, HASHTBLSZ * sizeof (thr_hash_table_t), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0)) == MAP_FAILED) thr_panic("cannot allocate thread hash table"); udp->thr_hash_table = htp = (thr_hash_table_t *)data; udp->hash_size = HASHTBLSZ; udp->hash_mask = HASHTBLSZ - 1; for (i = 0; i < HASHTBLSZ; i++, htp++) { htp->hash_lock.mutex_magic = MUTEX_MAGIC; htp->hash_cond.cond_magic = COND_MAGIC; } hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp); /* * Set up the SIGCANCEL handler for threads cancellation. */ init_sigcancel(); /* * Arrange to do special things on exit -- * - collect queue statistics from all remaining active threads. * - grab assert_lock to ensure that assertion failures * and a core dump take precedence over _exit(). * - dump queue statistics to stderr if _THREAD_QUEUE_DUMP is set. * (Functions are called in the reverse order of their registration.) */ (void) _atexit(dump_queue_statistics); (void) _atexit(grab_assert_lock); (void) _atexit(collect_queue_statistics); } /* * Used only by _postfork1_child(), below. */ static void mark_dead_and_buried(ulwp_t *ulwp) { ulwp->ul_dead = 1; ulwp->ul_lwpid = (lwpid_t)(-1); ulwp->ul_hash = NULL; ulwp->ul_ix = -1; ulwp->ul_schedctl = NULL; ulwp->ul_schedctl_called = NULL; } /* * This is called from fork1() in the child. * Reset our data structures to reflect one lwp. */ void _postfork1_child() { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *next; ulwp_t *ulwp; int i; /* daemon threads shouldn't call fork1(), but oh well... */ self->ul_usropts &= ~THR_DAEMON; udp->nthreads = 1; udp->ndaemons = 0; udp->uberflags.uf_mt = 0; __threaded = 0; for (i = 0; i < udp->hash_size; i++) udp->thr_hash_table[i].hash_bucket = NULL; self->ul_lwpid = __lwp_self(); hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp); /* no one in the child is on a sleep queue; reinitialize */ if (udp->queue_head) { (void) _private_memset(udp->queue_head, 0, 2 * QHASHSIZE * sizeof (queue_head_t)); for (i = 0; i < 2 * QHASHSIZE; i++) udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC; } /* * All lwps except ourself are gone. Mark them so. * First mark all of the lwps that have already been freed. * Then mark and free all of the active lwps except ourself. * Since we are single-threaded, no locks are required here. */ for (ulwp = udp->lwp_stacks; ulwp != NULL; ulwp = ulwp->ul_next) mark_dead_and_buried(ulwp); for (ulwp = udp->ulwp_freelist; ulwp != NULL; ulwp = ulwp->ul_next) mark_dead_and_buried(ulwp); for (ulwp = self->ul_forw; ulwp != self; ulwp = next) { next = ulwp->ul_forw; ulwp->ul_forw = ulwp->ul_back = NULL; mark_dead_and_buried(ulwp); tsd_free(ulwp); tls_free(ulwp); rwl_free(ulwp); ulwp_free(ulwp); } self->ul_forw = self->ul_back = udp->all_lwps = self; if (self != udp->ulwp_one) mark_dead_and_buried(udp->ulwp_one); if ((ulwp = udp->all_zombies) != NULL) { ASSERT(udp->nzombies != 0); do { next = ulwp->ul_forw; ulwp->ul_forw = ulwp->ul_back = NULL; mark_dead_and_buried(ulwp); udp->nzombies--; if (ulwp->ul_replace) { ulwp->ul_next = NULL; if (udp->ulwp_replace_free == NULL) { udp->ulwp_replace_free = udp->ulwp_replace_last = ulwp; } else { udp->ulwp_replace_last->ul_next = ulwp; udp->ulwp_replace_last = ulwp; } } } while ((ulwp = next) != udp->all_zombies); ASSERT(udp->nzombies == 0); udp->all_zombies = NULL; udp->nzombies = 0; } trim_stack_cache(0); } #pragma weak thr_setprio = _thr_setprio #pragma weak pthread_setschedprio = _thr_setprio #pragma weak _pthread_setschedprio = _thr_setprio int _thr_setprio(thread_t tid, int priority) { struct sched_param param; (void) _memset(¶m, 0, sizeof (param)); param.sched_priority = priority; return (_thread_setschedparam_main(tid, 0, ¶m, PRIO_SET_PRIO)); } #pragma weak thr_getprio = _thr_getprio int _thr_getprio(thread_t tid, int *priority) { uberdata_t *udp = curthread->ul_uberdata; ulwp_t *ulwp; int error = 0; if ((ulwp = find_lwp(tid)) == NULL) error = ESRCH; else { *priority = ulwp->ul_pri; ulwp_unlock(ulwp, udp); } return (error); } lwpid_t lwp_self(void) { return (curthread->ul_lwpid); } #pragma weak _ti_thr_self = _thr_self #pragma weak thr_self = _thr_self #pragma weak pthread_self = _thr_self #pragma weak _pthread_self = _thr_self thread_t _thr_self() { return (curthread->ul_lwpid); } #pragma weak thr_main = _thr_main int _thr_main() { ulwp_t *self = __curthread(); return ((self == NULL)? -1 : self->ul_main); } int _thrp_stksegment(ulwp_t *ulwp, stack_t *stk) { stk->ss_sp = (void *)ulwp->ul_stktop; stk->ss_size = ulwp->ul_stksiz; stk->ss_flags = 0; return (0); } #pragma weak thr_stksegment = _thr_stksegment int _thr_stksegment(stack_t *stk) { return (_thrp_stksegment(curthread, stk)); } void force_continue(ulwp_t *ulwp) { #if defined(THREAD_DEBUG) ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; #endif int error; timespec_t ts; ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self)); for (;;) { error = __lwp_continue(ulwp->ul_lwpid); if (error != 0 && error != EINTR) break; error = 0; if (ulwp->ul_stopping) { /* he is stopping himself */ ts.tv_sec = 0; /* give him a chance to run */ ts.tv_nsec = 100000; /* 100 usecs or clock tick */ (void) ___nanosleep(&ts, NULL); } if (!ulwp->ul_stopping) /* he is running now */ break; /* so we are done */ /* * He is marked as being in the process of stopping * himself. Loop around and continue him again. * He may not have been stopped the first time. */ } } /* * Suspend an lwp with lwp_suspend(), then move it to a safe * point, that is, to a point where ul_critical is zero. * On return, the ulwp_lock() is dropped as with ulwp_unlock(). * If 'link_dropped' is non-NULL, then 'link_lock' is held on entry. * If we have to drop link_lock, we store 1 through link_dropped. * If the lwp exits before it can be suspended, we return ESRCH. */ int safe_suspend(ulwp_t *ulwp, uchar_t whystopped, int *link_dropped) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; cond_t *cvp = ulwp_condvar(ulwp, udp); mutex_t *mp = ulwp_mutex(ulwp, udp); thread_t tid = ulwp->ul_lwpid; int ix = ulwp->ul_ix; int error = 0; ASSERT(whystopped == TSTP_REGULAR || whystopped == TSTP_MUTATOR || whystopped == TSTP_FORK); ASSERT(ulwp != self); ASSERT(!ulwp->ul_stop); ASSERT(MUTEX_OWNED(mp, self)); if (link_dropped != NULL) *link_dropped = 0; /* * We must grab the target's spin lock before suspending it. * See the comments below and in _thrp_suspend() for why. */ spin_lock_set(&ulwp->ul_spinlock); (void) ___lwp_suspend(tid); spin_lock_clear(&ulwp->ul_spinlock); top: if (ulwp->ul_critical == 0 || ulwp->ul_stopping) { /* thread is already safe */ ulwp->ul_stop |= whystopped; } else { /* * Setting ul_pleasestop causes the target thread to stop * itself in _thrp_suspend(), below, after we drop its lock. * We must continue the critical thread before dropping * link_lock because the critical thread may be holding * the queue lock for link_lock. This is delicate. */ ulwp->ul_pleasestop |= whystopped; force_continue(ulwp); if (link_dropped != NULL) { *link_dropped = 1; lmutex_unlock(&udp->link_lock); /* be sure to drop link_lock only once */ link_dropped = NULL; } /* * The thread may disappear by calling thr_exit() so we * cannot rely on the ulwp pointer after dropping the lock. * Instead, we search the hash table to find it again. * When we return, we may find that the thread has been * continued by some other thread. The suspend/continue * interfaces are prone to such race conditions by design. */ while (ulwp && !ulwp->ul_dead && !ulwp->ul_stop && (ulwp->ul_pleasestop & whystopped)) { (void) _cond_wait(cvp, mp); for (ulwp = udp->thr_hash_table[ix].hash_bucket; ulwp != NULL; ulwp = ulwp->ul_hash) { if (ulwp->ul_lwpid == tid) break; } } if (ulwp == NULL || ulwp->ul_dead) error = ESRCH; else { /* * Do another lwp_suspend() to make sure we don't * return until the target thread is fully stopped * in the kernel. Don't apply lwp_suspend() until * we know that the target is not holding any * queue locks, that is, that it has completed * ulwp_unlock(self) and has, or at least is * about to, call lwp_suspend() on itself. We do * this by grabbing the target's spin lock. */ ASSERT(ulwp->ul_lwpid == tid); spin_lock_set(&ulwp->ul_spinlock); (void) ___lwp_suspend(tid); spin_lock_clear(&ulwp->ul_spinlock); /* * If some other thread did a thr_continue() * on the target thread we have to start over. */ if (!ulwp->ul_stopping || !(ulwp->ul_stop & whystopped)) goto top; } } (void) cond_broadcast_internal(cvp); lmutex_unlock(mp); return (error); } int _thrp_suspend(thread_t tid, uchar_t whystopped) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *ulwp; int error = 0; ASSERT((whystopped & (TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) != 0); ASSERT((whystopped & ~(TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) == 0); /* * We can't suspend anyone except ourself while a fork is happening. * This also has the effect of allowing only one suspension at a time. */ if (tid != self->ul_lwpid) (void) fork_lock_enter(NULL); if ((ulwp = find_lwp(tid)) == NULL) error = ESRCH; else if (whystopped == TSTP_MUTATOR && !ulwp->ul_mutator) { ulwp_unlock(ulwp, udp); error = EINVAL; } else if (ulwp->ul_stop) { /* already stopped */ ulwp->ul_stop |= whystopped; ulwp_broadcast(ulwp); ulwp_unlock(ulwp, udp); } else if (ulwp != self) { /* * After suspending the other thread, move it out of a * critical section and deal with the schedctl mappings. * safe_suspend() suspends the other thread, calls * ulwp_broadcast(ulwp) and drops the ulwp lock. */ error = safe_suspend(ulwp, whystopped, NULL); } else { int schedctl_after_fork = 0; /* * We are suspending ourself. We must not take a signal * until we return from lwp_suspend() and clear ul_stopping. * This is to guard against siglongjmp(). */ enter_critical(self); self->ul_sp = stkptr(); _flush_windows(); /* sparc */ self->ul_pleasestop = 0; self->ul_stop |= whystopped; /* * Grab our spin lock before dropping ulwp_mutex(self). * This prevents the suspending thread from applying * lwp_suspend() to us before we emerge from * lmutex_unlock(mp) and have dropped mp's queue lock. */ spin_lock_set(&self->ul_spinlock); self->ul_stopping = 1; ulwp_broadcast(self); ulwp_unlock(self, udp); /* * From this point until we return from lwp_suspend(), * we must not call any function that might invoke the * dynamic linker, that is, we can only call functions * private to the library. * * Also, this is a nasty race condition for a process * that is undergoing a forkall() operation: * Once we clear our spinlock (below), we are vulnerable * to being suspended by the forkall() thread before * we manage to suspend ourself in ___lwp_suspend(). * See safe_suspend() and force_continue(). * * To avoid a SIGSEGV due to the disappearance * of the schedctl mappings in the child process, * which can happen in spin_lock_clear() if we * are suspended while we are in the middle of * its call to preempt(), we preemptively clear * our own schedctl pointer before dropping our * spinlock. We reinstate it, in both the parent * and (if this really is a forkall()) the child. */ if (whystopped & TSTP_FORK) { schedctl_after_fork = 1; self->ul_schedctl = NULL; self->ul_schedctl_called = &udp->uberflags; } spin_lock_clear(&self->ul_spinlock); (void) ___lwp_suspend(tid); /* * Somebody else continued us. * We can't grab ulwp_lock(self) * until after clearing ul_stopping. * force_continue() relies on this. */ self->ul_stopping = 0; self->ul_sp = 0; if (schedctl_after_fork) { self->ul_schedctl_called = NULL; self->ul_schedctl = NULL; (void) setup_schedctl(); } ulwp_lock(self, udp); ulwp_broadcast(self); ulwp_unlock(self, udp); exit_critical(self); } if (tid != self->ul_lwpid) fork_lock_exit(); return (error); } /* * Suspend all lwps other than ourself in preparation for fork. */ void suspend_fork() { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *ulwp; int link_dropped; top: lmutex_lock(&udp->link_lock); for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) { ulwp_lock(ulwp, udp); if (ulwp->ul_stop) { /* already stopped */ ulwp->ul_stop |= TSTP_FORK; ulwp_broadcast(ulwp); ulwp_unlock(ulwp, udp); } else { /* * Move the stopped lwp out of a critical section. */ if (safe_suspend(ulwp, TSTP_FORK, &link_dropped) || link_dropped) goto top; } } lmutex_unlock(&udp->link_lock); } void continue_fork(int child) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *ulwp; /* * Clear the schedctl pointers in the child of forkall(). */ if (child) { for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) { ulwp->ul_schedctl_called = ulwp->ul_dead? &udp->uberflags : NULL; ulwp->ul_schedctl = NULL; } } /* * Set all lwps that were stopped for fork() running again. */ lmutex_lock(&udp->link_lock); for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) { mutex_t *mp = ulwp_mutex(ulwp, udp); lmutex_lock(mp); ASSERT(ulwp->ul_stop & TSTP_FORK); ulwp->ul_stop &= ~TSTP_FORK; ulwp_broadcast(ulwp); if (!ulwp->ul_stop) force_continue(ulwp); lmutex_unlock(mp); } lmutex_unlock(&udp->link_lock); } int _thrp_continue(thread_t tid, uchar_t whystopped) { uberdata_t *udp = curthread->ul_uberdata; ulwp_t *ulwp; mutex_t *mp; int error = 0; ASSERT(whystopped == TSTP_REGULAR || whystopped == TSTP_MUTATOR); if ((ulwp = find_lwp(tid)) == NULL) return (ESRCH); mp = ulwp_mutex(ulwp, udp); if ((whystopped == TSTP_MUTATOR && !ulwp->ul_mutator)) { error = EINVAL; } else if (ulwp->ul_stop & whystopped) { ulwp->ul_stop &= ~whystopped; ulwp_broadcast(ulwp); if (!ulwp->ul_stop) { if (whystopped == TSTP_REGULAR && ulwp->ul_created) { ulwp->ul_sp = 0; ulwp->ul_created = 0; } force_continue(ulwp); } } lmutex_unlock(mp); return (error); } #pragma weak thr_suspend = _thr_suspend int _thr_suspend(thread_t tid) { return (_thrp_suspend(tid, TSTP_REGULAR)); } #pragma weak thr_continue = _thr_continue int _thr_continue(thread_t tid) { return (_thrp_continue(tid, TSTP_REGULAR)); } #pragma weak thr_yield = _thr_yield void _thr_yield() { lwp_yield(); } #pragma weak thr_kill = _thr_kill #pragma weak pthread_kill = _thr_kill #pragma weak _pthread_kill = _thr_kill int _thr_kill(thread_t tid, int sig) { if (sig == SIGCANCEL) return (EINVAL); return (__lwp_kill(tid, sig)); } /* * Exit a critical section, take deferred actions if necessary. */ void do_exit_critical() { ulwp_t *self = curthread; int sig; ASSERT(self->ul_critical == 0); if (self->ul_dead) return; while (self->ul_pleasestop || (self->ul_cursig != 0 && self->ul_sigdefer == 0)) { /* * Avoid a recursive call to exit_critical() in _thrp_suspend() * by keeping self->ul_critical == 1 here. */ self->ul_critical++; while (self->ul_pleasestop) { /* * Guard against suspending ourself while on a sleep * queue. See the comments in call_user_handler(). */ unsleep_self(); set_parking_flag(self, 0); (void) _thrp_suspend(self->ul_lwpid, self->ul_pleasestop); } self->ul_critical--; if ((sig = self->ul_cursig) != 0 && self->ul_sigdefer == 0) { /* * Clear ul_cursig before proceeding. * This protects us from the dynamic linker's * calls to bind_guard()/bind_clear() in the * event that it is invoked to resolve a symbol * like take_deferred_signal() below. */ self->ul_cursig = 0; take_deferred_signal(sig); ASSERT(self->ul_cursig == 0); } } ASSERT(self->ul_critical == 0); } int _ti_bind_guard(int bindflag) { ulwp_t *self = curthread; if ((self->ul_bindflags & bindflag) == bindflag) return (0); enter_critical(self); self->ul_bindflags |= bindflag; return (1); } int _ti_bind_clear(int bindflag) { ulwp_t *self = curthread; if ((self->ul_bindflags & bindflag) == 0) return (self->ul_bindflags); self->ul_bindflags &= ~bindflag; exit_critical(self); return (self->ul_bindflags); } /* * sigoff() and sigon() enable cond_wait() to behave (optionally) like * it does in the old libthread (see the comments in cond_wait_queue()). * Also, signals are deferred at thread startup until TLS constructors * have all been called, at which time _thr_setup() calls sigon(). * Also, _sigoff() and _sigon() are called from dbx's run-time checking * (librtc.so) to defer signals during its critical sections (not to be * confused with libc critical sections [see exit_critical() above]). */ void _sigoff(void) { sigoff(curthread); } void _sigon(void) { sigon(curthread); } void sigon(ulwp_t *self) { int sig; ASSERT(self->ul_sigdefer > 0); if (--self->ul_sigdefer == 0) { if ((sig = self->ul_cursig) != 0 && self->ul_critical == 0) { self->ul_cursig = 0; take_deferred_signal(sig); ASSERT(self->ul_cursig == 0); } } } #pragma weak thr_getconcurrency = _thr_getconcurrency int _thr_getconcurrency() { return (thr_concurrency); } #pragma weak pthread_getconcurrency = _pthread_getconcurrency int _pthread_getconcurrency() { return (pthread_concurrency); } #pragma weak thr_setconcurrency = _thr_setconcurrency int _thr_setconcurrency(int new_level) { uberdata_t *udp = curthread->ul_uberdata; if (new_level < 0) return (EINVAL); if (new_level > 65536) /* 65536 is totally arbitrary */ return (EAGAIN); lmutex_lock(&udp->link_lock); if (new_level > thr_concurrency) thr_concurrency = new_level; lmutex_unlock(&udp->link_lock); return (0); } #pragma weak pthread_setconcurrency = _pthread_setconcurrency int _pthread_setconcurrency(int new_level) { if (new_level < 0) return (EINVAL); if (new_level > 65536) /* 65536 is totally arbitrary */ return (EAGAIN); pthread_concurrency = new_level; return (0); } #pragma weak thr_min_stack = _thr_min_stack #pragma weak __pthread_min_stack = _thr_min_stack size_t _thr_min_stack(void) { return (MINSTACK); } int __nthreads(void) { return (curthread->ul_uberdata->nthreads); } /* * XXX * The remainder of this file implements the private interfaces to java for * garbage collection. It is no longer used, at least by java 1.2. * It can all go away once all old JVMs have disappeared. */ int suspendingallmutators; /* when non-zero, suspending all mutators. */ int suspendedallmutators; /* when non-zero, all mutators suspended. */ int mutatorsbarrier; /* when non-zero, mutators barrier imposed. */ mutex_t mutatorslock = DEFAULTMUTEX; /* used to enforce mutators barrier. */ cond_t mutatorscv = DEFAULTCV; /* where non-mutators sleep. */ /* * Get the available register state for the target thread. * Return non-volatile registers: TRS_NONVOLATILE */ #pragma weak thr_getstate = _thr_getstate int _thr_getstate(thread_t tid, int *flag, lwpid_t *lwp, stack_t *ss, gregset_t rs) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t **ulwpp; ulwp_t *ulwp; int error = 0; int trs_flag = TRS_LWPID; if (tid == 0 || self->ul_lwpid == tid) { ulwp = self; ulwp_lock(ulwp, udp); } else if ((ulwpp = find_lwpp(tid)) != NULL) { ulwp = *ulwpp; } else { if (flag) *flag = TRS_INVALID; return (ESRCH); } if (ulwp->ul_dead) { trs_flag = TRS_INVALID; } else if (!ulwp->ul_stop && !suspendedallmutators) { error = EINVAL; trs_flag = TRS_INVALID; } else if (ulwp->ul_stop) { trs_flag = TRS_NONVOLATILE; getgregs(ulwp, rs); } if (flag) *flag = trs_flag; if (lwp) *lwp = tid; if (ss != NULL) (void) _thrp_stksegment(ulwp, ss); ulwp_unlock(ulwp, udp); return (error); } /* * Set the appropriate register state for the target thread. * This is not used by java. It exists solely for the MSTC test suite. */ #pragma weak thr_setstate = _thr_setstate int _thr_setstate(thread_t tid, int flag, gregset_t rs) { uberdata_t *udp = curthread->ul_uberdata; ulwp_t *ulwp; int error = 0; if ((ulwp = find_lwp(tid)) == NULL) return (ESRCH); if (!ulwp->ul_stop && !suspendedallmutators) error = EINVAL; else if (rs != NULL) { switch (flag) { case TRS_NONVOLATILE: /* do /proc stuff here? */ if (ulwp->ul_stop) setgregs(ulwp, rs); else error = EINVAL; break; case TRS_LWPID: /* do /proc stuff here? */ default: error = EINVAL; break; } } ulwp_unlock(ulwp, udp); return (error); } int getlwpstatus(thread_t tid, struct lwpstatus *sp) { extern ssize_t _pread(int, void *, size_t, off_t); char buf[100]; int fd; /* "/proc/self/lwp/%u/lwpstatus" w/o stdio */ (void) strcpy(buf, "/proc/self/lwp/"); ultos((uint64_t)tid, 10, buf + strlen(buf)); (void) strcat(buf, "/lwpstatus"); if ((fd = _open(buf, O_RDONLY, 0)) >= 0) { while (_pread(fd, sp, sizeof (*sp), 0) == sizeof (*sp)) { if (sp->pr_flags & PR_STOPPED) { (void) _close(fd); return (0); } lwp_yield(); /* give him a chance to stop */ } (void) _close(fd); } return (-1); } int putlwpregs(thread_t tid, prgregset_t prp) { extern ssize_t _writev(int, const struct iovec *, int); char buf[100]; int fd; long dstop_sreg[2]; long run_null[2]; iovec_t iov[3]; /* "/proc/self/lwp/%u/lwpctl" w/o stdio */ (void) strcpy(buf, "/proc/self/lwp/"); ultos((uint64_t)tid, 10, buf + strlen(buf)); (void) strcat(buf, "/lwpctl"); if ((fd = _open(buf, O_WRONLY, 0)) >= 0) { dstop_sreg[0] = PCDSTOP; /* direct it to stop */ dstop_sreg[1] = PCSREG; /* set the registers */ iov[0].iov_base = (caddr_t)dstop_sreg; iov[0].iov_len = sizeof (dstop_sreg); iov[1].iov_base = (caddr_t)prp; /* from the register set */ iov[1].iov_len = sizeof (prgregset_t); run_null[0] = PCRUN; /* make it runnable again */ run_null[1] = 0; iov[2].iov_base = (caddr_t)run_null; iov[2].iov_len = sizeof (run_null); if (_writev(fd, iov, 3) >= 0) { (void) _close(fd); return (0); } (void) _close(fd); } return (-1); } static ulong_t gettsp_slow(thread_t tid) { char buf[100]; struct lwpstatus status; if (getlwpstatus(tid, &status) != 0) { /* "__gettsp(%u): can't read lwpstatus" w/o stdio */ (void) strcpy(buf, "__gettsp("); ultos((uint64_t)tid, 10, buf + strlen(buf)); (void) strcat(buf, "): can't read lwpstatus"); thr_panic(buf); } return (status.pr_reg[R_SP]); } ulong_t __gettsp(thread_t tid) { uberdata_t *udp = curthread->ul_uberdata; ulwp_t *ulwp; ulong_t result; if ((ulwp = find_lwp(tid)) == NULL) return (0); if (ulwp->ul_stop && (result = ulwp->ul_sp) != 0) { ulwp_unlock(ulwp, udp); return (result); } result = gettsp_slow(tid); ulwp_unlock(ulwp, udp); return (result); } /* * This tells java stack walkers how to find the ucontext * structure passed to signal handlers. */ #pragma weak thr_sighndlrinfo = _thr_sighndlrinfo void _thr_sighndlrinfo(void (**func)(), int *funcsize) { *func = &__sighndlr; *funcsize = (char *)&__sighndlrend - (char *)&__sighndlr; } /* * Mark a thread a mutator or reset a mutator to being a default, * non-mutator thread. */ #pragma weak thr_setmutator = _thr_setmutator int _thr_setmutator(thread_t tid, int enabled) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *ulwp; int error; enabled = enabled?1:0; top: if (tid == 0) { ulwp = self; ulwp_lock(ulwp, udp); } else if ((ulwp = find_lwp(tid)) == NULL) { return (ESRCH); } /* * The target thread should be the caller itself or a suspended thread. * This prevents the target from also changing its ul_mutator field. */ error = 0; if (ulwp != self && !ulwp->ul_stop && enabled) error = EINVAL; else if (ulwp->ul_mutator != enabled) { lmutex_lock(&mutatorslock); if (mutatorsbarrier) { ulwp_unlock(ulwp, udp); while (mutatorsbarrier) (void) _cond_wait(&mutatorscv, &mutatorslock); lmutex_unlock(&mutatorslock); goto top; } ulwp->ul_mutator = enabled; lmutex_unlock(&mutatorslock); } ulwp_unlock(ulwp, udp); return (error); } /* * Establish a barrier against new mutators. Any non-mutator trying * to become a mutator is suspended until the barrier is removed. */ #pragma weak thr_mutators_barrier = _thr_mutators_barrier void _thr_mutators_barrier(int enabled) { int oldvalue; lmutex_lock(&mutatorslock); /* * Wait if trying to set the barrier while it is already set. */ while (mutatorsbarrier && enabled) (void) _cond_wait(&mutatorscv, &mutatorslock); oldvalue = mutatorsbarrier; mutatorsbarrier = enabled; /* * Wakeup any blocked non-mutators when barrier is removed. */ if (oldvalue && !enabled) (void) cond_broadcast_internal(&mutatorscv); lmutex_unlock(&mutatorslock); } /* * Suspend the set of all mutators except for the caller. The list * of actively running threads is searched and only the mutators * in this list are suspended. Actively running non-mutators remain * running. Any other thread is suspended. */ #pragma weak thr_suspend_allmutators = _thr_suspend_allmutators int _thr_suspend_allmutators(void) { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *ulwp; int link_dropped; /* * We single-thread the entire thread suspend mechanism. */ (void) fork_lock_enter(NULL); top: lmutex_lock(&udp->link_lock); if (suspendingallmutators || suspendedallmutators) { lmutex_unlock(&udp->link_lock); fork_lock_exit(); return (EINVAL); } suspendingallmutators = 1; for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) { ulwp_lock(ulwp, udp); if (!ulwp->ul_mutator) { ulwp_unlock(ulwp, udp); } else if (ulwp->ul_stop) { /* already stopped */ ulwp->ul_stop |= TSTP_MUTATOR; ulwp_broadcast(ulwp); ulwp_unlock(ulwp, udp); } else { /* * Move the stopped lwp out of a critical section. */ if (safe_suspend(ulwp, TSTP_MUTATOR, &link_dropped) || link_dropped) { suspendingallmutators = 0; goto top; } } } suspendedallmutators = 1; suspendingallmutators = 0; lmutex_unlock(&udp->link_lock); fork_lock_exit(); return (0); } /* * Suspend the target mutator. The caller is permitted to suspend * itself. If a mutator barrier is enabled, the caller will suspend * itself as though it had been suspended by thr_suspend_allmutators(). * When the barrier is removed, this thread will be resumed. Any * suspended mutator, whether suspended by thr_suspend_mutator(), or by * thr_suspend_allmutators(), can be resumed by thr_continue_mutator(). */ #pragma weak thr_suspend_mutator = _thr_suspend_mutator int _thr_suspend_mutator(thread_t tid) { if (tid == 0) tid = curthread->ul_lwpid; return (_thrp_suspend(tid, TSTP_MUTATOR)); } /* * Resume the set of all suspended mutators. */ #pragma weak thr_continue_allmutators = _thr_continue_allmutators int _thr_continue_allmutators() { ulwp_t *self = curthread; uberdata_t *udp = self->ul_uberdata; ulwp_t *ulwp; lmutex_lock(&udp->link_lock); if (!suspendedallmutators) { lmutex_unlock(&udp->link_lock); return (EINVAL); } suspendedallmutators = 0; for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) { mutex_t *mp = ulwp_mutex(ulwp, udp); lmutex_lock(mp); if (ulwp->ul_stop & TSTP_MUTATOR) { ulwp->ul_stop &= ~TSTP_MUTATOR; ulwp_broadcast(ulwp); if (!ulwp->ul_stop) force_continue(ulwp); } lmutex_unlock(mp); } lmutex_unlock(&udp->link_lock); return (0); } /* * Resume a suspended mutator. */ #pragma weak thr_continue_mutator = _thr_continue_mutator int _thr_continue_mutator(thread_t tid) { return (_thrp_continue(tid, TSTP_MUTATOR)); } #pragma weak thr_wait_mutator = _thr_wait_mutator int _thr_wait_mutator(thread_t tid, int dontwait) { uberdata_t *udp = curthread->ul_uberdata; ulwp_t *ulwp; int error = 0; top: if ((ulwp = find_lwp(tid)) == NULL) return (ESRCH); if (!ulwp->ul_mutator) error = EINVAL; else if (dontwait) { if (!(ulwp->ul_stop & TSTP_MUTATOR)) error = EWOULDBLOCK; } else if (!(ulwp->ul_stop & TSTP_MUTATOR)) { cond_t *cvp = ulwp_condvar(ulwp, udp); mutex_t *mp = ulwp_mutex(ulwp, udp); (void) _cond_wait(cvp, mp); (void) lmutex_unlock(mp); goto top; } ulwp_unlock(ulwp, udp); return (error); } /* PROBE_SUPPORT begin */ void thr_probe_setup(void *data) { curthread->ul_tpdp = data; } static void * _thread_probe_getfunc() { return (curthread->ul_tpdp); } void * (*thr_probe_getfunc_addr)(void) = _thread_probe_getfunc; /* ARGSUSED */ void _resume(ulwp_t *ulwp, caddr_t sp, int dontsave) { /* never called */ } /* ARGSUSED */ void _resume_ret(ulwp_t *oldlwp) { /* never called */ } /* PROBE_SUPPORT end */