1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 * Copyright (c) 2017 by The MathWorks, Inc. All rights reserved.
26 * Copyright 2018 Joyent, Inc.
27 * Copyright 2025 MNX Cloud, Inc.
28 */
29
30 #include "lint.h"
31 #include "thr_uberdata.h"
32 #include <pthread.h>
33 #include <procfs.h>
34 #include <sys/uio.h>
35 #include <ctype.h>
36 #include "libc.h"
37
38 /*
39 * These symbols should not be exported from libc, but
40 * /lib/libm.so.2 references _thr_main. libm needs to be fixed.
41 * Also, some older versions of the Studio compiler/debugger
42 * components reference them. These need to be fixed, too.
43 */
44 #pragma weak _thr_main = thr_main
45 #pragma weak _thr_create = thr_create
46 #pragma weak _thr_join = thr_join
47 #pragma weak _thr_self = thr_self
48
49 #undef errno
50 extern int errno;
51
52 /*
53 * Between Solaris 2.5 and Solaris 9, __threaded was used to indicate
54 * "we are linked with libthread". The Sun Workshop 6 update 1 compilation
55 * system used it illegally (it is a consolidation private symbol).
56 * To accommodate this and possibly other abusers of the symbol,
57 * we make it always equal to 1 now that libthread has been folded
58 * into libc. The new __libc_threaded symbol is used to indicate
59 * the new meaning, "more than one thread exists".
60 */
61 int __threaded = 1; /* always equal to 1 */
62 int __libc_threaded = 0; /* zero until first thr_create() */
63
64 /*
65 * thr_concurrency and pthread_concurrency are not used by the library.
66 * They exist solely to hold and return the values set by calls to
67 * thr_setconcurrency() and pthread_setconcurrency().
68 * Because thr_concurrency is affected by the THR_NEW_LWP flag
69 * to thr_create(), thr_concurrency is protected by link_lock.
70 */
71 static int thr_concurrency = 1;
72 static int pthread_concurrency;
73
74 #define HASHTBLSZ 1024 /* must be a power of two */
75 #define TIDHASH(tid, udp) (tid & (udp)->hash_mask)
76
77 /* initial allocation, just enough for one lwp */
78 #pragma align 64(init_hash_table)
79 thr_hash_table_t init_hash_table[1] = {
80 { DEFAULTMUTEX, DEFAULTCV, NULL },
81 };
82
83 extern const Lc_interface rtld_funcs[];
84
85 /*
86 * The weak version is known to libc_db and mdb.
87 */
88 #pragma weak _uberdata = __uberdata
89 uberdata_t __uberdata = {
90 { DEFAULTMUTEX, 0, 0 }, /* link_lock */
91 { RECURSIVEMUTEX, 0, 0 }, /* ld_lock */
92 { RECURSIVEMUTEX, 0, 0 }, /* fork_lock */
93 { RECURSIVEMUTEX, 0, 0 }, /* atfork_lock */
94 { RECURSIVEMUTEX, 0, 0 }, /* callout_lock */
95 { DEFAULTMUTEX, 0, 0 }, /* tdb_hash_lock */
96 { 0, }, /* tdb_hash_lock_stats */
97 { { 0 }, }, /* siguaction[NSIG] */
98 {{ DEFAULTMUTEX, NULL, 0 }, /* bucket[NBUCKETS] */
99 { DEFAULTMUTEX, NULL, 0 },
100 { DEFAULTMUTEX, NULL, 0 },
101 { DEFAULTMUTEX, NULL, 0 },
102 { DEFAULTMUTEX, NULL, 0 },
103 { DEFAULTMUTEX, NULL, 0 },
104 { DEFAULTMUTEX, NULL, 0 },
105 { DEFAULTMUTEX, NULL, 0 },
106 { DEFAULTMUTEX, NULL, 0 },
107 { DEFAULTMUTEX, NULL, 0 }},
108 { RECURSIVEMUTEX, NULL, NULL }, /* atexit_root */
109 { RECURSIVEMUTEX, NULL }, /* quickexit_root */
110 { DEFAULTMUTEX, 0, 0, NULL }, /* tsd_metadata */
111 { DEFAULTMUTEX, {0, 0}, {0, 0} }, /* tls_metadata */
112 0, /* primary_map */
113 0, /* bucket_init */
114 0, /* pad[0] */
115 0, /* pad[1] */
116 { 0 }, /* uberflags */
117 NULL, /* queue_head */
118 init_hash_table, /* thr_hash_table */
119 1, /* hash_size: size of the hash table */
120 0, /* hash_mask: hash_size - 1 */
121 NULL, /* ulwp_one */
122 NULL, /* all_lwps */
123 NULL, /* all_zombies */
124 0, /* nthreads */
125 0, /* nzombies */
126 0, /* ndaemons */
127 0, /* pid */
128 sigacthandler, /* sigacthandler */
129 NULL, /* lwp_stacks */
130 NULL, /* lwp_laststack */
131 0, /* nfreestack */
132 10, /* thread_stack_cache */
133 NULL, /* ulwp_freelist */
134 NULL, /* ulwp_lastfree */
135 NULL, /* ulwp_replace_free */
136 NULL, /* ulwp_replace_last */
137 NULL, /* atforklist */
138 NULL, /* robustlocks */
139 NULL, /* robustlist */
140 NULL, /* progname */
141 NULL, /* ub_comm_page */
142 NULL, /* __tdb_bootstrap */
143 { /* tdb */
144 NULL, /* tdb_sync_addr_hash */
145 0, /* tdb_register_count */
146 0, /* tdb_hash_alloc_failed */
147 NULL, /* tdb_sync_addr_free */
148 NULL, /* tdb_sync_addr_last */
149 0, /* tdb_sync_alloc */
150 { 0, 0 }, /* tdb_ev_global_mask */
151 tdb_events, /* tdb_events array */
152 },
153 };
154
155 /*
156 * The weak version is known to libc_db and mdb.
157 */
158 #pragma weak _tdb_bootstrap = __tdb_bootstrap
159 uberdata_t **__tdb_bootstrap = NULL;
160
161 int thread_queue_fifo = 4;
162 int thread_queue_dump = 0;
163 int thread_cond_wait_defer = 0;
164 int thread_error_detection = 0;
165 int thread_async_safe = 0;
166 int thread_stack_cache = 10;
167 int thread_door_noreserve = 0;
168 int thread_locks_misaligned = 0;
169
170 static ulwp_t *ulwp_alloc(void);
171 static void ulwp_free(ulwp_t *);
172
173 /*
174 * Insert the lwp into the hash table.
175 */
176 void
hash_in_unlocked(ulwp_t * ulwp,int ix,uberdata_t * udp)177 hash_in_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
178 {
179 ulwp->ul_hash = udp->thr_hash_table[ix].hash_bucket;
180 udp->thr_hash_table[ix].hash_bucket = ulwp;
181 ulwp->ul_ix = ix;
182 }
183
184 void
hash_in(ulwp_t * ulwp,uberdata_t * udp)185 hash_in(ulwp_t *ulwp, uberdata_t *udp)
186 {
187 int ix = TIDHASH(ulwp->ul_lwpid, udp);
188 mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
189
190 lmutex_lock(mp);
191 hash_in_unlocked(ulwp, ix, udp);
192 lmutex_unlock(mp);
193 }
194
195 /*
196 * Delete the lwp from the hash table.
197 */
198 void
hash_out_unlocked(ulwp_t * ulwp,int ix,uberdata_t * udp)199 hash_out_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
200 {
201 ulwp_t **ulwpp;
202
203 for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
204 ulwp != *ulwpp;
205 ulwpp = &(*ulwpp)->ul_hash)
206 ;
207 *ulwpp = ulwp->ul_hash;
208 ulwp->ul_hash = NULL;
209 ulwp->ul_ix = -1;
210 }
211
212 void
hash_out(ulwp_t * ulwp,uberdata_t * udp)213 hash_out(ulwp_t *ulwp, uberdata_t *udp)
214 {
215 int ix;
216
217 if ((ix = ulwp->ul_ix) >= 0) {
218 mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
219
220 lmutex_lock(mp);
221 hash_out_unlocked(ulwp, ix, udp);
222 lmutex_unlock(mp);
223 }
224 }
225
226 /*
227 * Retain stack information for thread structures that are being recycled for
228 * new threads. All other members of the thread structure should be zeroed.
229 */
230 static void
ulwp_clean(ulwp_t * ulwp)231 ulwp_clean(ulwp_t *ulwp)
232 {
233 caddr_t stk = ulwp->ul_stk;
234 size_t mapsiz = ulwp->ul_mapsiz;
235 size_t guardsize = ulwp->ul_guardsize;
236 uintptr_t stktop = ulwp->ul_stktop;
237 size_t stksiz = ulwp->ul_stksiz;
238
239 (void) memset(ulwp, 0, sizeof (*ulwp));
240
241 ulwp->ul_stk = stk;
242 ulwp->ul_mapsiz = mapsiz;
243 ulwp->ul_guardsize = guardsize;
244 ulwp->ul_stktop = stktop;
245 ulwp->ul_stksiz = stksiz;
246 }
247
248 static int stackprot;
249
250 /*
251 * Answer the question, "Is the lwp in question really dead?"
252 * We must inquire of the operating system to be really sure
253 * because the lwp may have called lwp_exit() but it has not
254 * yet completed the exit.
255 */
256 static int
dead_and_buried(ulwp_t * ulwp)257 dead_and_buried(ulwp_t *ulwp)
258 {
259 if (ulwp->ul_lwpid == (lwpid_t)(-1))
260 return (1);
261 if (ulwp->ul_dead && ulwp->ul_detached &&
262 _lwp_kill(ulwp->ul_lwpid, 0) == ESRCH) {
263 ulwp->ul_lwpid = (lwpid_t)(-1);
264 return (1);
265 }
266 return (0);
267 }
268
269 /*
270 * Attempt to keep the stack cache within the specified cache limit.
271 */
272 static void
trim_stack_cache(int cache_limit)273 trim_stack_cache(int cache_limit)
274 {
275 ulwp_t *self = curthread;
276 uberdata_t *udp = self->ul_uberdata;
277 ulwp_t *prev = NULL;
278 ulwp_t **ulwpp = &udp->lwp_stacks;
279 ulwp_t *ulwp;
280
281 ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, self));
282
283 while (udp->nfreestack > cache_limit && (ulwp = *ulwpp) != NULL) {
284 if (dead_and_buried(ulwp)) {
285 *ulwpp = ulwp->ul_next;
286 if (ulwp == udp->lwp_laststack)
287 udp->lwp_laststack = prev;
288 hash_out(ulwp, udp);
289 udp->nfreestack--;
290 (void) munmap(ulwp->ul_stk, ulwp->ul_mapsiz);
291 /*
292 * Now put the free ulwp on the ulwp freelist.
293 */
294 ulwp->ul_mapsiz = 0;
295 ulwp->ul_next = NULL;
296 if (udp->ulwp_freelist == NULL)
297 udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
298 else {
299 udp->ulwp_lastfree->ul_next = ulwp;
300 udp->ulwp_lastfree = ulwp;
301 }
302 } else {
303 prev = ulwp;
304 ulwpp = &ulwp->ul_next;
305 }
306 }
307 }
308
309 /*
310 * Find an unused stack of the requested size
311 * or create a new stack of the requested size.
312 * Return a pointer to the ulwp_t structure referring to the stack, or NULL.
313 * thr_exit() stores 1 in the ul_dead member.
314 * thr_join() stores -1 in the ul_lwpid member.
315 */
316 static ulwp_t *
find_stack(size_t stksize,size_t guardsize)317 find_stack(size_t stksize, size_t guardsize)
318 {
319 static size_t pagesize = 0;
320
321 uberdata_t *udp = curthread->ul_uberdata;
322 size_t mapsize;
323 ulwp_t *prev;
324 ulwp_t *ulwp;
325 ulwp_t **ulwpp;
326 void *stk;
327
328 /*
329 * The stack is allocated PROT_READ|PROT_WRITE|PROT_EXEC
330 * unless overridden by the system's configuration.
331 */
332 if (stackprot == 0) { /* do this once */
333 long lprot = _sysconf(_SC_STACK_PROT);
334 if (lprot <= 0)
335 lprot = (PROT_READ|PROT_WRITE|PROT_EXEC);
336 stackprot = (int)lprot;
337 }
338 if (pagesize == 0) /* do this once */
339 pagesize = _sysconf(_SC_PAGESIZE);
340
341 /*
342 * One megabyte stacks by default, but subtract off
343 * two pages for the system-created red zones.
344 * Round up a non-zero stack size to a pagesize multiple.
345 */
346 if (stksize == 0)
347 stksize = DEFAULTSTACK - 2 * pagesize;
348 else
349 stksize = ((stksize + pagesize - 1) & -pagesize);
350
351 /*
352 * Round up the mapping size to a multiple of pagesize.
353 * Note: mmap() provides at least one page of red zone
354 * so we deduct that from the value of guardsize.
355 */
356 if (guardsize != 0)
357 guardsize = ((guardsize + pagesize - 1) & -pagesize) - pagesize;
358 mapsize = stksize + guardsize;
359
360 lmutex_lock(&udp->link_lock);
361 for (prev = NULL, ulwpp = &udp->lwp_stacks;
362 (ulwp = *ulwpp) != NULL;
363 prev = ulwp, ulwpp = &ulwp->ul_next) {
364 if (ulwp->ul_mapsiz == mapsize &&
365 ulwp->ul_guardsize == guardsize &&
366 dead_and_buried(ulwp)) {
367 /*
368 * The previous lwp is gone; reuse the stack.
369 * Remove the ulwp from the stack list.
370 */
371 *ulwpp = ulwp->ul_next;
372 ulwp->ul_next = NULL;
373 if (ulwp == udp->lwp_laststack)
374 udp->lwp_laststack = prev;
375 hash_out(ulwp, udp);
376 udp->nfreestack--;
377 lmutex_unlock(&udp->link_lock);
378 ulwp_clean(ulwp);
379 return (ulwp);
380 }
381 }
382
383 /*
384 * None of the cached stacks matched our mapping size.
385 * Reduce the stack cache to get rid of possibly
386 * very old stacks that will never be reused.
387 */
388 if (udp->nfreestack > udp->thread_stack_cache)
389 trim_stack_cache(udp->thread_stack_cache);
390 else if (udp->nfreestack > 0)
391 trim_stack_cache(udp->nfreestack - 1);
392 lmutex_unlock(&udp->link_lock);
393
394 /*
395 * Create a new stack.
396 */
397 if ((stk = mmap(NULL, mapsize, stackprot,
398 MAP_PRIVATE|MAP_NORESERVE|MAP_ANON, -1, (off_t)0)) != MAP_FAILED) {
399 /*
400 * We have allocated our stack. Now allocate the ulwp.
401 */
402 ulwp = ulwp_alloc();
403 if (ulwp == NULL)
404 (void) munmap(stk, mapsize);
405 else {
406 ulwp->ul_stk = stk;
407 ulwp->ul_mapsiz = mapsize;
408 ulwp->ul_guardsize = guardsize;
409 ulwp->ul_stktop = (uintptr_t)stk + mapsize;
410 ulwp->ul_stksiz = stksize;
411 if (guardsize) /* protect the extra red zone */
412 (void) mprotect(stk, guardsize, PROT_NONE);
413 }
414 }
415 return (ulwp);
416 }
417
418 /*
419 * Get a ulwp_t structure from the free list or allocate a new one.
420 * Such ulwp_t's do not have a stack allocated by the library.
421 */
422 static ulwp_t *
ulwp_alloc(void)423 ulwp_alloc(void)
424 {
425 ulwp_t *self = curthread;
426 uberdata_t *udp = self->ul_uberdata;
427 size_t tls_size;
428 ulwp_t *prev;
429 ulwp_t *ulwp;
430 ulwp_t **ulwpp;
431 caddr_t data;
432
433 lmutex_lock(&udp->link_lock);
434 for (prev = NULL, ulwpp = &udp->ulwp_freelist;
435 (ulwp = *ulwpp) != NULL;
436 prev = ulwp, ulwpp = &ulwp->ul_next) {
437 if (dead_and_buried(ulwp)) {
438 *ulwpp = ulwp->ul_next;
439 ulwp->ul_next = NULL;
440 if (ulwp == udp->ulwp_lastfree)
441 udp->ulwp_lastfree = prev;
442 hash_out(ulwp, udp);
443 lmutex_unlock(&udp->link_lock);
444 ulwp_clean(ulwp);
445 return (ulwp);
446 }
447 }
448 lmutex_unlock(&udp->link_lock);
449
450 tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
451 data = lmalloc(sizeof (*ulwp) + tls_size);
452 if (data != NULL) {
453 /* LINTED pointer cast may result in improper alignment */
454 ulwp = (ulwp_t *)(data + tls_size);
455 }
456 return (ulwp);
457 }
458
459 /*
460 * Free a ulwp structure.
461 * If there is an associated stack, put it on the stack list and
462 * munmap() previously freed stacks up to the residual cache limit.
463 * Else put it on the ulwp free list and never call lfree() on it.
464 */
465 static void
ulwp_free(ulwp_t * ulwp)466 ulwp_free(ulwp_t *ulwp)
467 {
468 uberdata_t *udp = curthread->ul_uberdata;
469
470 ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, curthread));
471 ulwp->ul_next = NULL;
472 if (ulwp == udp->ulwp_one) /* don't reuse the primoridal stack */
473 /*EMPTY*/;
474 else if (ulwp->ul_mapsiz != 0) {
475 if (udp->lwp_stacks == NULL)
476 udp->lwp_stacks = udp->lwp_laststack = ulwp;
477 else {
478 udp->lwp_laststack->ul_next = ulwp;
479 udp->lwp_laststack = ulwp;
480 }
481 if (++udp->nfreestack > udp->thread_stack_cache)
482 trim_stack_cache(udp->thread_stack_cache);
483 } else {
484 if (udp->ulwp_freelist == NULL)
485 udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
486 else {
487 udp->ulwp_lastfree->ul_next = ulwp;
488 udp->ulwp_lastfree = ulwp;
489 }
490 }
491 }
492
493 /*
494 * Find a named lwp and return a pointer to its hash list location.
495 * On success, returns with the hash lock held.
496 */
497 ulwp_t **
find_lwpp(thread_t tid)498 find_lwpp(thread_t tid)
499 {
500 uberdata_t *udp = curthread->ul_uberdata;
501 int ix = TIDHASH(tid, udp);
502 mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
503 ulwp_t *ulwp;
504 ulwp_t **ulwpp;
505
506 if (tid == 0)
507 return (NULL);
508
509 lmutex_lock(mp);
510 for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
511 (ulwp = *ulwpp) != NULL;
512 ulwpp = &ulwp->ul_hash) {
513 if (ulwp->ul_lwpid == tid)
514 return (ulwpp);
515 }
516 lmutex_unlock(mp);
517 return (NULL);
518 }
519
520 /*
521 * Wake up all lwps waiting on this lwp for some reason.
522 */
523 void
ulwp_broadcast(ulwp_t * ulwp)524 ulwp_broadcast(ulwp_t *ulwp)
525 {
526 ulwp_t *self = curthread;
527 uberdata_t *udp = self->ul_uberdata;
528
529 ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
530 (void) cond_broadcast(ulwp_condvar(ulwp, udp));
531 }
532
533 /*
534 * Find a named lwp and return a pointer to it.
535 * Returns with the hash lock held.
536 */
537 ulwp_t *
find_lwp(thread_t tid)538 find_lwp(thread_t tid)
539 {
540 ulwp_t *self = curthread;
541 uberdata_t *udp = self->ul_uberdata;
542 ulwp_t *ulwp = NULL;
543 ulwp_t **ulwpp;
544
545 if (self->ul_lwpid == tid) {
546 ulwp = self;
547 ulwp_lock(ulwp, udp);
548 } else if ((ulwpp = find_lwpp(tid)) != NULL) {
549 ulwp = *ulwpp;
550 }
551
552 if (ulwp && ulwp->ul_dead) {
553 ulwp_unlock(ulwp, udp);
554 ulwp = NULL;
555 }
556
557 return (ulwp);
558 }
559
560 int
_thrp_create(void * stk,size_t stksize,void * (* func)(void *),void * arg,long flags,thread_t * new_thread,size_t guardsize,const char * name)561 _thrp_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
562 long flags, thread_t *new_thread, size_t guardsize, const char *name)
563 {
564 ulwp_t *self = curthread;
565 uberdata_t *udp = self->ul_uberdata;
566 ucontext_t uc;
567 uint_t lwp_flags;
568 thread_t tid;
569 int error;
570 ulwp_t *ulwp;
571
572 /*
573 * Enforce the restriction of not creating any threads
574 * until the primary link map has been initialized.
575 * Also, disallow thread creation to a child of vfork().
576 */
577 if (!self->ul_primarymap || self->ul_vfork)
578 return (ENOTSUP);
579
580 if (udp->hash_size == 1)
581 finish_init();
582
583 if ((stk || stksize) && stksize < MINSTACK)
584 return (EINVAL);
585
586 if (stk == NULL) {
587 if ((ulwp = find_stack(stksize, guardsize)) == NULL)
588 return (ENOMEM);
589 stksize = ulwp->ul_mapsiz - ulwp->ul_guardsize;
590 } else {
591 /* initialize the private stack */
592 if ((ulwp = ulwp_alloc()) == NULL)
593 return (ENOMEM);
594 ulwp->ul_stk = stk;
595 ulwp->ul_stktop = (uintptr_t)stk + stksize;
596 ulwp->ul_stksiz = stksize;
597 }
598 /* ulwp is not in the hash table; make sure hash_out() doesn't fail */
599 ulwp->ul_ix = -1;
600 ulwp->ul_errnop = &ulwp->ul_errno;
601
602 lwp_flags = LWP_SUSPENDED;
603 if (flags & (THR_DETACHED|THR_DAEMON)) {
604 flags |= THR_DETACHED;
605 lwp_flags |= LWP_DETACHED;
606 }
607 if (flags & THR_DAEMON)
608 lwp_flags |= LWP_DAEMON;
609
610 /* creating a thread: enforce mt-correctness in mutex_lock() */
611 self->ul_async_safe = 1;
612
613 /* per-thread copies of global variables, for speed */
614 ulwp->ul_queue_fifo = self->ul_queue_fifo;
615 ulwp->ul_cond_wait_defer = self->ul_cond_wait_defer;
616 ulwp->ul_error_detection = self->ul_error_detection;
617 ulwp->ul_async_safe = self->ul_async_safe;
618 ulwp->ul_max_spinners = self->ul_max_spinners;
619 ulwp->ul_adaptive_spin = self->ul_adaptive_spin;
620 ulwp->ul_queue_spin = self->ul_queue_spin;
621 ulwp->ul_door_noreserve = self->ul_door_noreserve;
622 ulwp->ul_misaligned = self->ul_misaligned;
623
624 /* new thread inherits creating thread's scheduling parameters */
625 ulwp->ul_policy = self->ul_policy;
626 ulwp->ul_pri = (self->ul_epri? self->ul_epri : self->ul_pri);
627 ulwp->ul_cid = self->ul_cid;
628 ulwp->ul_rtclassid = self->ul_rtclassid;
629
630 ulwp->ul_primarymap = self->ul_primarymap;
631 ulwp->ul_self = ulwp;
632 ulwp->ul_uberdata = udp;
633
634 /* debugger support */
635 ulwp->ul_usropts = flags;
636
637 #ifdef __sparc
638 /*
639 * We cache several instructions in the thread structure for use
640 * by the fasttrap DTrace provider. When changing this, read the
641 * comment in fasttrap.h for the all the other places that must
642 * be changed.
643 */
644 ulwp->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
645 ulwp->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
646 ulwp->ul_dftret = 0x91d0203a; /* ta 0x3a */
647 ulwp->ul_dreturn = 0x81ca0000; /* return %o0 */
648 #endif
649
650 ulwp->ul_startpc = func;
651 ulwp->ul_startarg = arg;
652 _fpinherit(ulwp);
653 /*
654 * Defer signals on the new thread until its TLS constructors
655 * have been called. _thrp_setup() will call sigon() after
656 * it has called tls_setup().
657 */
658 ulwp->ul_sigdefer = 1;
659
660 error = setup_context(&uc, _thrp_setup, ulwp,
661 (caddr_t)ulwp->ul_stk + ulwp->ul_guardsize, stksize);
662 if (error != 0 && stk != NULL) /* inaccessible stack */
663 error = EFAULT;
664
665 /*
666 * Call enter_critical() to avoid being suspended until we
667 * have linked the new thread into the proper lists.
668 * This is necessary because forkall() and fork1() must
669 * suspend all threads and they must see a complete list.
670 */
671 enter_critical(self);
672 uc.uc_sigmask = ulwp->ul_sigmask = self->ul_sigmask;
673 if (error != 0 ||
674 (error = __lwp_create(&uc, lwp_flags, &tid)) != 0) {
675 exit_critical(self);
676 ulwp->ul_lwpid = (lwpid_t)(-1);
677 ulwp->ul_dead = 1;
678 ulwp->ul_detached = 1;
679 lmutex_lock(&udp->link_lock);
680 ulwp_free(ulwp);
681 lmutex_unlock(&udp->link_lock);
682 return (error);
683 }
684 self->ul_nocancel = 0; /* cancellation is now possible */
685 udp->uberflags.uf_mt = 1;
686 if (new_thread)
687 *new_thread = tid;
688 if (flags & THR_DETACHED)
689 ulwp->ul_detached = 1;
690 ulwp->ul_lwpid = tid;
691 ulwp->ul_stop = TSTP_REGULAR;
692 if (flags & THR_SUSPENDED)
693 ulwp->ul_created = 1;
694
695 lmutex_lock(&udp->link_lock);
696 ulwp->ul_forw = udp->all_lwps;
697 ulwp->ul_back = udp->all_lwps->ul_back;
698 ulwp->ul_back->ul_forw = ulwp;
699 ulwp->ul_forw->ul_back = ulwp;
700 hash_in(ulwp, udp);
701 udp->nthreads++;
702 if (flags & THR_DAEMON)
703 udp->ndaemons++;
704 if (flags & THR_NEW_LWP)
705 thr_concurrency++;
706 __libc_threaded = 1; /* inform stdio */
707 lmutex_unlock(&udp->link_lock);
708
709 if (__td_event_report(self, TD_CREATE, udp)) {
710 self->ul_td_evbuf.eventnum = TD_CREATE;
711 self->ul_td_evbuf.eventdata = (void *)(uintptr_t)tid;
712 tdb_event(TD_CREATE, udp);
713 }
714
715 exit_critical(self);
716
717 if (name != NULL && name[0] != '\0')
718 (void) pthread_setname_np(tid, name);
719
720 if (!(flags & THR_SUSPENDED))
721 (void) _thrp_continue(tid, TSTP_REGULAR);
722
723 return (0);
724 }
725
726 int
thr_create(void * stk,size_t stksize,void * (* func)(void *),void * arg,long flags,thread_t * new_thread)727 thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
728 long flags, thread_t *new_thread)
729 {
730 return (_thrp_create(stk, stksize, func, arg, flags, new_thread, 0,
731 NULL));
732 }
733
734 /*
735 * A special cancellation cleanup hook for DCE.
736 * cleanuphndlr, when it is not NULL, will contain a callback
737 * function to be called before a thread is terminated in
738 * thr_exit() as a result of being cancelled.
739 */
740 static void (*cleanuphndlr)(void) = NULL;
741
742 /*
743 * _pthread_setcleanupinit: sets the cleanup hook.
744 */
745 int
_pthread_setcleanupinit(void (* func)(void))746 _pthread_setcleanupinit(void (*func)(void))
747 {
748 cleanuphndlr = func;
749 return (0);
750 }
751
752 void
_thrp_exit()753 _thrp_exit()
754 {
755 ulwp_t *self = curthread;
756 uberdata_t *udp = self->ul_uberdata;
757 ulwp_t *replace = NULL;
758
759 if (__td_event_report(self, TD_DEATH, udp)) {
760 self->ul_td_evbuf.eventnum = TD_DEATH;
761 tdb_event(TD_DEATH, udp);
762 }
763
764 ASSERT(self->ul_sigdefer != 0);
765
766 lmutex_lock(&udp->link_lock);
767 udp->nthreads--;
768 if (self->ul_usropts & THR_NEW_LWP)
769 thr_concurrency--;
770 if (self->ul_usropts & THR_DAEMON)
771 udp->ndaemons--;
772 else if (udp->nthreads == udp->ndaemons) {
773 /*
774 * We are the last non-daemon thread exiting.
775 * Exit the process. We retain our TSD and TLS so
776 * that atexit() application functions can use them.
777 */
778 lmutex_unlock(&udp->link_lock);
779 exit(0);
780 }
781 lmutex_unlock(&udp->link_lock);
782
783 /*
784 * tsd_exit() may call its destructor free(), thus depending on
785 * tmem, therefore tmem_exit() needs to be called after tsd_exit()
786 * and tls_exit().
787 */
788 tsd_exit(); /* deallocate thread-specific data */
789 tls_exit(); /* deallocate thread-local storage */
790 tmem_exit(); /* deallocate tmem allocations */
791 heldlock_exit(); /* deal with left-over held locks */
792
793 /* block all signals to finish exiting */
794 block_all_signals(self);
795 /* also prevent ourself from being suspended */
796 enter_critical(self);
797 rwl_free(self);
798 lmutex_lock(&udp->link_lock);
799 ulwp_free(self);
800 (void) ulwp_lock(self, udp);
801
802 if (self->ul_mapsiz && !self->ul_detached) {
803 /*
804 * We want to free the stack for reuse but must keep
805 * the ulwp_t struct for the benefit of thr_join().
806 * For this purpose we allocate a replacement ulwp_t.
807 */
808 if ((replace = udp->ulwp_replace_free) == NULL)
809 replace = lmalloc(REPLACEMENT_SIZE);
810 else if ((udp->ulwp_replace_free = replace->ul_next) == NULL)
811 udp->ulwp_replace_last = NULL;
812 }
813
814 if (udp->all_lwps == self)
815 udp->all_lwps = self->ul_forw;
816 if (udp->all_lwps == self)
817 udp->all_lwps = NULL;
818 else {
819 self->ul_forw->ul_back = self->ul_back;
820 self->ul_back->ul_forw = self->ul_forw;
821 }
822 self->ul_forw = self->ul_back = NULL;
823 #if defined(DEBUG)
824 /* collect queue lock statistics before marking ourself dead */
825 record_spin_locks(self);
826 #endif
827 self->ul_dead = 1;
828 self->ul_pleasestop = 0;
829 if (replace != NULL) {
830 int ix = self->ul_ix; /* the hash index */
831 (void) memcpy(replace, self, REPLACEMENT_SIZE);
832 replace->ul_self = replace;
833 replace->ul_next = NULL; /* clone not on stack list */
834 replace->ul_mapsiz = 0; /* allows clone to be freed */
835 replace->ul_replace = 1; /* requires clone to be freed */
836 hash_out_unlocked(self, ix, udp);
837 hash_in_unlocked(replace, ix, udp);
838 ASSERT(!(self->ul_detached));
839 self->ul_detached = 1; /* this frees the stack */
840 self->ul_schedctl = NULL;
841 self->ul_schedctl_called = &udp->uberflags;
842 set_curthread(self = replace);
843 /*
844 * Having just changed the address of curthread, we
845 * must reset the ownership of the locks we hold so
846 * that assertions will not fire when we release them.
847 */
848 udp->link_lock.mutex_owner = (uintptr_t)self;
849 ulwp_mutex(self, udp)->mutex_owner = (uintptr_t)self;
850 /*
851 * NOTE:
852 * On i386, %gs still references the original, not the
853 * replacement, ulwp structure. Fetching the replacement
854 * curthread pointer via %gs:0 works correctly since the
855 * original ulwp structure will not be reallocated until
856 * this lwp has completed its lwp_exit() system call (see
857 * dead_and_buried()), but from here on out, we must make
858 * no references to %gs:<offset> other than %gs:0.
859 */
860 }
861 /*
862 * Put non-detached terminated threads in the all_zombies list.
863 */
864 if (!self->ul_detached) {
865 udp->nzombies++;
866 if (udp->all_zombies == NULL) {
867 ASSERT(udp->nzombies == 1);
868 udp->all_zombies = self->ul_forw = self->ul_back = self;
869 } else {
870 self->ul_forw = udp->all_zombies;
871 self->ul_back = udp->all_zombies->ul_back;
872 self->ul_back->ul_forw = self;
873 self->ul_forw->ul_back = self;
874 }
875 }
876 /*
877 * Notify everyone waiting for this thread.
878 */
879 ulwp_broadcast(self);
880 (void) ulwp_unlock(self, udp);
881 /*
882 * Prevent any more references to the schedctl data.
883 * We are exiting and continue_fork() may not find us.
884 * Do this just before dropping link_lock, since fork
885 * serializes on link_lock.
886 */
887 self->ul_schedctl = NULL;
888 self->ul_schedctl_called = &udp->uberflags;
889 lmutex_unlock(&udp->link_lock);
890
891 ASSERT(self->ul_critical == 1);
892 ASSERT(self->ul_preempt == 0);
893 _lwp_terminate(); /* never returns */
894 thr_panic("_thrp_exit(): _lwp_terminate() returned");
895 }
896
897 #if defined(DEBUG)
898 void
collect_queue_statistics()899 collect_queue_statistics()
900 {
901 uberdata_t *udp = curthread->ul_uberdata;
902 ulwp_t *ulwp;
903
904 if (thread_queue_dump) {
905 lmutex_lock(&udp->link_lock);
906 if ((ulwp = udp->all_lwps) != NULL) {
907 do {
908 record_spin_locks(ulwp);
909 } while ((ulwp = ulwp->ul_forw) != udp->all_lwps);
910 }
911 lmutex_unlock(&udp->link_lock);
912 }
913 }
914 #endif
915
916 static void __NORETURN
_thrp_exit_common(void * status,int unwind)917 _thrp_exit_common(void *status, int unwind)
918 {
919 ulwp_t *self = curthread;
920 int cancelled = (self->ul_cancel_pending && status == PTHREAD_CANCELED);
921
922 ASSERT(self->ul_critical == 0 && self->ul_preempt == 0);
923
924 /*
925 * Disable cancellation and call the special DCE cancellation
926 * cleanup hook if it is enabled. Do nothing else before calling
927 * the DCE cancellation cleanup hook; it may call longjmp() and
928 * never return here.
929 */
930 self->ul_cancel_disabled = 1;
931 self->ul_cancel_async = 0;
932 self->ul_save_async = 0;
933 self->ul_cancelable = 0;
934 self->ul_cancel_pending = 0;
935 set_cancel_pending_flag(self, 1);
936 if (cancelled && cleanuphndlr != NULL)
937 (*cleanuphndlr)();
938
939 /*
940 * Block application signals while we are exiting.
941 * We call out to C++, TSD, and TLS destructors while exiting
942 * and these are application-defined, so we cannot be assured
943 * that they won't reset the signal mask. We use sigoff() to
944 * defer any signals that may be received as a result of this
945 * bad behavior. Such signals will be lost to the process
946 * when the thread finishes exiting.
947 */
948 (void) thr_sigsetmask(SIG_SETMASK, &maskset, NULL);
949 sigoff(self);
950
951 self->ul_rval = status;
952
953 /*
954 * If thr_exit is being called from the places where
955 * C++ destructors are to be called such as cancellation
956 * points, then set this flag. It is checked in _t_cancel()
957 * to decide whether _ex_unwind() is to be called or not.
958 */
959 if (unwind)
960 self->ul_unwind = 1;
961
962 /*
963 * _thrp_unwind() will eventually call _thrp_exit().
964 * It never returns.
965 */
966 _thrp_unwind(NULL);
967 thr_panic("_thrp_exit_common(): _thrp_unwind() returned");
968 }
969
970 /*
971 * Called when a thread returns from its start function.
972 * We are at the top of the stack; no unwinding is necessary.
973 */
974 void
_thrp_terminate(void * status)975 _thrp_terminate(void *status)
976 {
977 _thrp_exit_common(status, 0);
978 }
979
980 #pragma weak pthread_exit = thr_exit
981 #pragma weak _thr_exit = thr_exit
982 void
thr_exit(void * status)983 thr_exit(void *status)
984 {
985 _thrp_exit_common(status, 1);
986 }
987
988 int
_thrp_join(thread_t tid,thread_t * departed,void ** status,int do_cancel)989 _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
990 {
991 uberdata_t *udp = curthread->ul_uberdata;
992 mutex_t *mp;
993 void *rval;
994 thread_t found;
995 ulwp_t *ulwp;
996 ulwp_t **ulwpp;
997 int replace;
998 int error;
999
1000 if (do_cancel)
1001 error = lwp_wait(tid, &found);
1002 else {
1003 while ((error = __lwp_wait(tid, &found)) == EINTR)
1004 ;
1005 }
1006 if (error)
1007 return (error);
1008
1009 /*
1010 * We must hold link_lock to avoid a race condition with find_stack().
1011 */
1012 lmutex_lock(&udp->link_lock);
1013 if ((ulwpp = find_lwpp(found)) == NULL) {
1014 /*
1015 * lwp_wait() found an lwp that the library doesn't know
1016 * about. It must have been created with _lwp_create().
1017 * Just return its lwpid; we can't know its status.
1018 */
1019 lmutex_unlock(&udp->link_lock);
1020 rval = NULL;
1021 } else {
1022 /*
1023 * Remove ulwp from the hash table.
1024 */
1025 ulwp = *ulwpp;
1026 *ulwpp = ulwp->ul_hash;
1027 ulwp->ul_hash = NULL;
1028 /*
1029 * Remove ulwp from all_zombies list.
1030 */
1031 ASSERT(udp->nzombies >= 1);
1032 if (udp->all_zombies == ulwp)
1033 udp->all_zombies = ulwp->ul_forw;
1034 if (udp->all_zombies == ulwp)
1035 udp->all_zombies = NULL;
1036 else {
1037 ulwp->ul_forw->ul_back = ulwp->ul_back;
1038 ulwp->ul_back->ul_forw = ulwp->ul_forw;
1039 }
1040 ulwp->ul_forw = ulwp->ul_back = NULL;
1041 udp->nzombies--;
1042 ASSERT(ulwp->ul_dead && !ulwp->ul_detached &&
1043 !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
1044 /*
1045 * We can't call ulwp_unlock(ulwp) after we set
1046 * ulwp->ul_ix = -1 so we have to get a pointer to the
1047 * ulwp's hash table mutex now in order to unlock it below.
1048 */
1049 mp = ulwp_mutex(ulwp, udp);
1050 ulwp->ul_lwpid = (lwpid_t)(-1);
1051 ulwp->ul_ix = -1;
1052 rval = ulwp->ul_rval;
1053 replace = ulwp->ul_replace;
1054 lmutex_unlock(mp);
1055 if (replace) {
1056 ulwp->ul_next = NULL;
1057 if (udp->ulwp_replace_free == NULL)
1058 udp->ulwp_replace_free =
1059 udp->ulwp_replace_last = ulwp;
1060 else {
1061 udp->ulwp_replace_last->ul_next = ulwp;
1062 udp->ulwp_replace_last = ulwp;
1063 }
1064 }
1065 lmutex_unlock(&udp->link_lock);
1066 }
1067
1068 if (departed != NULL)
1069 *departed = found;
1070 if (status != NULL)
1071 *status = rval;
1072 return (0);
1073 }
1074
1075 int
thr_join(thread_t tid,thread_t * departed,void ** status)1076 thr_join(thread_t tid, thread_t *departed, void **status)
1077 {
1078 int error = _thrp_join(tid, departed, status, 1);
1079 return ((error == EINVAL)? ESRCH : error);
1080 }
1081
1082 /*
1083 * pthread_join() differs from Solaris thr_join():
1084 * It does not return the departed thread's id
1085 * and hence does not have a "departed" argument.
1086 * It returns EINVAL if tid refers to a detached thread.
1087 */
1088 #pragma weak _pthread_join = pthread_join
1089 int
pthread_join(pthread_t tid,void ** status)1090 pthread_join(pthread_t tid, void **status)
1091 {
1092 return ((tid == 0)? ESRCH : _thrp_join(tid, NULL, status, 1));
1093 }
1094
1095 int
pthread_detach(pthread_t tid)1096 pthread_detach(pthread_t tid)
1097 {
1098 uberdata_t *udp = curthread->ul_uberdata;
1099 ulwp_t *ulwp;
1100 ulwp_t **ulwpp;
1101 int error = 0;
1102
1103 if ((ulwpp = find_lwpp(tid)) == NULL)
1104 return (ESRCH);
1105 ulwp = *ulwpp;
1106
1107 if (ulwp->ul_dead) {
1108 ulwp_unlock(ulwp, udp);
1109 error = _thrp_join(tid, NULL, NULL, 0);
1110 } else {
1111 error = __lwp_detach(tid);
1112 ulwp->ul_detached = 1;
1113 ulwp->ul_usropts |= THR_DETACHED;
1114 ulwp_unlock(ulwp, udp);
1115 }
1116 return (error);
1117 }
1118
1119 static const char *
ematch(const char * ev,const char * match)1120 ematch(const char *ev, const char *match)
1121 {
1122 int c;
1123
1124 while ((c = *match++) != '\0') {
1125 if (*ev++ != c)
1126 return (NULL);
1127 }
1128 if (*ev++ != '=')
1129 return (NULL);
1130 return (ev);
1131 }
1132
1133 static int
envvar(const char * ev,const char * match,int limit)1134 envvar(const char *ev, const char *match, int limit)
1135 {
1136 int val = -1;
1137 const char *ename;
1138
1139 if ((ename = ematch(ev, match)) != NULL) {
1140 int c;
1141 for (val = 0; (c = *ename) != '\0'; ename++) {
1142 if (!isdigit(c)) {
1143 val = -1;
1144 break;
1145 }
1146 val = val * 10 + (c - '0');
1147 if (val > limit) {
1148 val = limit;
1149 break;
1150 }
1151 }
1152 }
1153 return (val);
1154 }
1155
1156 static void
etest(const char * ev)1157 etest(const char *ev)
1158 {
1159 int value;
1160
1161 if ((value = envvar(ev, "QUEUE_SPIN", 1000000)) >= 0)
1162 thread_queue_spin = value;
1163 if ((value = envvar(ev, "ADAPTIVE_SPIN", 1000000)) >= 0)
1164 thread_adaptive_spin = value;
1165 if ((value = envvar(ev, "MAX_SPINNERS", 255)) >= 0)
1166 thread_max_spinners = value;
1167 if ((value = envvar(ev, "QUEUE_FIFO", 8)) >= 0)
1168 thread_queue_fifo = value;
1169 #if defined(DEBUG)
1170 if ((value = envvar(ev, "QUEUE_VERIFY", 1)) >= 0)
1171 thread_queue_verify = value;
1172 if ((value = envvar(ev, "QUEUE_DUMP", 1)) >= 0)
1173 thread_queue_dump = value;
1174 #endif
1175 if ((value = envvar(ev, "STACK_CACHE", 10000)) >= 0)
1176 thread_stack_cache = value;
1177 if ((value = envvar(ev, "COND_WAIT_DEFER", 1)) >= 0)
1178 thread_cond_wait_defer = value;
1179 if ((value = envvar(ev, "ERROR_DETECTION", 2)) >= 0)
1180 thread_error_detection = value;
1181 if ((value = envvar(ev, "ASYNC_SAFE", 1)) >= 0)
1182 thread_async_safe = value;
1183 if ((value = envvar(ev, "DOOR_NORESERVE", 1)) >= 0)
1184 thread_door_noreserve = value;
1185 if ((value = envvar(ev, "LOCKS_MISALIGNED", 1)) >= 0)
1186 thread_locks_misaligned = value;
1187 }
1188
1189 /*
1190 * Look for and evaluate environment variables of the form "_THREAD_*".
1191 * For compatibility with the past, we also look for environment
1192 * names of the form "LIBTHREAD_*".
1193 */
1194 static void
set_thread_vars()1195 set_thread_vars()
1196 {
1197 extern const char **_environ;
1198 const char **pev;
1199 const char *ev;
1200 char c;
1201
1202 if ((pev = _environ) == NULL)
1203 return;
1204 while ((ev = *pev++) != NULL) {
1205 c = *ev;
1206 if (c == '_' && strncmp(ev, "_THREAD_", 8) == 0)
1207 etest(ev + 8);
1208 if (c == 'L' && strncmp(ev, "LIBTHREAD_", 10) == 0)
1209 etest(ev + 10);
1210 }
1211 }
1212
1213 /* same as atexit() but private to the library */
1214 extern int _atexit(void (*)(void));
1215
1216 /* same as _cleanup() but private to the library */
1217 extern void __cleanup(void);
1218
1219 extern void atfork_init(void);
1220
1221 #ifdef __amd64
1222 extern void __proc64id(void);
1223 #endif
1224
1225 static void
init_auxv_data(uberdata_t * udp)1226 init_auxv_data(uberdata_t *udp)
1227 {
1228 Dl_argsinfo_t args;
1229
1230 udp->ub_comm_page = NULL;
1231 if (dlinfo(RTLD_SELF, RTLD_DI_ARGSINFO, &args) < 0)
1232 return;
1233
1234 while (args.dla_auxv->a_type != AT_NULL) {
1235 if (args.dla_auxv->a_type == AT_SUN_COMMPAGE) {
1236 udp->ub_comm_page = args.dla_auxv->a_un.a_ptr;
1237 }
1238 args.dla_auxv++;
1239 }
1240 }
1241
1242 /*
1243 * libc_init() is called by ld.so.1 for library initialization.
1244 * We perform minimal initialization; enough to work with the main thread.
1245 */
1246 void
libc_init(void)1247 libc_init(void)
1248 {
1249 uberdata_t *udp = &__uberdata;
1250 ulwp_t *oldself = __curthread();
1251 ucontext_t uc;
1252 ulwp_t *self;
1253 struct rlimit rl;
1254 caddr_t data;
1255 size_t tls_size;
1256 int setmask;
1257
1258 /*
1259 * For the initial stage of initialization, we must be careful
1260 * not to call any function that could possibly call _cerror().
1261 * For this purpose, we call only the raw system call wrappers.
1262 */
1263
1264 #ifdef __amd64
1265 /*
1266 * Gather information about cache layouts for optimized
1267 * AMD and Intel assembler strfoo() and memfoo() functions.
1268 */
1269 __proc64id();
1270 #endif
1271
1272 /*
1273 * Every libc, regardless of which link map, must register __cleanup().
1274 */
1275 (void) _atexit(__cleanup);
1276
1277 /*
1278 * Every libc, regardless of link map, needs to go through and check
1279 * its aux vectors. Doing so will indicate whether or not this has
1280 * been given a comm page (to optimize certain system actions).
1281 */
1282 init_auxv_data(udp);
1283
1284 /*
1285 * We keep our uberdata on one of (a) the first alternate link map
1286 * or (b) the primary link map. We switch to the primary link map
1287 * and stay there once we see it. All intermediate link maps are
1288 * subject to being unloaded at any time.
1289 */
1290 if (oldself != NULL && (oldself->ul_primarymap || !primary_link_map)) {
1291 __tdb_bootstrap = oldself->ul_uberdata->tdb_bootstrap;
1292 /*
1293 * Each link map has its own copy of the stack protector guard
1294 * and must always be initialized.
1295 */
1296 ssp_init();
1297 mutex_setup();
1298 atfork_init(); /* every link map needs atfork() processing */
1299 init_progname();
1300 return;
1301 }
1302
1303 /*
1304 * To establish the main stack information, we have to get our context.
1305 * This is also convenient to use for getting our signal mask.
1306 */
1307 uc.uc_flags = UC_ALL;
1308 (void) __getcontext(&uc);
1309 ASSERT(uc.uc_link == NULL);
1310
1311 tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
1312 ASSERT(primary_link_map || tls_size == 0);
1313 data = lmalloc(sizeof (ulwp_t) + tls_size);
1314 if (data == NULL)
1315 thr_panic("cannot allocate thread structure for main thread");
1316 /* LINTED pointer cast may result in improper alignment */
1317 self = (ulwp_t *)(data + tls_size);
1318 init_hash_table[0].hash_bucket = self;
1319
1320 self->ul_sigmask = uc.uc_sigmask;
1321 delete_reserved_signals(&self->ul_sigmask);
1322 /*
1323 * Are the old and new sets different?
1324 * (This can happen if we are currently blocking SIGCANCEL.)
1325 * If so, we must explicitly set our signal mask, below.
1326 */
1327 setmask =
1328 ((self->ul_sigmask.__sigbits[0] ^ uc.uc_sigmask.__sigbits[0]) |
1329 (self->ul_sigmask.__sigbits[1] ^ uc.uc_sigmask.__sigbits[1]) |
1330 (self->ul_sigmask.__sigbits[2] ^ uc.uc_sigmask.__sigbits[2]) |
1331 (self->ul_sigmask.__sigbits[3] ^ uc.uc_sigmask.__sigbits[3]));
1332
1333 #ifdef __sparc
1334 /*
1335 * We cache several instructions in the thread structure for use
1336 * by the fasttrap DTrace provider. When changing this, read the
1337 * comment in fasttrap.h for the all the other places that must
1338 * be changed.
1339 */
1340 self->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
1341 self->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
1342 self->ul_dftret = 0x91d0203a; /* ta 0x3a */
1343 self->ul_dreturn = 0x81ca0000; /* return %o0 */
1344 #endif
1345
1346 self->ul_stktop = (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
1347 (void) getrlimit(RLIMIT_STACK, &rl);
1348 self->ul_stksiz = rl.rlim_cur;
1349 self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz);
1350
1351 self->ul_forw = self->ul_back = self;
1352 self->ul_hash = NULL;
1353 self->ul_ix = 0;
1354 self->ul_lwpid = 1; /* _lwp_self() */
1355 self->ul_main = 1;
1356 self->ul_self = self;
1357 self->ul_policy = -1; /* initialize only when needed */
1358 self->ul_pri = 0;
1359 self->ul_cid = 0;
1360 self->ul_rtclassid = -1;
1361 self->ul_uberdata = udp;
1362 if (oldself != NULL) {
1363 int i;
1364
1365 ASSERT(primary_link_map);
1366 ASSERT(oldself->ul_main == 1);
1367 self->ul_stsd = oldself->ul_stsd;
1368 for (i = 0; i < TSD_NFAST; i++)
1369 self->ul_ftsd[i] = oldself->ul_ftsd[i];
1370 self->ul_tls = oldself->ul_tls;
1371 /*
1372 * Retrieve all pointers to uberdata allocated
1373 * while running on previous link maps.
1374 * We would like to do a structure assignment here, but
1375 * gcc turns structure assignments into calls to memcpy(),
1376 * a function exported from libc. We can't call any such
1377 * external functions until we establish curthread, below,
1378 * so we just call our private version of memcpy().
1379 */
1380 (void) memcpy(udp, oldself->ul_uberdata, sizeof (*udp));
1381 /*
1382 * These items point to global data on the primary link map.
1383 */
1384 udp->thr_hash_table = init_hash_table;
1385 udp->sigacthandler = sigacthandler;
1386 udp->tdb.tdb_events = tdb_events;
1387 ASSERT(udp->nthreads == 1 && !udp->uberflags.uf_mt);
1388 ASSERT(udp->lwp_stacks == NULL);
1389 ASSERT(udp->ulwp_freelist == NULL);
1390 ASSERT(udp->ulwp_replace_free == NULL);
1391 ASSERT(udp->hash_size == 1);
1392 }
1393 udp->all_lwps = self;
1394 udp->ulwp_one = self;
1395 udp->pid = getpid();
1396 udp->nthreads = 1;
1397 /*
1398 * In every link map, tdb_bootstrap points to the same piece of
1399 * allocated memory. When the primary link map is initialized,
1400 * the allocated memory is assigned a pointer to the one true
1401 * uberdata. This allows libc_db to initialize itself regardless
1402 * of which instance of libc it finds in the address space.
1403 */
1404 if (udp->tdb_bootstrap == NULL)
1405 udp->tdb_bootstrap = lmalloc(sizeof (uberdata_t *));
1406 __tdb_bootstrap = udp->tdb_bootstrap;
1407 if (primary_link_map) {
1408 self->ul_primarymap = 1;
1409 udp->primary_map = 1;
1410 *udp->tdb_bootstrap = udp;
1411 }
1412 /*
1413 * Cancellation can't happen until:
1414 * pthread_cancel() is called
1415 * or:
1416 * another thread is created
1417 * For now, as a single-threaded process, set the flag that tells
1418 * PROLOGUE/EPILOGUE (in scalls.c) that cancellation can't happen.
1419 */
1420 self->ul_nocancel = 1;
1421
1422 #if defined(__amd64)
1423 (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_FSBASE, self);
1424 #elif defined(__i386)
1425 (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_GSBASE, self);
1426 #endif /* __i386 || __amd64 */
1427 set_curthread(self); /* redundant on i386 */
1428 /*
1429 * Now curthread is established and it is safe to call any
1430 * function in libc except one that uses thread-local storage.
1431 */
1432 self->ul_errnop = &errno;
1433 if (oldself != NULL) {
1434 /* tls_size was zero when oldself was allocated */
1435 lfree(oldself, sizeof (ulwp_t));
1436 }
1437 ssp_init();
1438 mutex_setup();
1439 atfork_init();
1440 signal_init();
1441
1442 /*
1443 * If the stack is unlimited, we set the size to zero to disable
1444 * stack checking.
1445 * XXX: Work harder here. Get the stack size from /proc/self/rmap
1446 */
1447 if (self->ul_stksiz == RLIM_INFINITY) {
1448 self->ul_ustack.ss_sp = (void *)self->ul_stktop;
1449 self->ul_ustack.ss_size = 0;
1450 } else {
1451 self->ul_ustack.ss_sp = self->ul_stk;
1452 self->ul_ustack.ss_size = self->ul_stksiz;
1453 }
1454 self->ul_ustack.ss_flags = 0;
1455 (void) setustack(&self->ul_ustack);
1456
1457 /*
1458 * Get the variables that affect thread behavior from the environment.
1459 */
1460 set_thread_vars();
1461 udp->uberflags.uf_thread_error_detection = (char)thread_error_detection;
1462 udp->thread_stack_cache = thread_stack_cache;
1463
1464 /*
1465 * Make per-thread copies of global variables, for speed.
1466 */
1467 self->ul_queue_fifo = (char)thread_queue_fifo;
1468 self->ul_cond_wait_defer = (char)thread_cond_wait_defer;
1469 self->ul_error_detection = (char)thread_error_detection;
1470 self->ul_async_safe = (char)thread_async_safe;
1471 self->ul_door_noreserve = (char)thread_door_noreserve;
1472 self->ul_misaligned = (char)thread_locks_misaligned;
1473 self->ul_max_spinners = (uint8_t)thread_max_spinners;
1474 self->ul_adaptive_spin = thread_adaptive_spin;
1475 self->ul_queue_spin = thread_queue_spin;
1476
1477 #if defined(__sparc) && !defined(_LP64)
1478 if (self->ul_misaligned) {
1479 /*
1480 * Tell the kernel to fix up ldx/stx instructions that
1481 * refer to non-8-byte aligned data instead of giving
1482 * the process an alignment trap and generating SIGBUS.
1483 *
1484 * Programs compiled for 32-bit sparc with the Studio SS12
1485 * compiler get this done for them automatically (in _init()).
1486 * We do it here for the benefit of programs compiled with
1487 * other compilers, like gcc.
1488 *
1489 * This is necessary for the _THREAD_LOCKS_MISALIGNED=1
1490 * environment variable horrible hack to work.
1491 */
1492 extern void _do_fix_align(void);
1493 _do_fix_align();
1494 }
1495 #endif
1496
1497 /*
1498 * When we have initialized the primary link map, inform
1499 * the dynamic linker about our interface functions.
1500 * Set up our pointer to the program name.
1501 */
1502 if (self->ul_primarymap)
1503 _ld_libc((void *)rtld_funcs);
1504 init_progname();
1505
1506 /*
1507 * Defer signals until TLS constructors have been called.
1508 */
1509 sigoff(self);
1510 tls_setup();
1511 sigon(self);
1512 if (setmask)
1513 (void) restore_signals(self);
1514
1515 /*
1516 * Make private copies of __xpg4 and __xpg6 so libc can test
1517 * them after this point without invoking the dynamic linker.
1518 */
1519 libc__xpg4 = __xpg4;
1520 libc__xpg6 = __xpg6;
1521
1522 init_sigev_thread();
1523 init_aio();
1524
1525 /*
1526 * We need to reset __threaded dynamically at runtime, so that
1527 * __threaded can be bound to __threaded outside libc which may not
1528 * have initial value of 1 (without a copy relocation in a.out).
1529 */
1530 __threaded = 1;
1531 }
1532
1533 #pragma fini(libc_fini)
1534 void
libc_fini()1535 libc_fini()
1536 {
1537 /*
1538 * If we are doing fini processing for the instance of libc
1539 * on the first alternate link map (this happens only when
1540 * the dynamic linker rejects a bad audit library), then clear
1541 * __curthread(). We abandon whatever memory was allocated by
1542 * lmalloc() while running on this alternate link-map but we
1543 * don't care (and can't find the memory in any case); we just
1544 * want to protect the application from this bad audit library.
1545 * No fini processing is done by libc in the normal case.
1546 */
1547
1548 uberdata_t *udp = curthread->ul_uberdata;
1549
1550 if (udp->primary_map == 0 && udp == &__uberdata)
1551 set_curthread(NULL);
1552 }
1553
1554 /*
1555 * finish_init is called when we are about to become multi-threaded,
1556 * that is, on the first call to thr_create().
1557 */
1558 void
finish_init()1559 finish_init()
1560 {
1561 ulwp_t *self = curthread;
1562 uberdata_t *udp = self->ul_uberdata;
1563 thr_hash_table_t *htp;
1564 void *data;
1565 int i;
1566
1567 /*
1568 * No locks needed here; we are single-threaded on the first call.
1569 * We can be called only after the primary link map has been set up.
1570 */
1571 ASSERT(self->ul_primarymap);
1572 ASSERT(self == udp->ulwp_one);
1573 ASSERT(!udp->uberflags.uf_mt);
1574 ASSERT(udp->hash_size == 1);
1575
1576 /*
1577 * Initialize self->ul_policy, self->ul_cid, and self->ul_pri.
1578 */
1579 update_sched(self);
1580
1581 /*
1582 * Allocate the queue_head array if not already allocated.
1583 */
1584 if (udp->queue_head == NULL)
1585 queue_alloc();
1586
1587 /*
1588 * Now allocate the thread hash table.
1589 */
1590 if ((data = mmap(NULL, HASHTBLSZ * sizeof (thr_hash_table_t),
1591 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0))
1592 == MAP_FAILED)
1593 thr_panic("cannot allocate thread hash table");
1594
1595 udp->thr_hash_table = htp = (thr_hash_table_t *)data;
1596 udp->hash_size = HASHTBLSZ;
1597 udp->hash_mask = HASHTBLSZ - 1;
1598
1599 for (i = 0; i < HASHTBLSZ; i++, htp++) {
1600 htp->hash_lock.mutex_flag = LOCK_INITED;
1601 htp->hash_lock.mutex_magic = MUTEX_MAGIC;
1602 htp->hash_cond.cond_magic = COND_MAGIC;
1603 }
1604 hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1605
1606 /*
1607 * Set up the SIGCANCEL handler for threads cancellation.
1608 */
1609 setup_cancelsig(SIGCANCEL);
1610
1611 /*
1612 * Arrange to do special things on exit --
1613 * - collect queue statistics from all remaining active threads.
1614 * - dump queue statistics to stderr if _THREAD_QUEUE_DUMP is set.
1615 * - grab assert_lock to ensure that assertion failures
1616 * and a core dump take precedence over _exit().
1617 * (Functions are called in the reverse order of their registration.)
1618 */
1619 (void) _atexit(grab_assert_lock);
1620 #if defined(DEBUG)
1621 (void) _atexit(dump_queue_statistics);
1622 (void) _atexit(collect_queue_statistics);
1623 #endif
1624 }
1625
1626 /*
1627 * Used only by postfork1_child(), below.
1628 */
1629 static void
mark_dead_and_buried(ulwp_t * ulwp)1630 mark_dead_and_buried(ulwp_t *ulwp)
1631 {
1632 ulwp->ul_dead = 1;
1633 ulwp->ul_lwpid = (lwpid_t)(-1);
1634 ulwp->ul_hash = NULL;
1635 ulwp->ul_ix = -1;
1636 ulwp->ul_schedctl = NULL;
1637 ulwp->ul_schedctl_called = NULL;
1638 }
1639
1640 /*
1641 * This is called from fork1() in the child.
1642 * Reset our data structures to reflect one lwp.
1643 */
1644 void
postfork1_child()1645 postfork1_child()
1646 {
1647 ulwp_t *self = curthread;
1648 uberdata_t *udp = self->ul_uberdata;
1649 queue_head_t *qp;
1650 ulwp_t *next;
1651 ulwp_t *ulwp;
1652 int i;
1653
1654 /* daemon threads shouldn't call fork1(), but oh well... */
1655 self->ul_usropts &= ~THR_DAEMON;
1656 udp->nthreads = 1;
1657 udp->ndaemons = 0;
1658 udp->uberflags.uf_mt = 0;
1659 __libc_threaded = 0;
1660 for (i = 0; i < udp->hash_size; i++)
1661 udp->thr_hash_table[i].hash_bucket = NULL;
1662 self->ul_lwpid = _lwp_self();
1663 hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1664
1665 /*
1666 * Some thread in the parent might have been suspended
1667 * while holding udp->callout_lock or udp->ld_lock.
1668 * Reinitialize the child's copies.
1669 */
1670 (void) mutex_init(&udp->callout_lock,
1671 USYNC_THREAD | LOCK_RECURSIVE, NULL);
1672 (void) mutex_init(&udp->ld_lock,
1673 USYNC_THREAD | LOCK_RECURSIVE, NULL);
1674
1675 /* no one in the child is on a sleep queue; reinitialize */
1676 if ((qp = udp->queue_head) != NULL) {
1677 (void) memset(qp, 0, 2 * QHASHSIZE * sizeof (queue_head_t));
1678 for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
1679 qp->qh_type = (i < QHASHSIZE)? MX : CV;
1680 qp->qh_lock.mutex_flag = LOCK_INITED;
1681 qp->qh_lock.mutex_magic = MUTEX_MAGIC;
1682 qp->qh_hlist = &qp->qh_def_root;
1683 #if defined(DEBUG)
1684 qp->qh_hlen = 1;
1685 qp->qh_hmax = 1;
1686 #endif
1687 }
1688 }
1689
1690 /*
1691 * Do post-fork1 processing for subsystems that need it.
1692 * We need to do this before unmapping all of the abandoned
1693 * threads' stacks, below(), because the post-fork1 actions
1694 * might require access to those stacks.
1695 */
1696 postfork1_child_sigev_aio();
1697 postfork1_child_sigev_mq();
1698 postfork1_child_sigev_timer();
1699 postfork1_child_aio();
1700 /*
1701 * The above subsystems use thread pools, so this action
1702 * must be performed after those actions.
1703 */
1704 postfork1_child_tpool();
1705
1706 /*
1707 * All lwps except ourself are gone. Mark them so.
1708 * First mark all of the lwps that have already been freed.
1709 * Then mark and free all of the active lwps except ourself.
1710 * Since we are single-threaded, no locks are required here.
1711 */
1712 for (ulwp = udp->lwp_stacks; ulwp != NULL; ulwp = ulwp->ul_next)
1713 mark_dead_and_buried(ulwp);
1714 for (ulwp = udp->ulwp_freelist; ulwp != NULL; ulwp = ulwp->ul_next)
1715 mark_dead_and_buried(ulwp);
1716 for (ulwp = self->ul_forw; ulwp != self; ulwp = next) {
1717 next = ulwp->ul_forw;
1718 ulwp->ul_forw = ulwp->ul_back = NULL;
1719 mark_dead_and_buried(ulwp);
1720 tsd_free(ulwp);
1721 tls_free(ulwp);
1722 rwl_free(ulwp);
1723 heldlock_free(ulwp);
1724 ulwp_free(ulwp);
1725 }
1726 self->ul_forw = self->ul_back = udp->all_lwps = self;
1727 if (self != udp->ulwp_one)
1728 mark_dead_and_buried(udp->ulwp_one);
1729 if ((ulwp = udp->all_zombies) != NULL) {
1730 ASSERT(udp->nzombies != 0);
1731 do {
1732 next = ulwp->ul_forw;
1733 ulwp->ul_forw = ulwp->ul_back = NULL;
1734 mark_dead_and_buried(ulwp);
1735 udp->nzombies--;
1736 if (ulwp->ul_replace) {
1737 ulwp->ul_next = NULL;
1738 if (udp->ulwp_replace_free == NULL) {
1739 udp->ulwp_replace_free =
1740 udp->ulwp_replace_last = ulwp;
1741 } else {
1742 udp->ulwp_replace_last->ul_next = ulwp;
1743 udp->ulwp_replace_last = ulwp;
1744 }
1745 }
1746 } while ((ulwp = next) != udp->all_zombies);
1747 ASSERT(udp->nzombies == 0);
1748 udp->all_zombies = NULL;
1749 udp->nzombies = 0;
1750 }
1751 trim_stack_cache(0);
1752 }
1753
1754 lwpid_t
lwp_self(void)1755 lwp_self(void)
1756 {
1757 return (curthread->ul_lwpid);
1758 }
1759
1760 #pragma weak _ti_thr_self = thr_self
1761 #pragma weak pthread_self = thr_self
1762 thread_t
thr_self()1763 thr_self()
1764 {
1765 return (curthread->ul_lwpid);
1766 }
1767
1768 int
thr_main()1769 thr_main()
1770 {
1771 ulwp_t *self = __curthread();
1772
1773 return ((self == NULL)? -1 : self->ul_main);
1774 }
1775
1776 int
_thrp_cancelled(void)1777 _thrp_cancelled(void)
1778 {
1779 return (curthread->ul_rval == PTHREAD_CANCELED);
1780 }
1781
1782 int
_thrp_stksegment(ulwp_t * ulwp,stack_t * stk)1783 _thrp_stksegment(ulwp_t *ulwp, stack_t *stk)
1784 {
1785 stk->ss_sp = (void *)ulwp->ul_stktop;
1786 stk->ss_size = ulwp->ul_stksiz;
1787 stk->ss_flags = 0;
1788 return (0);
1789 }
1790
1791 #pragma weak _thr_stksegment = thr_stksegment
1792 int
thr_stksegment(stack_t * stk)1793 thr_stksegment(stack_t *stk)
1794 {
1795 return (_thrp_stksegment(curthread, stk));
1796 }
1797
1798 void
force_continue(ulwp_t * ulwp)1799 force_continue(ulwp_t *ulwp)
1800 {
1801 #if defined(DEBUG)
1802 ulwp_t *self = curthread;
1803 uberdata_t *udp = self->ul_uberdata;
1804 #endif
1805 int error;
1806 timespec_t ts;
1807
1808 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1809 ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
1810
1811 for (;;) {
1812 error = _lwp_continue(ulwp->ul_lwpid);
1813 if (error != 0 && error != EINTR)
1814 break;
1815 error = 0;
1816 if (ulwp->ul_stopping) { /* it is stopping itself */
1817 ts.tv_sec = 0; /* give it a chance to run */
1818 ts.tv_nsec = 100000; /* 100 usecs or clock tick */
1819 (void) __nanosleep(&ts, NULL);
1820 }
1821 if (!ulwp->ul_stopping) /* it is running now */
1822 break; /* so we are done */
1823 /*
1824 * It is marked as being in the process of stopping
1825 * itself. Loop around and continue it again.
1826 * It may not have been stopped the first time.
1827 */
1828 }
1829 }
1830
1831 /*
1832 * Suspend an lwp with lwp_suspend(), then move it to a safe point,
1833 * that is, to a point where ul_critical and ul_rtld are both zero.
1834 * On return, the ulwp_lock() is dropped as with ulwp_unlock().
1835 * If 'link_dropped' is non-NULL, then 'link_lock' is held on entry.
1836 * If we have to drop link_lock, we store 1 through link_dropped.
1837 * If the lwp exits before it can be suspended, we return ESRCH.
1838 */
1839 int
safe_suspend(ulwp_t * ulwp,uchar_t whystopped,int * link_dropped)1840 safe_suspend(ulwp_t *ulwp, uchar_t whystopped, int *link_dropped)
1841 {
1842 ulwp_t *self = curthread;
1843 uberdata_t *udp = self->ul_uberdata;
1844 cond_t *cvp = ulwp_condvar(ulwp, udp);
1845 mutex_t *mp = ulwp_mutex(ulwp, udp);
1846 thread_t tid = ulwp->ul_lwpid;
1847 int ix = ulwp->ul_ix;
1848 int error = 0;
1849
1850 ASSERT(whystopped == TSTP_REGULAR ||
1851 whystopped == TSTP_MUTATOR ||
1852 whystopped == TSTP_FORK);
1853 ASSERT(ulwp != self);
1854 ASSERT(!ulwp->ul_stop);
1855 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1856 ASSERT(MUTEX_OWNED(mp, self));
1857
1858 if (link_dropped != NULL)
1859 *link_dropped = 0;
1860
1861 /*
1862 * We must grab the target's spin lock before suspending it.
1863 * See the comments below and in _thrp_suspend() for why.
1864 */
1865 spin_lock_set(&ulwp->ul_spinlock);
1866 (void) ___lwp_suspend(tid);
1867 spin_lock_clear(&ulwp->ul_spinlock);
1868
1869 top:
1870 if ((ulwp->ul_critical == 0 && ulwp->ul_rtld == 0) ||
1871 ulwp->ul_stopping) {
1872 /* thread is already safe */
1873 ulwp->ul_stop |= whystopped;
1874 } else {
1875 /*
1876 * Setting ul_pleasestop causes the target thread to stop
1877 * itself in _thrp_suspend(), below, after we drop its lock.
1878 * We must continue the critical thread before dropping
1879 * link_lock because the critical thread may be holding
1880 * the queue lock for link_lock. This is delicate.
1881 */
1882 ulwp->ul_pleasestop |= whystopped;
1883 force_continue(ulwp);
1884 if (link_dropped != NULL) {
1885 *link_dropped = 1;
1886 lmutex_unlock(&udp->link_lock);
1887 /* be sure to drop link_lock only once */
1888 link_dropped = NULL;
1889 }
1890
1891 /*
1892 * The thread may disappear by calling thr_exit() so we
1893 * cannot rely on the ulwp pointer after dropping the lock.
1894 * Instead, we search the hash table to find it again.
1895 * When we return, we may find that the thread has been
1896 * continued by some other thread. The suspend/continue
1897 * interfaces are prone to such race conditions by design.
1898 */
1899 while (ulwp && !ulwp->ul_dead && !ulwp->ul_stop &&
1900 (ulwp->ul_pleasestop & whystopped)) {
1901 (void) __cond_wait(cvp, mp);
1902 for (ulwp = udp->thr_hash_table[ix].hash_bucket;
1903 ulwp != NULL; ulwp = ulwp->ul_hash) {
1904 if (ulwp->ul_lwpid == tid)
1905 break;
1906 }
1907 }
1908
1909 if (ulwp == NULL || ulwp->ul_dead)
1910 error = ESRCH;
1911 else {
1912 /*
1913 * Do another lwp_suspend() to make sure we don't
1914 * return until the target thread is fully stopped
1915 * in the kernel. Don't apply lwp_suspend() until
1916 * we know that the target is not holding any
1917 * queue locks, that is, that it has completed
1918 * ulwp_unlock(self) and has, or at least is
1919 * about to, call lwp_suspend() on itself. We do
1920 * this by grabbing the target's spin lock.
1921 */
1922 ASSERT(ulwp->ul_lwpid == tid);
1923 spin_lock_set(&ulwp->ul_spinlock);
1924 (void) ___lwp_suspend(tid);
1925 spin_lock_clear(&ulwp->ul_spinlock);
1926 /*
1927 * If some other thread did a thr_continue()
1928 * on the target thread we have to start over.
1929 */
1930 if (!ulwp->ul_stopping || !(ulwp->ul_stop & whystopped))
1931 goto top;
1932 }
1933 }
1934
1935 (void) cond_broadcast(cvp);
1936 lmutex_unlock(mp);
1937 return (error);
1938 }
1939
1940 int
_thrp_suspend(thread_t tid,uchar_t whystopped)1941 _thrp_suspend(thread_t tid, uchar_t whystopped)
1942 {
1943 ulwp_t *self = curthread;
1944 uberdata_t *udp = self->ul_uberdata;
1945 ulwp_t *ulwp;
1946 int error = 0;
1947
1948 ASSERT((whystopped & (TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) != 0);
1949 ASSERT((whystopped & ~(TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) == 0);
1950
1951 /*
1952 * We can't suspend anyone except ourself while
1953 * some other thread is performing a fork.
1954 * This also allows only one suspension at a time.
1955 */
1956 if (tid != self->ul_lwpid)
1957 fork_lock_enter();
1958
1959 if ((ulwp = find_lwp(tid)) == NULL)
1960 error = ESRCH;
1961 else if (whystopped == TSTP_MUTATOR && !ulwp->ul_mutator) {
1962 ulwp_unlock(ulwp, udp);
1963 error = EINVAL;
1964 } else if (ulwp->ul_stop) { /* already stopped */
1965 ulwp->ul_stop |= whystopped;
1966 ulwp_broadcast(ulwp);
1967 ulwp_unlock(ulwp, udp);
1968 } else if (ulwp != self) {
1969 /*
1970 * After suspending the other thread, move it out of a
1971 * critical section and deal with the schedctl mappings.
1972 * safe_suspend() suspends the other thread, calls
1973 * ulwp_broadcast(ulwp) and drops the ulwp lock.
1974 */
1975 error = safe_suspend(ulwp, whystopped, NULL);
1976 } else {
1977 int schedctl_after_fork = 0;
1978
1979 /*
1980 * We are suspending ourself. We must not take a signal
1981 * until we return from lwp_suspend() and clear ul_stopping.
1982 * This is to guard against siglongjmp().
1983 */
1984 enter_critical(self);
1985 self->ul_sp = stkptr();
1986 _flush_windows(); /* sparc */
1987 self->ul_pleasestop = 0;
1988 self->ul_stop |= whystopped;
1989 /*
1990 * Grab our spin lock before dropping ulwp_mutex(self).
1991 * This prevents the suspending thread from applying
1992 * lwp_suspend() to us before we emerge from
1993 * lmutex_unlock(mp) and have dropped mp's queue lock.
1994 */
1995 spin_lock_set(&self->ul_spinlock);
1996 self->ul_stopping = 1;
1997 ulwp_broadcast(self);
1998 ulwp_unlock(self, udp);
1999 /*
2000 * From this point until we return from lwp_suspend(),
2001 * we must not call any function that might invoke the
2002 * dynamic linker, that is, we can only call functions
2003 * private to the library.
2004 *
2005 * Also, this is a nasty race condition for a process
2006 * that is undergoing a forkall() operation:
2007 * Once we clear our spinlock (below), we are vulnerable
2008 * to being suspended by the forkall() thread before
2009 * we manage to suspend ourself in ___lwp_suspend().
2010 * See safe_suspend() and force_continue().
2011 *
2012 * To avoid a SIGSEGV due to the disappearance
2013 * of the schedctl mappings in the child process,
2014 * which can happen in spin_lock_clear() if we
2015 * are suspended while we are in the middle of
2016 * its call to preempt(), we preemptively clear
2017 * our own schedctl pointer before dropping our
2018 * spinlock. We reinstate it, in both the parent
2019 * and (if this really is a forkall()) the child.
2020 */
2021 if (whystopped & TSTP_FORK) {
2022 schedctl_after_fork = 1;
2023 self->ul_schedctl = NULL;
2024 self->ul_schedctl_called = &udp->uberflags;
2025 }
2026 spin_lock_clear(&self->ul_spinlock);
2027 (void) ___lwp_suspend(tid);
2028 /*
2029 * Somebody else continued us.
2030 * We can't grab ulwp_lock(self)
2031 * until after clearing ul_stopping.
2032 * force_continue() relies on this.
2033 */
2034 self->ul_stopping = 0;
2035 self->ul_sp = 0;
2036 if (schedctl_after_fork) {
2037 self->ul_schedctl_called = NULL;
2038 self->ul_schedctl = NULL;
2039 (void) setup_schedctl();
2040 }
2041 ulwp_lock(self, udp);
2042 ulwp_broadcast(self);
2043 ulwp_unlock(self, udp);
2044 exit_critical(self);
2045 }
2046
2047 if (tid != self->ul_lwpid)
2048 fork_lock_exit();
2049
2050 return (error);
2051 }
2052
2053 /*
2054 * Suspend all lwps other than ourself in preparation for fork.
2055 */
2056 void
suspend_fork()2057 suspend_fork()
2058 {
2059 ulwp_t *self = curthread;
2060 uberdata_t *udp = self->ul_uberdata;
2061 ulwp_t *ulwp;
2062 int link_dropped;
2063
2064 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2065 top:
2066 lmutex_lock(&udp->link_lock);
2067
2068 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2069 ulwp_lock(ulwp, udp);
2070 if (ulwp->ul_stop) { /* already stopped */
2071 ulwp->ul_stop |= TSTP_FORK;
2072 ulwp_broadcast(ulwp);
2073 ulwp_unlock(ulwp, udp);
2074 } else {
2075 /*
2076 * Move the stopped lwp out of a critical section.
2077 */
2078 if (safe_suspend(ulwp, TSTP_FORK, &link_dropped) ||
2079 link_dropped)
2080 goto top;
2081 }
2082 }
2083
2084 lmutex_unlock(&udp->link_lock);
2085 }
2086
2087 void
continue_fork(int child)2088 continue_fork(int child)
2089 {
2090 ulwp_t *self = curthread;
2091 uberdata_t *udp = self->ul_uberdata;
2092 ulwp_t *ulwp;
2093
2094 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2095
2096 /*
2097 * Clear the schedctl pointers in the child of forkall().
2098 */
2099 if (child) {
2100 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2101 ulwp->ul_schedctl_called =
2102 ulwp->ul_dead? &udp->uberflags : NULL;
2103 ulwp->ul_schedctl = NULL;
2104 }
2105 }
2106
2107 /*
2108 * Set all lwps that were stopped for fork() running again.
2109 */
2110 lmutex_lock(&udp->link_lock);
2111 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2112 mutex_t *mp = ulwp_mutex(ulwp, udp);
2113 lmutex_lock(mp);
2114 ASSERT(ulwp->ul_stop & TSTP_FORK);
2115 ulwp->ul_stop &= ~TSTP_FORK;
2116 ulwp_broadcast(ulwp);
2117 if (!ulwp->ul_stop)
2118 force_continue(ulwp);
2119 lmutex_unlock(mp);
2120 }
2121 lmutex_unlock(&udp->link_lock);
2122 }
2123
2124 int
_thrp_continue(thread_t tid,uchar_t whystopped)2125 _thrp_continue(thread_t tid, uchar_t whystopped)
2126 {
2127 uberdata_t *udp = curthread->ul_uberdata;
2128 ulwp_t *ulwp;
2129 mutex_t *mp;
2130 int error = 0;
2131
2132 ASSERT(whystopped == TSTP_REGULAR ||
2133 whystopped == TSTP_MUTATOR);
2134
2135 /*
2136 * We single-thread the entire thread suspend/continue mechanism.
2137 */
2138 fork_lock_enter();
2139
2140 if ((ulwp = find_lwp(tid)) == NULL) {
2141 fork_lock_exit();
2142 return (ESRCH);
2143 }
2144
2145 mp = ulwp_mutex(ulwp, udp);
2146 if ((whystopped == TSTP_MUTATOR && !ulwp->ul_mutator)) {
2147 error = EINVAL;
2148 } else if (ulwp->ul_stop & whystopped) {
2149 ulwp->ul_stop &= ~whystopped;
2150 ulwp_broadcast(ulwp);
2151 if (!ulwp->ul_stop) {
2152 if (whystopped == TSTP_REGULAR && ulwp->ul_created) {
2153 ulwp->ul_sp = 0;
2154 ulwp->ul_created = 0;
2155 }
2156 force_continue(ulwp);
2157 }
2158 }
2159 lmutex_unlock(mp);
2160
2161 fork_lock_exit();
2162 return (error);
2163 }
2164
2165 int
thr_suspend(thread_t tid)2166 thr_suspend(thread_t tid)
2167 {
2168 return (_thrp_suspend(tid, TSTP_REGULAR));
2169 }
2170
2171 int
thr_continue(thread_t tid)2172 thr_continue(thread_t tid)
2173 {
2174 return (_thrp_continue(tid, TSTP_REGULAR));
2175 }
2176
2177 void
thr_yield()2178 thr_yield()
2179 {
2180 yield();
2181 }
2182
2183 #pragma weak pthread_kill = thr_kill
2184 #pragma weak _thr_kill = thr_kill
2185 int
thr_kill(thread_t tid,int sig)2186 thr_kill(thread_t tid, int sig)
2187 {
2188 if (sig == SIGCANCEL)
2189 return (EINVAL);
2190 return (_lwp_kill(tid, sig));
2191 }
2192
2193 /*
2194 * Exit a critical section, take deferred actions if necessary.
2195 * Called from exit_critical() and from sigon().
2196 */
2197 void
do_exit_critical()2198 do_exit_critical()
2199 {
2200 ulwp_t *self = curthread;
2201 int sig;
2202
2203 ASSERT(self->ul_critical == 0);
2204
2205 /*
2206 * Don't suspend ourself or take a deferred signal while dying
2207 * or while executing inside the dynamic linker (ld.so.1).
2208 */
2209 if (self->ul_dead || self->ul_rtld)
2210 return;
2211
2212 while (self->ul_pleasestop ||
2213 (self->ul_cursig != 0 && self->ul_sigdefer == 0)) {
2214 /*
2215 * Avoid a recursive call to exit_critical() in _thrp_suspend()
2216 * by keeping self->ul_critical == 1 here.
2217 */
2218 self->ul_critical++;
2219 while (self->ul_pleasestop) {
2220 /*
2221 * Guard against suspending ourself while on a sleep
2222 * queue. See the comments in call_user_handler().
2223 */
2224 unsleep_self();
2225 set_parking_flag(self, 0);
2226 (void) _thrp_suspend(self->ul_lwpid,
2227 self->ul_pleasestop);
2228 }
2229 self->ul_critical--;
2230
2231 if ((sig = self->ul_cursig) != 0 && self->ul_sigdefer == 0) {
2232 /*
2233 * Clear ul_cursig before proceeding.
2234 * This protects us from the dynamic linker's
2235 * calls to bind_guard()/bind_clear() in the
2236 * event that it is invoked to resolve a symbol
2237 * like take_deferred_signal() below.
2238 */
2239 self->ul_cursig = 0;
2240 take_deferred_signal(sig);
2241 ASSERT(self->ul_cursig == 0);
2242 }
2243 }
2244 ASSERT(self->ul_critical == 0);
2245 }
2246
2247 /*
2248 * _ti_bind_guard() and _ti_bind_clear() are called by the dynamic linker
2249 * (ld.so.1) when it has do do something, like resolve a symbol to be called
2250 * by the application or one of its libraries. _ti_bind_guard() is called
2251 * on entry to ld.so.1, _ti_bind_clear() on exit from ld.so.1 back to the
2252 * application. The dynamic linker gets special dispensation from libc to
2253 * run in a critical region (all signals deferred and no thread suspension
2254 * or forking allowed), and to be immune from cancellation for the duration.
2255 */
2256 int
_ti_bind_guard(int flags)2257 _ti_bind_guard(int flags)
2258 {
2259 ulwp_t *self = curthread;
2260 uberdata_t *udp = self->ul_uberdata;
2261 int bindflag = (flags & THR_FLG_RTLD);
2262
2263 if ((self->ul_bindflags & bindflag) == bindflag)
2264 return (0);
2265 self->ul_bindflags |= bindflag;
2266 if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2267 sigoff(self); /* see no signals while holding ld_lock */
2268 self->ul_rtld++; /* don't suspend while in ld.so.1 */
2269 (void) mutex_lock(&udp->ld_lock);
2270 }
2271 enter_critical(self);
2272 self->ul_save_state = self->ul_cancel_disabled;
2273 self->ul_cancel_disabled = 1;
2274 set_cancel_pending_flag(self, 0);
2275 return (1);
2276 }
2277
2278 int
_ti_bind_clear(int flags)2279 _ti_bind_clear(int flags)
2280 {
2281 ulwp_t *self = curthread;
2282 uberdata_t *udp = self->ul_uberdata;
2283 int bindflag = (flags & THR_FLG_RTLD);
2284
2285 if ((self->ul_bindflags & bindflag) == 0)
2286 return (self->ul_bindflags);
2287 self->ul_bindflags &= ~bindflag;
2288 self->ul_cancel_disabled = self->ul_save_state;
2289 set_cancel_pending_flag(self, 0);
2290 exit_critical(self);
2291 if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2292 if (MUTEX_OWNED(&udp->ld_lock, self)) {
2293 (void) mutex_unlock(&udp->ld_lock);
2294 self->ul_rtld--;
2295 sigon(self); /* reenable signals */
2296 }
2297 }
2298 return (self->ul_bindflags);
2299 }
2300
2301 /*
2302 * Tell the dynamic linker (ld.so.1) whether or not it was entered from
2303 * a critical region in libc. Return zero if not, else return non-zero.
2304 */
2305 int
_ti_critical(void)2306 _ti_critical(void)
2307 {
2308 ulwp_t *self = curthread;
2309 int level = self->ul_critical;
2310
2311 if ((self->ul_bindflags & THR_FLG_RTLD) == 0 || level == 0)
2312 return (level); /* ld.so.1 hasn't (yet) called enter() */
2313 return (level - 1);
2314 }
2315
2316 /*
2317 * sigoff() and sigon() enable cond_wait() to behave (optionally) like
2318 * it does in the old libthread (see the comments in cond_wait_queue()).
2319 * Also, signals are deferred at thread startup until TLS constructors
2320 * have all been called, at which time _thrp_setup() calls sigon().
2321 *
2322 * _sigoff() and _sigon() are external consolidation-private interfaces to
2323 * sigoff() and sigon(), respectively, in libc. These are used in libnsl.
2324 * Also, _sigoff() and _sigon() are called from dbx's run-time checking
2325 * (librtc.so) to defer signals during its critical sections (not to be
2326 * confused with libc critical sections [see exit_critical() above]).
2327 */
2328 void
_sigoff(void)2329 _sigoff(void)
2330 {
2331 ulwp_t *self = curthread;
2332
2333 sigoff(self);
2334 }
2335
2336 void
_sigon(void)2337 _sigon(void)
2338 {
2339 ulwp_t *self = curthread;
2340
2341 ASSERT(self->ul_sigdefer > 0);
2342 sigon(self);
2343 }
2344
2345 int
thr_getconcurrency()2346 thr_getconcurrency()
2347 {
2348 return (thr_concurrency);
2349 }
2350
2351 int
pthread_getconcurrency()2352 pthread_getconcurrency()
2353 {
2354 return (pthread_concurrency);
2355 }
2356
2357 int
thr_setconcurrency(int new_level)2358 thr_setconcurrency(int new_level)
2359 {
2360 uberdata_t *udp = curthread->ul_uberdata;
2361
2362 if (new_level < 0)
2363 return (EINVAL);
2364 if (new_level > 65536) /* 65536 is totally arbitrary */
2365 return (EAGAIN);
2366 lmutex_lock(&udp->link_lock);
2367 if (new_level > thr_concurrency)
2368 thr_concurrency = new_level;
2369 lmutex_unlock(&udp->link_lock);
2370 return (0);
2371 }
2372
2373 int
pthread_setconcurrency(int new_level)2374 pthread_setconcurrency(int new_level)
2375 {
2376 if (new_level < 0)
2377 return (EINVAL);
2378 if (new_level > 65536) /* 65536 is totally arbitrary */
2379 return (EAGAIN);
2380 pthread_concurrency = new_level;
2381 return (0);
2382 }
2383
2384 size_t
thr_min_stack(void)2385 thr_min_stack(void)
2386 {
2387 return (MINSTACK);
2388 }
2389
2390 int
__nthreads(void)2391 __nthreads(void)
2392 {
2393 return (curthread->ul_uberdata->nthreads);
2394 }
2395
2396 /* "/proc/self/lwp/%u/lwpname" w/o stdio */
2397 static void
lwpname_path(pthread_t tid,char * buf,size_t bufsize)2398 lwpname_path(pthread_t tid, char *buf, size_t bufsize)
2399 {
2400 (void) strlcpy(buf, "/proc/self/lwp/", bufsize);
2401 ultos((uint64_t)tid, 10, buf + strlen(buf));
2402 (void) strlcat(buf, "/lwpname", bufsize);
2403 }
2404
2405 #pragma weak pthread_setname_np = thr_setname
2406 int
thr_setname(pthread_t tid,const char * name)2407 thr_setname(pthread_t tid, const char *name)
2408 {
2409 extern ssize_t __write(int, const void *, size_t);
2410 char path[PATH_MAX];
2411 int saved_errno;
2412 size_t len;
2413 ssize_t n;
2414 int fd;
2415
2416 if (name == NULL)
2417 name = "";
2418
2419 len = strlen(name) + 1;
2420 if (len > THREAD_NAME_MAX)
2421 return (ERANGE);
2422
2423 lwpname_path(tid, path, sizeof (path));
2424
2425 if ((fd = __open(path, O_WRONLY, 0)) < 0) {
2426 if (errno == ENOENT)
2427 errno = ESRCH;
2428 return (errno);
2429 }
2430
2431 n = __write(fd, name, len);
2432 saved_errno = errno;
2433 (void) __close(fd);
2434
2435 if (n < 0)
2436 return (saved_errno);
2437 if (n != len)
2438 return (EFAULT);
2439 return (0);
2440 }
2441
2442 #pragma weak pthread_getname_np = thr_getname
2443 int
thr_getname(pthread_t tid,char * buf,size_t bufsize)2444 thr_getname(pthread_t tid, char *buf, size_t bufsize)
2445 {
2446 extern ssize_t __read(int, void *, size_t);
2447 char name[THREAD_NAME_MAX];
2448 char path[PATH_MAX];
2449 int saved_errno;
2450 ssize_t n;
2451 int fd;
2452
2453 if (buf == NULL)
2454 return (EINVAL);
2455
2456 lwpname_path(tid, path, sizeof (path));
2457
2458 if ((fd = __open(path, O_RDONLY, 0)) < 0) {
2459 if (errno == ENOENT)
2460 errno = ESRCH;
2461 return (errno);
2462 }
2463
2464 n = __read(fd, name, sizeof (name));
2465 saved_errno = errno;
2466 (void) __close(fd);
2467
2468 if (n < 0)
2469 return (saved_errno);
2470 if (n != sizeof (name))
2471 return (EFAULT);
2472 if (strlcpy(buf, name, bufsize) >= bufsize)
2473 return (ERANGE);
2474 return (0);
2475 }
2476
2477 /*
2478 * XXX
2479 * The remainder of this file implements the private interfaces to java for
2480 * garbage collection. It is no longer used, at least by java 1.2.
2481 * It can all go away once all old JVMs have disappeared.
2482 */
2483
2484 int suspendingallmutators; /* when non-zero, suspending all mutators. */
2485 int suspendedallmutators; /* when non-zero, all mutators suspended. */
2486 int mutatorsbarrier; /* when non-zero, mutators barrier imposed. */
2487 mutex_t mutatorslock = DEFAULTMUTEX; /* used to enforce mutators barrier. */
2488 cond_t mutatorscv = DEFAULTCV; /* where non-mutators sleep. */
2489
2490 /*
2491 * Get the available register state for the target thread.
2492 * Return non-volatile registers: TRS_NONVOLATILE
2493 */
2494 #pragma weak _thr_getstate = thr_getstate
2495 int
thr_getstate(thread_t tid,int * flag,lwpid_t * lwp,stack_t * ss,gregset_t rs)2496 thr_getstate(thread_t tid, int *flag, lwpid_t *lwp, stack_t *ss, gregset_t rs)
2497 {
2498 ulwp_t *self = curthread;
2499 uberdata_t *udp = self->ul_uberdata;
2500 ulwp_t **ulwpp;
2501 ulwp_t *ulwp;
2502 int error = 0;
2503 int trs_flag = TRS_LWPID;
2504
2505 if (tid == 0 || self->ul_lwpid == tid) {
2506 ulwp = self;
2507 ulwp_lock(ulwp, udp);
2508 } else if ((ulwpp = find_lwpp(tid)) != NULL) {
2509 ulwp = *ulwpp;
2510 } else {
2511 if (flag)
2512 *flag = TRS_INVALID;
2513 return (ESRCH);
2514 }
2515
2516 if (ulwp->ul_dead) {
2517 trs_flag = TRS_INVALID;
2518 } else if (!ulwp->ul_stop && !suspendedallmutators) {
2519 error = EINVAL;
2520 trs_flag = TRS_INVALID;
2521 } else if (ulwp->ul_stop) {
2522 trs_flag = TRS_NONVOLATILE;
2523 getgregs(ulwp, rs);
2524 }
2525
2526 if (flag)
2527 *flag = trs_flag;
2528 if (lwp)
2529 *lwp = tid;
2530 if (ss != NULL)
2531 (void) _thrp_stksegment(ulwp, ss);
2532
2533 ulwp_unlock(ulwp, udp);
2534 return (error);
2535 }
2536
2537 /*
2538 * Set the appropriate register state for the target thread.
2539 * This is not used by java. It exists solely for the MSTC test suite.
2540 */
2541 #pragma weak _thr_setstate = thr_setstate
2542 int
thr_setstate(thread_t tid,int flag,gregset_t rs)2543 thr_setstate(thread_t tid, int flag, gregset_t rs)
2544 {
2545 uberdata_t *udp = curthread->ul_uberdata;
2546 ulwp_t *ulwp;
2547 int error = 0;
2548
2549 if ((ulwp = find_lwp(tid)) == NULL)
2550 return (ESRCH);
2551
2552 if (!ulwp->ul_stop && !suspendedallmutators)
2553 error = EINVAL;
2554 else if (rs != NULL) {
2555 switch (flag) {
2556 case TRS_NONVOLATILE:
2557 /* do /proc stuff here? */
2558 if (ulwp->ul_stop)
2559 setgregs(ulwp, rs);
2560 else
2561 error = EINVAL;
2562 break;
2563 case TRS_LWPID: /* do /proc stuff here? */
2564 default:
2565 error = EINVAL;
2566 break;
2567 }
2568 }
2569
2570 ulwp_unlock(ulwp, udp);
2571 return (error);
2572 }
2573
2574 int
getlwpstatus(thread_t tid,struct lwpstatus * sp)2575 getlwpstatus(thread_t tid, struct lwpstatus *sp)
2576 {
2577 extern ssize_t __pread(int, void *, size_t, off_t);
2578 char buf[100];
2579 int fd;
2580
2581 /* "/proc/self/lwp/%u/lwpstatus" w/o stdio */
2582 (void) strcpy(buf, "/proc/self/lwp/");
2583 ultos((uint64_t)tid, 10, buf + strlen(buf));
2584 (void) strcat(buf, "/lwpstatus");
2585 if ((fd = __open(buf, O_RDONLY, 0)) >= 0) {
2586 while (__pread(fd, sp, sizeof (*sp), 0) == sizeof (*sp)) {
2587 if (sp->pr_flags & PR_STOPPED) {
2588 (void) __close(fd);
2589 return (0);
2590 }
2591 yield(); /* give it a chance to stop */
2592 }
2593 (void) __close(fd);
2594 }
2595 return (-1);
2596 }
2597
2598 int
putlwpregs(thread_t tid,prgregset_t prp)2599 putlwpregs(thread_t tid, prgregset_t prp)
2600 {
2601 extern ssize_t __writev(int, const struct iovec *, int);
2602 char buf[100];
2603 int fd;
2604 long dstop_sreg[2];
2605 long run_null[2];
2606 iovec_t iov[3];
2607
2608 /* "/proc/self/lwp/%u/lwpctl" w/o stdio */
2609 (void) strcpy(buf, "/proc/self/lwp/");
2610 ultos((uint64_t)tid, 10, buf + strlen(buf));
2611 (void) strcat(buf, "/lwpctl");
2612 if ((fd = __open(buf, O_WRONLY, 0)) >= 0) {
2613 dstop_sreg[0] = PCDSTOP; /* direct it to stop */
2614 dstop_sreg[1] = PCSREG; /* set the registers */
2615 iov[0].iov_base = (caddr_t)dstop_sreg;
2616 iov[0].iov_len = sizeof (dstop_sreg);
2617 iov[1].iov_base = (caddr_t)prp; /* from the register set */
2618 iov[1].iov_len = sizeof (prgregset_t);
2619 run_null[0] = PCRUN; /* make it runnable again */
2620 run_null[1] = 0;
2621 iov[2].iov_base = (caddr_t)run_null;
2622 iov[2].iov_len = sizeof (run_null);
2623 if (__writev(fd, iov, 3) >= 0) {
2624 (void) __close(fd);
2625 return (0);
2626 }
2627 (void) __close(fd);
2628 }
2629 return (-1);
2630 }
2631
2632 static ulong_t
gettsp_slow(thread_t tid)2633 gettsp_slow(thread_t tid)
2634 {
2635 char buf[100];
2636 struct lwpstatus status;
2637
2638 if (getlwpstatus(tid, &status) != 0) {
2639 /* "__gettsp(%u): can't read lwpstatus" w/o stdio */
2640 (void) strcpy(buf, "__gettsp(");
2641 ultos((uint64_t)tid, 10, buf + strlen(buf));
2642 (void) strcat(buf, "): can't read lwpstatus");
2643 thr_panic(buf);
2644 }
2645 return (status.pr_reg[R_SP]);
2646 }
2647
2648 ulong_t
__gettsp(thread_t tid)2649 __gettsp(thread_t tid)
2650 {
2651 uberdata_t *udp = curthread->ul_uberdata;
2652 ulwp_t *ulwp;
2653 ulong_t result;
2654
2655 if ((ulwp = find_lwp(tid)) == NULL)
2656 return (0);
2657
2658 if (ulwp->ul_stop && (result = ulwp->ul_sp) != 0) {
2659 ulwp_unlock(ulwp, udp);
2660 return (result);
2661 }
2662
2663 result = gettsp_slow(tid);
2664 ulwp_unlock(ulwp, udp);
2665 return (result);
2666 }
2667
2668 /*
2669 * This tells java stack walkers how to find the ucontext
2670 * structure passed to signal handlers.
2671 */
2672 #pragma weak _thr_sighndlrinfo = thr_sighndlrinfo
2673 void
thr_sighndlrinfo(void (** func)(),int * funcsize)2674 thr_sighndlrinfo(void (**func)(), int *funcsize)
2675 {
2676 *func = &__sighndlr;
2677 *funcsize = (char *)&__sighndlrend - (char *)&__sighndlr;
2678 }
2679
2680 /*
2681 * Mark a thread a mutator or reset a mutator to being a default,
2682 * non-mutator thread.
2683 */
2684 #pragma weak _thr_setmutator = thr_setmutator
2685 int
thr_setmutator(thread_t tid,int enabled)2686 thr_setmutator(thread_t tid, int enabled)
2687 {
2688 ulwp_t *self = curthread;
2689 uberdata_t *udp = self->ul_uberdata;
2690 ulwp_t *ulwp;
2691 int error;
2692 int cancel_state;
2693
2694 enabled = enabled? 1 : 0;
2695 top:
2696 if (tid == 0) {
2697 ulwp = self;
2698 ulwp_lock(ulwp, udp);
2699 } else if ((ulwp = find_lwp(tid)) == NULL) {
2700 return (ESRCH);
2701 }
2702
2703 /*
2704 * The target thread should be the caller itself or a suspended thread.
2705 * This prevents the target from also changing its ul_mutator field.
2706 */
2707 error = 0;
2708 if (ulwp != self && !ulwp->ul_stop && enabled)
2709 error = EINVAL;
2710 else if (ulwp->ul_mutator != enabled) {
2711 lmutex_lock(&mutatorslock);
2712 if (mutatorsbarrier) {
2713 ulwp_unlock(ulwp, udp);
2714 (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
2715 &cancel_state);
2716 while (mutatorsbarrier)
2717 (void) cond_wait(&mutatorscv, &mutatorslock);
2718 (void) pthread_setcancelstate(cancel_state, NULL);
2719 lmutex_unlock(&mutatorslock);
2720 goto top;
2721 }
2722 ulwp->ul_mutator = enabled;
2723 lmutex_unlock(&mutatorslock);
2724 }
2725
2726 ulwp_unlock(ulwp, udp);
2727 return (error);
2728 }
2729
2730 /*
2731 * Establish a barrier against new mutators. Any non-mutator trying
2732 * to become a mutator is suspended until the barrier is removed.
2733 */
2734 #pragma weak _thr_mutators_barrier = thr_mutators_barrier
2735 void
thr_mutators_barrier(int enabled)2736 thr_mutators_barrier(int enabled)
2737 {
2738 int oldvalue;
2739 int cancel_state;
2740
2741 lmutex_lock(&mutatorslock);
2742
2743 /*
2744 * Wait if trying to set the barrier while it is already set.
2745 */
2746 (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2747 while (mutatorsbarrier && enabled)
2748 (void) cond_wait(&mutatorscv, &mutatorslock);
2749 (void) pthread_setcancelstate(cancel_state, NULL);
2750
2751 oldvalue = mutatorsbarrier;
2752 mutatorsbarrier = enabled;
2753 /*
2754 * Wakeup any blocked non-mutators when barrier is removed.
2755 */
2756 if (oldvalue && !enabled)
2757 (void) cond_broadcast(&mutatorscv);
2758 lmutex_unlock(&mutatorslock);
2759 }
2760
2761 /*
2762 * Suspend the set of all mutators except for the caller. The list
2763 * of actively running threads is searched and only the mutators
2764 * in this list are suspended. Actively running non-mutators remain
2765 * running. Any other thread is suspended.
2766 */
2767 #pragma weak _thr_suspend_allmutators = thr_suspend_allmutators
2768 int
thr_suspend_allmutators(void)2769 thr_suspend_allmutators(void)
2770 {
2771 ulwp_t *self = curthread;
2772 uberdata_t *udp = self->ul_uberdata;
2773 ulwp_t *ulwp;
2774 int link_dropped;
2775
2776 /*
2777 * We single-thread the entire thread suspend/continue mechanism.
2778 */
2779 fork_lock_enter();
2780
2781 top:
2782 lmutex_lock(&udp->link_lock);
2783
2784 if (suspendingallmutators || suspendedallmutators) {
2785 lmutex_unlock(&udp->link_lock);
2786 fork_lock_exit();
2787 return (EINVAL);
2788 }
2789 suspendingallmutators = 1;
2790
2791 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2792 ulwp_lock(ulwp, udp);
2793 if (!ulwp->ul_mutator) {
2794 ulwp_unlock(ulwp, udp);
2795 } else if (ulwp->ul_stop) { /* already stopped */
2796 ulwp->ul_stop |= TSTP_MUTATOR;
2797 ulwp_broadcast(ulwp);
2798 ulwp_unlock(ulwp, udp);
2799 } else {
2800 /*
2801 * Move the stopped lwp out of a critical section.
2802 */
2803 if (safe_suspend(ulwp, TSTP_MUTATOR, &link_dropped) ||
2804 link_dropped) {
2805 suspendingallmutators = 0;
2806 goto top;
2807 }
2808 }
2809 }
2810
2811 suspendedallmutators = 1;
2812 suspendingallmutators = 0;
2813 lmutex_unlock(&udp->link_lock);
2814 fork_lock_exit();
2815 return (0);
2816 }
2817
2818 /*
2819 * Suspend the target mutator. The caller is permitted to suspend
2820 * itself. If a mutator barrier is enabled, the caller will suspend
2821 * itself as though it had been suspended by thr_suspend_allmutators().
2822 * When the barrier is removed, this thread will be resumed. Any
2823 * suspended mutator, whether suspended by thr_suspend_mutator(), or by
2824 * thr_suspend_allmutators(), can be resumed by thr_continue_mutator().
2825 */
2826 #pragma weak _thr_suspend_mutator = thr_suspend_mutator
2827 int
thr_suspend_mutator(thread_t tid)2828 thr_suspend_mutator(thread_t tid)
2829 {
2830 if (tid == 0)
2831 tid = curthread->ul_lwpid;
2832 return (_thrp_suspend(tid, TSTP_MUTATOR));
2833 }
2834
2835 /*
2836 * Resume the set of all suspended mutators.
2837 */
2838 #pragma weak _thr_continue_allmutators = thr_continue_allmutators
2839 int
thr_continue_allmutators()2840 thr_continue_allmutators()
2841 {
2842 ulwp_t *self = curthread;
2843 uberdata_t *udp = self->ul_uberdata;
2844 ulwp_t *ulwp;
2845
2846 /*
2847 * We single-thread the entire thread suspend/continue mechanism.
2848 */
2849 fork_lock_enter();
2850
2851 lmutex_lock(&udp->link_lock);
2852 if (!suspendedallmutators) {
2853 lmutex_unlock(&udp->link_lock);
2854 fork_lock_exit();
2855 return (EINVAL);
2856 }
2857 suspendedallmutators = 0;
2858
2859 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2860 mutex_t *mp = ulwp_mutex(ulwp, udp);
2861 lmutex_lock(mp);
2862 if (ulwp->ul_stop & TSTP_MUTATOR) {
2863 ulwp->ul_stop &= ~TSTP_MUTATOR;
2864 ulwp_broadcast(ulwp);
2865 if (!ulwp->ul_stop)
2866 force_continue(ulwp);
2867 }
2868 lmutex_unlock(mp);
2869 }
2870
2871 lmutex_unlock(&udp->link_lock);
2872 fork_lock_exit();
2873 return (0);
2874 }
2875
2876 /*
2877 * Resume a suspended mutator.
2878 */
2879 #pragma weak _thr_continue_mutator = thr_continue_mutator
2880 int
thr_continue_mutator(thread_t tid)2881 thr_continue_mutator(thread_t tid)
2882 {
2883 return (_thrp_continue(tid, TSTP_MUTATOR));
2884 }
2885
2886 #pragma weak _thr_wait_mutator = thr_wait_mutator
2887 int
thr_wait_mutator(thread_t tid,int dontwait)2888 thr_wait_mutator(thread_t tid, int dontwait)
2889 {
2890 uberdata_t *udp = curthread->ul_uberdata;
2891 ulwp_t *ulwp;
2892 int cancel_state;
2893 int error = 0;
2894
2895 (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2896 top:
2897 if ((ulwp = find_lwp(tid)) == NULL) {
2898 (void) pthread_setcancelstate(cancel_state, NULL);
2899 return (ESRCH);
2900 }
2901
2902 if (!ulwp->ul_mutator)
2903 error = EINVAL;
2904 else if (dontwait) {
2905 if (!(ulwp->ul_stop & TSTP_MUTATOR))
2906 error = EWOULDBLOCK;
2907 } else if (!(ulwp->ul_stop & TSTP_MUTATOR)) {
2908 cond_t *cvp = ulwp_condvar(ulwp, udp);
2909 mutex_t *mp = ulwp_mutex(ulwp, udp);
2910
2911 (void) cond_wait(cvp, mp);
2912 (void) lmutex_unlock(mp);
2913 goto top;
2914 }
2915
2916 ulwp_unlock(ulwp, udp);
2917 (void) pthread_setcancelstate(cancel_state, NULL);
2918 return (error);
2919 }
2920
2921 /* PROBE_SUPPORT begin */
2922
2923 void
thr_probe_setup(void * data)2924 thr_probe_setup(void *data)
2925 {
2926 curthread->ul_tpdp = data;
2927 }
2928
2929 static void *
_thread_probe_getfunc()2930 _thread_probe_getfunc()
2931 {
2932 return (curthread->ul_tpdp);
2933 }
2934
2935 void * (*thr_probe_getfunc_addr)(void) = _thread_probe_getfunc;
2936
2937 /* ARGSUSED */
2938 void
_resume(ulwp_t * ulwp,caddr_t sp,int dontsave)2939 _resume(ulwp_t *ulwp, caddr_t sp, int dontsave)
2940 {
2941 /* never called */
2942 }
2943
2944 /* ARGSUSED */
2945 void
_resume_ret(ulwp_t * oldlwp)2946 _resume_ret(ulwp_t *oldlwp)
2947 {
2948 /* never called */
2949 }
2950
2951 /* PROBE_SUPPORT end */
2952