1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * Copyright 2016 Joyent, Inc. 28 * Copyright 2018 Nexenta Systems, Inc. 29 * Copyright 2026 Oxide Computer Company 30 */ 31 32 #ifndef _THR_UBERDATA_H 33 #define _THR_UBERDATA_H 34 35 #include <stdlib.h> 36 #include <unistd.h> 37 #include <sys/types.h> 38 #include <fcntl.h> 39 #include <string.h> 40 #include <signal.h> 41 #include <ucontext.h> 42 #include <thread.h> 43 #include <pthread.h> 44 #include <atomic.h> 45 #include <link.h> 46 #include <sys/resource.h> 47 #include <sys/lwp.h> 48 #include <errno.h> 49 #include <sys/asm_linkage.h> 50 #include <sys/regset.h> 51 #include <sys/fcntl.h> 52 #include <sys/mman.h> 53 #include <synch.h> 54 #include <door.h> 55 #include <limits.h> 56 #include <sys/synch32.h> 57 #include <schedctl.h> 58 #include <sys/priocntl.h> 59 #include <thread_db.h> 60 #include <setjmp.h> 61 #include <sys/thread.h> 62 #include <sys/debug.h> 63 #include "libc_int.h" 64 #include "tdb_agent.h" 65 66 /* 67 * This is an implementation-specific include file for threading support. 68 * It is not to be seen by the clients of the library. 69 * 70 * This file also describes uberdata in libc. 71 * 72 * The term "uberdata" refers to data that is unique and visible across 73 * all link maps. The name is meant to imply that such data is truly 74 * global, not just locally global to a particular link map. 75 * 76 * See the Linker and Libraries Guide for a full description of alternate 77 * link maps and how they are set up and used. 78 * 79 * Alternate link maps implement multiple global namespaces within a single 80 * process. There may be multiple instances of identical dynamic libraries 81 * loaded in a process's address space at the same time, each on a different 82 * link map (as determined by the dynamic linker), each with its own set of 83 * global variables. Which particular instance of a global variable is seen 84 * by a thread running in the process is determined by the link map on which 85 * the thread happens to be executing at the time. 86 * 87 * However, there are aspects of a process that are unique across all 88 * link maps, in particular the structures used to implement threads 89 * of control (in Sparc terminology, there is only one %g7 regardless 90 * of the link map on which the thread is executing). 91 * 92 * All uberdata is referenced from a base pointer in the thread's ulwp_t 93 * structure (which is also uberdata). All allocations and deallocations 94 * of uberdata are made via the uberdata-aware lmalloc() and lfree() 95 * interfaces (malloc() and free() are simply locally-global). 96 */ 97 98 /* 99 * Special libc-private access to errno. 100 * We do this so that references to errno do not invoke the dynamic linker. 101 */ 102 #undef errno 103 #define errno (*curthread->ul_errnop) 104 105 /* 106 * See <sys/synch32.h> for the reasons for these values 107 * and why they are different for sparc and intel. 108 */ 109 #if defined(__sparc) 110 111 /* lock.lock64.pad[x] 4 5 6 7 */ 112 #define LOCKMASK 0xff000000 113 #define WAITERMASK 0x000000ff 114 #define SPINNERMASK 0x00ff0000 115 #define SPINNERSHIFT 16 116 #define WAITER 0x00000001 117 #define LOCKSET 0xff 118 #define LOCKCLEAR 0 119 120 #define PIDSHIFT 32 121 #define LOCKMASK64 0xffffffffff000000ULL 122 #define LOCKBYTE64 0x00000000ff000000ULL 123 #define WAITERMASK64 0x00000000000000ffULL 124 #define SPINNERMASK64 0x0000000000ff0000ULL 125 126 #elif defined(__x86) 127 128 /* lock.lock64.pad[x] 7 6 5 4 */ 129 #define LOCKMASK 0xff000000 130 #define WAITERMASK 0x00ff0000 131 #define SPINNERMASK 0x0000ff00 132 #define SPINNERSHIFT 8 133 #define WAITER 0x00010000 134 #define LOCKSET 0x01 135 #define LOCKCLEAR 0 136 137 #define PIDSHIFT 0 138 #define LOCKMASK64 0xff000000ffffffffULL 139 #define LOCKBYTE64 0x0100000000000000ULL 140 #define WAITERMASK64 0x00ff000000000000ULL 141 #define SPINNERMASK64 0x0000ff0000000000ULL 142 143 #else 144 #error "neither __sparc nor __x86 is defined" 145 #endif 146 147 /* 148 * Fetch the owner of a USYNC_THREAD mutex. 149 * Don't use this with process-shared mutexes; 150 * the owing thread may be in a different process. 151 */ 152 #define MUTEX_OWNER(mp) ((ulwp_t *)(uintptr_t)(mp)->mutex_owner) 153 154 /* 155 * Test if a thread owns a process-private (USYNC_THREAD) mutex. 156 * This is inappropriate for a process-shared (USYNC_PROCESS) mutex. 157 * The 'mp' argument must not have side-effects since it is evaluated twice. 158 */ 159 #define MUTEX_OWNED(mp, thrp) \ 160 ((mp)->mutex_lockw != 0 && MUTEX_OWNER(mp) == thrp) 161 162 163 /* 164 * uberflags.uf_tdb_register_sync is an interface with libc_db to enable the 165 * collection of lock statistics by a debugger or other collecting tool. 166 * 167 * uberflags.uf_thread_error_detection is set by an environment variable: 168 * _THREAD_ERROR_DETECTION 169 * 0 == no detection of locking primitive errors. 170 * 1 == detect errors and issue a warning message. 171 * 2 == detect errors, issue a warning message, and dump core. 172 * 173 * We bundle these together in uberflags.uf_trs_ted to make a test of either 174 * being non-zero a single memory reference (for speed of mutex_lock(), etc). 175 * 176 * uberflags.uf_mt is set non-zero when the first thread (in addition 177 * to the main thread) is created. 178 * 179 * We bundle all these flags together in uberflags.uf_all to make a test 180 * of any being non-zero a single memory reference (again, for speed). 181 */ 182 typedef union { 183 int uf_all; /* combined all flags */ 184 struct { 185 short h_pad; 186 short h_trs_ted; /* combined reg sync & error detect */ 187 } uf_h; 188 struct { 189 char x_mt; 190 char x_pad; 191 char x_tdb_register_sync; 192 char x_thread_error_detection; 193 } uf_x; 194 } uberflags_t; 195 196 #define uf_mt uf_x.x_mt 197 #define uf_tdb_register_sync uf_x.x_tdb_register_sync 198 #define uf_thread_error_detection uf_x.x_thread_error_detection 199 #define uf_trs_ted uf_h.h_trs_ted /* both of the above */ 200 201 /* 202 * NOTE WELL: 203 * To enable further optimization, the "ul_schedctl_called" member 204 * of the ulwp_t structure (below) serves double-duty: 205 * 1. If NULL, it means that the thread must call __schedctl() 206 * to set up its schedctl mappings before acquiring a mutex. 207 * This is required by the implementation of adaptive mutex locking. 208 * 2. If non-NULL, it points to uberdata.uberflags, so that tests of 209 * uberflags can be made without additional memory references. 210 * This allows the common case of _mutex_lock() and _mutex_unlock() for 211 * USYNC_THREAD mutexes with no error detection and no lock statistics 212 * to be optimized for speed. 213 */ 214 215 /* double the default stack size for 64-bit processes */ 216 #ifdef _LP64 217 #define MINSTACK (8 * 1024) 218 #define DEFAULTSTACK (2 * 1024 * 1024) 219 #else 220 #define MINSTACK (4 * 1024) 221 #define DEFAULTSTACK (1024 * 1024) 222 #endif 223 224 #define MUTEX_TRY 0 225 #define MUTEX_LOCK 1 226 #define MUTEX_NOCEIL 0x40 227 228 #if defined(__x86) 229 230 typedef struct { /* structure returned by fnstenv */ 231 int fctrl; /* control word */ 232 int fstat; /* status word (flags, etc) */ 233 int ftag; /* tag of which regs busy */ 234 int misc[4]; /* other stuff, 28 bytes total */ 235 } fpuenv_t; 236 237 #ifdef _SYSCALL32 238 typedef fpuenv_t fpuenv32_t; 239 #endif /* _SYSCALL32 */ 240 241 #elif defined(__sparc) 242 243 typedef struct { /* fp state structure */ 244 greg_t fsr; 245 greg_t fpu_en; 246 } fpuenv_t; 247 248 #ifdef _SYSCALL32 249 typedef struct { 250 greg32_t fsr; 251 greg32_t fpu_en; 252 } fpuenv32_t; 253 #endif /* _SYSCALL32 */ 254 255 #endif /* __x86 */ 256 257 #if defined(__x86) 258 extern void ht_pause(void); /* "pause" instruction */ 259 #define SMT_PAUSE() ht_pause() 260 #elif defined(SMT_PAUSE_FUNCTION) 261 extern void SMT_PAUSE_FUNCTION(void); 262 #define SMT_PAUSE() SMT_PAUSE_FUNCTION() 263 #else 264 #define SMT_PAUSE() smt_pause() 265 #endif /* __x86 */ 266 267 /* 268 * Cleanup handler related data. 269 * This structure is exported as _cleanup_t in pthread.h. 270 * pthread.h exports only the size of this structure, so check 271 * _cleanup_t in pthread.h before making any change here. 272 */ 273 typedef struct __cleanup { 274 struct __cleanup *next; /* pointer to next handler */ 275 caddr_t fp; /* current frame pointer */ 276 void (*func)(void *); /* cleanup handler address */ 277 void *arg; /* handler's argument */ 278 } __cleanup_t; 279 280 /* 281 * Thread-Specific Data (TSD) 282 * TSD_NFAST includes the invalid key zero, so there 283 * are really only (TSD_NFAST - 1) fast key slots. 284 */ 285 typedef void (*PFrV)(void *); 286 #define TSD_UNALLOCATED ((PFrV)1) 287 #define TSD_NFAST 9 288 289 /* 290 * The tsd union is designed to burn a little memory (9 words) to make 291 * lookups blindingly fast. Note that tsd_nalloc could be placed at the 292 * end of the pad region to increase the likelihood that it falls on the 293 * same cache line as the data. 294 */ 295 typedef union tsd { 296 uint_t tsd_nalloc; /* Amount of allocated storage */ 297 void *tsd_pad[TSD_NFAST]; 298 void *tsd_data[1]; 299 } tsd_t; 300 301 typedef struct { 302 mutex_t tsdm_lock; /* Lock protecting the data */ 303 uint_t tsdm_nkeys; /* Number of allocated keys */ 304 uint_t tsdm_nused; /* Number of used keys */ 305 PFrV *tsdm_destro; /* Per-key destructors */ 306 char tsdm_pad[64 - /* pad to 64 bytes */ 307 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (PFrV *))]; 308 } tsd_metadata_t; 309 310 #ifdef _SYSCALL32 311 typedef union tsd32 { 312 uint_t tsd_nalloc; /* Amount of allocated storage */ 313 caddr32_t tsd_pad[TSD_NFAST]; 314 caddr32_t tsd_data[1]; 315 } tsd32_t; 316 317 typedef struct { 318 mutex_t tsdm_lock; /* Lock protecting the data */ 319 uint_t tsdm_nkeys; /* Number of allocated keys */ 320 uint_t tsdm_nused; /* Number of used keys */ 321 caddr32_t tsdm_destro; /* Per-key destructors */ 322 char tsdm_pad[64 - /* pad to 64 bytes */ 323 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (caddr32_t))]; 324 } tsd_metadata32_t; 325 #endif /* _SYSCALL32 */ 326 327 328 /* 329 * Thread-Local Storage (TLS) 330 */ 331 typedef struct { 332 void *tls_data; 333 size_t tls_size; 334 } tls_t; 335 336 typedef struct { 337 mutex_t tls_lock; /* Lock protecting the data */ 338 tls_t tls_modinfo; /* Root of all TLS_modinfo data */ 339 tls_t static_tls; /* Template for static TLS */ 340 char tls_pad[64 - /* pad to 64 bytes */ 341 (sizeof (mutex_t) + 2 * sizeof (tls_t))]; 342 } tls_metadata_t; 343 344 #ifdef _SYSCALL32 345 typedef struct { 346 caddr32_t tls_data; 347 size32_t tls_size; 348 } tls32_t; 349 350 typedef struct { 351 mutex_t tls_lock; /* Lock protecting the data */ 352 tls32_t tls_modinfo; /* Root of all TLS_modinfo data */ 353 tls32_t static_tls; /* Template for static TLS */ 354 char tls_pad[64 - /* pad to 64 bytes */ 355 (sizeof (mutex_t) + 2 * sizeof (tls32_t))]; 356 } tls_metadata32_t; 357 #endif /* _SYSCALL32 */ 358 359 360 /* 361 * Sleep queue root for USYNC_THREAD condvars and mutexes. 362 * There is a default queue root for each queue head (see below). 363 * Also, each ulwp_t contains a queue root that can be used 364 * when the thread is enqueued on the queue, if necessary 365 * (when more than one wchan hashes to the same queue head). 366 */ 367 typedef struct queue_root { 368 struct queue_root *qr_next; 369 struct queue_root *qr_prev; 370 struct ulwp *qr_head; 371 struct ulwp *qr_tail; 372 void *qr_wchan; 373 uint32_t qr_rtcount; 374 uint32_t qr_qlen; 375 uint32_t qr_qmax; 376 } queue_root_t; 377 378 #ifdef _SYSCALL32 379 typedef struct queue_root32 { 380 caddr32_t qr_next; 381 caddr32_t qr_prev; 382 caddr32_t qr_head; 383 caddr32_t qr_tail; 384 caddr32_t qr_wchan; 385 uint32_t qr_rtcount; 386 uint32_t qr_qlen; 387 uint32_t qr_qmax; 388 } queue_root32_t; 389 #endif 390 391 /* 392 * Sleep queue heads for USYNC_THREAD condvars and mutexes. 393 * The size and alignment is 128 bytes to reduce cache conflicts. 394 * Each queue head points to a list of queue roots, defined above. 395 * Each queue head contains a default queue root for use when only one 396 * is needed. It is always at the tail of the queue root hash chain. 397 */ 398 typedef union { 399 uint64_t qh_64[16]; 400 struct { 401 mutex_t q_lock; 402 uint8_t q_qcnt; 403 uint8_t q_type; /* MX or CV */ 404 uint8_t q_pad1[2]; 405 uint32_t q_lockcount; 406 uint32_t q_qlen; 407 uint32_t q_qmax; 408 void *q_wchan; /* valid only while locked */ 409 struct queue_root *q_root; /* valid only while locked */ 410 struct queue_root *q_hlist; 411 #if !defined(_LP64) 412 caddr_t q_pad2[3]; 413 #endif 414 queue_root_t q_def_root; 415 uint32_t q_hlen; 416 uint32_t q_hmax; 417 } qh_qh; 418 } queue_head_t; 419 420 #define qh_lock qh_qh.q_lock 421 #define qh_qcnt qh_qh.q_qcnt 422 #define qh_type qh_qh.q_type 423 #if defined(DEBUG) 424 #define qh_lockcount qh_qh.q_lockcount 425 #define qh_qlen qh_qh.q_qlen 426 #define qh_qmax qh_qh.q_qmax 427 #endif 428 #define qh_wchan qh_qh.q_wchan 429 #define qh_root qh_qh.q_root 430 #define qh_hlist qh_qh.q_hlist 431 #define qh_def_root qh_qh.q_def_root 432 #define qh_hlen qh_qh.q_hlen 433 #define qh_hmax qh_qh.q_hmax 434 435 /* queue types passed to queue_lock() */ 436 #define MX 0 437 #define CV 1 438 #define QHASHSHIFT 9 /* number of hashing bits */ 439 #define QHASHSIZE (1 << QHASHSHIFT) /* power of 2 (1<<9 == 512) */ 440 #define QUEUE_HASH(wchan, type) ((uint_t) \ 441 ((((uintptr_t)(wchan) >> 3) \ 442 ^ ((uintptr_t)(wchan) >> (QHASHSHIFT + 3))) \ 443 & (QHASHSIZE - 1)) + (((type) == MX)? 0 : QHASHSIZE)) 444 445 extern queue_head_t *queue_lock(void *, int); 446 extern void queue_unlock(queue_head_t *); 447 extern void enqueue(queue_head_t *, struct ulwp *, int); 448 extern struct ulwp *dequeue(queue_head_t *, int *); 449 extern struct ulwp **queue_slot(queue_head_t *, struct ulwp **, int *); 450 extern struct ulwp *queue_waiter(queue_head_t *); 451 extern int dequeue_self(queue_head_t *); 452 extern void queue_unlink(queue_head_t *, 453 struct ulwp **, struct ulwp *); 454 extern void unsleep_self(void); 455 extern void spin_lock_set(mutex_t *); 456 extern void spin_lock_clear(mutex_t *); 457 458 /* 459 * Scheduling class information structure. 460 */ 461 typedef struct { 462 short pcc_state; 463 short pcc_policy; 464 pri_t pcc_primin; 465 pri_t pcc_primax; 466 pcinfo_t pcc_info; 467 } pcclass_t; 468 469 /* 470 * Memory block for chain of owned ceiling mutexes. 471 */ 472 typedef struct mxchain { 473 struct mxchain *mxchain_next; 474 mutex_t *mxchain_mx; 475 } mxchain_t; 476 477 /* 478 * Pointer to an rwlock that is held for reading. 479 * Used in rw_rdlock() to allow a thread that already holds a read 480 * lock to acquire another read lock on the same rwlock even if 481 * there are writers waiting. This to avoid deadlock when acquiring 482 * a read lock more than once in the presence of pending writers. 483 * POSIX mandates this behavior. 484 */ 485 typedef struct { 486 void *rd_rwlock; /* the rwlock held for reading */ 487 size_t rd_count; /* count of read locks applied */ 488 } readlock_t; 489 490 #ifdef _SYSCALL32 491 typedef struct { 492 caddr32_t rd_rwlock; 493 size32_t rd_count; 494 } readlock32_t; 495 #endif /* _SYSCALL32 */ 496 497 /* 498 * As part of per-thread caching libumem (ptcumem), we add a small amount to the 499 * thread's uberdata to facilitate it. The tm_roots are the roots of linked 500 * lists which is used by libumem to chain together allocations. tm_size is used 501 * to track the total amount of data stored across those linked lists. For more 502 * information, see libumem's big theory statement. 503 */ 504 #define NTMEMBASE 16 505 506 typedef struct { 507 size_t tm_size; 508 void *tm_roots[NTMEMBASE]; 509 } tumem_t; 510 511 #ifdef _SYSCALL32 512 typedef struct { 513 uint32_t tm_size; 514 caddr32_t tm_roots[NTMEMBASE]; 515 } tumem32_t; 516 #endif 517 518 typedef void (*tmem_func_t)(void *, int); 519 520 /* 521 * Maximum number of read locks allowed for one thread on one rwlock. 522 * This could be as large as INT_MAX, but the SUSV3 test suite would 523 * take an inordinately long time to complete. This is big enough. 524 */ 525 #define READ_LOCK_MAX 100000 526 527 #define ul_tlsent ul_tls.tls_data /* array of pointers to dynamic TLS */ 528 #define ul_ntlsent ul_tls.tls_size /* number of entries in ul_tlsent */ 529 530 /* 531 * Round up an integral value to a multiple of 64 532 */ 533 #define roundup64(x) (-(-(x) & -64)) 534 535 /* 536 * NOTE: Whatever changes are made to ulwp_t must be 537 * reflected in $SRC/cmd/mdb/common/modules/libc/libc.c 538 * 539 * NOTE: ulwp32_t (defined later in this file) should be kept in sync 540 * with ulwp_t 541 * 542 * NOTE: ul_self *must* be the first member of ulwp_t on x86 543 * Low-level x86 code relies on this. 544 */ 545 typedef struct ulwp { 546 /* 547 * These members always need to come first on sparc. 548 * For dtrace, a ulwp_t must be aligned on a 64-byte boundary. 549 */ 550 #if defined(__sparc) 551 uint32_t ul_dinstr; /* scratch space for dtrace */ 552 uint32_t ul_padsparc0[15]; 553 uint32_t ul_dsave; /* dtrace: save %g1, %g0, %sp */ 554 uint32_t ul_drestore; /* dtrace: restore %g0, %g0, %g0 */ 555 uint32_t ul_dftret; /* dtrace: return probe fasttrap */ 556 uint32_t ul_dreturn; /* dtrace: return %o0 */ 557 #endif 558 struct ulwp *ul_self; /* pointer to self */ 559 #if defined(__i386) 560 uint8_t ul_dinstr[40]; /* scratch space for dtrace */ 561 #elif defined(__amd64) 562 uint8_t ul_dinstr[56]; /* scratch space for dtrace */ 563 #endif 564 struct uberdata *ul_uberdata; /* uber (super-global) data */ 565 tls_t ul_tls; /* dynamic thread-local storage base */ 566 struct ulwp *ul_forw; /* forw, back all_lwps list, */ 567 struct ulwp *ul_back; /* protected by link_lock */ 568 struct ulwp *ul_next; /* list to keep track of stacks */ 569 struct ulwp *ul_hash; /* hash chain linked list */ 570 void *ul_rval; /* return value from thr_exit() */ 571 caddr_t ul_stk; /* mapping base of the stack */ 572 size_t ul_mapsiz; /* mapping size of the stack */ 573 size_t ul_guardsize; /* normally _lpagesize */ 574 uintptr_t ul_stktop; /* broken thr_stksegment() interface */ 575 size_t ul_stksiz; /* broken thr_stksegment() interface */ 576 stack_t ul_ustack; /* current stack boundaries */ 577 int ul_ix; /* hash index */ 578 lwpid_t ul_lwpid; /* thread id, aka the lwp id */ 579 pri_t ul_pri; /* scheduling priority */ 580 pri_t ul_epri; /* real-time ceiling priority */ 581 char ul_policy; /* scheduling policy */ 582 char ul_cid; /* scheduling class id */ 583 union { 584 struct { 585 char cursig; /* deferred signal number */ 586 char pleasestop; /* lwp requested to stop itself */ 587 } s; 588 short curplease; /* for testing both at once */ 589 } ul_cp; 590 char ul_stop; /* reason for stopping */ 591 char ul_signalled; /* this lwp was cond_signal()d */ 592 char ul_dead; /* this lwp has called thr_exit */ 593 char ul_unwind; /* posix: unwind C++ stack */ 594 char ul_detached; /* THR_DETACHED at thread_create() */ 595 /* or pthread_detach() was called */ 596 char ul_writer; /* sleeping in rw_wrlock() */ 597 char ul_stopping; /* set by curthread: stopping self */ 598 char ul_cancel_prologue; /* for _cancel_prologue() */ 599 short ul_preempt; /* no_preempt()/preempt() */ 600 short ul_savpreempt; /* pre-existing preempt value */ 601 char ul_sigsuspend; /* thread is in sigsuspend/pollsys */ 602 char ul_main; /* thread is the main thread */ 603 char ul_fork; /* thread is performing a fork */ 604 char ul_primarymap; /* primary link-map is initialized */ 605 /* per-thread copies of the corresponding global variables */ 606 uint8_t ul_max_spinners; /* thread_max_spinners */ 607 char ul_door_noreserve; /* thread_door_noreserve */ 608 char ul_queue_fifo; /* thread_queue_fifo */ 609 char ul_cond_wait_defer; /* thread_cond_wait_defer */ 610 char ul_error_detection; /* thread_error_detection */ 611 char ul_async_safe; /* thread_async_safe */ 612 char ul_rt; /* found on an RT queue */ 613 char ul_rtqueued; /* was RT when queued */ 614 char ul_misaligned; /* thread_locks_misaligned */ 615 char ul_pad[3]; 616 int ul_adaptive_spin; /* thread_adaptive_spin */ 617 int ul_queue_spin; /* thread_queue_spin */ 618 volatile int ul_critical; /* non-zero == in a critical region */ 619 int ul_sigdefer; /* non-zero == defer signals */ 620 int ul_vfork; /* thread is the child of vfork() */ 621 int ul_cancelable; /* _cancelon()/_canceloff() */ 622 char ul_cancel_pending; /* pthread_cancel() was called */ 623 char ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */ 624 char ul_cancel_async; /* PTHREAD_CANCEL_ASYNCHRONOUS */ 625 char ul_save_async; /* saved copy of ul_cancel_async */ 626 char ul_mutator; /* lwp is a mutator (java interface) */ 627 char ul_created; /* created suspended */ 628 char ul_replace; /* replacement; must be free()d */ 629 uchar_t ul_nocancel; /* cancellation can't happen */ 630 int ul_errno; /* per-thread errno */ 631 int *ul_errnop; /* pointer to errno or self->ul_errno */ 632 __cleanup_t *ul_clnup_hdr; /* head of cleanup handlers list */ 633 uberflags_t *ul_schedctl_called; /* ul_schedctl is set up */ 634 volatile sc_shared_t *ul_schedctl; /* schedctl data */ 635 int ul_bindflags; /* bind_guard() interface to ld.so.1 */ 636 uint_t ul_libc_locks; /* count of cancel_safe_mutex_lock()s */ 637 tsd_t *ul_stsd; /* slow TLS for keys >= TSD_NFAST */ 638 void *ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */ 639 td_evbuf_t ul_td_evbuf; /* event buffer */ 640 char ul_td_events_enable; /* event mechanism enabled */ 641 char ul_sync_obj_reg; /* tdb_sync_obj_register() */ 642 char ul_qtype; /* MX or CV */ 643 char ul_cv_wake; /* != 0: just wake up, don't requeue */ 644 int ul_rtld; /* thread is running inside ld.so.1 */ 645 int ul_usropts; /* flags given to thr_create() */ 646 void *(*ul_startpc)(void *); /* start func (thr_create()) */ 647 void *ul_startarg; /* argument for start function */ 648 void *ul_wchan; /* synch object when sleeping */ 649 struct ulwp *ul_link; /* sleep queue link */ 650 queue_head_t *ul_sleepq; /* sleep queue thread is waiting on */ 651 mutex_t *ul_cvmutex; /* mutex dropped when waiting on a cv */ 652 mxchain_t *ul_mxchain; /* chain of owned ceiling mutexes */ 653 int ul_save_state; /* bind_guard() interface to ld.so.1 */ 654 uint_t ul_rdlockcnt; /* # entries in ul_readlock array */ 655 /* 0 means there is but a single entry */ 656 union { /* single entry or pointer to array */ 657 readlock_t single; 658 readlock_t *array; 659 } ul_readlock; 660 uint_t ul_heldlockcnt; /* # entries in ul_heldlocks array */ 661 /* 0 means there is but a single entry */ 662 union { /* single entry or pointer to array */ 663 mutex_t *single; 664 mutex_t **array; 665 } ul_heldlocks; 666 /* PROBE_SUPPORT begin */ 667 void *ul_tpdp; 668 /* PROBE_SUPPORT end */ 669 ucontext_t *ul_siglink; /* pointer to previous context */ 670 uint_t ul_spin_lock_spin; /* spin lock statistics */ 671 uint_t ul_spin_lock_spin2; 672 uint_t ul_spin_lock_sleep; 673 uint_t ul_spin_lock_wakeup; 674 queue_root_t ul_queue_root; /* root of a sleep queue */ 675 id_t ul_rtclassid; /* real-time class id */ 676 uint_t ul_pilocks; /* count of PI locks held */ 677 /* the following members *must* be last in the structure */ 678 /* they are discarded when ulwp is replaced on thr_exit() */ 679 sigset_t ul_sigmask; /* thread's current signal mask */ 680 sigset_t ul_tmpmask; /* signal mask for sigsuspend/pollsys */ 681 siginfo_t ul_siginfo; /* deferred siginfo */ 682 mutex_t ul_spinlock; /* used when suspending/continuing */ 683 fpuenv_t ul_fpuenv; /* floating point state */ 684 uintptr_t ul_sp; /* stack pointer when blocked */ 685 void *ul_ex_unwind; /* address of _ex_unwind() or -1 */ 686 #if defined(sparc) 687 void *ul_unwind_ret; /* used only by _ex_clnup_handler() */ 688 #endif 689 tumem_t ul_tmem; /* used only by umem */ 690 uint_t ul_ptinherit; /* pthreads sched inherit value */ 691 char ul_ntoabuf[18]; /* thread-specific inet_ntoa buffer */ 692 } ulwp_t; 693 694 #define ul_cursig ul_cp.s.cursig /* deferred signal number */ 695 #define ul_pleasestop ul_cp.s.pleasestop /* lwp requested to stop */ 696 #define ul_curplease ul_cp.curplease /* for testing both at once */ 697 698 /* 699 * This is the size of a replacement ulwp, retained only for the benefit 700 * of thr_join(). The trailing members are unneeded for this purpose. 701 */ 702 #define REPLACEMENT_SIZE ((size_t)&((ulwp_t *)NULL)->ul_sigmask) 703 704 /* 705 * Definitions for static initialization of signal sets, 706 * plus some sneaky optimizations in various places. 707 */ 708 709 #define SIGMASK(sig) ((uint32_t)1 << (((sig) - 1) & (32 - 1))) 710 711 #if (MAXSIG > (2 * 32) && MAXSIG <= (3 * 32)) 712 #define FILLSET0 0xffffffffu 713 #define FILLSET1 0xffffffffu 714 #define FILLSET2 ((1u << (MAXSIG - 64)) - 1) 715 #define FILLSET3 0 716 #else 717 #error "fix me: MAXSIG out of bounds" 718 #endif 719 720 #define CANTMASK0 (SIGMASK(SIGKILL) | SIGMASK(SIGSTOP)) 721 #define CANTMASK1 0 722 #define CANTMASK2 0 723 #define CANTMASK3 0 724 725 #define MASKSET0 (FILLSET0 & ~CANTMASK0) 726 #define MASKSET1 (FILLSET1 & ~CANTMASK1) 727 #define MASKSET2 (FILLSET2 & ~CANTMASK2) 728 #define MASKSET3 (FILLSET3 & ~CANTMASK3) 729 730 extern const sigset_t maskset; /* set of all maskable signals */ 731 732 extern int thread_adaptive_spin; 733 extern uint_t thread_max_spinners; 734 extern int thread_queue_spin; 735 extern int thread_queue_fifo; 736 extern int thread_queue_dump; 737 extern int thread_cond_wait_defer; 738 extern int thread_async_safe; 739 extern int thread_queue_verify; 740 741 /* 742 * pthread_atfork() related data, used to store atfork handlers. 743 */ 744 typedef struct atfork { 745 struct atfork *forw; /* forward pointer */ 746 struct atfork *back; /* backward pointer */ 747 void (*prepare)(void); /* pre-fork handler */ 748 void (*parent)(void); /* post-fork parent handler */ 749 void (*child)(void); /* post-fork child handler */ 750 } atfork_t; 751 752 /* 753 * Element in the table and in the list of registered process 754 * robust locks. We keep track of these to make sure that we 755 * only call ___lwp_mutex_register() once for each such lock 756 * after it is first mapped in (or newly mapped in). 757 */ 758 typedef struct robust { 759 struct robust *robust_next; /* hash table list */ 760 struct robust *robust_list; /* global list */ 761 mutex_t *robust_lock; 762 } robust_t; 763 764 /* 765 * Invalid address, used to mark an unused element in the hash table. 766 */ 767 #define INVALID_ADDR ((void *)(uintptr_t)(-1L)) 768 769 /* 770 * Parameters of the lock registration hash table. 771 */ 772 #define LOCKSHIFT 15 /* number of hashing bits */ 773 #define LOCKHASHSZ (1 << LOCKSHIFT) /* power of 2 (1<<15 == 32K) */ 774 #define LOCK_HASH(addr) (uint_t) \ 775 ((((uintptr_t)(addr) >> 3) \ 776 ^ ((uintptr_t)(addr) >> (LOCKSHIFT + 3))) \ 777 & (LOCKHASHSZ - 1)) 778 779 /* 780 * Make our hot locks reside on private cache lines (64 bytes). 781 */ 782 typedef struct { 783 mutex_t pad_lock; 784 char pad_pad[64 - sizeof (mutex_t)]; 785 } pad_lock_t; 786 787 /* 788 * Make our semi-hot locks reside on semi-private cache lines (32 bytes). 789 */ 790 typedef struct { 791 mutex_t pad_lock; 792 char pad_pad[32 - sizeof (mutex_t)]; 793 } pad32_lock_t; 794 795 /* 796 * The threads hash table is used for fast lookup and locking of an active 797 * thread structure (ulwp_t) given a thread-id. It is an N-element array of 798 * thr_hash_table_t structures, where N == 1 before the main thread creates 799 * the first additional thread and N == 1024 afterwards. Each element of the 800 * table is 64 bytes in size and alignment to reduce cache conflicts. 801 */ 802 typedef struct { 803 mutex_t hash_lock; /* lock per bucket */ 804 cond_t hash_cond; /* convar per bucket */ 805 ulwp_t *hash_bucket; /* hash bucket points to the list of ulwps */ 806 char hash_pad[64 - /* pad out to 64 bytes */ 807 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (ulwp_t *))]; 808 } thr_hash_table_t; 809 810 #ifdef _SYSCALL32 811 typedef struct { 812 mutex_t hash_lock; 813 cond_t hash_cond; 814 caddr32_t hash_bucket; 815 char hash_pad[64 - 816 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (caddr32_t))]; 817 } thr_hash_table32_t; 818 #endif /* _SYSCALL32 */ 819 820 821 /* 822 * siguaction members have 128-byte size and 64-byte alignment. 823 * We know that sizeof (struct sigaction) is 32 bytes for both 824 * _ILP32 and _LP64 and that sizeof (rwlock_t) is 64 bytes. 825 */ 826 typedef struct { 827 rwlock_t sig_lock; 828 struct sigaction sig_uaction; 829 char sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction)]; 830 } siguaction_t; 831 832 #ifdef _SYSCALL32 833 typedef struct { 834 rwlock_t sig_lock; 835 struct sigaction32 sig_uaction; 836 char sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction32)]; 837 } siguaction32_t; 838 #endif /* _SYSCALL32 */ 839 840 841 /* 842 * Bucket structures, used by lmalloc()/lfree(). 843 * See port/threads/alloc.c for details. 844 * A bucket's size and alignment is 64 bytes. 845 */ 846 typedef struct { 847 mutex_t bucket_lock; /* protects the free list allocations */ 848 void *free_list; /* LIFO list of blocks to allocate/free */ 849 size_t chunks; /* number of 64K blocks mmap()ed last time */ 850 char pad64[64 - /* pad out to 64 bytes */ 851 (sizeof (mutex_t) + sizeof (void *) + sizeof (size_t))]; 852 } bucket_t; 853 854 #ifdef _SYSCALL32 855 typedef struct { 856 mutex_t bucket_lock; 857 caddr32_t free_list; 858 size32_t chunks; 859 char pad64[64 - /* pad out to 64 bytes */ 860 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (size32_t))]; 861 } bucket32_t; 862 #endif /* _SYSCALL32 */ 863 864 #define NBUCKETS 10 /* sizes ranging from 64 to 32768 */ 865 866 867 /* 868 * atexit() data structures. 869 * See port/gen/atexit.c for details. 870 */ 871 typedef void (*_exithdlr_func_t) (void*); 872 873 typedef struct _exthdlr { 874 struct _exthdlr *next; /* next in handler list */ 875 _exithdlr_func_t hdlr; /* handler itself */ 876 void *arg; /* argument to handler */ 877 void *dso; /* DSO associated with handler */ 878 } _exthdlr_t; 879 880 typedef struct { 881 mutex_t exitfns_lock; 882 _exthdlr_t *head; 883 /* 884 * exit_frame_monitor is part of a private contract between libc and 885 * the Sun C++ runtime. 886 * 887 * It should be NULL until exit() is called, and thereafter hold the 888 * frame pointer of the function implementing our exit processing. 889 */ 890 void *exit_frame_monitor; 891 char exit_pad[64 - /* pad out to 64 bytes */ 892 (sizeof (mutex_t) + sizeof (_exthdlr_t *) + sizeof (void *))]; 893 } atexit_root_t; 894 895 #ifdef _SYSCALL32 896 typedef struct { 897 mutex_t exitfns_lock; 898 caddr32_t head; 899 caddr32_t exit_frame_monitor; 900 char exit_pad[64 - /* pad out to 64 bytes */ 901 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (caddr32_t))]; 902 } atexit_root32_t; 903 #endif /* _SYSCALL32 */ 904 905 /* 906 * at_quick_exit() and quick_exit() data structures. The ISO/IEC C11 odd 907 * siblings of atexit() 908 */ 909 typedef void (*_quick_exithdlr_func_t)(void); 910 911 typedef struct _qexthdlr { 912 struct _qexthdlr *next; /* next in handler list */ 913 _quick_exithdlr_func_t hdlr; /* handler itself */ 914 } _qexthdlr_t; 915 916 /* 917 * We add a pad on 32-bit systems to allow us to always have the structure size 918 * be 32-bytes which helps us deal with the compiler's alignment when building 919 * in ILP32 / LP64 systems. 920 */ 921 typedef struct { 922 mutex_t exitfns_lock; 923 _qexthdlr_t *head; 924 #if !defined(_LP64) 925 uint32_t pad; 926 #endif 927 } quickexit_root_t; 928 929 #ifdef _SYSCALL32 930 typedef struct { 931 mutex_t exitfns_lock; 932 caddr32_t head; 933 uint32_t pad; 934 } quickexit_root32_t; 935 #endif /* _SYSCALL32 */ 936 937 /* 938 * This is data that is global to all link maps (uberdata, aka super-global). 939 * Note: When changing this, please be sure to keep the 32-bit variant of 940 * this in sync. (see uberdata32_t below) 941 */ 942 typedef struct uberdata { 943 pad_lock_t _link_lock; 944 pad_lock_t _ld_lock; 945 pad_lock_t _fork_lock; 946 pad_lock_t _atfork_lock; 947 pad32_lock_t _callout_lock; 948 pad32_lock_t _tdb_hash_lock; 949 tdb_sync_stats_t tdb_hash_lock_stats; 950 siguaction_t siguaction[NSIG]; 951 bucket_t bucket[NBUCKETS]; 952 atexit_root_t atexit_root; 953 quickexit_root_t quickexit_root; 954 tsd_metadata_t tsd_metadata; 955 tls_metadata_t tls_metadata; 956 /* 957 * Every object before this point has size and alignment of 64 bytes. 958 * Don't add any other type of data before this point. 959 */ 960 char primary_map; /* set when primary link map is initialized */ 961 char bucket_init; /* set when bucket[NBUCKETS] is initialized */ 962 char pad[2]; 963 uberflags_t uberflags; 964 queue_head_t *queue_head; 965 thr_hash_table_t *thr_hash_table; 966 uint_t hash_size; /* # of entries in thr_hash_table[] */ 967 uint_t hash_mask; /* hash_size - 1 */ 968 ulwp_t *ulwp_one; /* main thread */ 969 ulwp_t *all_lwps; /* circular ul_forw/ul_back list of live lwps */ 970 ulwp_t *all_zombies; /* circular ul_forw/ul_back list of zombies */ 971 int nthreads; /* total number of live threads/lwps */ 972 int nzombies; /* total number of zombie threads */ 973 int ndaemons; /* total number of THR_DAEMON threads/lwps */ 974 pid_t pid; /* the current process's pid */ 975 void (*sigacthandler)(int, siginfo_t *, void *); 976 ulwp_t *lwp_stacks; 977 ulwp_t *lwp_laststack; 978 int nfreestack; 979 int thread_stack_cache; 980 ulwp_t *ulwp_freelist; 981 ulwp_t *ulwp_lastfree; 982 ulwp_t *ulwp_replace_free; 983 ulwp_t *ulwp_replace_last; 984 atfork_t *atforklist; /* circular Q for fork handlers */ 985 robust_t **robustlocks; /* table of registered robust locks */ 986 robust_t *robustlist; /* list of registered robust locks */ 987 char *progname; /* the basename of the program, from argv[0] */ 988 void *ub_comm_page; /* arch-specific comm page of kernel data */ 989 struct uberdata **tdb_bootstrap; 990 tdb_t tdb; /* thread debug interfaces (for libc_db) */ 991 } uberdata_t; 992 993 #define link_lock _link_lock.pad_lock 994 #define ld_lock _ld_lock.pad_lock 995 #define fork_lock _fork_lock.pad_lock 996 #define atfork_lock _atfork_lock.pad_lock 997 #define callout_lock _callout_lock.pad_lock 998 #define tdb_hash_lock _tdb_hash_lock.pad_lock 999 1000 #pragma align 64(__uberdata) 1001 extern uberdata_t __uberdata; 1002 extern uberdata_t **__tdb_bootstrap; /* known to libc_db and mdb */ 1003 extern int primary_link_map; 1004 1005 #define ulwp_mutex(ulwp, udp) \ 1006 (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_lock) 1007 #define ulwp_condvar(ulwp, udp) \ 1008 (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_cond) 1009 1010 /* 1011 * Grab and release the hash table lock for the specified lwp. 1012 */ 1013 #define ulwp_lock(ulwp, udp) lmutex_lock(ulwp_mutex(ulwp, udp)) 1014 #define ulwp_unlock(ulwp, udp) lmutex_unlock(ulwp_mutex(ulwp, udp)) 1015 1016 #ifdef _SYSCALL32 /* needed by libc_db */ 1017 1018 typedef struct ulwp32 { 1019 #if defined(__sparc) 1020 uint32_t ul_dinstr; /* scratch space for dtrace */ 1021 uint32_t ul_padsparc0[15]; 1022 uint32_t ul_dsave; /* dtrace: save %g1, %g0, %sp */ 1023 uint32_t ul_drestore; /* dtrace: restore %g0, %g0, %g0 */ 1024 uint32_t ul_dftret; /* dtrace: return probe fasttrap */ 1025 uint32_t ul_dreturn; /* dtrace: return %o0 */ 1026 #endif 1027 caddr32_t ul_self; /* pointer to self */ 1028 #if defined(__x86) 1029 uint8_t ul_dinstr[40]; /* scratch space for dtrace */ 1030 #endif 1031 caddr32_t ul_uberdata; /* uber (super-global) data */ 1032 tls32_t ul_tls; /* dynamic thread-local storage base */ 1033 caddr32_t ul_forw; /* forw, back all_lwps list, */ 1034 caddr32_t ul_back; /* protected by link_lock */ 1035 caddr32_t ul_next; /* list to keep track of stacks */ 1036 caddr32_t ul_hash; /* hash chain linked list */ 1037 caddr32_t ul_rval; /* return value from thr_exit() */ 1038 caddr32_t ul_stk; /* mapping base of the stack */ 1039 size32_t ul_mapsiz; /* mapping size of the stack */ 1040 size32_t ul_guardsize; /* normally _lpagesize */ 1041 caddr32_t ul_stktop; /* broken thr_stksegment() interface */ 1042 size32_t ul_stksiz; /* broken thr_stksegment() interface */ 1043 stack32_t ul_ustack; /* current stack boundaries */ 1044 int ul_ix; /* hash index */ 1045 lwpid_t ul_lwpid; /* thread id, aka the lwp id */ 1046 pri_t ul_pri; /* scheduling priority */ 1047 pri_t ul_epri; /* real-time ceiling priority */ 1048 char ul_policy; /* scheduling policy */ 1049 char ul_cid; /* scheduling class id */ 1050 union { 1051 struct { 1052 char cursig; /* deferred signal number */ 1053 char pleasestop; /* lwp requested to stop itself */ 1054 } s; 1055 short curplease; /* for testing both at once */ 1056 } ul_cp; 1057 char ul_stop; /* reason for stopping */ 1058 char ul_signalled; /* this lwp was cond_signal()d */ 1059 char ul_dead; /* this lwp has called thr_exit */ 1060 char ul_unwind; /* posix: unwind C++ stack */ 1061 char ul_detached; /* THR_DETACHED at thread_create() */ 1062 /* or pthread_detach() was called */ 1063 char ul_writer; /* sleeping in rw_wrlock() */ 1064 char ul_stopping; /* set by curthread: stopping self */ 1065 char ul_cancel_prologue; /* for _cancel_prologue() */ 1066 short ul_preempt; /* no_preempt()/preempt() */ 1067 short ul_savpreempt; /* pre-existing preempt value */ 1068 char ul_sigsuspend; /* thread is in sigsuspend/pollsys */ 1069 char ul_main; /* thread is the main thread */ 1070 char ul_fork; /* thread is performing a fork */ 1071 char ul_primarymap; /* primary link-map is initialized */ 1072 /* per-thread copies of the corresponding global variables */ 1073 uint8_t ul_max_spinners; /* thread_max_spinners */ 1074 char ul_door_noreserve; /* thread_door_noreserve */ 1075 char ul_queue_fifo; /* thread_queue_fifo */ 1076 char ul_cond_wait_defer; /* thread_cond_wait_defer */ 1077 char ul_error_detection; /* thread_error_detection */ 1078 char ul_async_safe; /* thread_async_safe */ 1079 char ul_rt; /* found on an RT queue */ 1080 char ul_rtqueued; /* was RT when queued */ 1081 char ul_misaligned; /* thread_locks_misaligned */ 1082 char ul_pad[3]; 1083 int ul_adaptive_spin; /* thread_adaptive_spin */ 1084 int ul_queue_spin; /* thread_queue_spin */ 1085 int ul_critical; /* non-zero == in a critical region */ 1086 int ul_sigdefer; /* non-zero == defer signals */ 1087 int ul_vfork; /* thread is the child of vfork() */ 1088 int ul_cancelable; /* _cancelon()/_canceloff() */ 1089 char ul_cancel_pending; /* pthread_cancel() was called */ 1090 char ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */ 1091 char ul_cancel_async; /* PTHREAD_CANCEL_ASYNCHRONOUS */ 1092 char ul_save_async; /* saved copy of ul_cancel_async */ 1093 char ul_mutator; /* lwp is a mutator (java interface) */ 1094 char ul_created; /* created suspended */ 1095 char ul_replace; /* replacement; must be free()d */ 1096 uchar_t ul_nocancel; /* cancellation can't happen */ 1097 int ul_errno; /* per-thread errno */ 1098 caddr32_t ul_errnop; /* pointer to errno or self->ul_errno */ 1099 caddr32_t ul_clnup_hdr; /* head of cleanup handlers list */ 1100 caddr32_t ul_schedctl_called; /* ul_schedctl is set up */ 1101 caddr32_t ul_schedctl; /* schedctl data */ 1102 int ul_bindflags; /* bind_guard() interface to ld.so.1 */ 1103 uint_t ul_libc_locks; /* count of cancel_safe_mutex_lock()s */ 1104 caddr32_t ul_stsd; /* slow TLS for keys >= TSD_NFAST */ 1105 caddr32_t ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */ 1106 td_evbuf32_t ul_td_evbuf; /* event buffer */ 1107 char ul_td_events_enable; /* event mechanism enabled */ 1108 char ul_sync_obj_reg; /* tdb_sync_obj_register() */ 1109 char ul_qtype; /* MX or CV */ 1110 char ul_cv_wake; /* != 0: just wake up, don't requeue */ 1111 int ul_rtld; /* thread is running inside ld.so.1 */ 1112 int ul_usropts; /* flags given to thr_create() */ 1113 caddr32_t ul_startpc; /* start func (thr_create()) */ 1114 caddr32_t ul_startarg; /* argument for start function */ 1115 caddr32_t ul_wchan; /* synch object when sleeping */ 1116 caddr32_t ul_link; /* sleep queue link */ 1117 caddr32_t ul_sleepq; /* sleep queue thread is waiting on */ 1118 caddr32_t ul_cvmutex; /* mutex dropped when waiting on a cv */ 1119 caddr32_t ul_mxchain; /* chain of owned ceiling mutexes */ 1120 int ul_save_state; /* bind_guard() interface to ld.so.1 */ 1121 uint_t ul_rdlockcnt; /* # entries in ul_readlock array */ 1122 /* 0 means there is but a single entry */ 1123 union { /* single entry or pointer to array */ 1124 readlock32_t single; 1125 caddr32_t array; 1126 } ul_readlock; 1127 uint_t ul_heldlockcnt; /* # entries in ul_heldlocks array */ 1128 /* 0 means there is but a single entry */ 1129 union { /* single entry or pointer to array */ 1130 caddr32_t single; 1131 caddr32_t array; 1132 } ul_heldlocks; 1133 /* PROBE_SUPPORT begin */ 1134 caddr32_t ul_tpdp; 1135 /* PROBE_SUPPORT end */ 1136 caddr32_t ul_siglink; /* pointer to previous context */ 1137 uint_t ul_spin_lock_spin; /* spin lock statistics */ 1138 uint_t ul_spin_lock_spin2; 1139 uint_t ul_spin_lock_sleep; 1140 uint_t ul_spin_lock_wakeup; 1141 queue_root32_t ul_queue_root; /* root of a sleep queue */ 1142 id_t ul_rtclassid; /* real-time class id */ 1143 uint_t ul_pilocks; /* count of PI locks held */ 1144 /* the following members *must* be last in the structure */ 1145 /* they are discarded when ulwp is replaced on thr_exit() */ 1146 sigset_t ul_sigmask; /* thread's current signal mask */ 1147 sigset_t ul_tmpmask; /* signal mask for sigsuspend/pollsys */ 1148 siginfo32_t ul_siginfo; /* deferred siginfo */ 1149 mutex_t ul_spinlock; /* used when suspending/continuing */ 1150 fpuenv32_t ul_fpuenv; /* floating point state */ 1151 caddr32_t ul_sp; /* stack pointer when blocked */ 1152 #if defined(sparc) 1153 caddr32_t ul_unwind_ret; /* used only by _ex_clnup_handler() */ 1154 #endif 1155 tumem32_t ul_tmem; /* used only by umem */ 1156 uint_t ul_ptinherit; /* pthreads sched inherit value */ 1157 char ul_ntoabuf[18]; /* thread-specific inet_ntoa buffer */ 1158 } ulwp32_t; 1159 1160 #define REPLACEMENT_SIZE32 ((size_t)&((ulwp32_t *)NULL)->ul_sigmask) 1161 1162 typedef struct uberdata32 { 1163 pad_lock_t _link_lock; 1164 pad_lock_t _ld_lock; 1165 pad_lock_t _fork_lock; 1166 pad_lock_t _atfork_lock; 1167 pad32_lock_t _callout_lock; 1168 pad32_lock_t _tdb_hash_lock; 1169 tdb_sync_stats_t tdb_hash_lock_stats; 1170 siguaction32_t siguaction[NSIG]; 1171 bucket32_t bucket[NBUCKETS]; 1172 atexit_root32_t atexit_root; 1173 quickexit_root32_t quickexit_root; 1174 tsd_metadata32_t tsd_metadata; 1175 tls_metadata32_t tls_metadata; 1176 char primary_map; 1177 char bucket_init; 1178 char pad[2]; 1179 uberflags_t uberflags; 1180 caddr32_t queue_head; 1181 caddr32_t thr_hash_table; 1182 uint_t hash_size; 1183 uint_t hash_mask; 1184 caddr32_t ulwp_one; 1185 caddr32_t all_lwps; 1186 caddr32_t all_zombies; 1187 int nthreads; 1188 int nzombies; 1189 int ndaemons; 1190 int pid; 1191 caddr32_t sigacthandler; 1192 caddr32_t lwp_stacks; 1193 caddr32_t lwp_laststack; 1194 int nfreestack; 1195 int thread_stack_cache; 1196 caddr32_t ulwp_freelist; 1197 caddr32_t ulwp_lastfree; 1198 caddr32_t ulwp_replace_free; 1199 caddr32_t ulwp_replace_last; 1200 caddr32_t atforklist; 1201 caddr32_t robustlocks; 1202 caddr32_t robustlist; 1203 caddr32_t progname; 1204 caddr32_t ub_comm_page; 1205 caddr32_t tdb_bootstrap; 1206 tdb32_t tdb; 1207 } uberdata32_t; 1208 1209 #endif /* _SYSCALL32 */ 1210 1211 /* ul_stop values */ 1212 #define TSTP_REGULAR 0x01 /* Stopped by thr_suspend() */ 1213 #define TSTP_MUTATOR 0x08 /* stopped by thr_suspend_*mutator*() */ 1214 #define TSTP_FORK 0x20 /* stopped by suspend_fork() */ 1215 1216 /* 1217 * Implementation-specific attribute types for pthread_mutexattr_init() etc. 1218 */ 1219 1220 typedef struct _cvattr { 1221 int pshared; 1222 clockid_t clockid; 1223 } cvattr_t; 1224 1225 typedef struct _mattr { 1226 int pshared; 1227 int protocol; 1228 int prioceiling; 1229 int type; 1230 int robustness; 1231 } mattr_t; 1232 1233 typedef struct _thrattr { 1234 size_t stksize; 1235 void *stkaddr; 1236 int detachstate; 1237 int daemonstate; 1238 int scope; 1239 int prio; 1240 int policy; 1241 int inherit; 1242 size_t guardsize; 1243 char name[THREAD_NAME_MAX]; 1244 } thrattr_t; 1245 1246 typedef struct _rwlattr { 1247 int pshared; 1248 } rwlattr_t; 1249 1250 /* _curthread() is inline for speed */ 1251 extern ulwp_t *_curthread(void); 1252 #define curthread (_curthread()) 1253 1254 /* this version (also inline) can be tested for NULL */ 1255 extern ulwp_t *__curthread(void); 1256 1257 /* get the current stack pointer (also inline) */ 1258 extern greg_t stkptr(void); 1259 1260 /* 1261 * Suppress __attribute__((...)) if we are not compiling with gcc 1262 */ 1263 #if !defined(__GNUC__) 1264 #define __attribute__(string) 1265 #endif 1266 1267 /* Fetch the dispatch (kernel) priority of a thread */ 1268 #define real_priority(ulwp) \ 1269 ((ulwp)->ul_schedctl? (ulwp)->ul_schedctl->sc_priority : 0) 1270 1271 /* 1272 * Implementation functions. Not visible outside of the library itself. 1273 */ 1274 extern int __nanosleep(const timespec_t *, timespec_t *); 1275 extern void getgregs(ulwp_t *, gregset_t); 1276 extern void setgregs(ulwp_t *, gregset_t); 1277 extern void thr_panic(const char *) __NORETURN; 1278 extern void mutex_panic(mutex_t *, const char *) __NORETURN; 1279 extern ulwp_t *find_lwp(thread_t); 1280 extern void finish_init(void); 1281 extern void update_sched(ulwp_t *); 1282 extern void queue_alloc(void); 1283 extern void tmem_exit(void); 1284 extern void tsd_exit(void); 1285 extern void tsd_free(ulwp_t *); 1286 extern void tls_setup(void); 1287 extern void tls_exit(void); 1288 extern void tls_free(ulwp_t *); 1289 extern void rwl_free(ulwp_t *); 1290 extern void heldlock_exit(void); 1291 extern void heldlock_free(ulwp_t *); 1292 extern void sigacthandler(int, siginfo_t *, void *); 1293 extern void signal_init(void); 1294 extern int sigequalset(const sigset_t *, const sigset_t *); 1295 extern void mutex_setup(void); 1296 extern void take_deferred_signal(int); 1297 extern void *setup_top_frame(void *, size_t, ulwp_t *); 1298 extern int setup_context(ucontext_t *, void *(*func)(ulwp_t *), 1299 ulwp_t *ulwp, caddr_t stk, size_t stksize); 1300 extern volatile sc_shared_t *setup_schedctl(void); 1301 extern void *lmalloc(size_t); 1302 extern void lfree(void *, size_t); 1303 extern void *libc_malloc(size_t); 1304 extern void *libc_realloc(void *, size_t); 1305 extern void libc_free(void *); 1306 extern char *libc_strdup(const char *); 1307 extern void ultos(uint64_t, int, char *); 1308 extern void lock_error(const mutex_t *, const char *, void *, const char *); 1309 extern void rwlock_error(const rwlock_t *, const char *, const char *); 1310 extern void thread_error(const char *); 1311 extern void grab_assert_lock(void); 1312 extern void dump_queue_statistics(void); 1313 extern void collect_queue_statistics(void); 1314 extern void record_spin_locks(ulwp_t *); 1315 extern void remember_lock(mutex_t *); 1316 extern void forget_lock(mutex_t *); 1317 extern void register_lock(mutex_t *); 1318 extern void unregister_locks(void); 1319 #if defined(__sparc) 1320 extern void _flush_windows(void); 1321 #else 1322 #define _flush_windows() 1323 #endif 1324 extern void set_curthread(void *); 1325 extern void ssp_init(void); 1326 extern locale_t __curlocale(void); 1327 1328 extern int sema_relclockwait(sema_t *, clockid_t, const timespec_t *); 1329 extern int sema_clockwait(sema_t *, clockid_t, const timespec_t *); 1330 1331 1332 /* 1333 * Utility function used when waking up many threads (more than MAXLWPS) 1334 * all at once. See mutex_wakeup_all(), cond_broadcast(), and rw_unlock(). 1335 */ 1336 #define MAXLWPS 128 /* max remembered lwpids before overflow */ 1337 #define NEWLWPS 2048 /* max remembered lwpids at first overflow */ 1338 extern lwpid_t *alloc_lwpids(lwpid_t *, int *, int *); 1339 1340 /* enter a critical section */ 1341 #define enter_critical(self) (self->ul_critical++) 1342 1343 /* exit a critical section, take deferred actions if necessary */ 1344 extern void do_exit_critical(void); 1345 #define exit_critical(self) \ 1346 (void) (self->ul_critical--, \ 1347 ((self->ul_curplease && self->ul_critical == 0)? \ 1348 (do_exit_critical(), 0) : 0)) 1349 1350 /* 1351 * Like enter_critical()/exit_critical() but just for deferring signals. 1352 * Unlike enter_critical()/exit_critical(), ul_sigdefer may be set while 1353 * calling application functions like constructors and destructors. 1354 * Care must be taken if the application function attempts to set 1355 * the signal mask while a deferred signal is present; the setting 1356 * of the signal mask must also be deferred. 1357 */ 1358 #define sigoff(self) (self->ul_sigdefer++) 1359 #define sigon(self) \ 1360 (void) ((--self->ul_sigdefer == 0 && \ 1361 self->ul_curplease && self->ul_critical == 0)? \ 1362 (do_exit_critical(), 0) : 0) 1363 1364 /* these are exported functions */ 1365 extern void _sigoff(void); 1366 extern void _sigon(void); 1367 1368 #define sigorset(s1, s2) \ 1369 (((s1)->__sigbits[0] |= (s2)->__sigbits[0]), \ 1370 ((s1)->__sigbits[1] |= (s2)->__sigbits[1]), \ 1371 ((s1)->__sigbits[2] |= (s2)->__sigbits[2]), \ 1372 ((s1)->__sigbits[3] |= (s2)->__sigbits[3])) 1373 1374 #define sigandset(s1, s2) \ 1375 (((s1)->__sigbits[0] &= (s2)->__sigbits[0]), \ 1376 ((s1)->__sigbits[1] &= (s2)->__sigbits[1]), \ 1377 ((s1)->__sigbits[2] &= (s2)->__sigbits[2]), \ 1378 ((s1)->__sigbits[3] &= (s2)->__sigbits[3])) 1379 1380 #define sigdiffset(s1, s2) \ 1381 (((s1)->__sigbits[0] &= ~(s2)->__sigbits[0]), \ 1382 ((s1)->__sigbits[1] &= ~(s2)->__sigbits[1]), \ 1383 ((s1)->__sigbits[2] &= ~(s2)->__sigbits[2]), \ 1384 ((s1)->__sigbits[3] &= ~(s2)->__sigbits[3])) 1385 1386 #define delete_reserved_signals(s) \ 1387 (((s)->__sigbits[0] &= MASKSET0), \ 1388 ((s)->__sigbits[1] &= (MASKSET1 & ~SIGMASK(SIGCANCEL))),\ 1389 ((s)->__sigbits[2] &= MASKSET2), \ 1390 ((s)->__sigbits[3] &= MASKSET3)) 1391 1392 extern void block_all_signals(ulwp_t *self); 1393 1394 /* 1395 * When restoring the signal mask after having previously called 1396 * block_all_signals(), if we have a deferred signal present then 1397 * do nothing other than ASSERT() that we are in a critical region. 1398 * The signal mask will be set when we emerge from the critical region 1399 * and call take_deferred_signal(). There is no race condition here 1400 * because the kernel currently has all signals blocked for this thread. 1401 */ 1402 #define restore_signals(self) \ 1403 ((void) ((self)->ul_cursig? \ 1404 (ASSERT((self)->ul_critical + (self)->ul_sigdefer != 0), 0) : \ 1405 __lwp_sigmask(SIG_SETMASK, &(self)->ul_sigmask))) 1406 1407 extern void set_cancel_pending_flag(ulwp_t *, int); 1408 extern void set_cancel_eintr_flag(ulwp_t *); 1409 extern void set_parking_flag(ulwp_t *, int); 1410 extern int cancel_active(void); 1411 1412 extern void *_thrp_setup(ulwp_t *); 1413 extern void _fpinherit(ulwp_t *); 1414 extern void _lwp_start(void); 1415 extern void _lwp_terminate(void); 1416 extern void lmutex_lock(mutex_t *); 1417 extern void lmutex_unlock(mutex_t *); 1418 extern void lrw_rdlock(rwlock_t *); 1419 extern void lrw_wrlock(rwlock_t *); 1420 extern void lrw_unlock(rwlock_t *); 1421 extern void sig_mutex_lock(mutex_t *); 1422 extern void sig_mutex_unlock(mutex_t *); 1423 extern int sig_mutex_trylock(mutex_t *); 1424 extern int sig_cond_wait(cond_t *, mutex_t *); 1425 extern int sig_cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *); 1426 extern void cancel_safe_mutex_lock(mutex_t *); 1427 extern void cancel_safe_mutex_unlock(mutex_t *); 1428 extern int cancel_safe_mutex_trylock(mutex_t *); 1429 extern void _prefork_handler(void); 1430 extern void _postfork_parent_handler(void); 1431 extern void _postfork_child_handler(void); 1432 extern void postfork1_child(void); 1433 extern void postfork1_child_aio(void); 1434 extern void postfork1_child_sigev_aio(void); 1435 extern void postfork1_child_sigev_mq(void); 1436 extern void postfork1_child_sigev_timer(void); 1437 extern void postfork1_child_tpool(void); 1438 extern void fork_lock_enter(void); 1439 extern void fork_lock_exit(void); 1440 extern void suspend_fork(void); 1441 extern void continue_fork(int); 1442 extern void do_sigcancel(void); 1443 extern void setup_cancelsig(int); 1444 extern void init_sigev_thread(void); 1445 extern void init_aio(void); 1446 extern void init_progname(void); 1447 extern void _cancelon(void); 1448 extern void _canceloff(void); 1449 extern void _canceloff_nocancel(void); 1450 extern void _cancel_prologue(void); 1451 extern void _cancel_epilogue(void); 1452 extern void no_preempt(ulwp_t *); 1453 extern void preempt(ulwp_t *); 1454 extern void _thrp_unwind(void *); 1455 1456 extern pid_t __forkx(int); 1457 extern pid_t __forkallx(int); 1458 extern int __open(const char *, int, mode_t); 1459 extern int __open64(const char *, int, mode_t); 1460 extern int __openat(int, const char *, int, mode_t); 1461 extern int __openat64(int, const char *, int, mode_t); 1462 extern int __close(int); 1463 extern ssize_t __read(int, void *, size_t); 1464 extern ssize_t __write(int, const void *, size_t); 1465 extern int __fcntl(int, int, ...); 1466 extern int __lwp_continue(lwpid_t); 1467 extern int __lwp_create(ucontext_t *, uint_t, lwpid_t *); 1468 extern int ___lwp_suspend(lwpid_t); 1469 extern int lwp_wait(lwpid_t, lwpid_t *); 1470 extern int __lwp_wait(lwpid_t, lwpid_t *); 1471 extern int __lwp_detach(lwpid_t); 1472 extern sc_shared_t *__schedctl(void); 1473 1474 /* actual system call traps */ 1475 extern int __setcontext(const ucontext_t *); 1476 extern int __getcontext(ucontext_t *); 1477 extern int __clock_gettime(clockid_t, timespec_t *); 1478 extern void abstime_to_reltime(clockid_t, const timespec_t *, timespec_t *); 1479 extern void hrt2ts(hrtime_t, timespec_t *); 1480 1481 extern int __sigaction(int, const struct sigaction *, struct sigaction *); 1482 extern int __sigprocmask(int, const sigset_t *, sigset_t *); 1483 extern int __lwp_sigmask(int, const sigset_t *); 1484 extern void __sighndlr(int, siginfo_t *, ucontext_t *, void (*)()); 1485 extern caddr_t __sighndlrend; 1486 1487 /* belongs in <pthread.h> */ 1488 #define PTHREAD_CREATE_DAEMON_NP 0x100 /* = THR_DAEMON */ 1489 #define PTHREAD_CREATE_NONDAEMON_NP 0 1490 extern int pthread_attr_setdaemonstate_np(pthread_attr_t *, int); 1491 extern int pthread_attr_getdaemonstate_np(const pthread_attr_t *, int *); 1492 1493 extern int mutex_held(mutex_t *); 1494 extern int mutex_lock_internal(mutex_t *, timespec_t *, int); 1495 extern int mutex_unlock_internal(mutex_t *, int); 1496 1497 /* not cancellation points: */ 1498 extern int __cond_wait(cond_t *, mutex_t *); 1499 extern int __cond_timedwait(cond_t *, mutex_t *, clockid_t, 1500 const timespec_t *); 1501 extern int __cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *); 1502 1503 extern int rw_read_held(rwlock_t *); 1504 extern int rw_write_held(rwlock_t *); 1505 1506 extern int _thrp_create(void *, size_t, void *(*)(void *), void *, long, 1507 thread_t *, size_t, const char *); 1508 extern int _thrp_suspend(thread_t, uchar_t); 1509 extern int _thrp_continue(thread_t, uchar_t); 1510 1511 extern void _thrp_terminate(void *); 1512 extern void _thrp_exit(void); 1513 1514 extern const pcclass_t *get_info_by_class(id_t); 1515 extern const pcclass_t *get_info_by_policy(int); 1516 extern const thrattr_t *def_thrattr(void); 1517 extern id_t setparam(idtype_t, id_t, int, int); 1518 extern id_t setprio(idtype_t, id_t, int, int *); 1519 extern id_t getparam(idtype_t, id_t, int *, struct sched_param *); 1520 struct kspawn_sched; 1521 extern int spawn_sched_resolve(int, int, int, struct kspawn_sched *); 1522 1523 /* 1524 * System call wrappers (direct interfaces to the kernel) 1525 */ 1526 extern int ___lwp_mutex_register(mutex_t *, mutex_t **); 1527 extern int ___lwp_mutex_trylock(mutex_t *, ulwp_t *); 1528 extern int ___lwp_mutex_timedlock(mutex_t *, timespec_t *, ulwp_t *); 1529 extern int ___lwp_mutex_unlock(mutex_t *); 1530 extern int ___lwp_mutex_wakeup(mutex_t *, int); 1531 extern int ___lwp_cond_wait(cond_t *, mutex_t *, timespec_t *, int); 1532 extern int ___lwp_sema_timedwait(lwp_sema_t *, timespec_t *, int); 1533 extern int __lwp_rwlock_rdlock(rwlock_t *, timespec_t *); 1534 extern int __lwp_rwlock_wrlock(rwlock_t *, timespec_t *); 1535 extern int __lwp_rwlock_tryrdlock(rwlock_t *); 1536 extern int __lwp_rwlock_trywrlock(rwlock_t *); 1537 extern int __lwp_rwlock_unlock(rwlock_t *); 1538 extern int __lwp_park(timespec_t *, lwpid_t); 1539 extern int __lwp_unpark(lwpid_t); 1540 extern int __lwp_unpark_all(lwpid_t *, int); 1541 #if defined(__x86) 1542 extern int ___lwp_private(int, int, void *); 1543 #endif /* __x86 */ 1544 1545 /* 1546 * inlines 1547 */ 1548 extern int set_lock_byte(volatile uint8_t *); 1549 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t); 1550 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t); 1551 extern void atomic_inc_32(volatile uint32_t *); 1552 extern void atomic_dec_32(volatile uint32_t *); 1553 extern void atomic_and_32(volatile uint32_t *, uint32_t); 1554 extern void atomic_or_32(volatile uint32_t *, uint32_t); 1555 #if defined(__sparc) 1556 extern ulong_t caller(void); 1557 extern ulong_t getfp(void); 1558 #endif /* __sparc */ 1559 1560 #include "thr_inlines.h" 1561 1562 #endif /* _THR_UBERDATA_H */ 1563