1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/param.h> 30 #include <sys/sysmacros.h> 31 #include <sys/signal.h> 32 #include <sys/stack.h> 33 #include <sys/pcb.h> 34 #include <sys/user.h> 35 #include <sys/systm.h> 36 #include <sys/sysinfo.h> 37 #include <sys/var.h> 38 #include <sys/errno.h> 39 #include <sys/cmn_err.h> 40 #include <sys/cred.h> 41 #include <sys/resource.h> 42 #include <sys/task.h> 43 #include <sys/project.h> 44 #include <sys/proc.h> 45 #include <sys/debug.h> 46 #include <sys/inline.h> 47 #include <sys/disp.h> 48 #include <sys/class.h> 49 #include <vm/seg_kmem.h> 50 #include <vm/seg_kp.h> 51 #include <sys/machlock.h> 52 #include <sys/kmem.h> 53 #include <sys/varargs.h> 54 #include <sys/turnstile.h> 55 #include <sys/poll.h> 56 #include <sys/vtrace.h> 57 #include <sys/callb.h> 58 #include <c2/audit.h> 59 #include <sys/tnf.h> 60 #include <sys/sobject.h> 61 #include <sys/cpupart.h> 62 #include <sys/pset.h> 63 #include <sys/door.h> 64 #include <sys/spl.h> 65 #include <sys/copyops.h> 66 #include <sys/rctl.h> 67 #include <sys/pool.h> 68 #include <sys/zone.h> 69 #include <sys/tsol/label.h> 70 #include <sys/tsol/tndb.h> 71 #include <sys/cpc_impl.h> 72 #include <sys/sdt.h> 73 #include <sys/reboot.h> 74 #include <sys/kdi.h> 75 76 struct kmem_cache *thread_cache; /* cache of free threads */ 77 struct kmem_cache *lwp_cache; /* cache of free lwps */ 78 struct kmem_cache *turnstile_cache; /* cache of free turnstiles */ 79 80 /* 81 * allthreads is only for use by kmem_readers. All kernel loops can use 82 * the current thread as a start/end point. 83 */ 84 static kthread_t *allthreads = &t0; /* circular list of all threads */ 85 86 static kcondvar_t reaper_cv; /* synchronization var */ 87 kthread_t *thread_deathrow; /* circular list of reapable threads */ 88 kthread_t *lwp_deathrow; /* circular list of reapable threads */ 89 kmutex_t reaplock; /* protects lwp and thread deathrows */ 90 kmutex_t thread_free_lock; /* protects clock from reaper */ 91 int thread_reapcnt = 0; /* number of threads on deathrow */ 92 int lwp_reapcnt = 0; /* number of lwps on deathrow */ 93 int reaplimit = 16; /* delay reaping until reaplimit */ 94 95 extern int nthread; 96 97 id_t syscid; /* system scheduling class ID */ 98 void *segkp_thread; /* cookie for segkp pool */ 99 100 int lwp_cache_sz = 32; 101 int t_cache_sz = 8; 102 static kt_did_t next_t_id = 1; 103 104 /* 105 * Min/Max stack sizes for stack size parameters 106 */ 107 #define MAX_STKSIZE (32 * DEFAULTSTKSZ) 108 #define MIN_STKSIZE DEFAULTSTKSZ 109 110 /* 111 * default_stksize overrides lwp_default_stksize if it is set. 112 */ 113 int default_stksize; 114 int lwp_default_stksize; 115 116 static zone_key_t zone_thread_key; 117 118 /* 119 * forward declarations for internal thread specific data (tsd) 120 */ 121 static void *tsd_realloc(void *, size_t, size_t); 122 123 /*ARGSUSED*/ 124 static int 125 turnstile_constructor(void *buf, void *cdrarg, int kmflags) 126 { 127 bzero(buf, sizeof (turnstile_t)); 128 return (0); 129 } 130 131 /*ARGSUSED*/ 132 static void 133 turnstile_destructor(void *buf, void *cdrarg) 134 { 135 turnstile_t *ts = buf; 136 137 ASSERT(ts->ts_free == NULL); 138 ASSERT(ts->ts_waiters == 0); 139 ASSERT(ts->ts_inheritor == NULL); 140 ASSERT(ts->ts_sleepq[0].sq_first == NULL); 141 ASSERT(ts->ts_sleepq[1].sq_first == NULL); 142 } 143 144 void 145 thread_init(void) 146 { 147 kthread_t *tp; 148 extern char sys_name[]; 149 extern void idle(); 150 struct cpu *cpu = CPU; 151 152 mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL)); 153 154 #if defined(__i386) || defined(__amd64) 155 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t), 156 PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); 157 158 /* 159 * "struct _klwp" includes a "struct pcb", which includes a 160 * "struct fpu", which needs to be 16-byte aligned on amd64 161 * (and even on i386 for fxsave/fxrstor). 162 */ 163 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t), 164 16, NULL, NULL, NULL, NULL, NULL, 0); 165 #else 166 /* 167 * Allocate thread structures from static_arena. This prevents 168 * issues where a thread tries to relocate its own thread 169 * structure and touches it after the mapping has been suspended. 170 */ 171 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t), 172 PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0); 173 174 lwp_stk_cache_init(); 175 176 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t), 177 0, NULL, NULL, NULL, NULL, NULL, 0); 178 #endif 179 180 turnstile_cache = kmem_cache_create("turnstile_cache", 181 sizeof (turnstile_t), 0, 182 turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0); 183 184 label_init(); 185 cred_init(); 186 187 rctl_init(); 188 project_init(); 189 zone_init(); 190 task_init(); 191 tcache_init(); 192 pool_init(); 193 194 curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP); 195 196 /* 197 * Originally, we had two parameters to set default stack 198 * size: one for lwp's (lwp_default_stksize), and one for 199 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz). 200 * Now we have a third parameter that overrides both if it is 201 * set to a legal stack size, called default_stksize. 202 */ 203 204 if (default_stksize == 0) { 205 default_stksize = DEFAULTSTKSZ; 206 } else if (default_stksize % PAGESIZE != 0 || 207 default_stksize > MAX_STKSIZE || 208 default_stksize < MIN_STKSIZE) { 209 cmn_err(CE_WARN, "Illegal stack size. Using %d", 210 (int)DEFAULTSTKSZ); 211 default_stksize = DEFAULTSTKSZ; 212 } else { 213 lwp_default_stksize = default_stksize; 214 } 215 216 if (lwp_default_stksize == 0) { 217 lwp_default_stksize = default_stksize; 218 } else if (lwp_default_stksize % PAGESIZE != 0 || 219 lwp_default_stksize > MAX_STKSIZE || 220 lwp_default_stksize < MIN_STKSIZE) { 221 cmn_err(CE_WARN, "Illegal stack size. Using %d", 222 default_stksize); 223 lwp_default_stksize = default_stksize; 224 } 225 226 segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz, 227 lwp_default_stksize, 228 (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED)); 229 230 segkp_thread = segkp_cache_init(segkp, t_cache_sz, 231 default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON); 232 233 (void) getcid(sys_name, &syscid); 234 curthread->t_cid = syscid; /* current thread is t0 */ 235 236 /* 237 * Set up the first CPU's idle thread. 238 * It runs whenever the CPU has nothing worthwhile to do. 239 */ 240 tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1); 241 cpu->cpu_idle_thread = tp; 242 tp->t_preempt = 1; 243 tp->t_disp_queue = cpu->cpu_disp; 244 ASSERT(tp->t_disp_queue != NULL); 245 tp->t_bound_cpu = cpu; 246 tp->t_affinitycnt = 1; 247 248 /* 249 * Registering a thread in the callback table is usually 250 * done in the initialization code of the thread. In this 251 * case, we do it right after thread creation to avoid 252 * blocking idle thread while registering itself. It also 253 * avoids the possibility of reregistration in case a CPU 254 * restarts its idle thread. 255 */ 256 CALLB_CPR_INIT_SAFE(tp, "idle"); 257 258 /* 259 * Finish initializing the kernel memory allocator now that 260 * thread_create() is available. 261 */ 262 kmem_thread_init(); 263 264 if (boothowto & RB_DEBUG) 265 kdi_dvec_thravail(); 266 } 267 268 /* 269 * Create a thread. 270 * 271 * thread_create() blocks for memory if necessary. It never fails. 272 * 273 * If stk is NULL, the thread is created at the base of the stack 274 * and cannot be swapped. 275 */ 276 kthread_t * 277 thread_create( 278 caddr_t stk, 279 size_t stksize, 280 void (*proc)(), 281 void *arg, 282 size_t len, 283 proc_t *pp, 284 int state, 285 pri_t pri) 286 { 287 kthread_t *t; 288 extern struct classfuncs sys_classfuncs; 289 turnstile_t *ts; 290 291 /* 292 * Every thread keeps a turnstile around in case it needs to block. 293 * The only reason the turnstile is not simply part of the thread 294 * structure is that we may have to break the association whenever 295 * more than one thread blocks on a given synchronization object. 296 * From a memory-management standpoint, turnstiles are like the 297 * "attached mblks" that hang off dblks in the streams allocator. 298 */ 299 ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP); 300 301 if (stk == NULL) { 302 /* 303 * alloc both thread and stack in segkp chunk 304 */ 305 306 if (stksize < default_stksize) 307 stksize = default_stksize; 308 309 if (stksize == default_stksize) { 310 stk = (caddr_t)segkp_cache_get(segkp_thread); 311 } else { 312 stksize = roundup(stksize, PAGESIZE); 313 stk = (caddr_t)segkp_get(segkp, stksize, 314 (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED)); 315 } 316 317 ASSERT(stk != NULL); 318 319 /* 320 * The machine-dependent mutex code may require that 321 * thread pointers (since they may be used for mutex owner 322 * fields) have certain alignment requirements. 323 * PTR24_ALIGN is the size of the alignment quanta. 324 * XXX - assumes stack grows toward low addresses. 325 */ 326 if (stksize <= sizeof (kthread_t) + PTR24_ALIGN) 327 cmn_err(CE_PANIC, "thread_create: proposed stack size" 328 " too small to hold thread."); 329 #ifdef STACK_GROWTH_DOWN 330 stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1); 331 stksize &= -PTR24_ALIGN; /* make thread aligned */ 332 t = (kthread_t *)(stk + stksize); 333 bzero(t, sizeof (kthread_t)); 334 #ifdef C2_AUDIT 335 if (audit_active) 336 audit_thread_create(t); 337 #endif 338 t->t_stk = stk + stksize; 339 t->t_stkbase = stk; 340 #else /* stack grows to larger addresses */ 341 stksize -= SA(sizeof (kthread_t)); 342 t = (kthread_t *)(stk); 343 bzero(t, sizeof (kthread_t)); 344 t->t_stk = stk + sizeof (kthread_t); 345 t->t_stkbase = stk + stksize + sizeof (kthread_t); 346 #endif /* STACK_GROWTH_DOWN */ 347 t->t_flag |= T_TALLOCSTK; 348 t->t_swap = stk; 349 } else { 350 t = kmem_cache_alloc(thread_cache, KM_SLEEP); 351 bzero(t, sizeof (kthread_t)); 352 ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0); 353 #ifdef C2_AUDIT 354 if (audit_active) 355 audit_thread_create(t); 356 #endif 357 /* 358 * Initialize t_stk to the kernel stack pointer to use 359 * upon entry to the kernel 360 */ 361 #ifdef STACK_GROWTH_DOWN 362 t->t_stk = stk + stksize; 363 t->t_stkbase = stk; 364 #else 365 t->t_stk = stk; /* 3b2-like */ 366 t->t_stkbase = stk + stksize; 367 #endif /* STACK_GROWTH_DOWN */ 368 } 369 370 /* set default stack flag */ 371 if (stksize == lwp_default_stksize) 372 t->t_flag |= T_DFLTSTK; 373 374 t->t_ts = ts; 375 376 /* 377 * p_cred could be NULL if it thread_create is called before cred_init 378 * is called in main. 379 */ 380 mutex_enter(&pp->p_crlock); 381 if (pp->p_cred) 382 crhold(t->t_cred = pp->p_cred); 383 mutex_exit(&pp->p_crlock); 384 t->t_start = gethrestime_sec(); 385 t->t_startpc = proc; 386 t->t_procp = pp; 387 t->t_clfuncs = &sys_classfuncs.thread; 388 t->t_cid = syscid; 389 t->t_pri = pri; 390 t->t_stime = lbolt; 391 t->t_schedflag = TS_LOAD | TS_DONT_SWAP; 392 t->t_bind_cpu = PBIND_NONE; 393 t->t_bind_pset = PS_NONE; 394 t->t_plockp = &pp->p_lock; 395 t->t_copyops = NULL; 396 t->t_taskq = NULL; 397 t->t_anttime = 0; 398 t->t_hatdepth = 0; 399 400 t->t_dtrace_vtime = 1; /* assure vtimestamp is always non-zero */ 401 402 CPU_STATS_ADDQ(CPU, sys, nthreads, 1); 403 #ifndef NPROBE 404 /* Kernel probe */ 405 tnf_thread_create(t); 406 #endif /* NPROBE */ 407 LOCK_INIT_CLEAR(&t->t_lock); 408 409 /* 410 * Callers who give us a NULL proc must do their own 411 * stack initialization. e.g. lwp_create() 412 */ 413 if (proc != NULL) { 414 t->t_stk = thread_stk_init(t->t_stk); 415 thread_load(t, proc, arg, len); 416 } 417 418 /* 419 * Put a hold on project0. If this thread is actually in a 420 * different project, then t_proj will be changed later in 421 * lwp_create(). All kernel-only threads must be in project 0. 422 */ 423 t->t_proj = project_hold(proj0p); 424 425 lgrp_affinity_init(&t->t_lgrp_affinity); 426 427 mutex_enter(&pidlock); 428 nthread++; 429 t->t_did = next_t_id++; 430 t->t_prev = curthread->t_prev; 431 t->t_next = curthread; 432 433 /* 434 * Add the thread to the list of all threads, and initialize 435 * its t_cpu pointer. We need to block preemption since 436 * cpu_offline walks the thread list looking for threads 437 * with t_cpu pointing to the CPU being offlined. We want 438 * to make sure that the list is consistent and that if t_cpu 439 * is set, the thread is on the list. 440 */ 441 kpreempt_disable(); 442 curthread->t_prev->t_next = t; 443 curthread->t_prev = t; 444 445 /* 446 * Threads should never have a NULL t_cpu pointer so assign it 447 * here. If the thread is being created with state TS_RUN a 448 * better CPU may be chosen when it is placed on the run queue. 449 * 450 * We need to keep kernel preemption disabled when setting all 451 * three fields to keep them in sync. Also, always create in 452 * the default partition since that's where kernel threads go 453 * (if this isn't a kernel thread, t_cpupart will be changed 454 * in lwp_create before setting the thread runnable). 455 */ 456 t->t_cpupart = &cp_default; 457 458 /* 459 * For now, affiliate this thread with the root lgroup. 460 * Since the kernel does not (presently) allocate its memory 461 * in a locality aware fashion, the root is an appropriate home. 462 * If this thread is later associated with an lwp, it will have 463 * it's lgroup re-assigned at that time. 464 */ 465 lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1); 466 467 /* 468 * Inherit the current cpu. If this cpu isn't part of the chosen 469 * lgroup, a new cpu will be chosen by cpu_choose when the thread 470 * is ready to run. 471 */ 472 if (CPU->cpu_part == &cp_default) 473 t->t_cpu = CPU; 474 else 475 t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t->t_lpl, 476 t->t_pri, NULL); 477 478 t->t_disp_queue = t->t_cpu->cpu_disp; 479 kpreempt_enable(); 480 481 /* 482 * Initialize thread state and the dispatcher lock pointer. 483 * Need to hold onto pidlock to block allthreads walkers until 484 * the state is set. 485 */ 486 switch (state) { 487 case TS_RUN: 488 curthread->t_oldspl = splhigh(); /* get dispatcher spl */ 489 THREAD_SET_STATE(t, TS_STOPPED, &transition_lock); 490 CL_SETRUN(t); 491 thread_unlock(t); 492 break; 493 494 case TS_ONPROC: 495 THREAD_ONPROC(t, t->t_cpu); 496 break; 497 498 case TS_FREE: 499 /* 500 * Free state will be used for intr threads. 501 * The interrupt routine must set the thread dispatcher 502 * lock pointer (t_lockp) if starting on a CPU 503 * other than the current one. 504 */ 505 THREAD_FREEINTR(t, CPU); 506 break; 507 508 case TS_STOPPED: 509 THREAD_SET_STATE(t, TS_STOPPED, &stop_lock); 510 break; 511 512 default: /* TS_SLEEP, TS_ZOMB or TS_TRANS */ 513 cmn_err(CE_PANIC, "thread_create: invalid state %d", state); 514 } 515 mutex_exit(&pidlock); 516 return (t); 517 } 518 519 /* 520 * Move thread to project0 and take care of project reference counters. 521 */ 522 void 523 thread_rele(kthread_t *t) 524 { 525 kproject_t *kpj; 526 527 thread_lock(t); 528 529 ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0); 530 kpj = ttoproj(t); 531 t->t_proj = proj0p; 532 533 thread_unlock(t); 534 535 if (kpj != proj0p) { 536 project_rele(kpj); 537 (void) project_hold(proj0p); 538 } 539 } 540 541 542 void (*ip_cleanup_func)(void); 543 544 void 545 thread_exit() 546 { 547 kthread_t *t = curthread; 548 549 if ((t->t_proc_flag & TP_ZTHREAD) != 0) 550 cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called"); 551 552 if (ip_cleanup_func != NULL) 553 (*ip_cleanup_func)(); 554 555 tsd_exit(); /* Clean up this thread's TSD */ 556 557 kcpc_passivate(); /* clean up performance counter state */ 558 559 /* 560 * No kernel thread should have called poll() without arranging 561 * calling pollcleanup() here. 562 */ 563 ASSERT(t->t_pollstate == NULL); 564 ASSERT(t->t_schedctl == NULL); 565 if (t->t_door) 566 door_slam(); /* in case thread did an upcall */ 567 568 #ifndef NPROBE 569 /* Kernel probe */ 570 if (t->t_tnf_tpdp) 571 tnf_thread_exit(); 572 #endif /* NPROBE */ 573 574 thread_rele(t); 575 t->t_preempt++; 576 577 /* 578 * remove thread from the all threads list so that 579 * death-row can use the same pointers. 580 */ 581 mutex_enter(&pidlock); 582 t->t_next->t_prev = t->t_prev; 583 t->t_prev->t_next = t->t_next; 584 ASSERT(allthreads != t); /* t0 never exits */ 585 cv_broadcast(&t->t_joincv); /* wake up anyone in thread_join */ 586 mutex_exit(&pidlock); 587 588 if (t->t_ctx != NULL) 589 exitctx(t); 590 if (t->t_procp->p_pctx != NULL) 591 exitpctx(t->t_procp); 592 593 t->t_state = TS_ZOMB; /* set zombie thread */ 594 595 swtch_from_zombie(); /* give up the CPU */ 596 /* NOTREACHED */ 597 } 598 599 /* 600 * Check to see if the specified thread is active (defined as being on 601 * the thread list). This is certainly a slow way to do this; if there's 602 * ever a reason to speed it up, we could maintain a hash table of active 603 * threads indexed by their t_did. 604 */ 605 static kthread_t * 606 did_to_thread(kt_did_t tid) 607 { 608 kthread_t *t; 609 610 ASSERT(MUTEX_HELD(&pidlock)); 611 for (t = curthread->t_next; t != curthread; t = t->t_next) { 612 if (t->t_did == tid) 613 break; 614 } 615 if (t->t_did == tid) 616 return (t); 617 else 618 return (NULL); 619 } 620 621 /* 622 * Wait for specified thread to exit. Returns immediately if the thread 623 * could not be found, meaning that it has either already exited or never 624 * existed. 625 */ 626 void 627 thread_join(kt_did_t tid) 628 { 629 kthread_t *t; 630 631 ASSERT(tid != curthread->t_did); 632 ASSERT(tid != t0.t_did); 633 634 mutex_enter(&pidlock); 635 /* 636 * Make sure we check that the thread is on the thread list 637 * before blocking on it; otherwise we could end up blocking on 638 * a cv that's already been freed. In other words, don't cache 639 * the thread pointer across calls to cv_wait. 640 * 641 * The choice of loop invariant means that whenever a thread 642 * is taken off the allthreads list, a cv_broadcast must be 643 * performed on that thread's t_joincv to wake up any waiters. 644 * The broadcast doesn't have to happen right away, but it 645 * shouldn't be postponed indefinitely (e.g., by doing it in 646 * thread_free which may only be executed when the deathrow 647 * queue is processed. 648 */ 649 while (t = did_to_thread(tid)) 650 cv_wait(&t->t_joincv, &pidlock); 651 mutex_exit(&pidlock); 652 } 653 654 void 655 thread_free(kthread_t *t) 656 { 657 ASSERT(t != &t0 && t->t_state == TS_FREE); 658 ASSERT(t->t_door == NULL); 659 ASSERT(t->t_schedctl == NULL); 660 ASSERT(t->t_pollstate == NULL); 661 662 t->t_pri = 0; 663 t->t_pc = 0; 664 t->t_sp = 0; 665 t->t_wchan0 = NULL; 666 t->t_wchan = NULL; 667 if (t->t_cred != NULL) { 668 crfree(t->t_cred); 669 t->t_cred = 0; 670 } 671 if (t->t_pdmsg) { 672 kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1); 673 t->t_pdmsg = NULL; 674 } 675 #ifdef C2_AUDIT 676 if (audit_active) 677 audit_thread_free(t); 678 #endif 679 #ifndef NPROBE 680 if (t->t_tnf_tpdp) 681 tnf_thread_free(t); 682 #endif /* NPROBE */ 683 if (t->t_cldata) { 684 CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata); 685 } 686 if (t->t_rprof != NULL) { 687 kmem_free(t->t_rprof, sizeof (*t->t_rprof)); 688 t->t_rprof = NULL; 689 } 690 t->t_lockp = NULL; /* nothing should try to lock this thread now */ 691 if (t->t_lwp) 692 lwp_freeregs(t->t_lwp, 0); 693 if (t->t_ctx) 694 freectx(t, 0); 695 if (t->t_procp->p_pctx) 696 freepctx(t->t_procp, 0); 697 t->t_stk = NULL; 698 if (t->t_lwp) 699 lwp_stk_fini(t->t_lwp); 700 lock_clear(&t->t_lock); 701 702 if (t->t_ts->ts_waiters > 0) 703 panic("thread_free: turnstile still active"); 704 705 kmem_cache_free(turnstile_cache, t->t_ts); 706 707 free_afd(&t->t_activefd); 708 709 /* 710 * Barrier for clock thread. The clock holds this lock to 711 * keep the thread from going away while it's looking at it. 712 */ 713 mutex_enter(&thread_free_lock); 714 mutex_exit(&thread_free_lock); 715 716 ASSERT(ttoproj(t) == proj0p); 717 project_rele(ttoproj(t)); 718 719 lgrp_affinity_free(&t->t_lgrp_affinity); 720 721 /* 722 * Free thread struct and its stack. 723 */ 724 if (t->t_flag & T_TALLOCSTK) { 725 /* thread struct is embedded in stack */ 726 segkp_release(segkp, t->t_swap); 727 mutex_enter(&pidlock); 728 nthread--; 729 mutex_exit(&pidlock); 730 } else { 731 if (t->t_swap) { 732 segkp_release(segkp, t->t_swap); 733 t->t_swap = NULL; 734 } 735 if (t->t_lwp) { 736 kmem_cache_free(lwp_cache, t->t_lwp); 737 t->t_lwp = NULL; 738 } 739 mutex_enter(&pidlock); 740 nthread--; 741 mutex_exit(&pidlock); 742 kmem_cache_free(thread_cache, t); 743 } 744 } 745 746 /* 747 * Removes threads associated with the given zone from a deathrow queue. 748 * tp is a pointer to the head of the deathrow queue, and countp is a 749 * pointer to the current deathrow count. Returns a linked list of 750 * threads removed from the list. 751 */ 752 static kthread_t * 753 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid) 754 { 755 kthread_t *tmp, *list = NULL; 756 cred_t *cr; 757 758 ASSERT(MUTEX_HELD(&reaplock)); 759 while (*tp != NULL) { 760 if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) { 761 tmp = *tp; 762 *tp = tmp->t_forw; 763 tmp->t_forw = list; 764 list = tmp; 765 (*countp)--; 766 } else { 767 tp = &(*tp)->t_forw; 768 } 769 } 770 return (list); 771 } 772 773 static void 774 thread_reap_list(kthread_t *t) 775 { 776 kthread_t *next; 777 778 while (t != NULL) { 779 next = t->t_forw; 780 thread_free(t); 781 t = next; 782 } 783 } 784 785 /* ARGSUSED */ 786 static void 787 thread_zone_destroy(zoneid_t zoneid, void *unused) 788 { 789 kthread_t *t, *l; 790 791 mutex_enter(&reaplock); 792 /* 793 * Pull threads and lwps associated with zone off deathrow lists. 794 */ 795 t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid); 796 l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid); 797 mutex_exit(&reaplock); 798 799 /* 800 * Reap threads 801 */ 802 thread_reap_list(t); 803 804 /* 805 * Reap lwps 806 */ 807 thread_reap_list(l); 808 } 809 810 /* 811 * cleanup zombie threads that are on deathrow. 812 */ 813 void 814 thread_reaper() 815 { 816 kthread_t *t, *l; 817 callb_cpr_t cprinfo; 818 819 /* 820 * Register callback to clean up threads when zone is destroyed. 821 */ 822 zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy); 823 824 CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper"); 825 for (;;) { 826 mutex_enter(&reaplock); 827 while (thread_deathrow == NULL && lwp_deathrow == NULL) { 828 CALLB_CPR_SAFE_BEGIN(&cprinfo); 829 cv_wait(&reaper_cv, &reaplock); 830 CALLB_CPR_SAFE_END(&cprinfo, &reaplock); 831 } 832 t = thread_deathrow; 833 l = lwp_deathrow; 834 thread_deathrow = NULL; 835 lwp_deathrow = NULL; 836 thread_reapcnt = 0; 837 lwp_reapcnt = 0; 838 mutex_exit(&reaplock); 839 840 /* 841 * Reap threads 842 */ 843 thread_reap_list(t); 844 845 /* 846 * Reap lwps 847 */ 848 thread_reap_list(l); 849 } 850 } 851 852 /* 853 * This is called by resume() to put a zombie thread onto deathrow. 854 * The thread's state is changed to TS_FREE to indicate that is reapable. 855 * This is called from the idle thread so it must not block (just spin). 856 */ 857 void 858 reapq_add(kthread_t *t) 859 { 860 mutex_enter(&reaplock); 861 862 /* 863 * lwp_deathrow contains only threads with lwp linkage 864 * that are of the default stacksize. Anything else goes 865 * on thread_deathrow. 866 */ 867 if (ttolwp(t) && (t->t_flag & T_DFLTSTK)) { 868 t->t_forw = lwp_deathrow; 869 lwp_deathrow = t; 870 lwp_reapcnt++; 871 } else { 872 t->t_forw = thread_deathrow; 873 thread_deathrow = t; 874 thread_reapcnt++; 875 } 876 if (lwp_reapcnt + thread_reapcnt > reaplimit) 877 cv_signal(&reaper_cv); /* wake the reaper */ 878 t->t_state = TS_FREE; 879 lock_clear(&t->t_lock); 880 mutex_exit(&reaplock); 881 } 882 883 /* 884 * Install thread context ops for the current thread. 885 */ 886 void 887 installctx( 888 kthread_t *t, 889 void *arg, 890 void (*save)(void *), 891 void (*restore)(void *), 892 void (*fork)(void *, void *), 893 void (*lwp_create)(void *, void *), 894 void (*exit)(void *), 895 void (*free)(void *, int)) 896 { 897 struct ctxop *ctx; 898 899 ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP); 900 ctx->save_op = save; 901 ctx->restore_op = restore; 902 ctx->fork_op = fork; 903 ctx->lwp_create_op = lwp_create; 904 ctx->exit_op = exit; 905 ctx->free_op = free; 906 ctx->arg = arg; 907 ctx->next = t->t_ctx; 908 t->t_ctx = ctx; 909 } 910 911 /* 912 * Remove thread context ops from the current thread. 913 * (Or allow the agent thread to remove thread context ops from another 914 * thread in the same process) 915 */ 916 int 917 removectx( 918 kthread_t *t, 919 void *arg, 920 void (*save)(void *), 921 void (*restore)(void *), 922 void (*fork)(void *, void *), 923 void (*lwp_create)(void *, void *), 924 void (*exit)(void *), 925 void (*free)(void *, int)) 926 { 927 struct ctxop *ctx, *prev_ctx; 928 929 ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL || 930 ttoproc(t)->p_agenttp == curthread); 931 932 /* 933 * There's a potential race for t_ctx between the agent thread 934 * and the target thread when lwps are exiting (for example, 935 * when the process is reacting to having been killed). At 936 * other times, the target thread will be TS_STOPPED whilst the 937 * agent thread is inside this function. However, from the 938 * perspective of the cost of locking, it seems cheaper to take 939 * a thread-specific lock everytime we come through here. 940 */ 941 mutex_enter(&t->t_ctx_lock); 942 prev_ctx = NULL; 943 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) { 944 if (ctx->save_op == save && ctx->restore_op == restore && 945 ctx->fork_op == fork && ctx->lwp_create_op == lwp_create && 946 ctx->exit_op == exit && ctx->free_op == free && 947 ctx->arg == arg) { 948 if (prev_ctx) 949 prev_ctx->next = ctx->next; 950 else 951 t->t_ctx = ctx->next; 952 mutex_exit(&t->t_ctx_lock); 953 if (ctx->free_op != NULL) 954 (ctx->free_op)(ctx->arg, 0); 955 kmem_free(ctx, sizeof (struct ctxop)); 956 return (1); 957 } 958 prev_ctx = ctx; 959 } 960 mutex_exit(&t->t_ctx_lock); 961 962 return (0); 963 } 964 965 void 966 savectx(kthread_t *t) 967 { 968 struct ctxop *ctx; 969 970 ASSERT(t == curthread); 971 for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next) 972 if (ctx->save_op != NULL) 973 (ctx->save_op)(ctx->arg); 974 } 975 976 void 977 restorectx(kthread_t *t) 978 { 979 struct ctxop *ctx; 980 981 ASSERT(t == curthread); 982 for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next) 983 if (ctx->restore_op != NULL) 984 (ctx->restore_op)(ctx->arg); 985 } 986 987 void 988 forkctx(kthread_t *t, kthread_t *ct) 989 { 990 struct ctxop *ctx; 991 992 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) 993 if (ctx->fork_op != NULL) 994 (ctx->fork_op)(t, ct); 995 } 996 997 /* 998 * Note that this operator is only invoked via the _lwp_create 999 * system call. The system may have other reasons to create lwps 1000 * e.g. the agent lwp or the doors unreferenced lwp. 1001 */ 1002 void 1003 lwp_createctx(kthread_t *t, kthread_t *ct) 1004 { 1005 struct ctxop *ctx; 1006 1007 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) 1008 if (ctx->lwp_create_op != NULL) 1009 (ctx->lwp_create_op)(t, ct); 1010 } 1011 1012 /* 1013 * exitctx is called from thread_exit() and lwp_exit() to perform any actions 1014 * needed when the thread/LWP leaves the processor for the last time. This 1015 * routine is not intended to deal with freeing memory; freectx() is used for 1016 * that purpose during thread_free(). This routine is provided to allow for 1017 * clean-up that can't wait until thread_free(). 1018 */ 1019 void 1020 exitctx(kthread_t *t) 1021 { 1022 struct ctxop *ctx; 1023 1024 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) 1025 if (ctx->exit_op != NULL) 1026 (ctx->exit_op)(t); 1027 } 1028 1029 /* 1030 * freectx is called from thread_free() and exec() to get 1031 * rid of old thread context ops. 1032 */ 1033 void 1034 freectx(kthread_t *t, int isexec) 1035 { 1036 struct ctxop *ctx; 1037 1038 while ((ctx = t->t_ctx) != NULL) { 1039 t->t_ctx = ctx->next; 1040 if (ctx->free_op != NULL) 1041 (ctx->free_op)(ctx->arg, isexec); 1042 kmem_free(ctx, sizeof (struct ctxop)); 1043 } 1044 } 1045 1046 /* 1047 * Set the thread running; arrange for it to be swapped in if necessary. 1048 */ 1049 void 1050 setrun_locked(kthread_t *t) 1051 { 1052 ASSERT(THREAD_LOCK_HELD(t)); 1053 if (t->t_state == TS_SLEEP) { 1054 /* 1055 * Take off sleep queue. 1056 */ 1057 SOBJ_UNSLEEP(t->t_sobj_ops, t); 1058 } else if (t->t_state & (TS_RUN | TS_ONPROC)) { 1059 /* 1060 * Already on dispatcher queue. 1061 */ 1062 return; 1063 } else if (t->t_state == TS_STOPPED) { 1064 /* 1065 * All of the sending of SIGCONT (TC_XSTART) and /proc 1066 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have 1067 * requested that the thread be run. 1068 * Just calling setrun() is not sufficient to set a stopped 1069 * thread running. TP_TXSTART is always set if the thread 1070 * is not stopped by a jobcontrol stop signal. 1071 * TP_TPSTART is always set if /proc is not controlling it. 1072 * TP_TCSTART is always set if lwp_suspend() didn't stop it. 1073 * The thread won't be stopped unless one of these 1074 * three mechanisms did it. 1075 * 1076 * These flags must be set before calling setrun_locked(t). 1077 * They can't be passed as arguments because the streams 1078 * code calls setrun() indirectly and the mechanism for 1079 * doing so admits only one argument. Note that the 1080 * thread must be locked in order to change t_schedflags. 1081 */ 1082 if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART) 1083 return; 1084 /* 1085 * Process is no longer stopped (a thread is running). 1086 */ 1087 t->t_whystop = 0; 1088 t->t_whatstop = 0; 1089 /* 1090 * Strictly speaking, we do not have to clear these 1091 * flags here; they are cleared on entry to stop(). 1092 * However, they are confusing when doing kernel 1093 * debugging or when they are revealed by ps(1). 1094 */ 1095 t->t_schedflag &= ~TS_ALLSTART; 1096 THREAD_TRANSITION(t); /* drop stopped-thread lock */ 1097 ASSERT(t->t_lockp == &transition_lock); 1098 ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL); 1099 /* 1100 * Let the class put the process on the dispatcher queue. 1101 */ 1102 CL_SETRUN(t); 1103 } 1104 1105 1106 } 1107 1108 void 1109 setrun(kthread_t *t) 1110 { 1111 thread_lock(t); 1112 setrun_locked(t); 1113 thread_unlock(t); 1114 } 1115 1116 /* 1117 * Unpin an interrupted thread. 1118 * When an interrupt occurs, the interrupt is handled on the stack 1119 * of an interrupt thread, taken from a pool linked to the CPU structure. 1120 * 1121 * When swtch() is switching away from an interrupt thread because it 1122 * blocked or was preempted, this routine is called to complete the 1123 * saving of the interrupted thread state, and returns the interrupted 1124 * thread pointer so it may be resumed. 1125 * 1126 * Called by swtch() only at high spl. 1127 */ 1128 kthread_t * 1129 thread_unpin() 1130 { 1131 kthread_t *t = curthread; /* current thread */ 1132 kthread_t *itp; /* interrupted thread */ 1133 int i; /* interrupt level */ 1134 extern int intr_passivate(); 1135 1136 ASSERT(t->t_intr != NULL); 1137 1138 itp = t->t_intr; /* interrupted thread */ 1139 t->t_intr = NULL; /* clear interrupt ptr */ 1140 1141 /* 1142 * Get state from interrupt thread for the one 1143 * it interrupted. 1144 */ 1145 1146 i = intr_passivate(t, itp); 1147 1148 TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE, 1149 "intr_passivate:level %d curthread %p (%T) ithread %p (%T)", 1150 i, t, t, itp, itp); 1151 1152 /* 1153 * Dissociate the current thread from the interrupted thread's LWP. 1154 */ 1155 t->t_lwp = NULL; 1156 1157 /* 1158 * Interrupt handlers above the level that spinlocks block must 1159 * not block. 1160 */ 1161 #if DEBUG 1162 if (i < 0 || i > LOCK_LEVEL) 1163 cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i); 1164 #endif 1165 1166 /* 1167 * Compute the CPU's base interrupt level based on the active 1168 * interrupts. 1169 */ 1170 ASSERT(CPU->cpu_intr_actv & (1 << i)); 1171 set_base_spl(); 1172 1173 return (itp); 1174 } 1175 1176 /* 1177 * Create and initialize an interrupt thread. 1178 * Returns non-zero on error. 1179 * Called at spl7() or better. 1180 */ 1181 void 1182 thread_create_intr(struct cpu *cp) 1183 { 1184 kthread_t *tp; 1185 1186 tp = thread_create(NULL, 0, 1187 (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0); 1188 1189 /* 1190 * Set the thread in the TS_FREE state. The state will change 1191 * to TS_ONPROC only while the interrupt is active. Think of these 1192 * as being on a private free list for the CPU. Being TS_FREE keeps 1193 * inactive interrupt threads out of debugger thread lists. 1194 * 1195 * We cannot call thread_create with TS_FREE because of the current 1196 * checks there for ONPROC. Fix this when thread_create takes flags. 1197 */ 1198 THREAD_FREEINTR(tp, cp); 1199 1200 /* 1201 * Nobody should ever reference the credentials of an interrupt 1202 * thread so make it NULL to catch any such references. 1203 */ 1204 tp->t_cred = NULL; 1205 tp->t_flag |= T_INTR_THREAD; 1206 tp->t_cpu = cp; 1207 tp->t_bound_cpu = cp; 1208 tp->t_disp_queue = cp->cpu_disp; 1209 tp->t_affinitycnt = 1; 1210 tp->t_preempt = 1; 1211 1212 /* 1213 * Don't make a user-requested binding on this thread so that 1214 * the processor can be offlined. 1215 */ 1216 tp->t_bind_cpu = PBIND_NONE; /* no USER-requested binding */ 1217 tp->t_bind_pset = PS_NONE; 1218 1219 #if defined(__i386) || defined(__amd64) 1220 tp->t_stk -= STACK_ALIGN; 1221 *(tp->t_stk) = 0; /* terminate intr thread stack */ 1222 #endif 1223 1224 /* 1225 * Link onto CPU's interrupt pool. 1226 */ 1227 tp->t_link = cp->cpu_intr_thread; 1228 cp->cpu_intr_thread = tp; 1229 } 1230 1231 /* 1232 * TSD -- THREAD SPECIFIC DATA 1233 */ 1234 static kmutex_t tsd_mutex; /* linked list spin lock */ 1235 static uint_t tsd_nkeys; /* size of destructor array */ 1236 /* per-key destructor funcs */ 1237 static void (**tsd_destructor)(void *); 1238 /* list of tsd_thread's */ 1239 static struct tsd_thread *tsd_list; 1240 1241 /* 1242 * Default destructor 1243 * Needed because NULL destructor means that the key is unused 1244 */ 1245 /* ARGSUSED */ 1246 void 1247 tsd_defaultdestructor(void *value) 1248 {} 1249 1250 /* 1251 * Create a key (index into per thread array) 1252 * Locks out tsd_create, tsd_destroy, and tsd_exit 1253 * May allocate memory with lock held 1254 */ 1255 void 1256 tsd_create(uint_t *keyp, void (*destructor)(void *)) 1257 { 1258 int i; 1259 uint_t nkeys; 1260 1261 /* 1262 * if key is allocated, do nothing 1263 */ 1264 mutex_enter(&tsd_mutex); 1265 if (*keyp) { 1266 mutex_exit(&tsd_mutex); 1267 return; 1268 } 1269 /* 1270 * find an unused key 1271 */ 1272 if (destructor == NULL) 1273 destructor = tsd_defaultdestructor; 1274 1275 for (i = 0; i < tsd_nkeys; ++i) 1276 if (tsd_destructor[i] == NULL) 1277 break; 1278 1279 /* 1280 * if no unused keys, increase the size of the destructor array 1281 */ 1282 if (i == tsd_nkeys) { 1283 if ((nkeys = (tsd_nkeys << 1)) == 0) 1284 nkeys = 1; 1285 tsd_destructor = 1286 (void (**)(void *))tsd_realloc((void *)tsd_destructor, 1287 (size_t)(tsd_nkeys * sizeof (void (*)(void *))), 1288 (size_t)(nkeys * sizeof (void (*)(void *)))); 1289 tsd_nkeys = nkeys; 1290 } 1291 1292 /* 1293 * allocate the next available unused key 1294 */ 1295 tsd_destructor[i] = destructor; 1296 *keyp = i + 1; 1297 mutex_exit(&tsd_mutex); 1298 } 1299 1300 /* 1301 * Destroy a key -- this is for unloadable modules 1302 * 1303 * Assumes that the caller is preventing tsd_set and tsd_get 1304 * Locks out tsd_create, tsd_destroy, and tsd_exit 1305 * May free memory with lock held 1306 */ 1307 void 1308 tsd_destroy(uint_t *keyp) 1309 { 1310 uint_t key; 1311 struct tsd_thread *tsd; 1312 1313 /* 1314 * protect the key namespace and our destructor lists 1315 */ 1316 mutex_enter(&tsd_mutex); 1317 key = *keyp; 1318 *keyp = 0; 1319 1320 ASSERT(key <= tsd_nkeys); 1321 1322 /* 1323 * if the key is valid 1324 */ 1325 if (key != 0) { 1326 uint_t k = key - 1; 1327 /* 1328 * for every thread with TSD, call key's destructor 1329 */ 1330 for (tsd = tsd_list; tsd; tsd = tsd->ts_next) { 1331 /* 1332 * no TSD for key in this thread 1333 */ 1334 if (key > tsd->ts_nkeys) 1335 continue; 1336 /* 1337 * call destructor for key 1338 */ 1339 if (tsd->ts_value[k] && tsd_destructor[k]) 1340 (*tsd_destructor[k])(tsd->ts_value[k]); 1341 /* 1342 * reset value for key 1343 */ 1344 tsd->ts_value[k] = NULL; 1345 } 1346 /* 1347 * actually free the key (NULL destructor == unused) 1348 */ 1349 tsd_destructor[k] = NULL; 1350 } 1351 1352 mutex_exit(&tsd_mutex); 1353 } 1354 1355 /* 1356 * Quickly return the per thread value that was stored with the specified key 1357 * Assumes the caller is protecting key from tsd_create and tsd_destroy 1358 */ 1359 void * 1360 tsd_get(uint_t key) 1361 { 1362 return (tsd_agent_get(curthread, key)); 1363 } 1364 1365 /* 1366 * Set a per thread value indexed with the specified key 1367 */ 1368 int 1369 tsd_set(uint_t key, void *value) 1370 { 1371 return (tsd_agent_set(curthread, key, value)); 1372 } 1373 1374 /* 1375 * Like tsd_get(), except that the agent lwp can get the tsd of 1376 * another thread in the same process (the agent thread only runs when the 1377 * process is completely stopped by /proc), or syslwp is creating a new lwp. 1378 */ 1379 void * 1380 tsd_agent_get(kthread_t *t, uint_t key) 1381 { 1382 struct tsd_thread *tsd = t->t_tsd; 1383 1384 ASSERT(t == curthread || 1385 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED); 1386 1387 if (key && tsd != NULL && key <= tsd->ts_nkeys) 1388 return (tsd->ts_value[key - 1]); 1389 return (NULL); 1390 } 1391 1392 /* 1393 * Like tsd_set(), except that the agent lwp can set the tsd of 1394 * another thread in the same process, or syslwp can set the tsd 1395 * of a thread it's in the middle of creating. 1396 * 1397 * Assumes the caller is protecting key from tsd_create and tsd_destroy 1398 * May lock out tsd_destroy (and tsd_create), may allocate memory with 1399 * lock held 1400 */ 1401 int 1402 tsd_agent_set(kthread_t *t, uint_t key, void *value) 1403 { 1404 struct tsd_thread *tsd = t->t_tsd; 1405 1406 ASSERT(t == curthread || 1407 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED); 1408 1409 if (key == 0) 1410 return (EINVAL); 1411 if (tsd == NULL) 1412 tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP); 1413 if (key <= tsd->ts_nkeys) { 1414 tsd->ts_value[key - 1] = value; 1415 return (0); 1416 } 1417 1418 ASSERT(key <= tsd_nkeys); 1419 1420 /* 1421 * lock out tsd_destroy() 1422 */ 1423 mutex_enter(&tsd_mutex); 1424 if (tsd->ts_nkeys == 0) { 1425 /* 1426 * Link onto list of threads with TSD 1427 */ 1428 if ((tsd->ts_next = tsd_list) != NULL) 1429 tsd_list->ts_prev = tsd; 1430 tsd_list = tsd; 1431 } 1432 1433 /* 1434 * Allocate thread local storage and set the value for key 1435 */ 1436 tsd->ts_value = tsd_realloc(tsd->ts_value, 1437 tsd->ts_nkeys * sizeof (void *), 1438 key * sizeof (void *)); 1439 tsd->ts_nkeys = key; 1440 tsd->ts_value[key - 1] = value; 1441 mutex_exit(&tsd_mutex); 1442 1443 return (0); 1444 } 1445 1446 1447 /* 1448 * Return the per thread value that was stored with the specified key 1449 * If necessary, create the key and the value 1450 * Assumes the caller is protecting *keyp from tsd_destroy 1451 */ 1452 void * 1453 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void)) 1454 { 1455 void *value; 1456 uint_t key = *keyp; 1457 struct tsd_thread *tsd = curthread->t_tsd; 1458 1459 if (tsd == NULL) 1460 tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP); 1461 if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1])) 1462 return (value); 1463 if (key == 0) 1464 tsd_create(keyp, destroy); 1465 (void) tsd_set(*keyp, value = (*allocate)()); 1466 1467 return (value); 1468 } 1469 1470 /* 1471 * Called from thread_exit() to run the destructor function for each tsd 1472 * Locks out tsd_create and tsd_destroy 1473 * Assumes that the destructor *DOES NOT* use tsd 1474 */ 1475 void 1476 tsd_exit(void) 1477 { 1478 int i; 1479 struct tsd_thread *tsd = curthread->t_tsd; 1480 1481 if (tsd == NULL) 1482 return; 1483 1484 if (tsd->ts_nkeys == 0) { 1485 kmem_free(tsd, sizeof (*tsd)); 1486 curthread->t_tsd = NULL; 1487 return; 1488 } 1489 1490 /* 1491 * lock out tsd_create and tsd_destroy, call 1492 * the destructor, and mark the value as destroyed. 1493 */ 1494 mutex_enter(&tsd_mutex); 1495 1496 for (i = 0; i < tsd->ts_nkeys; i++) { 1497 if (tsd->ts_value[i] && tsd_destructor[i]) 1498 (*tsd_destructor[i])(tsd->ts_value[i]); 1499 tsd->ts_value[i] = NULL; 1500 } 1501 1502 /* 1503 * remove from linked list of threads with TSD 1504 */ 1505 if (tsd->ts_next) 1506 tsd->ts_next->ts_prev = tsd->ts_prev; 1507 if (tsd->ts_prev) 1508 tsd->ts_prev->ts_next = tsd->ts_next; 1509 if (tsd_list == tsd) 1510 tsd_list = tsd->ts_next; 1511 1512 mutex_exit(&tsd_mutex); 1513 1514 /* 1515 * free up the TSD 1516 */ 1517 kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *)); 1518 kmem_free(tsd, sizeof (struct tsd_thread)); 1519 curthread->t_tsd = NULL; 1520 } 1521 1522 /* 1523 * realloc 1524 */ 1525 static void * 1526 tsd_realloc(void *old, size_t osize, size_t nsize) 1527 { 1528 void *new; 1529 1530 new = kmem_zalloc(nsize, KM_SLEEP); 1531 if (old) { 1532 bcopy(old, new, osize); 1533 kmem_free(old, osize); 1534 } 1535 return (new); 1536 } 1537 1538 /* 1539 * Check to see if an interrupt thread might be active at a given ipl. 1540 * If so return true. 1541 * We must be conservative--it is ok to give a false yes, but a false no 1542 * will cause disaster. (But if the situation changes after we check it is 1543 * ok--the caller is trying to ensure that an interrupt routine has been 1544 * exited). 1545 * This is used when trying to remove an interrupt handler from an autovector 1546 * list in avintr.c. 1547 */ 1548 int 1549 intr_active(struct cpu *cp, int level) 1550 { 1551 if (level <= LOCK_LEVEL) 1552 return (cp->cpu_thread != cp->cpu_dispthread); 1553 else 1554 return (CPU_ON_INTR(cp)); 1555 } 1556 1557 /* 1558 * Return non-zero if an interrupt is being serviced. 1559 */ 1560 int 1561 servicing_interrupt() 1562 { 1563 int onintr = 0; 1564 1565 /* Are we an interrupt thread */ 1566 if (curthread->t_flag & T_INTR_THREAD) 1567 return (1); 1568 /* Are we servicing a high level interrupt? */ 1569 if (CPU_ON_INTR(CPU)) { 1570 kpreempt_disable(); 1571 onintr = CPU_ON_INTR(CPU); 1572 kpreempt_enable(); 1573 } 1574 return (onintr); 1575 } 1576 1577 1578 /* 1579 * Change the dispatch priority of a thread in the system. 1580 * Used when raising or lowering a thread's priority. 1581 * (E.g., priority inheritance) 1582 * 1583 * Since threads are queued according to their priority, we 1584 * we must check the thread's state to determine whether it 1585 * is on a queue somewhere. If it is, we've got to: 1586 * 1587 * o Dequeue the thread. 1588 * o Change its effective priority. 1589 * o Enqueue the thread. 1590 * 1591 * Assumptions: The thread whose priority we wish to change 1592 * must be locked before we call thread_change_(e)pri(). 1593 * The thread_change(e)pri() function doesn't drop the thread 1594 * lock--that must be done by its caller. 1595 */ 1596 void 1597 thread_change_epri(kthread_t *t, pri_t disp_pri) 1598 { 1599 uint_t state; 1600 1601 ASSERT(THREAD_LOCK_HELD(t)); 1602 1603 /* 1604 * If the inherited priority hasn't actually changed, 1605 * just return. 1606 */ 1607 if (t->t_epri == disp_pri) 1608 return; 1609 1610 state = t->t_state; 1611 1612 /* 1613 * If it's not on a queue, change the priority with 1614 * impunity. 1615 */ 1616 if ((state & (TS_SLEEP | TS_RUN)) == 0) { 1617 t->t_epri = disp_pri; 1618 1619 if (state == TS_ONPROC) { 1620 cpu_t *cp = t->t_disp_queue->disp_cpu; 1621 1622 if (t == cp->cpu_dispthread) 1623 cp->cpu_dispatch_pri = DISP_PRIO(t); 1624 } 1625 return; 1626 } 1627 1628 /* 1629 * It's either on a sleep queue or a run queue. 1630 */ 1631 if (state == TS_SLEEP) { 1632 1633 /* 1634 * Take the thread out of its sleep queue. 1635 * Change the inherited priority. 1636 * Re-enqueue the thread. 1637 * Each synchronization object exports a function 1638 * to do this in an appropriate manner. 1639 */ 1640 SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri); 1641 } else { 1642 /* 1643 * The thread is on a run queue. 1644 * Note: setbackdq() may not put the thread 1645 * back on the same run queue where it originally 1646 * resided. 1647 */ 1648 (void) dispdeq(t); 1649 t->t_epri = disp_pri; 1650 setbackdq(t); 1651 } 1652 } /* end of thread_change_epri */ 1653 1654 /* 1655 * Function: Change the t_pri field of a thread. 1656 * Side Effects: Adjust the thread ordering on a run queue 1657 * or sleep queue, if necessary. 1658 * Returns: 1 if the thread was on a run queue, else 0. 1659 */ 1660 int 1661 thread_change_pri(kthread_t *t, pri_t disp_pri, int front) 1662 { 1663 uint_t state; 1664 int on_rq = 0; 1665 1666 ASSERT(THREAD_LOCK_HELD(t)); 1667 1668 state = t->t_state; 1669 THREAD_WILLCHANGE_PRI(t, disp_pri); 1670 1671 /* 1672 * If it's not on a queue, change the priority with 1673 * impunity. 1674 */ 1675 if ((state & (TS_SLEEP | TS_RUN)) == 0) { 1676 t->t_pri = disp_pri; 1677 1678 if (state == TS_ONPROC) { 1679 cpu_t *cp = t->t_disp_queue->disp_cpu; 1680 1681 if (t == cp->cpu_dispthread) 1682 cp->cpu_dispatch_pri = DISP_PRIO(t); 1683 } 1684 return (0); 1685 } 1686 1687 /* 1688 * It's either on a sleep queue or a run queue. 1689 */ 1690 if (state == TS_SLEEP) { 1691 /* 1692 * If the priority has changed, take the thread out of 1693 * its sleep queue and change the priority. 1694 * Re-enqueue the thread. 1695 * Each synchronization object exports a function 1696 * to do this in an appropriate manner. 1697 */ 1698 if (disp_pri != t->t_pri) 1699 SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri); 1700 } else { 1701 /* 1702 * The thread is on a run queue. 1703 * Note: setbackdq() may not put the thread 1704 * back on the same run queue where it originally 1705 * resided. 1706 * 1707 * We still requeue the thread even if the priority 1708 * is unchanged to preserve round-robin (and other) 1709 * effects between threads of the same priority. 1710 */ 1711 on_rq = dispdeq(t); 1712 ASSERT(on_rq); 1713 t->t_pri = disp_pri; 1714 if (front) { 1715 setfrontdq(t); 1716 } else { 1717 setbackdq(t); 1718 } 1719 } 1720 return (on_rq); 1721 } 1722