xref: /titanic_51/usr/src/uts/common/disp/thread.c (revision 8edda6281c84e0632a22f9c8dbf0d6f1558878eb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/sysmacros.h>
31 #include <sys/signal.h>
32 #include <sys/stack.h>
33 #include <sys/pcb.h>
34 #include <sys/user.h>
35 #include <sys/systm.h>
36 #include <sys/sysinfo.h>
37 #include <sys/var.h>
38 #include <sys/errno.h>
39 #include <sys/cmn_err.h>
40 #include <sys/cred.h>
41 #include <sys/resource.h>
42 #include <sys/task.h>
43 #include <sys/project.h>
44 #include <sys/proc.h>
45 #include <sys/debug.h>
46 #include <sys/inline.h>
47 #include <sys/disp.h>
48 #include <sys/class.h>
49 #include <vm/seg_kmem.h>
50 #include <vm/seg_kp.h>
51 #include <sys/machlock.h>
52 #include <sys/kmem.h>
53 #include <sys/varargs.h>
54 #include <sys/turnstile.h>
55 #include <sys/poll.h>
56 #include <sys/vtrace.h>
57 #include <sys/callb.h>
58 #include <c2/audit.h>
59 #include <sys/tnf.h>
60 #include <sys/sobject.h>
61 #include <sys/cpupart.h>
62 #include <sys/pset.h>
63 #include <sys/door.h>
64 #include <sys/spl.h>
65 #include <sys/copyops.h>
66 #include <sys/rctl.h>
67 #include <sys/brand.h>
68 #include <sys/pool.h>
69 #include <sys/zone.h>
70 #include <sys/tsol/label.h>
71 #include <sys/tsol/tndb.h>
72 #include <sys/cpc_impl.h>
73 #include <sys/sdt.h>
74 #include <sys/reboot.h>
75 #include <sys/kdi.h>
76 #include <sys/waitq.h>
77 #include <sys/cpucaps.h>
78 #include <inet/ip.h>
79 #include <inet/ip_if.h>
80 
81 struct kmem_cache *thread_cache;	/* cache of free threads */
82 struct kmem_cache *lwp_cache;		/* cache of free lwps */
83 struct kmem_cache *turnstile_cache;	/* cache of free turnstiles */
84 
85 /*
86  * allthreads is only for use by kmem_readers.  All kernel loops can use
87  * the current thread as a start/end point.
88  */
89 static kthread_t *allthreads = &t0;	/* circular list of all threads */
90 
91 static kcondvar_t reaper_cv;		/* synchronization var */
92 kthread_t	*thread_deathrow;	/* circular list of reapable threads */
93 kthread_t	*lwp_deathrow;		/* circular list of reapable threads */
94 kmutex_t	reaplock;		/* protects lwp and thread deathrows */
95 kmutex_t	thread_free_lock;	/* protects clock from reaper */
96 int	thread_reapcnt = 0;		/* number of threads on deathrow */
97 int	lwp_reapcnt = 0;		/* number of lwps on deathrow */
98 int	reaplimit = 16;			/* delay reaping until reaplimit */
99 
100 extern int nthread;
101 
102 id_t	syscid;				/* system scheduling class ID */
103 void	*segkp_thread;			/* cookie for segkp pool */
104 
105 int lwp_cache_sz = 32;
106 int t_cache_sz = 8;
107 static kt_did_t next_t_id = 1;
108 
109 /*
110  * Min/Max stack sizes for stack size parameters
111  */
112 #define	MAX_STKSIZE	(32 * DEFAULTSTKSZ)
113 #define	MIN_STKSIZE	DEFAULTSTKSZ
114 
115 /*
116  * default_stksize overrides lwp_default_stksize if it is set.
117  */
118 int	default_stksize;
119 int	lwp_default_stksize;
120 
121 static zone_key_t zone_thread_key;
122 
123 /*
124  * forward declarations for internal thread specific data (tsd)
125  */
126 static void *tsd_realloc(void *, size_t, size_t);
127 
128 void thread_reaper(void);
129 
130 /*ARGSUSED*/
131 static int
132 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
133 {
134 	bzero(buf, sizeof (turnstile_t));
135 	return (0);
136 }
137 
138 /*ARGSUSED*/
139 static void
140 turnstile_destructor(void *buf, void *cdrarg)
141 {
142 	turnstile_t *ts = buf;
143 
144 	ASSERT(ts->ts_free == NULL);
145 	ASSERT(ts->ts_waiters == 0);
146 	ASSERT(ts->ts_inheritor == NULL);
147 	ASSERT(ts->ts_sleepq[0].sq_first == NULL);
148 	ASSERT(ts->ts_sleepq[1].sq_first == NULL);
149 }
150 
151 void
152 thread_init(void)
153 {
154 	kthread_t *tp;
155 	extern char sys_name[];
156 	extern void idle();
157 	struct cpu *cpu = CPU;
158 
159 	mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
160 
161 #if defined(__i386) || defined(__amd64)
162 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
163 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
164 
165 	/*
166 	 * "struct _klwp" includes a "struct pcb", which includes a
167 	 * "struct fpu", which needs to be 16-byte aligned on amd64
168 	 * (and even on i386 for fxsave/fxrstor).
169 	 */
170 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
171 	    16, NULL, NULL, NULL, NULL, NULL, 0);
172 #else
173 	/*
174 	 * Allocate thread structures from static_arena.  This prevents
175 	 * issues where a thread tries to relocate its own thread
176 	 * structure and touches it after the mapping has been suspended.
177 	 */
178 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
179 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
180 
181 	lwp_stk_cache_init();
182 
183 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
184 	    0, NULL, NULL, NULL, NULL, NULL, 0);
185 #endif
186 
187 	turnstile_cache = kmem_cache_create("turnstile_cache",
188 	    sizeof (turnstile_t), 0,
189 	    turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
190 
191 	label_init();
192 	cred_init();
193 
194 	/*
195 	 * Initialize various resource management facilities.
196 	 */
197 	rctl_init();
198 	cpucaps_init();
199 	/*
200 	 * Zone_init() should be called before project_init() so that project ID
201 	 * for the first project is initialized correctly.
202 	 */
203 	zone_init();
204 	project_init();
205 	brand_init();
206 	task_init();
207 	tcache_init();
208 	pool_init();
209 
210 	curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
211 
212 	/*
213 	 * Originally, we had two parameters to set default stack
214 	 * size: one for lwp's (lwp_default_stksize), and one for
215 	 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
216 	 * Now we have a third parameter that overrides both if it is
217 	 * set to a legal stack size, called default_stksize.
218 	 */
219 
220 	if (default_stksize == 0) {
221 		default_stksize = DEFAULTSTKSZ;
222 	} else if (default_stksize % PAGESIZE != 0 ||
223 	    default_stksize > MAX_STKSIZE ||
224 	    default_stksize < MIN_STKSIZE) {
225 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
226 		    (int)DEFAULTSTKSZ);
227 		default_stksize = DEFAULTSTKSZ;
228 	} else {
229 		lwp_default_stksize = default_stksize;
230 	}
231 
232 	if (lwp_default_stksize == 0) {
233 		lwp_default_stksize = default_stksize;
234 	} else if (lwp_default_stksize % PAGESIZE != 0 ||
235 	    lwp_default_stksize > MAX_STKSIZE ||
236 	    lwp_default_stksize < MIN_STKSIZE) {
237 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
238 		    default_stksize);
239 		lwp_default_stksize = default_stksize;
240 	}
241 
242 	segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
243 	    lwp_default_stksize,
244 	    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
245 
246 	segkp_thread = segkp_cache_init(segkp, t_cache_sz,
247 	    default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
248 
249 	(void) getcid(sys_name, &syscid);
250 	curthread->t_cid = syscid;	/* current thread is t0 */
251 
252 	/*
253 	 * Set up the first CPU's idle thread.
254 	 * It runs whenever the CPU has nothing worthwhile to do.
255 	 */
256 	tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
257 	cpu->cpu_idle_thread = tp;
258 	tp->t_preempt = 1;
259 	tp->t_disp_queue = cpu->cpu_disp;
260 	ASSERT(tp->t_disp_queue != NULL);
261 	tp->t_bound_cpu = cpu;
262 	tp->t_affinitycnt = 1;
263 
264 	/*
265 	 * Registering a thread in the callback table is usually
266 	 * done in the initialization code of the thread. In this
267 	 * case, we do it right after thread creation to avoid
268 	 * blocking idle thread while registering itself. It also
269 	 * avoids the possibility of reregistration in case a CPU
270 	 * restarts its idle thread.
271 	 */
272 	CALLB_CPR_INIT_SAFE(tp, "idle");
273 
274 	/*
275 	 * Create the thread_reaper daemon. From this point on, exited
276 	 * threads will get reaped.
277 	 */
278 	(void) thread_create(NULL, 0, (void (*)())thread_reaper,
279 	    NULL, 0, &p0, TS_RUN, minclsyspri);
280 
281 	/*
282 	 * Finish initializing the kernel memory allocator now that
283 	 * thread_create() is available.
284 	 */
285 	kmem_thread_init();
286 
287 	if (boothowto & RB_DEBUG)
288 		kdi_dvec_thravail();
289 }
290 
291 /*
292  * Create a thread.
293  *
294  * thread_create() blocks for memory if necessary.  It never fails.
295  *
296  * If stk is NULL, the thread is created at the base of the stack
297  * and cannot be swapped.
298  */
299 kthread_t *
300 thread_create(
301 	caddr_t	stk,
302 	size_t	stksize,
303 	void	(*proc)(),
304 	void	*arg,
305 	size_t	len,
306 	proc_t	 *pp,
307 	int	state,
308 	pri_t	pri)
309 {
310 	kthread_t *t;
311 	extern struct classfuncs sys_classfuncs;
312 	turnstile_t *ts;
313 
314 	/*
315 	 * Every thread keeps a turnstile around in case it needs to block.
316 	 * The only reason the turnstile is not simply part of the thread
317 	 * structure is that we may have to break the association whenever
318 	 * more than one thread blocks on a given synchronization object.
319 	 * From a memory-management standpoint, turnstiles are like the
320 	 * "attached mblks" that hang off dblks in the streams allocator.
321 	 */
322 	ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
323 
324 	if (stk == NULL) {
325 		/*
326 		 * alloc both thread and stack in segkp chunk
327 		 */
328 
329 		if (stksize < default_stksize)
330 			stksize = default_stksize;
331 
332 		if (stksize == default_stksize) {
333 			stk = (caddr_t)segkp_cache_get(segkp_thread);
334 		} else {
335 			stksize = roundup(stksize, PAGESIZE);
336 			stk = (caddr_t)segkp_get(segkp, stksize,
337 			    (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
338 		}
339 
340 		ASSERT(stk != NULL);
341 
342 		/*
343 		 * The machine-dependent mutex code may require that
344 		 * thread pointers (since they may be used for mutex owner
345 		 * fields) have certain alignment requirements.
346 		 * PTR24_ALIGN is the size of the alignment quanta.
347 		 * XXX - assumes stack grows toward low addresses.
348 		 */
349 		if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
350 			cmn_err(CE_PANIC, "thread_create: proposed stack size"
351 			    " too small to hold thread.");
352 #ifdef STACK_GROWTH_DOWN
353 		stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
354 		stksize &= -PTR24_ALIGN;	/* make thread aligned */
355 		t = (kthread_t *)(stk + stksize);
356 		bzero(t, sizeof (kthread_t));
357 #ifdef	C2_AUDIT
358 		if (audit_active)
359 			audit_thread_create(t);
360 #endif
361 		t->t_stk = stk + stksize;
362 		t->t_stkbase = stk;
363 #else	/* stack grows to larger addresses */
364 		stksize -= SA(sizeof (kthread_t));
365 		t = (kthread_t *)(stk);
366 		bzero(t, sizeof (kthread_t));
367 		t->t_stk = stk + sizeof (kthread_t);
368 		t->t_stkbase = stk + stksize + sizeof (kthread_t);
369 #endif	/* STACK_GROWTH_DOWN */
370 		t->t_flag |= T_TALLOCSTK;
371 		t->t_swap = stk;
372 	} else {
373 		t = kmem_cache_alloc(thread_cache, KM_SLEEP);
374 		bzero(t, sizeof (kthread_t));
375 		ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
376 #ifdef	C2_AUDIT
377 		if (audit_active)
378 			audit_thread_create(t);
379 #endif
380 		/*
381 		 * Initialize t_stk to the kernel stack pointer to use
382 		 * upon entry to the kernel
383 		 */
384 #ifdef STACK_GROWTH_DOWN
385 		t->t_stk = stk + stksize;
386 		t->t_stkbase = stk;
387 #else
388 		t->t_stk = stk;			/* 3b2-like */
389 		t->t_stkbase = stk + stksize;
390 #endif /* STACK_GROWTH_DOWN */
391 	}
392 
393 	/* set default stack flag */
394 	if (stksize == lwp_default_stksize)
395 		t->t_flag |= T_DFLTSTK;
396 
397 	t->t_ts = ts;
398 
399 	/*
400 	 * p_cred could be NULL if it thread_create is called before cred_init
401 	 * is called in main.
402 	 */
403 	mutex_enter(&pp->p_crlock);
404 	if (pp->p_cred)
405 		crhold(t->t_cred = pp->p_cred);
406 	mutex_exit(&pp->p_crlock);
407 	t->t_start = gethrestime_sec();
408 	t->t_startpc = proc;
409 	t->t_procp = pp;
410 	t->t_clfuncs = &sys_classfuncs.thread;
411 	t->t_cid = syscid;
412 	t->t_pri = pri;
413 	t->t_stime = lbolt;
414 	t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
415 	t->t_bind_cpu = PBIND_NONE;
416 	t->t_bind_pset = PS_NONE;
417 	t->t_plockp = &pp->p_lock;
418 	t->t_copyops = NULL;
419 	t->t_taskq = NULL;
420 	t->t_anttime = 0;
421 	t->t_hatdepth = 0;
422 
423 	t->t_dtrace_vtime = 1;	/* assure vtimestamp is always non-zero */
424 
425 	CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
426 #ifndef NPROBE
427 	/* Kernel probe */
428 	tnf_thread_create(t);
429 #endif /* NPROBE */
430 	LOCK_INIT_CLEAR(&t->t_lock);
431 
432 	/*
433 	 * Callers who give us a NULL proc must do their own
434 	 * stack initialization.  e.g. lwp_create()
435 	 */
436 	if (proc != NULL) {
437 		t->t_stk = thread_stk_init(t->t_stk);
438 		thread_load(t, proc, arg, len);
439 	}
440 
441 	/*
442 	 * Put a hold on project0. If this thread is actually in a
443 	 * different project, then t_proj will be changed later in
444 	 * lwp_create().  All kernel-only threads must be in project 0.
445 	 */
446 	t->t_proj = project_hold(proj0p);
447 
448 	lgrp_affinity_init(&t->t_lgrp_affinity);
449 
450 	mutex_enter(&pidlock);
451 	nthread++;
452 	t->t_did = next_t_id++;
453 	t->t_prev = curthread->t_prev;
454 	t->t_next = curthread;
455 
456 	/*
457 	 * Add the thread to the list of all threads, and initialize
458 	 * its t_cpu pointer.  We need to block preemption since
459 	 * cpu_offline walks the thread list looking for threads
460 	 * with t_cpu pointing to the CPU being offlined.  We want
461 	 * to make sure that the list is consistent and that if t_cpu
462 	 * is set, the thread is on the list.
463 	 */
464 	kpreempt_disable();
465 	curthread->t_prev->t_next = t;
466 	curthread->t_prev = t;
467 
468 	/*
469 	 * Threads should never have a NULL t_cpu pointer so assign it
470 	 * here.  If the thread is being created with state TS_RUN a
471 	 * better CPU may be chosen when it is placed on the run queue.
472 	 *
473 	 * We need to keep kernel preemption disabled when setting all
474 	 * three fields to keep them in sync.  Also, always create in
475 	 * the default partition since that's where kernel threads go
476 	 * (if this isn't a kernel thread, t_cpupart will be changed
477 	 * in lwp_create before setting the thread runnable).
478 	 */
479 	t->t_cpupart = &cp_default;
480 
481 	/*
482 	 * For now, affiliate this thread with the root lgroup.
483 	 * Since the kernel does not (presently) allocate its memory
484 	 * in a locality aware fashion, the root is an appropriate home.
485 	 * If this thread is later associated with an lwp, it will have
486 	 * it's lgroup re-assigned at that time.
487 	 */
488 	lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
489 
490 	/*
491 	 * Inherit the current cpu.  If this cpu isn't part of the chosen
492 	 * lgroup, a new cpu will be chosen by cpu_choose when the thread
493 	 * is ready to run.
494 	 */
495 	if (CPU->cpu_part == &cp_default)
496 		t->t_cpu = CPU;
497 	else
498 		t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t->t_lpl,
499 		    t->t_pri, NULL);
500 
501 	t->t_disp_queue = t->t_cpu->cpu_disp;
502 	kpreempt_enable();
503 
504 	/*
505 	 * Initialize thread state and the dispatcher lock pointer.
506 	 * Need to hold onto pidlock to block allthreads walkers until
507 	 * the state is set.
508 	 */
509 	switch (state) {
510 	case TS_RUN:
511 		curthread->t_oldspl = splhigh();	/* get dispatcher spl */
512 		THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
513 		CL_SETRUN(t);
514 		thread_unlock(t);
515 		break;
516 
517 	case TS_ONPROC:
518 		THREAD_ONPROC(t, t->t_cpu);
519 		break;
520 
521 	case TS_FREE:
522 		/*
523 		 * Free state will be used for intr threads.
524 		 * The interrupt routine must set the thread dispatcher
525 		 * lock pointer (t_lockp) if starting on a CPU
526 		 * other than the current one.
527 		 */
528 		THREAD_FREEINTR(t, CPU);
529 		break;
530 
531 	case TS_STOPPED:
532 		THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
533 		break;
534 
535 	default:			/* TS_SLEEP, TS_ZOMB or TS_TRANS */
536 		cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
537 	}
538 	mutex_exit(&pidlock);
539 	return (t);
540 }
541 
542 /*
543  * Move thread to project0 and take care of project reference counters.
544  */
545 void
546 thread_rele(kthread_t *t)
547 {
548 	kproject_t *kpj;
549 
550 	thread_lock(t);
551 
552 	ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
553 	kpj = ttoproj(t);
554 	t->t_proj = proj0p;
555 
556 	thread_unlock(t);
557 
558 	if (kpj != proj0p) {
559 		project_rele(kpj);
560 		(void) project_hold(proj0p);
561 	}
562 }
563 
564 /*
565  * This is a function which is called from thread_exit
566  * that can be used to debug reference count issues in IP.
567  */
568 void (*ip_cleanup_func)(void);
569 
570 void
571 thread_exit()
572 {
573 	kthread_t *t = curthread;
574 
575 	if ((t->t_proc_flag & TP_ZTHREAD) != 0)
576 		cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
577 
578 	if (ip_cleanup_func != NULL)
579 		(*ip_cleanup_func)();
580 
581 	tsd_exit();		/* Clean up this thread's TSD */
582 
583 	kcpc_passivate();	/* clean up performance counter state */
584 
585 	/*
586 	 * No kernel thread should have called poll() without arranging
587 	 * calling pollcleanup() here.
588 	 */
589 	ASSERT(t->t_pollstate == NULL);
590 	ASSERT(t->t_schedctl == NULL);
591 	if (t->t_door)
592 		door_slam();	/* in case thread did an upcall */
593 
594 #ifndef NPROBE
595 	/* Kernel probe */
596 	if (t->t_tnf_tpdp)
597 		tnf_thread_exit();
598 #endif /* NPROBE */
599 
600 	thread_rele(t);
601 	t->t_preempt++;
602 
603 	/*
604 	 * remove thread from the all threads list so that
605 	 * death-row can use the same pointers.
606 	 */
607 	mutex_enter(&pidlock);
608 	t->t_next->t_prev = t->t_prev;
609 	t->t_prev->t_next = t->t_next;
610 	ASSERT(allthreads != t);	/* t0 never exits */
611 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
612 	mutex_exit(&pidlock);
613 
614 	if (t->t_ctx != NULL)
615 		exitctx(t);
616 	if (t->t_procp->p_pctx != NULL)
617 		exitpctx(t->t_procp);
618 
619 	t->t_state = TS_ZOMB;	/* set zombie thread */
620 
621 	swtch_from_zombie();	/* give up the CPU */
622 	/* NOTREACHED */
623 }
624 
625 /*
626  * Check to see if the specified thread is active (defined as being on
627  * the thread list).  This is certainly a slow way to do this; if there's
628  * ever a reason to speed it up, we could maintain a hash table of active
629  * threads indexed by their t_did.
630  */
631 static kthread_t *
632 did_to_thread(kt_did_t tid)
633 {
634 	kthread_t *t;
635 
636 	ASSERT(MUTEX_HELD(&pidlock));
637 	for (t = curthread->t_next; t != curthread; t = t->t_next) {
638 		if (t->t_did == tid)
639 			break;
640 	}
641 	if (t->t_did == tid)
642 		return (t);
643 	else
644 		return (NULL);
645 }
646 
647 /*
648  * Wait for specified thread to exit.  Returns immediately if the thread
649  * could not be found, meaning that it has either already exited or never
650  * existed.
651  */
652 void
653 thread_join(kt_did_t tid)
654 {
655 	kthread_t *t;
656 
657 	ASSERT(tid != curthread->t_did);
658 	ASSERT(tid != t0.t_did);
659 
660 	mutex_enter(&pidlock);
661 	/*
662 	 * Make sure we check that the thread is on the thread list
663 	 * before blocking on it; otherwise we could end up blocking on
664 	 * a cv that's already been freed.  In other words, don't cache
665 	 * the thread pointer across calls to cv_wait.
666 	 *
667 	 * The choice of loop invariant means that whenever a thread
668 	 * is taken off the allthreads list, a cv_broadcast must be
669 	 * performed on that thread's t_joincv to wake up any waiters.
670 	 * The broadcast doesn't have to happen right away, but it
671 	 * shouldn't be postponed indefinitely (e.g., by doing it in
672 	 * thread_free which may only be executed when the deathrow
673 	 * queue is processed.
674 	 */
675 	while (t = did_to_thread(tid))
676 		cv_wait(&t->t_joincv, &pidlock);
677 	mutex_exit(&pidlock);
678 }
679 
680 void
681 thread_free(kthread_t *t)
682 {
683 	ASSERT(t != &t0 && t->t_state == TS_FREE);
684 	ASSERT(t->t_door == NULL);
685 	ASSERT(t->t_schedctl == NULL);
686 	ASSERT(t->t_pollstate == NULL);
687 
688 	t->t_pri = 0;
689 	t->t_pc = 0;
690 	t->t_sp = 0;
691 	t->t_wchan0 = NULL;
692 	t->t_wchan = NULL;
693 	if (t->t_cred != NULL) {
694 		crfree(t->t_cred);
695 		t->t_cred = 0;
696 	}
697 	if (t->t_pdmsg) {
698 		kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
699 		t->t_pdmsg = NULL;
700 	}
701 #ifdef	C2_AUDIT
702 	if (audit_active)
703 		audit_thread_free(t);
704 #endif
705 #ifndef NPROBE
706 	if (t->t_tnf_tpdp)
707 		tnf_thread_free(t);
708 #endif /* NPROBE */
709 	if (t->t_cldata) {
710 		CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
711 	}
712 	if (t->t_rprof != NULL) {
713 		kmem_free(t->t_rprof, sizeof (*t->t_rprof));
714 		t->t_rprof = NULL;
715 	}
716 	t->t_lockp = NULL;	/* nothing should try to lock this thread now */
717 	if (t->t_lwp)
718 		lwp_freeregs(t->t_lwp, 0);
719 	if (t->t_ctx)
720 		freectx(t, 0);
721 	t->t_stk = NULL;
722 	if (t->t_lwp)
723 		lwp_stk_fini(t->t_lwp);
724 	lock_clear(&t->t_lock);
725 
726 	if (t->t_ts->ts_waiters > 0)
727 		panic("thread_free: turnstile still active");
728 
729 	kmem_cache_free(turnstile_cache, t->t_ts);
730 
731 	free_afd(&t->t_activefd);
732 
733 	/*
734 	 * Barrier for clock thread.  The clock holds this lock to
735 	 * keep the thread from going away while it's looking at it.
736 	 */
737 	mutex_enter(&thread_free_lock);
738 	mutex_exit(&thread_free_lock);
739 
740 	ASSERT(ttoproj(t) == proj0p);
741 	project_rele(ttoproj(t));
742 
743 	lgrp_affinity_free(&t->t_lgrp_affinity);
744 
745 	/*
746 	 * Free thread struct and its stack.
747 	 */
748 	if (t->t_flag & T_TALLOCSTK) {
749 		/* thread struct is embedded in stack */
750 		segkp_release(segkp, t->t_swap);
751 		mutex_enter(&pidlock);
752 		nthread--;
753 		mutex_exit(&pidlock);
754 	} else {
755 		if (t->t_swap) {
756 			segkp_release(segkp, t->t_swap);
757 			t->t_swap = NULL;
758 		}
759 		if (t->t_lwp) {
760 			kmem_cache_free(lwp_cache, t->t_lwp);
761 			t->t_lwp = NULL;
762 		}
763 		mutex_enter(&pidlock);
764 		nthread--;
765 		mutex_exit(&pidlock);
766 		kmem_cache_free(thread_cache, t);
767 	}
768 }
769 
770 /*
771  * Removes threads associated with the given zone from a deathrow queue.
772  * tp is a pointer to the head of the deathrow queue, and countp is a
773  * pointer to the current deathrow count.  Returns a linked list of
774  * threads removed from the list.
775  */
776 static kthread_t *
777 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
778 {
779 	kthread_t *tmp, *list = NULL;
780 	cred_t *cr;
781 
782 	ASSERT(MUTEX_HELD(&reaplock));
783 	while (*tp != NULL) {
784 		if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
785 			tmp = *tp;
786 			*tp = tmp->t_forw;
787 			tmp->t_forw = list;
788 			list = tmp;
789 			(*countp)--;
790 		} else {
791 			tp = &(*tp)->t_forw;
792 		}
793 	}
794 	return (list);
795 }
796 
797 static void
798 thread_reap_list(kthread_t *t)
799 {
800 	kthread_t *next;
801 
802 	while (t != NULL) {
803 		next = t->t_forw;
804 		thread_free(t);
805 		t = next;
806 	}
807 }
808 
809 /* ARGSUSED */
810 static void
811 thread_zone_destroy(zoneid_t zoneid, void *unused)
812 {
813 	kthread_t *t, *l;
814 
815 	mutex_enter(&reaplock);
816 	/*
817 	 * Pull threads and lwps associated with zone off deathrow lists.
818 	 */
819 	t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
820 	l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
821 	mutex_exit(&reaplock);
822 
823 	/*
824 	 * Reap threads
825 	 */
826 	thread_reap_list(t);
827 
828 	/*
829 	 * Reap lwps
830 	 */
831 	thread_reap_list(l);
832 }
833 
834 /*
835  * cleanup zombie threads that are on deathrow.
836  */
837 void
838 thread_reaper()
839 {
840 	kthread_t *t, *l;
841 	callb_cpr_t cprinfo;
842 
843 	/*
844 	 * Register callback to clean up threads when zone is destroyed.
845 	 */
846 	zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
847 
848 	CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
849 	for (;;) {
850 		mutex_enter(&reaplock);
851 		while (thread_deathrow == NULL && lwp_deathrow == NULL) {
852 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
853 			cv_wait(&reaper_cv, &reaplock);
854 			CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
855 		}
856 		t = thread_deathrow;
857 		l = lwp_deathrow;
858 		thread_deathrow = NULL;
859 		lwp_deathrow = NULL;
860 		thread_reapcnt = 0;
861 		lwp_reapcnt = 0;
862 		mutex_exit(&reaplock);
863 
864 		/*
865 		 * Reap threads
866 		 */
867 		thread_reap_list(t);
868 
869 		/*
870 		 * Reap lwps
871 		 */
872 		thread_reap_list(l);
873 	}
874 }
875 
876 /*
877  * This is called by resume() to put a zombie thread onto deathrow.
878  * The thread's state is changed to TS_FREE to indicate that is reapable.
879  * This is called from the idle thread so it must not block (just spin).
880  */
881 void
882 reapq_add(kthread_t *t)
883 {
884 	mutex_enter(&reaplock);
885 
886 	/*
887 	 * lwp_deathrow contains only threads with lwp linkage
888 	 * that are of the default stacksize. Anything else goes
889 	 * on thread_deathrow.
890 	 */
891 	if (ttolwp(t) && (t->t_flag & T_DFLTSTK)) {
892 		t->t_forw = lwp_deathrow;
893 		lwp_deathrow = t;
894 		lwp_reapcnt++;
895 	} else {
896 		t->t_forw = thread_deathrow;
897 		thread_deathrow = t;
898 		thread_reapcnt++;
899 	}
900 	if (lwp_reapcnt + thread_reapcnt > reaplimit)
901 		cv_signal(&reaper_cv);	/* wake the reaper */
902 	t->t_state = TS_FREE;
903 	lock_clear(&t->t_lock);
904 
905 	/*
906 	 * Before we return, we need to grab and drop the thread lock for
907 	 * the dead thread.  At this point, the current thread is the idle
908 	 * thread, and the dead thread's CPU lock points to the current
909 	 * CPU -- and we must grab and drop the lock to synchronize with
910 	 * a racing thread walking a blocking chain that the zombie thread
911 	 * was recently in.  By this point, that blocking chain is (by
912 	 * definition) stale:  the dead thread is not holding any locks, and
913 	 * is therefore not in any blocking chains -- but if we do not regrab
914 	 * our lock before freeing the dead thread's data structures, the
915 	 * thread walking the (stale) blocking chain will die on memory
916 	 * corruption when it attempts to drop the dead thread's lock.  We
917 	 * only need do this once because there is no way for the dead thread
918 	 * to ever again be on a blocking chain:  once we have grabbed and
919 	 * dropped the thread lock, we are guaranteed that anyone that could
920 	 * have seen this thread in a blocking chain can no longer see it.
921 	 */
922 	thread_lock(t);
923 	thread_unlock(t);
924 
925 	mutex_exit(&reaplock);
926 }
927 
928 /*
929  * Install thread context ops for the current thread.
930  */
931 void
932 installctx(
933 	kthread_t *t,
934 	void	*arg,
935 	void	(*save)(void *),
936 	void	(*restore)(void *),
937 	void	(*fork)(void *, void *),
938 	void	(*lwp_create)(void *, void *),
939 	void	(*exit)(void *),
940 	void	(*free)(void *, int))
941 {
942 	struct ctxop *ctx;
943 
944 	ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
945 	ctx->save_op = save;
946 	ctx->restore_op = restore;
947 	ctx->fork_op = fork;
948 	ctx->lwp_create_op = lwp_create;
949 	ctx->exit_op = exit;
950 	ctx->free_op = free;
951 	ctx->arg = arg;
952 	ctx->next = t->t_ctx;
953 	t->t_ctx = ctx;
954 }
955 
956 /*
957  * Remove the thread context ops from a thread.
958  */
959 int
960 removectx(
961 	kthread_t *t,
962 	void	*arg,
963 	void	(*save)(void *),
964 	void	(*restore)(void *),
965 	void	(*fork)(void *, void *),
966 	void	(*lwp_create)(void *, void *),
967 	void	(*exit)(void *),
968 	void	(*free)(void *, int))
969 {
970 	struct ctxop *ctx, *prev_ctx;
971 
972 	/*
973 	 * The incoming kthread_t (which is the thread for which the
974 	 * context ops will be removed) should be one of the following:
975 	 *
976 	 * a) the current thread,
977 	 *
978 	 * b) a thread of a process that's being forked (SIDL),
979 	 *
980 	 * c) a thread that belongs to the same process as the current
981 	 *    thread and for which the current thread is the agent thread,
982 	 *
983 	 * d) a thread that is TS_STOPPED which is indicative of it
984 	 *    being (if curthread is not an agent) a thread being created
985 	 *    as part of an lwp creation.
986 	 */
987 	ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
988 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
989 
990 	/*
991 	 * Serialize modifications to t->t_ctx to prevent the agent thread
992 	 * and the target thread from racing with each other during lwp exit.
993 	 */
994 	mutex_enter(&t->t_ctx_lock);
995 	prev_ctx = NULL;
996 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) {
997 		if (ctx->save_op == save && ctx->restore_op == restore &&
998 		    ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
999 		    ctx->exit_op == exit && ctx->free_op == free &&
1000 		    ctx->arg == arg) {
1001 			if (prev_ctx)
1002 				prev_ctx->next = ctx->next;
1003 			else
1004 				t->t_ctx = ctx->next;
1005 			mutex_exit(&t->t_ctx_lock);
1006 			if (ctx->free_op != NULL)
1007 				(ctx->free_op)(ctx->arg, 0);
1008 			kmem_free(ctx, sizeof (struct ctxop));
1009 			return (1);
1010 		}
1011 		prev_ctx = ctx;
1012 	}
1013 	mutex_exit(&t->t_ctx_lock);
1014 
1015 	return (0);
1016 }
1017 
1018 void
1019 savectx(kthread_t *t)
1020 {
1021 	struct ctxop *ctx;
1022 
1023 	ASSERT(t == curthread);
1024 	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1025 		if (ctx->save_op != NULL)
1026 			(ctx->save_op)(ctx->arg);
1027 }
1028 
1029 void
1030 restorectx(kthread_t *t)
1031 {
1032 	struct ctxop *ctx;
1033 
1034 	ASSERT(t == curthread);
1035 	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1036 		if (ctx->restore_op != NULL)
1037 			(ctx->restore_op)(ctx->arg);
1038 }
1039 
1040 void
1041 forkctx(kthread_t *t, kthread_t *ct)
1042 {
1043 	struct ctxop *ctx;
1044 
1045 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1046 		if (ctx->fork_op != NULL)
1047 			(ctx->fork_op)(t, ct);
1048 }
1049 
1050 /*
1051  * Note that this operator is only invoked via the _lwp_create
1052  * system call.  The system may have other reasons to create lwps
1053  * e.g. the agent lwp or the doors unreferenced lwp.
1054  */
1055 void
1056 lwp_createctx(kthread_t *t, kthread_t *ct)
1057 {
1058 	struct ctxop *ctx;
1059 
1060 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1061 		if (ctx->lwp_create_op != NULL)
1062 			(ctx->lwp_create_op)(t, ct);
1063 }
1064 
1065 /*
1066  * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1067  * needed when the thread/LWP leaves the processor for the last time. This
1068  * routine is not intended to deal with freeing memory; freectx() is used for
1069  * that purpose during thread_free(). This routine is provided to allow for
1070  * clean-up that can't wait until thread_free().
1071  */
1072 void
1073 exitctx(kthread_t *t)
1074 {
1075 	struct ctxop *ctx;
1076 
1077 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1078 		if (ctx->exit_op != NULL)
1079 			(ctx->exit_op)(t);
1080 }
1081 
1082 /*
1083  * freectx is called from thread_free() and exec() to get
1084  * rid of old thread context ops.
1085  */
1086 void
1087 freectx(kthread_t *t, int isexec)
1088 {
1089 	struct ctxop *ctx;
1090 
1091 	while ((ctx = t->t_ctx) != NULL) {
1092 		t->t_ctx = ctx->next;
1093 		if (ctx->free_op != NULL)
1094 			(ctx->free_op)(ctx->arg, isexec);
1095 		kmem_free(ctx, sizeof (struct ctxop));
1096 	}
1097 }
1098 
1099 /*
1100  * Set the thread running; arrange for it to be swapped in if necessary.
1101  */
1102 void
1103 setrun_locked(kthread_t *t)
1104 {
1105 	ASSERT(THREAD_LOCK_HELD(t));
1106 	if (t->t_state == TS_SLEEP) {
1107 		/*
1108 		 * Take off sleep queue.
1109 		 */
1110 		SOBJ_UNSLEEP(t->t_sobj_ops, t);
1111 	} else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1112 		/*
1113 		 * Already on dispatcher queue.
1114 		 */
1115 		return;
1116 	} else if (t->t_state == TS_WAIT) {
1117 		waitq_setrun(t);
1118 	} else if (t->t_state == TS_STOPPED) {
1119 		/*
1120 		 * All of the sending of SIGCONT (TC_XSTART) and /proc
1121 		 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1122 		 * requested that the thread be run.
1123 		 * Just calling setrun() is not sufficient to set a stopped
1124 		 * thread running.  TP_TXSTART is always set if the thread
1125 		 * is not stopped by a jobcontrol stop signal.
1126 		 * TP_TPSTART is always set if /proc is not controlling it.
1127 		 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1128 		 * The thread won't be stopped unless one of these
1129 		 * three mechanisms did it.
1130 		 *
1131 		 * These flags must be set before calling setrun_locked(t).
1132 		 * They can't be passed as arguments because the streams
1133 		 * code calls setrun() indirectly and the mechanism for
1134 		 * doing so admits only one argument.  Note that the
1135 		 * thread must be locked in order to change t_schedflags.
1136 		 */
1137 		if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1138 			return;
1139 		/*
1140 		 * Process is no longer stopped (a thread is running).
1141 		 */
1142 		t->t_whystop = 0;
1143 		t->t_whatstop = 0;
1144 		/*
1145 		 * Strictly speaking, we do not have to clear these
1146 		 * flags here; they are cleared on entry to stop().
1147 		 * However, they are confusing when doing kernel
1148 		 * debugging or when they are revealed by ps(1).
1149 		 */
1150 		t->t_schedflag &= ~TS_ALLSTART;
1151 		THREAD_TRANSITION(t);	/* drop stopped-thread lock */
1152 		ASSERT(t->t_lockp == &transition_lock);
1153 		ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1154 		/*
1155 		 * Let the class put the process on the dispatcher queue.
1156 		 */
1157 		CL_SETRUN(t);
1158 	}
1159 }
1160 
1161 void
1162 setrun(kthread_t *t)
1163 {
1164 	thread_lock(t);
1165 	setrun_locked(t);
1166 	thread_unlock(t);
1167 }
1168 
1169 /*
1170  * Unpin an interrupted thread.
1171  *	When an interrupt occurs, the interrupt is handled on the stack
1172  *	of an interrupt thread, taken from a pool linked to the CPU structure.
1173  *
1174  *	When swtch() is switching away from an interrupt thread because it
1175  *	blocked or was preempted, this routine is called to complete the
1176  *	saving of the interrupted thread state, and returns the interrupted
1177  *	thread pointer so it may be resumed.
1178  *
1179  *	Called by swtch() only at high spl.
1180  */
1181 kthread_t *
1182 thread_unpin()
1183 {
1184 	kthread_t	*t = curthread;	/* current thread */
1185 	kthread_t	*itp;		/* interrupted thread */
1186 	int		i;		/* interrupt level */
1187 	extern int	intr_passivate();
1188 
1189 	ASSERT(t->t_intr != NULL);
1190 
1191 	itp = t->t_intr;		/* interrupted thread */
1192 	t->t_intr = NULL;		/* clear interrupt ptr */
1193 
1194 	/*
1195 	 * Get state from interrupt thread for the one
1196 	 * it interrupted.
1197 	 */
1198 
1199 	i = intr_passivate(t, itp);
1200 
1201 	TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1202 	    "intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1203 	    i, t, t, itp, itp);
1204 
1205 	/*
1206 	 * Dissociate the current thread from the interrupted thread's LWP.
1207 	 */
1208 	t->t_lwp = NULL;
1209 
1210 	/*
1211 	 * Interrupt handlers above the level that spinlocks block must
1212 	 * not block.
1213 	 */
1214 #if DEBUG
1215 	if (i < 0 || i > LOCK_LEVEL)
1216 		cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1217 #endif
1218 
1219 	/*
1220 	 * Compute the CPU's base interrupt level based on the active
1221 	 * interrupts.
1222 	 */
1223 	ASSERT(CPU->cpu_intr_actv & (1 << i));
1224 	set_base_spl();
1225 
1226 	return (itp);
1227 }
1228 
1229 /*
1230  * Create and initialize an interrupt thread.
1231  *	Returns non-zero on error.
1232  *	Called at spl7() or better.
1233  */
1234 void
1235 thread_create_intr(struct cpu *cp)
1236 {
1237 	kthread_t *tp;
1238 
1239 	tp = thread_create(NULL, 0,
1240 	    (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1241 
1242 	/*
1243 	 * Set the thread in the TS_FREE state.  The state will change
1244 	 * to TS_ONPROC only while the interrupt is active.  Think of these
1245 	 * as being on a private free list for the CPU.  Being TS_FREE keeps
1246 	 * inactive interrupt threads out of debugger thread lists.
1247 	 *
1248 	 * We cannot call thread_create with TS_FREE because of the current
1249 	 * checks there for ONPROC.  Fix this when thread_create takes flags.
1250 	 */
1251 	THREAD_FREEINTR(tp, cp);
1252 
1253 	/*
1254 	 * Nobody should ever reference the credentials of an interrupt
1255 	 * thread so make it NULL to catch any such references.
1256 	 */
1257 	tp->t_cred = NULL;
1258 	tp->t_flag |= T_INTR_THREAD;
1259 	tp->t_cpu = cp;
1260 	tp->t_bound_cpu = cp;
1261 	tp->t_disp_queue = cp->cpu_disp;
1262 	tp->t_affinitycnt = 1;
1263 	tp->t_preempt = 1;
1264 
1265 	/*
1266 	 * Don't make a user-requested binding on this thread so that
1267 	 * the processor can be offlined.
1268 	 */
1269 	tp->t_bind_cpu = PBIND_NONE;	/* no USER-requested binding */
1270 	tp->t_bind_pset = PS_NONE;
1271 
1272 #if defined(__i386) || defined(__amd64)
1273 	tp->t_stk -= STACK_ALIGN;
1274 	*(tp->t_stk) = 0;		/* terminate intr thread stack */
1275 #endif
1276 
1277 	/*
1278 	 * Link onto CPU's interrupt pool.
1279 	 */
1280 	tp->t_link = cp->cpu_intr_thread;
1281 	cp->cpu_intr_thread = tp;
1282 }
1283 
1284 /*
1285  * TSD -- THREAD SPECIFIC DATA
1286  */
1287 static kmutex_t		tsd_mutex;	 /* linked list spin lock */
1288 static uint_t		tsd_nkeys;	 /* size of destructor array */
1289 /* per-key destructor funcs */
1290 static void 		(**tsd_destructor)(void *);
1291 /* list of tsd_thread's */
1292 static struct tsd_thread	*tsd_list;
1293 
1294 /*
1295  * Default destructor
1296  *	Needed because NULL destructor means that the key is unused
1297  */
1298 /* ARGSUSED */
1299 void
1300 tsd_defaultdestructor(void *value)
1301 {}
1302 
1303 /*
1304  * Create a key (index into per thread array)
1305  *	Locks out tsd_create, tsd_destroy, and tsd_exit
1306  *	May allocate memory with lock held
1307  */
1308 void
1309 tsd_create(uint_t *keyp, void (*destructor)(void *))
1310 {
1311 	int	i;
1312 	uint_t	nkeys;
1313 
1314 	/*
1315 	 * if key is allocated, do nothing
1316 	 */
1317 	mutex_enter(&tsd_mutex);
1318 	if (*keyp) {
1319 		mutex_exit(&tsd_mutex);
1320 		return;
1321 	}
1322 	/*
1323 	 * find an unused key
1324 	 */
1325 	if (destructor == NULL)
1326 		destructor = tsd_defaultdestructor;
1327 
1328 	for (i = 0; i < tsd_nkeys; ++i)
1329 		if (tsd_destructor[i] == NULL)
1330 			break;
1331 
1332 	/*
1333 	 * if no unused keys, increase the size of the destructor array
1334 	 */
1335 	if (i == tsd_nkeys) {
1336 		if ((nkeys = (tsd_nkeys << 1)) == 0)
1337 			nkeys = 1;
1338 		tsd_destructor =
1339 		    (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1340 		    (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1341 		    (size_t)(nkeys * sizeof (void (*)(void *))));
1342 		tsd_nkeys = nkeys;
1343 	}
1344 
1345 	/*
1346 	 * allocate the next available unused key
1347 	 */
1348 	tsd_destructor[i] = destructor;
1349 	*keyp = i + 1;
1350 	mutex_exit(&tsd_mutex);
1351 }
1352 
1353 /*
1354  * Destroy a key -- this is for unloadable modules
1355  *
1356  * Assumes that the caller is preventing tsd_set and tsd_get
1357  * Locks out tsd_create, tsd_destroy, and tsd_exit
1358  * May free memory with lock held
1359  */
1360 void
1361 tsd_destroy(uint_t *keyp)
1362 {
1363 	uint_t key;
1364 	struct tsd_thread *tsd;
1365 
1366 	/*
1367 	 * protect the key namespace and our destructor lists
1368 	 */
1369 	mutex_enter(&tsd_mutex);
1370 	key = *keyp;
1371 	*keyp = 0;
1372 
1373 	ASSERT(key <= tsd_nkeys);
1374 
1375 	/*
1376 	 * if the key is valid
1377 	 */
1378 	if (key != 0) {
1379 		uint_t k = key - 1;
1380 		/*
1381 		 * for every thread with TSD, call key's destructor
1382 		 */
1383 		for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1384 			/*
1385 			 * no TSD for key in this thread
1386 			 */
1387 			if (key > tsd->ts_nkeys)
1388 				continue;
1389 			/*
1390 			 * call destructor for key
1391 			 */
1392 			if (tsd->ts_value[k] && tsd_destructor[k])
1393 				(*tsd_destructor[k])(tsd->ts_value[k]);
1394 			/*
1395 			 * reset value for key
1396 			 */
1397 			tsd->ts_value[k] = NULL;
1398 		}
1399 		/*
1400 		 * actually free the key (NULL destructor == unused)
1401 		 */
1402 		tsd_destructor[k] = NULL;
1403 	}
1404 
1405 	mutex_exit(&tsd_mutex);
1406 }
1407 
1408 /*
1409  * Quickly return the per thread value that was stored with the specified key
1410  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1411  */
1412 void *
1413 tsd_get(uint_t key)
1414 {
1415 	return (tsd_agent_get(curthread, key));
1416 }
1417 
1418 /*
1419  * Set a per thread value indexed with the specified key
1420  */
1421 int
1422 tsd_set(uint_t key, void *value)
1423 {
1424 	return (tsd_agent_set(curthread, key, value));
1425 }
1426 
1427 /*
1428  * Like tsd_get(), except that the agent lwp can get the tsd of
1429  * another thread in the same process (the agent thread only runs when the
1430  * process is completely stopped by /proc), or syslwp is creating a new lwp.
1431  */
1432 void *
1433 tsd_agent_get(kthread_t *t, uint_t key)
1434 {
1435 	struct tsd_thread *tsd = t->t_tsd;
1436 
1437 	ASSERT(t == curthread ||
1438 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1439 
1440 	if (key && tsd != NULL && key <= tsd->ts_nkeys)
1441 		return (tsd->ts_value[key - 1]);
1442 	return (NULL);
1443 }
1444 
1445 /*
1446  * Like tsd_set(), except that the agent lwp can set the tsd of
1447  * another thread in the same process, or syslwp can set the tsd
1448  * of a thread it's in the middle of creating.
1449  *
1450  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1451  * May lock out tsd_destroy (and tsd_create), may allocate memory with
1452  * lock held
1453  */
1454 int
1455 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1456 {
1457 	struct tsd_thread *tsd = t->t_tsd;
1458 
1459 	ASSERT(t == curthread ||
1460 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1461 
1462 	if (key == 0)
1463 		return (EINVAL);
1464 	if (tsd == NULL)
1465 		tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1466 	if (key <= tsd->ts_nkeys) {
1467 		tsd->ts_value[key - 1] = value;
1468 		return (0);
1469 	}
1470 
1471 	ASSERT(key <= tsd_nkeys);
1472 
1473 	/*
1474 	 * lock out tsd_destroy()
1475 	 */
1476 	mutex_enter(&tsd_mutex);
1477 	if (tsd->ts_nkeys == 0) {
1478 		/*
1479 		 * Link onto list of threads with TSD
1480 		 */
1481 		if ((tsd->ts_next = tsd_list) != NULL)
1482 			tsd_list->ts_prev = tsd;
1483 		tsd_list = tsd;
1484 	}
1485 
1486 	/*
1487 	 * Allocate thread local storage and set the value for key
1488 	 */
1489 	tsd->ts_value = tsd_realloc(tsd->ts_value,
1490 	    tsd->ts_nkeys * sizeof (void *),
1491 	    key * sizeof (void *));
1492 	tsd->ts_nkeys = key;
1493 	tsd->ts_value[key - 1] = value;
1494 	mutex_exit(&tsd_mutex);
1495 
1496 	return (0);
1497 }
1498 
1499 
1500 /*
1501  * Return the per thread value that was stored with the specified key
1502  *	If necessary, create the key and the value
1503  *	Assumes the caller is protecting *keyp from tsd_destroy
1504  */
1505 void *
1506 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1507 {
1508 	void *value;
1509 	uint_t key = *keyp;
1510 	struct tsd_thread *tsd = curthread->t_tsd;
1511 
1512 	if (tsd == NULL)
1513 		tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1514 	if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1515 		return (value);
1516 	if (key == 0)
1517 		tsd_create(keyp, destroy);
1518 	(void) tsd_set(*keyp, value = (*allocate)());
1519 
1520 	return (value);
1521 }
1522 
1523 /*
1524  * Called from thread_exit() to run the destructor function for each tsd
1525  *	Locks out tsd_create and tsd_destroy
1526  *	Assumes that the destructor *DOES NOT* use tsd
1527  */
1528 void
1529 tsd_exit(void)
1530 {
1531 	int i;
1532 	struct tsd_thread *tsd = curthread->t_tsd;
1533 
1534 	if (tsd == NULL)
1535 		return;
1536 
1537 	if (tsd->ts_nkeys == 0) {
1538 		kmem_free(tsd, sizeof (*tsd));
1539 		curthread->t_tsd = NULL;
1540 		return;
1541 	}
1542 
1543 	/*
1544 	 * lock out tsd_create and tsd_destroy, call
1545 	 * the destructor, and mark the value as destroyed.
1546 	 */
1547 	mutex_enter(&tsd_mutex);
1548 
1549 	for (i = 0; i < tsd->ts_nkeys; i++) {
1550 		if (tsd->ts_value[i] && tsd_destructor[i])
1551 			(*tsd_destructor[i])(tsd->ts_value[i]);
1552 		tsd->ts_value[i] = NULL;
1553 	}
1554 
1555 	/*
1556 	 * remove from linked list of threads with TSD
1557 	 */
1558 	if (tsd->ts_next)
1559 		tsd->ts_next->ts_prev = tsd->ts_prev;
1560 	if (tsd->ts_prev)
1561 		tsd->ts_prev->ts_next = tsd->ts_next;
1562 	if (tsd_list == tsd)
1563 		tsd_list = tsd->ts_next;
1564 
1565 	mutex_exit(&tsd_mutex);
1566 
1567 	/*
1568 	 * free up the TSD
1569 	 */
1570 	kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1571 	kmem_free(tsd, sizeof (struct tsd_thread));
1572 	curthread->t_tsd = NULL;
1573 }
1574 
1575 /*
1576  * realloc
1577  */
1578 static void *
1579 tsd_realloc(void *old, size_t osize, size_t nsize)
1580 {
1581 	void *new;
1582 
1583 	new = kmem_zalloc(nsize, KM_SLEEP);
1584 	if (old) {
1585 		bcopy(old, new, osize);
1586 		kmem_free(old, osize);
1587 	}
1588 	return (new);
1589 }
1590 
1591 /*
1592  * Check to see if an interrupt thread might be active at a given ipl.
1593  * If so return true.
1594  * We must be conservative--it is ok to give a false yes, but a false no
1595  * will cause disaster.  (But if the situation changes after we check it is
1596  * ok--the caller is trying to ensure that an interrupt routine has been
1597  * exited).
1598  * This is used when trying to remove an interrupt handler from an autovector
1599  * list in avintr.c.
1600  */
1601 int
1602 intr_active(struct cpu *cp, int level)
1603 {
1604 	if (level <= LOCK_LEVEL)
1605 		return (cp->cpu_thread != cp->cpu_dispthread);
1606 	else
1607 		return (CPU_ON_INTR(cp));
1608 }
1609 
1610 /*
1611  * Return non-zero if an interrupt is being serviced.
1612  */
1613 int
1614 servicing_interrupt()
1615 {
1616 	int onintr = 0;
1617 
1618 	/* Are we an interrupt thread */
1619 	if (curthread->t_flag & T_INTR_THREAD)
1620 		return (1);
1621 	/* Are we servicing a high level interrupt? */
1622 	if (CPU_ON_INTR(CPU)) {
1623 		kpreempt_disable();
1624 		onintr = CPU_ON_INTR(CPU);
1625 		kpreempt_enable();
1626 	}
1627 	return (onintr);
1628 }
1629 
1630 
1631 /*
1632  * Change the dispatch priority of a thread in the system.
1633  * Used when raising or lowering a thread's priority.
1634  * (E.g., priority inheritance)
1635  *
1636  * Since threads are queued according to their priority, we
1637  * we must check the thread's state to determine whether it
1638  * is on a queue somewhere. If it is, we've got to:
1639  *
1640  *	o Dequeue the thread.
1641  *	o Change its effective priority.
1642  *	o Enqueue the thread.
1643  *
1644  * Assumptions: The thread whose priority we wish to change
1645  * must be locked before we call thread_change_(e)pri().
1646  * The thread_change(e)pri() function doesn't drop the thread
1647  * lock--that must be done by its caller.
1648  */
1649 void
1650 thread_change_epri(kthread_t *t, pri_t disp_pri)
1651 {
1652 	uint_t	state;
1653 
1654 	ASSERT(THREAD_LOCK_HELD(t));
1655 
1656 	/*
1657 	 * If the inherited priority hasn't actually changed,
1658 	 * just return.
1659 	 */
1660 	if (t->t_epri == disp_pri)
1661 		return;
1662 
1663 	state = t->t_state;
1664 
1665 	/*
1666 	 * If it's not on a queue, change the priority with
1667 	 * impunity.
1668 	 */
1669 	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1670 		t->t_epri = disp_pri;
1671 
1672 		if (state == TS_ONPROC) {
1673 			cpu_t *cp = t->t_disp_queue->disp_cpu;
1674 
1675 			if (t == cp->cpu_dispthread)
1676 				cp->cpu_dispatch_pri = DISP_PRIO(t);
1677 		}
1678 		return;
1679 	}
1680 
1681 	/*
1682 	 * It's either on a sleep queue or a run queue.
1683 	 */
1684 	if (state == TS_SLEEP) {
1685 		/*
1686 		 * Take the thread out of its sleep queue.
1687 		 * Change the inherited priority.
1688 		 * Re-enqueue the thread.
1689 		 * Each synchronization object exports a function
1690 		 * to do this in an appropriate manner.
1691 		 */
1692 		SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1693 	} else if (state == TS_WAIT) {
1694 		/*
1695 		 * Re-enqueue a thread on the wait queue if its
1696 		 * effective priority needs to change.
1697 		 */
1698 		if (disp_pri != t->t_epri)
1699 			waitq_change_pri(t, disp_pri);
1700 	} else {
1701 		/*
1702 		 * The thread is on a run queue.
1703 		 * Note: setbackdq() may not put the thread
1704 		 * back on the same run queue where it originally
1705 		 * resided.
1706 		 */
1707 		(void) dispdeq(t);
1708 		t->t_epri = disp_pri;
1709 		setbackdq(t);
1710 	}
1711 }	/* end of thread_change_epri */
1712 
1713 /*
1714  * Function: Change the t_pri field of a thread.
1715  * Side Effects: Adjust the thread ordering on a run queue
1716  *		 or sleep queue, if necessary.
1717  * Returns: 1 if the thread was on a run queue, else 0.
1718  */
1719 int
1720 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1721 {
1722 	uint_t	state;
1723 	int	on_rq = 0;
1724 
1725 	ASSERT(THREAD_LOCK_HELD(t));
1726 
1727 	state = t->t_state;
1728 	THREAD_WILLCHANGE_PRI(t, disp_pri);
1729 
1730 	/*
1731 	 * If it's not on a queue, change the priority with
1732 	 * impunity.
1733 	 */
1734 	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1735 		t->t_pri = disp_pri;
1736 
1737 		if (state == TS_ONPROC) {
1738 			cpu_t *cp = t->t_disp_queue->disp_cpu;
1739 
1740 			if (t == cp->cpu_dispthread)
1741 				cp->cpu_dispatch_pri = DISP_PRIO(t);
1742 		}
1743 		return (0);
1744 	}
1745 
1746 	/*
1747 	 * It's either on a sleep queue or a run queue.
1748 	 */
1749 	if (state == TS_SLEEP) {
1750 		/*
1751 		 * If the priority has changed, take the thread out of
1752 		 * its sleep queue and change the priority.
1753 		 * Re-enqueue the thread.
1754 		 * Each synchronization object exports a function
1755 		 * to do this in an appropriate manner.
1756 		 */
1757 		if (disp_pri != t->t_pri)
1758 			SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1759 	} else if (state == TS_WAIT) {
1760 		/*
1761 		 * Re-enqueue a thread on the wait queue if its
1762 		 * priority needs to change.
1763 		 */
1764 		if (disp_pri != t->t_pri)
1765 			waitq_change_pri(t, disp_pri);
1766 	} else {
1767 		/*
1768 		 * The thread is on a run queue.
1769 		 * Note: setbackdq() may not put the thread
1770 		 * back on the same run queue where it originally
1771 		 * resided.
1772 		 *
1773 		 * We still requeue the thread even if the priority
1774 		 * is unchanged to preserve round-robin (and other)
1775 		 * effects between threads of the same priority.
1776 		 */
1777 		on_rq = dispdeq(t);
1778 		ASSERT(on_rq);
1779 		t->t_pri = disp_pri;
1780 		if (front) {
1781 			setfrontdq(t);
1782 		} else {
1783 			setbackdq(t);
1784 		}
1785 	}
1786 	return (on_rq);
1787 }
1788