xref: /illumos-gate/usr/src/uts/common/disp/thread.c (revision 0ccf9e790d232720597416743840df88825a9317)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/sysmacros.h>
31 #include <sys/signal.h>
32 #include <sys/stack.h>
33 #include <sys/pcb.h>
34 #include <sys/user.h>
35 #include <sys/systm.h>
36 #include <sys/sysinfo.h>
37 #include <sys/var.h>
38 #include <sys/errno.h>
39 #include <sys/cmn_err.h>
40 #include <sys/cred.h>
41 #include <sys/resource.h>
42 #include <sys/task.h>
43 #include <sys/project.h>
44 #include <sys/proc.h>
45 #include <sys/debug.h>
46 #include <sys/inline.h>
47 #include <sys/disp.h>
48 #include <sys/class.h>
49 #include <vm/seg_kmem.h>
50 #include <vm/seg_kp.h>
51 #include <sys/machlock.h>
52 #include <sys/kmem.h>
53 #include <sys/varargs.h>
54 #include <sys/turnstile.h>
55 #include <sys/poll.h>
56 #include <sys/vtrace.h>
57 #include <sys/callb.h>
58 #include <c2/audit.h>
59 #include <sys/tnf.h>
60 #include <sys/sobject.h>
61 #include <sys/cpupart.h>
62 #include <sys/pset.h>
63 #include <sys/door.h>
64 #include <sys/spl.h>
65 #include <sys/copyops.h>
66 #include <sys/rctl.h>
67 #include <sys/pool.h>
68 #include <sys/zone.h>
69 #include <sys/tsol/label.h>
70 #include <sys/tsol/tndb.h>
71 #include <sys/cpc_impl.h>
72 #include <sys/sdt.h>
73 #include <sys/reboot.h>
74 #include <sys/kdi.h>
75 
76 struct kmem_cache *thread_cache;	/* cache of free threads */
77 struct kmem_cache *lwp_cache;		/* cache of free lwps */
78 struct kmem_cache *turnstile_cache;	/* cache of free turnstiles */
79 
80 /*
81  * allthreads is only for use by kmem_readers.  All kernel loops can use
82  * the current thread as a start/end point.
83  */
84 static kthread_t *allthreads = &t0;	/* circular list of all threads */
85 
86 static kcondvar_t reaper_cv;		/* synchronization var */
87 kthread_t	*thread_deathrow;	/* circular list of reapable threads */
88 kthread_t	*lwp_deathrow;		/* circular list of reapable threads */
89 kmutex_t	reaplock;		/* protects lwp and thread deathrows */
90 kmutex_t	thread_free_lock;	/* protects clock from reaper */
91 int	thread_reapcnt = 0;		/* number of threads on deathrow */
92 int	lwp_reapcnt = 0;		/* number of lwps on deathrow */
93 int	reaplimit = 16;			/* delay reaping until reaplimit */
94 
95 extern int nthread;
96 
97 id_t	syscid;				/* system scheduling class ID */
98 void	*segkp_thread;			/* cookie for segkp pool */
99 
100 int lwp_cache_sz = 32;
101 int t_cache_sz = 8;
102 static kt_did_t next_t_id = 1;
103 
104 /*
105  * Min/Max stack sizes for stack size parameters
106  */
107 #define	MAX_STKSIZE	(32 * DEFAULTSTKSZ)
108 #define	MIN_STKSIZE	DEFAULTSTKSZ
109 
110 /*
111  * default_stksize overrides lwp_default_stksize if it is set.
112  */
113 int	default_stksize;
114 int	lwp_default_stksize;
115 
116 static zone_key_t zone_thread_key;
117 
118 /*
119  * forward declarations for internal thread specific data (tsd)
120  */
121 static void *tsd_realloc(void *, size_t, size_t);
122 
123 /*ARGSUSED*/
124 static int
125 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
126 {
127 	bzero(buf, sizeof (turnstile_t));
128 	return (0);
129 }
130 
131 /*ARGSUSED*/
132 static void
133 turnstile_destructor(void *buf, void *cdrarg)
134 {
135 	turnstile_t *ts = buf;
136 
137 	ASSERT(ts->ts_free == NULL);
138 	ASSERT(ts->ts_waiters == 0);
139 	ASSERT(ts->ts_inheritor == NULL);
140 	ASSERT(ts->ts_sleepq[0].sq_first == NULL);
141 	ASSERT(ts->ts_sleepq[1].sq_first == NULL);
142 }
143 
144 void
145 thread_init(void)
146 {
147 	kthread_t *tp;
148 	extern char sys_name[];
149 	extern void idle();
150 	struct cpu *cpu = CPU;
151 
152 	mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
153 
154 #if defined(__i386) || defined(__amd64)
155 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
156 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
157 
158 	/*
159 	 * "struct _klwp" includes a "struct pcb", which includes a
160 	 * "struct fpu", which needs to be 16-byte aligned on amd64
161 	 * (and even on i386 for fxsave/fxrstor).
162 	 */
163 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
164 	    16, NULL, NULL, NULL, NULL, NULL, 0);
165 #else
166 	/*
167 	 * Allocate thread structures from static_arena.  This prevents
168 	 * issues where a thread tries to relocate its own thread
169 	 * structure and touches it after the mapping has been suspended.
170 	 */
171 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
172 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
173 
174 	lwp_stk_cache_init();
175 
176 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
177 	    0, NULL, NULL, NULL, NULL, NULL, 0);
178 #endif
179 
180 	turnstile_cache = kmem_cache_create("turnstile_cache",
181 	    sizeof (turnstile_t), 0,
182 	    turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
183 
184 	label_init();
185 	cred_init();
186 
187 	rctl_init();
188 	project_init();
189 	zone_init();
190 	task_init();
191 	tcache_init();
192 	pool_init();
193 
194 	curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
195 
196 	/*
197 	 * Originally, we had two parameters to set default stack
198 	 * size: one for lwp's (lwp_default_stksize), and one for
199 	 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
200 	 * Now we have a third parameter that overrides both if it is
201 	 * set to a legal stack size, called default_stksize.
202 	 */
203 
204 	if (default_stksize == 0) {
205 		default_stksize = DEFAULTSTKSZ;
206 	} else if (default_stksize % PAGESIZE != 0 ||
207 	    default_stksize > MAX_STKSIZE ||
208 	    default_stksize < MIN_STKSIZE) {
209 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
210 		    (int)DEFAULTSTKSZ);
211 		default_stksize = DEFAULTSTKSZ;
212 	} else {
213 		lwp_default_stksize = default_stksize;
214 	}
215 
216 	if (lwp_default_stksize == 0) {
217 		lwp_default_stksize = default_stksize;
218 	} else if (lwp_default_stksize % PAGESIZE != 0 ||
219 	    lwp_default_stksize > MAX_STKSIZE ||
220 	    lwp_default_stksize < MIN_STKSIZE) {
221 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
222 		    default_stksize);
223 		lwp_default_stksize = default_stksize;
224 	}
225 
226 	segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
227 	    lwp_default_stksize,
228 	    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
229 
230 	segkp_thread = segkp_cache_init(segkp, t_cache_sz,
231 	    default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
232 
233 	(void) getcid(sys_name, &syscid);
234 	curthread->t_cid = syscid;	/* current thread is t0 */
235 
236 	/*
237 	 * Set up the first CPU's idle thread.
238 	 * It runs whenever the CPU has nothing worthwhile to do.
239 	 */
240 	tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
241 	cpu->cpu_idle_thread = tp;
242 	tp->t_preempt = 1;
243 	tp->t_disp_queue = cpu->cpu_disp;
244 	ASSERT(tp->t_disp_queue != NULL);
245 	tp->t_bound_cpu = cpu;
246 	tp->t_affinitycnt = 1;
247 
248 	/*
249 	 * Registering a thread in the callback table is usually
250 	 * done in the initialization code of the thread. In this
251 	 * case, we do it right after thread creation to avoid
252 	 * blocking idle thread while registering itself. It also
253 	 * avoids the possibility of reregistration in case a CPU
254 	 * restarts its idle thread.
255 	 */
256 	CALLB_CPR_INIT_SAFE(tp, "idle");
257 
258 	/*
259 	 * Finish initializing the kernel memory allocator now that
260 	 * thread_create() is available.
261 	 */
262 	kmem_thread_init();
263 
264 	if (boothowto & RB_DEBUG)
265 		kdi_dvec_thravail();
266 }
267 
268 /*
269  * Create a thread.
270  *
271  * thread_create() blocks for memory if necessary.  It never fails.
272  *
273  * If stk is NULL, the thread is created at the base of the stack
274  * and cannot be swapped.
275  */
276 kthread_t *
277 thread_create(
278 	caddr_t	stk,
279 	size_t	stksize,
280 	void	(*proc)(),
281 	void	*arg,
282 	size_t	len,
283 	proc_t	 *pp,
284 	int	state,
285 	pri_t	pri)
286 {
287 	kthread_t *t;
288 	extern struct classfuncs sys_classfuncs;
289 	turnstile_t *ts;
290 
291 	/*
292 	 * Every thread keeps a turnstile around in case it needs to block.
293 	 * The only reason the turnstile is not simply part of the thread
294 	 * structure is that we may have to break the association whenever
295 	 * more than one thread blocks on a given synchronization object.
296 	 * From a memory-management standpoint, turnstiles are like the
297 	 * "attached mblks" that hang off dblks in the streams allocator.
298 	 */
299 	ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
300 
301 	if (stk == NULL) {
302 		/*
303 		 * alloc both thread and stack in segkp chunk
304 		 */
305 
306 		if (stksize < default_stksize)
307 			stksize = default_stksize;
308 
309 		if (stksize == default_stksize) {
310 			stk = (caddr_t)segkp_cache_get(segkp_thread);
311 		} else {
312 			stksize = roundup(stksize, PAGESIZE);
313 			stk = (caddr_t)segkp_get(segkp, stksize,
314 			    (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
315 		}
316 
317 		ASSERT(stk != NULL);
318 
319 		/*
320 		 * The machine-dependent mutex code may require that
321 		 * thread pointers (since they may be used for mutex owner
322 		 * fields) have certain alignment requirements.
323 		 * PTR24_ALIGN is the size of the alignment quanta.
324 		 * XXX - assumes stack grows toward low addresses.
325 		 */
326 		if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
327 			cmn_err(CE_PANIC, "thread_create: proposed stack size"
328 			    " too small to hold thread.");
329 #ifdef STACK_GROWTH_DOWN
330 		stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
331 		stksize &= -PTR24_ALIGN;	/* make thread aligned */
332 		t = (kthread_t *)(stk + stksize);
333 		bzero(t, sizeof (kthread_t));
334 #ifdef	C2_AUDIT
335 		if (audit_active)
336 			audit_thread_create(t);
337 #endif
338 		t->t_stk = stk + stksize;
339 		t->t_stkbase = stk;
340 #else	/* stack grows to larger addresses */
341 		stksize -= SA(sizeof (kthread_t));
342 		t = (kthread_t *)(stk);
343 		bzero(t, sizeof (kthread_t));
344 		t->t_stk = stk + sizeof (kthread_t);
345 		t->t_stkbase = stk + stksize + sizeof (kthread_t);
346 #endif	/* STACK_GROWTH_DOWN */
347 		t->t_flag |= T_TALLOCSTK;
348 		t->t_swap = stk;
349 	} else {
350 		t = kmem_cache_alloc(thread_cache, KM_SLEEP);
351 		bzero(t, sizeof (kthread_t));
352 		ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
353 #ifdef	C2_AUDIT
354 		if (audit_active)
355 			audit_thread_create(t);
356 #endif
357 		/*
358 		 * Initialize t_stk to the kernel stack pointer to use
359 		 * upon entry to the kernel
360 		 */
361 #ifdef STACK_GROWTH_DOWN
362 		t->t_stk = stk + stksize;
363 		t->t_stkbase = stk;
364 #else
365 		t->t_stk = stk;			/* 3b2-like */
366 		t->t_stkbase = stk + stksize;
367 #endif /* STACK_GROWTH_DOWN */
368 	}
369 
370 	/* set default stack flag */
371 	if (stksize == lwp_default_stksize)
372 		t->t_flag |= T_DFLTSTK;
373 
374 	t->t_ts = ts;
375 
376 	/*
377 	 * p_cred could be NULL if it thread_create is called before cred_init
378 	 * is called in main.
379 	 */
380 	mutex_enter(&pp->p_crlock);
381 	if (pp->p_cred)
382 		crhold(t->t_cred = pp->p_cred);
383 	mutex_exit(&pp->p_crlock);
384 	t->t_start = gethrestime_sec();
385 	t->t_startpc = proc;
386 	t->t_procp = pp;
387 	t->t_clfuncs = &sys_classfuncs.thread;
388 	t->t_cid = syscid;
389 	t->t_pri = pri;
390 	t->t_stime = lbolt;
391 	t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
392 	t->t_bind_cpu = PBIND_NONE;
393 	t->t_bind_pset = PS_NONE;
394 	t->t_plockp = &pp->p_lock;
395 	t->t_copyops = NULL;
396 	t->t_taskq = NULL;
397 	t->t_anttime = 0;
398 	t->t_hatdepth = 0;
399 
400 	t->t_dtrace_vtime = 1;	/* assure vtimestamp is always non-zero */
401 
402 	CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
403 #ifndef NPROBE
404 	/* Kernel probe */
405 	tnf_thread_create(t);
406 #endif /* NPROBE */
407 	LOCK_INIT_CLEAR(&t->t_lock);
408 
409 	/*
410 	 * Callers who give us a NULL proc must do their own
411 	 * stack initialization.  e.g. lwp_create()
412 	 */
413 	if (proc != NULL) {
414 		t->t_stk = thread_stk_init(t->t_stk);
415 		thread_load(t, proc, arg, len);
416 	}
417 
418 	/*
419 	 * Put a hold on project0. If this thread is actually in a
420 	 * different project, then t_proj will be changed later in
421 	 * lwp_create().  All kernel-only threads must be in project 0.
422 	 */
423 	t->t_proj = project_hold(proj0p);
424 
425 	lgrp_affinity_init(&t->t_lgrp_affinity);
426 
427 	mutex_enter(&pidlock);
428 	nthread++;
429 	t->t_did = next_t_id++;
430 	t->t_prev = curthread->t_prev;
431 	t->t_next = curthread;
432 
433 	/*
434 	 * Add the thread to the list of all threads, and initialize
435 	 * its t_cpu pointer.  We need to block preemption since
436 	 * cpu_offline walks the thread list looking for threads
437 	 * with t_cpu pointing to the CPU being offlined.  We want
438 	 * to make sure that the list is consistent and that if t_cpu
439 	 * is set, the thread is on the list.
440 	 */
441 	kpreempt_disable();
442 	curthread->t_prev->t_next = t;
443 	curthread->t_prev = t;
444 
445 	/*
446 	 * Threads should never have a NULL t_cpu pointer so assign it
447 	 * here.  If the thread is being created with state TS_RUN a
448 	 * better CPU may be chosen when it is placed on the run queue.
449 	 *
450 	 * We need to keep kernel preemption disabled when setting all
451 	 * three fields to keep them in sync.  Also, always create in
452 	 * the default partition since that's where kernel threads go
453 	 * (if this isn't a kernel thread, t_cpupart will be changed
454 	 * in lwp_create before setting the thread runnable).
455 	 */
456 	t->t_cpupart = &cp_default;
457 
458 	/*
459 	 * For now, affiliate this thread with the root lgroup.
460 	 * Since the kernel does not (presently) allocate its memory
461 	 * in a locality aware fashion, the root is an appropriate home.
462 	 * If this thread is later associated with an lwp, it will have
463 	 * it's lgroup re-assigned at that time.
464 	 */
465 	lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
466 
467 	/*
468 	 * Inherit the current cpu.  If this cpu isn't part of the chosen
469 	 * lgroup, a new cpu will be chosen by cpu_choose when the thread
470 	 * is ready to run.
471 	 */
472 	if (CPU->cpu_part == &cp_default)
473 		t->t_cpu = CPU;
474 	else
475 		t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t->t_lpl,
476 		    t->t_pri, NULL);
477 
478 	t->t_disp_queue = t->t_cpu->cpu_disp;
479 	kpreempt_enable();
480 
481 	/*
482 	 * Initialize thread state and the dispatcher lock pointer.
483 	 * Need to hold onto pidlock to block allthreads walkers until
484 	 * the state is set.
485 	 */
486 	switch (state) {
487 	case TS_RUN:
488 		curthread->t_oldspl = splhigh();	/* get dispatcher spl */
489 		THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
490 		CL_SETRUN(t);
491 		thread_unlock(t);
492 		break;
493 
494 	case TS_ONPROC:
495 		THREAD_ONPROC(t, t->t_cpu);
496 		break;
497 
498 	case TS_FREE:
499 		/*
500 		 * Free state will be used for intr threads.
501 		 * The interrupt routine must set the thread dispatcher
502 		 * lock pointer (t_lockp) if starting on a CPU
503 		 * other than the current one.
504 		 */
505 		THREAD_FREEINTR(t, CPU);
506 		break;
507 
508 	case TS_STOPPED:
509 		THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
510 		break;
511 
512 	default:			/* TS_SLEEP, TS_ZOMB or TS_TRANS */
513 		cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
514 	}
515 	mutex_exit(&pidlock);
516 	return (t);
517 }
518 
519 /*
520  * Move thread to project0 and take care of project reference counters.
521  */
522 void
523 thread_rele(kthread_t *t)
524 {
525 	kproject_t *kpj;
526 
527 	thread_lock(t);
528 
529 	ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
530 	kpj = ttoproj(t);
531 	t->t_proj = proj0p;
532 
533 	thread_unlock(t);
534 
535 	if (kpj != proj0p) {
536 		project_rele(kpj);
537 		(void) project_hold(proj0p);
538 	}
539 }
540 
541 
542 void	(*ip_cleanup_func)(void);
543 
544 void
545 thread_exit()
546 {
547 	kthread_t *t = curthread;
548 
549 	if ((t->t_proc_flag & TP_ZTHREAD) != 0)
550 		cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
551 
552 	if (ip_cleanup_func != NULL)
553 		(*ip_cleanup_func)();
554 
555 	tsd_exit();		/* Clean up this thread's TSD */
556 
557 	kcpc_passivate();	/* clean up performance counter state */
558 
559 	/*
560 	 * No kernel thread should have called poll() without arranging
561 	 * calling pollcleanup() here.
562 	 */
563 	ASSERT(t->t_pollstate == NULL);
564 	ASSERT(t->t_schedctl == NULL);
565 	if (t->t_door)
566 		door_slam();	/* in case thread did an upcall */
567 
568 #ifndef NPROBE
569 	/* Kernel probe */
570 	if (t->t_tnf_tpdp)
571 		tnf_thread_exit();
572 #endif /* NPROBE */
573 
574 	thread_rele(t);
575 	t->t_preempt++;
576 
577 	/*
578 	 * remove thread from the all threads list so that
579 	 * death-row can use the same pointers.
580 	 */
581 	mutex_enter(&pidlock);
582 	t->t_next->t_prev = t->t_prev;
583 	t->t_prev->t_next = t->t_next;
584 	ASSERT(allthreads != t);	/* t0 never exits */
585 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
586 	mutex_exit(&pidlock);
587 
588 	if (t->t_ctx != NULL)
589 		exitctx(t);
590 	if (t->t_procp->p_pctx != NULL)
591 		exitpctx(t->t_procp);
592 
593 	t->t_state = TS_ZOMB;	/* set zombie thread */
594 
595 	swtch_from_zombie();	/* give up the CPU */
596 	/* NOTREACHED */
597 }
598 
599 /*
600  * Check to see if the specified thread is active (defined as being on
601  * the thread list).  This is certainly a slow way to do this; if there's
602  * ever a reason to speed it up, we could maintain a hash table of active
603  * threads indexed by their t_did.
604  */
605 static kthread_t *
606 did_to_thread(kt_did_t tid)
607 {
608 	kthread_t *t;
609 
610 	ASSERT(MUTEX_HELD(&pidlock));
611 	for (t = curthread->t_next; t != curthread; t = t->t_next) {
612 		if (t->t_did == tid)
613 			break;
614 	}
615 	if (t->t_did == tid)
616 		return (t);
617 	else
618 		return (NULL);
619 }
620 
621 /*
622  * Wait for specified thread to exit.  Returns immediately if the thread
623  * could not be found, meaning that it has either already exited or never
624  * existed.
625  */
626 void
627 thread_join(kt_did_t tid)
628 {
629 	kthread_t *t;
630 
631 	ASSERT(tid != curthread->t_did);
632 	ASSERT(tid != t0.t_did);
633 
634 	mutex_enter(&pidlock);
635 	/*
636 	 * Make sure we check that the thread is on the thread list
637 	 * before blocking on it; otherwise we could end up blocking on
638 	 * a cv that's already been freed.  In other words, don't cache
639 	 * the thread pointer across calls to cv_wait.
640 	 *
641 	 * The choice of loop invariant means that whenever a thread
642 	 * is taken off the allthreads list, a cv_broadcast must be
643 	 * performed on that thread's t_joincv to wake up any waiters.
644 	 * The broadcast doesn't have to happen right away, but it
645 	 * shouldn't be postponed indefinitely (e.g., by doing it in
646 	 * thread_free which may only be executed when the deathrow
647 	 * queue is processed.
648 	 */
649 	while (t = did_to_thread(tid))
650 		cv_wait(&t->t_joincv, &pidlock);
651 	mutex_exit(&pidlock);
652 }
653 
654 void
655 thread_free(kthread_t *t)
656 {
657 	ASSERT(t != &t0 && t->t_state == TS_FREE);
658 	ASSERT(t->t_door == NULL);
659 	ASSERT(t->t_schedctl == NULL);
660 	ASSERT(t->t_pollstate == NULL);
661 
662 	t->t_pri = 0;
663 	t->t_pc = 0;
664 	t->t_sp = 0;
665 	t->t_wchan0 = NULL;
666 	t->t_wchan = NULL;
667 	if (t->t_cred != NULL) {
668 		crfree(t->t_cred);
669 		t->t_cred = 0;
670 	}
671 	if (t->t_pdmsg) {
672 		kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
673 		t->t_pdmsg = NULL;
674 	}
675 #ifdef	C2_AUDIT
676 	if (audit_active)
677 		audit_thread_free(t);
678 #endif
679 #ifndef NPROBE
680 	if (t->t_tnf_tpdp)
681 		tnf_thread_free(t);
682 #endif /* NPROBE */
683 	if (t->t_cldata) {
684 		CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
685 	}
686 	if (t->t_rprof != NULL) {
687 		kmem_free(t->t_rprof, sizeof (*t->t_rprof));
688 		t->t_rprof = NULL;
689 	}
690 	t->t_lockp = NULL;	/* nothing should try to lock this thread now */
691 	if (t->t_lwp)
692 		lwp_freeregs(t->t_lwp, 0);
693 	if (t->t_ctx)
694 		freectx(t, 0);
695 	if (t->t_procp->p_pctx)
696 		freepctx(t->t_procp, 0);
697 	t->t_stk = NULL;
698 	if (t->t_lwp)
699 		lwp_stk_fini(t->t_lwp);
700 	lock_clear(&t->t_lock);
701 
702 	if (t->t_ts->ts_waiters > 0)
703 		panic("thread_free: turnstile still active");
704 
705 	kmem_cache_free(turnstile_cache, t->t_ts);
706 
707 	free_afd(&t->t_activefd);
708 
709 	/*
710 	 * Barrier for clock thread.  The clock holds this lock to
711 	 * keep the thread from going away while it's looking at it.
712 	 */
713 	mutex_enter(&thread_free_lock);
714 	mutex_exit(&thread_free_lock);
715 
716 	ASSERT(ttoproj(t) == proj0p);
717 	project_rele(ttoproj(t));
718 
719 	lgrp_affinity_free(&t->t_lgrp_affinity);
720 
721 	/*
722 	 * Free thread struct and its stack.
723 	 */
724 	if (t->t_flag & T_TALLOCSTK) {
725 		/* thread struct is embedded in stack */
726 		segkp_release(segkp, t->t_swap);
727 		mutex_enter(&pidlock);
728 		nthread--;
729 		mutex_exit(&pidlock);
730 	} else {
731 		if (t->t_swap) {
732 			segkp_release(segkp, t->t_swap);
733 			t->t_swap = NULL;
734 		}
735 		if (t->t_lwp) {
736 			kmem_cache_free(lwp_cache, t->t_lwp);
737 			t->t_lwp = NULL;
738 		}
739 		mutex_enter(&pidlock);
740 		nthread--;
741 		mutex_exit(&pidlock);
742 		kmem_cache_free(thread_cache, t);
743 	}
744 }
745 
746 /*
747  * Removes threads associated with the given zone from a deathrow queue.
748  * tp is a pointer to the head of the deathrow queue, and countp is a
749  * pointer to the current deathrow count.  Returns a linked list of
750  * threads removed from the list.
751  */
752 static kthread_t *
753 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
754 {
755 	kthread_t *tmp, *list = NULL;
756 	cred_t *cr;
757 
758 	ASSERT(MUTEX_HELD(&reaplock));
759 	while (*tp != NULL) {
760 		if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
761 			tmp = *tp;
762 			*tp = tmp->t_forw;
763 			tmp->t_forw = list;
764 			list = tmp;
765 			(*countp)--;
766 		} else {
767 			tp = &(*tp)->t_forw;
768 		}
769 	}
770 	return (list);
771 }
772 
773 static void
774 thread_reap_list(kthread_t *t)
775 {
776 	kthread_t *next;
777 
778 	while (t != NULL) {
779 		next = t->t_forw;
780 		thread_free(t);
781 		t = next;
782 	}
783 }
784 
785 /* ARGSUSED */
786 static void
787 thread_zone_destroy(zoneid_t zoneid, void *unused)
788 {
789 	kthread_t *t, *l;
790 
791 	mutex_enter(&reaplock);
792 	/*
793 	 * Pull threads and lwps associated with zone off deathrow lists.
794 	 */
795 	t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
796 	l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
797 	mutex_exit(&reaplock);
798 
799 	/*
800 	 * Reap threads
801 	 */
802 	thread_reap_list(t);
803 
804 	/*
805 	 * Reap lwps
806 	 */
807 	thread_reap_list(l);
808 }
809 
810 /*
811  * cleanup zombie threads that are on deathrow.
812  */
813 void
814 thread_reaper()
815 {
816 	kthread_t *t, *l;
817 	callb_cpr_t cprinfo;
818 
819 	/*
820 	 * Register callback to clean up threads when zone is destroyed.
821 	 */
822 	zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
823 
824 	CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
825 	for (;;) {
826 		mutex_enter(&reaplock);
827 		while (thread_deathrow == NULL && lwp_deathrow == NULL) {
828 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
829 			cv_wait(&reaper_cv, &reaplock);
830 			CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
831 		}
832 		t = thread_deathrow;
833 		l = lwp_deathrow;
834 		thread_deathrow = NULL;
835 		lwp_deathrow = NULL;
836 		thread_reapcnt = 0;
837 		lwp_reapcnt = 0;
838 		mutex_exit(&reaplock);
839 
840 		/*
841 		 * Reap threads
842 		 */
843 		thread_reap_list(t);
844 
845 		/*
846 		 * Reap lwps
847 		 */
848 		thread_reap_list(l);
849 	}
850 }
851 
852 /*
853  * This is called by resume() to put a zombie thread onto deathrow.
854  * The thread's state is changed to TS_FREE to indicate that is reapable.
855  * This is called from the idle thread so it must not block (just spin).
856  */
857 void
858 reapq_add(kthread_t *t)
859 {
860 	mutex_enter(&reaplock);
861 
862 	/*
863 	 * lwp_deathrow contains only threads with lwp linkage
864 	 * that are of the default stacksize. Anything else goes
865 	 * on thread_deathrow.
866 	 */
867 	if (ttolwp(t) && (t->t_flag & T_DFLTSTK)) {
868 		t->t_forw = lwp_deathrow;
869 		lwp_deathrow = t;
870 		lwp_reapcnt++;
871 	} else {
872 		t->t_forw = thread_deathrow;
873 		thread_deathrow = t;
874 		thread_reapcnt++;
875 	}
876 	if (lwp_reapcnt + thread_reapcnt > reaplimit)
877 		cv_signal(&reaper_cv);	/* wake the reaper */
878 	t->t_state = TS_FREE;
879 	lock_clear(&t->t_lock);
880 	mutex_exit(&reaplock);
881 }
882 
883 /*
884  * Install thread context ops for the current thread.
885  */
886 void
887 installctx(
888 	kthread_t *t,
889 	void	*arg,
890 	void	(*save)(void *),
891 	void	(*restore)(void *),
892 	void	(*fork)(void *, void *),
893 	void	(*lwp_create)(void *, void *),
894 	void	(*exit)(void *),
895 	void	(*free)(void *, int))
896 {
897 	struct ctxop *ctx;
898 
899 	ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
900 	ctx->save_op = save;
901 	ctx->restore_op = restore;
902 	ctx->fork_op = fork;
903 	ctx->lwp_create_op = lwp_create;
904 	ctx->exit_op = exit;
905 	ctx->free_op = free;
906 	ctx->arg = arg;
907 	ctx->next = t->t_ctx;
908 	t->t_ctx = ctx;
909 }
910 
911 /*
912  * Remove thread context ops from the current thread.
913  * (Or allow the agent thread to remove thread context ops from another
914  * thread in the same process)
915  */
916 int
917 removectx(
918 	kthread_t *t,
919 	void	*arg,
920 	void	(*save)(void *),
921 	void	(*restore)(void *),
922 	void	(*fork)(void *, void *),
923 	void	(*lwp_create)(void *, void *),
924 	void	(*exit)(void *),
925 	void	(*free)(void *, int))
926 {
927 	struct ctxop *ctx, *prev_ctx;
928 
929 	ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
930 	    ttoproc(t)->p_agenttp == curthread);
931 
932 	/*
933 	 * There's a potential race for t_ctx between the agent thread
934 	 * and the target thread when lwps are exiting (for example,
935 	 * when the process is reacting to having been killed).  At
936 	 * other times, the target thread will be TS_STOPPED whilst the
937 	 * agent thread is inside this function.  However, from the
938 	 * perspective of the cost of locking, it seems cheaper to take
939 	 * a thread-specific lock everytime we come through here.
940 	 */
941 	mutex_enter(&t->t_ctx_lock);
942 	prev_ctx = NULL;
943 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) {
944 		if (ctx->save_op == save && ctx->restore_op == restore &&
945 		    ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
946 		    ctx->exit_op == exit && ctx->free_op == free &&
947 		    ctx->arg == arg) {
948 			if (prev_ctx)
949 				prev_ctx->next = ctx->next;
950 			else
951 				t->t_ctx = ctx->next;
952 			mutex_exit(&t->t_ctx_lock);
953 			if (ctx->free_op != NULL)
954 				(ctx->free_op)(ctx->arg, 0);
955 			kmem_free(ctx, sizeof (struct ctxop));
956 			return (1);
957 		}
958 		prev_ctx = ctx;
959 	}
960 	mutex_exit(&t->t_ctx_lock);
961 
962 	return (0);
963 }
964 
965 void
966 savectx(kthread_t *t)
967 {
968 	struct ctxop *ctx;
969 
970 	ASSERT(t == curthread);
971 	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
972 		if (ctx->save_op != NULL)
973 			(ctx->save_op)(ctx->arg);
974 }
975 
976 void
977 restorectx(kthread_t *t)
978 {
979 	struct ctxop *ctx;
980 
981 	ASSERT(t == curthread);
982 	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
983 		if (ctx->restore_op != NULL)
984 			(ctx->restore_op)(ctx->arg);
985 }
986 
987 void
988 forkctx(kthread_t *t, kthread_t *ct)
989 {
990 	struct ctxop *ctx;
991 
992 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
993 		if (ctx->fork_op != NULL)
994 			(ctx->fork_op)(t, ct);
995 }
996 
997 /*
998  * Note that this operator is only invoked via the _lwp_create
999  * system call.  The system may have other reasons to create lwps
1000  * e.g. the agent lwp or the doors unreferenced lwp.
1001  */
1002 void
1003 lwp_createctx(kthread_t *t, kthread_t *ct)
1004 {
1005 	struct ctxop *ctx;
1006 
1007 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1008 		if (ctx->lwp_create_op != NULL)
1009 			(ctx->lwp_create_op)(t, ct);
1010 }
1011 
1012 /*
1013  * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1014  * needed when the thread/LWP leaves the processor for the last time. This
1015  * routine is not intended to deal with freeing memory; freectx() is used for
1016  * that purpose during thread_free(). This routine is provided to allow for
1017  * clean-up that can't wait until thread_free().
1018  */
1019 void
1020 exitctx(kthread_t *t)
1021 {
1022 	struct ctxop *ctx;
1023 
1024 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1025 		if (ctx->exit_op != NULL)
1026 			(ctx->exit_op)(t);
1027 }
1028 
1029 /*
1030  * freectx is called from thread_free() and exec() to get
1031  * rid of old thread context ops.
1032  */
1033 void
1034 freectx(kthread_t *t, int isexec)
1035 {
1036 	struct ctxop *ctx;
1037 
1038 	while ((ctx = t->t_ctx) != NULL) {
1039 		t->t_ctx = ctx->next;
1040 		if (ctx->free_op != NULL)
1041 			(ctx->free_op)(ctx->arg, isexec);
1042 		kmem_free(ctx, sizeof (struct ctxop));
1043 	}
1044 }
1045 
1046 /*
1047  * Set the thread running; arrange for it to be swapped in if necessary.
1048  */
1049 void
1050 setrun_locked(kthread_t *t)
1051 {
1052 	ASSERT(THREAD_LOCK_HELD(t));
1053 	if (t->t_state == TS_SLEEP) {
1054 		/*
1055 		 * Take off sleep queue.
1056 		 */
1057 		SOBJ_UNSLEEP(t->t_sobj_ops, t);
1058 	} else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1059 		/*
1060 		 * Already on dispatcher queue.
1061 		 */
1062 		return;
1063 	} else if (t->t_state == TS_STOPPED) {
1064 		/*
1065 		 * All of the sending of SIGCONT (TC_XSTART) and /proc
1066 		 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1067 		 * requested that the thread be run.
1068 		 * Just calling setrun() is not sufficient to set a stopped
1069 		 * thread running.  TP_TXSTART is always set if the thread
1070 		 * is not stopped by a jobcontrol stop signal.
1071 		 * TP_TPSTART is always set if /proc is not controlling it.
1072 		 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1073 		 * The thread won't be stopped unless one of these
1074 		 * three mechanisms did it.
1075 		 *
1076 		 * These flags must be set before calling setrun_locked(t).
1077 		 * They can't be passed as arguments because the streams
1078 		 * code calls setrun() indirectly and the mechanism for
1079 		 * doing so admits only one argument.  Note that the
1080 		 * thread must be locked in order to change t_schedflags.
1081 		 */
1082 		if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1083 			return;
1084 		/*
1085 		 * Process is no longer stopped (a thread is running).
1086 		 */
1087 		t->t_whystop = 0;
1088 		t->t_whatstop = 0;
1089 		/*
1090 		 * Strictly speaking, we do not have to clear these
1091 		 * flags here; they are cleared on entry to stop().
1092 		 * However, they are confusing when doing kernel
1093 		 * debugging or when they are revealed by ps(1).
1094 		 */
1095 		t->t_schedflag &= ~TS_ALLSTART;
1096 		THREAD_TRANSITION(t);	/* drop stopped-thread lock */
1097 		ASSERT(t->t_lockp == &transition_lock);
1098 		ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1099 		/*
1100 		 * Let the class put the process on the dispatcher queue.
1101 		 */
1102 		CL_SETRUN(t);
1103 	}
1104 
1105 
1106 }
1107 
1108 void
1109 setrun(kthread_t *t)
1110 {
1111 	thread_lock(t);
1112 	setrun_locked(t);
1113 	thread_unlock(t);
1114 }
1115 
1116 /*
1117  * Unpin an interrupted thread.
1118  *	When an interrupt occurs, the interrupt is handled on the stack
1119  *	of an interrupt thread, taken from a pool linked to the CPU structure.
1120  *
1121  *	When swtch() is switching away from an interrupt thread because it
1122  *	blocked or was preempted, this routine is called to complete the
1123  *	saving of the interrupted thread state, and returns the interrupted
1124  *	thread pointer so it may be resumed.
1125  *
1126  *	Called by swtch() only at high spl.
1127  */
1128 kthread_t *
1129 thread_unpin()
1130 {
1131 	kthread_t	*t = curthread;	/* current thread */
1132 	kthread_t	*itp;		/* interrupted thread */
1133 	int		i;		/* interrupt level */
1134 	extern int	intr_passivate();
1135 
1136 	ASSERT(t->t_intr != NULL);
1137 
1138 	itp = t->t_intr;		/* interrupted thread */
1139 	t->t_intr = NULL;		/* clear interrupt ptr */
1140 
1141 	/*
1142 	 * Get state from interrupt thread for the one
1143 	 * it interrupted.
1144 	 */
1145 
1146 	i = intr_passivate(t, itp);
1147 
1148 	TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1149 		"intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1150 		i, t, t, itp, itp);
1151 
1152 	/*
1153 	 * Dissociate the current thread from the interrupted thread's LWP.
1154 	 */
1155 	t->t_lwp = NULL;
1156 
1157 	/*
1158 	 * Interrupt handlers above the level that spinlocks block must
1159 	 * not block.
1160 	 */
1161 #if DEBUG
1162 	if (i < 0 || i > LOCK_LEVEL)
1163 		cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1164 #endif
1165 
1166 	/*
1167 	 * Compute the CPU's base interrupt level based on the active
1168 	 * interrupts.
1169 	 */
1170 	ASSERT(CPU->cpu_intr_actv & (1 << i));
1171 	set_base_spl();
1172 
1173 	return (itp);
1174 }
1175 
1176 /*
1177  * Create and initialize an interrupt thread.
1178  *	Returns non-zero on error.
1179  *	Called at spl7() or better.
1180  */
1181 void
1182 thread_create_intr(struct cpu *cp)
1183 {
1184 	kthread_t *tp;
1185 
1186 	tp = thread_create(NULL, 0,
1187 	    (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1188 
1189 	/*
1190 	 * Set the thread in the TS_FREE state.  The state will change
1191 	 * to TS_ONPROC only while the interrupt is active.  Think of these
1192 	 * as being on a private free list for the CPU.  Being TS_FREE keeps
1193 	 * inactive interrupt threads out of debugger thread lists.
1194 	 *
1195 	 * We cannot call thread_create with TS_FREE because of the current
1196 	 * checks there for ONPROC.  Fix this when thread_create takes flags.
1197 	 */
1198 	THREAD_FREEINTR(tp, cp);
1199 
1200 	/*
1201 	 * Nobody should ever reference the credentials of an interrupt
1202 	 * thread so make it NULL to catch any such references.
1203 	 */
1204 	tp->t_cred = NULL;
1205 	tp->t_flag |= T_INTR_THREAD;
1206 	tp->t_cpu = cp;
1207 	tp->t_bound_cpu = cp;
1208 	tp->t_disp_queue = cp->cpu_disp;
1209 	tp->t_affinitycnt = 1;
1210 	tp->t_preempt = 1;
1211 
1212 	/*
1213 	 * Don't make a user-requested binding on this thread so that
1214 	 * the processor can be offlined.
1215 	 */
1216 	tp->t_bind_cpu = PBIND_NONE;	/* no USER-requested binding */
1217 	tp->t_bind_pset = PS_NONE;
1218 
1219 #if defined(__i386) || defined(__amd64)
1220 	tp->t_stk -= STACK_ALIGN;
1221 	*(tp->t_stk) = 0;		/* terminate intr thread stack */
1222 #endif
1223 
1224 	/*
1225 	 * Link onto CPU's interrupt pool.
1226 	 */
1227 	tp->t_link = cp->cpu_intr_thread;
1228 	cp->cpu_intr_thread = tp;
1229 }
1230 
1231 /*
1232  * TSD -- THREAD SPECIFIC DATA
1233  */
1234 static kmutex_t		tsd_mutex;	 /* linked list spin lock */
1235 static uint_t		tsd_nkeys;	 /* size of destructor array */
1236 /* per-key destructor funcs */
1237 static void 		(**tsd_destructor)(void *);
1238 /* list of tsd_thread's */
1239 static struct tsd_thread	*tsd_list;
1240 
1241 /*
1242  * Default destructor
1243  *	Needed because NULL destructor means that the key is unused
1244  */
1245 /* ARGSUSED */
1246 void
1247 tsd_defaultdestructor(void *value)
1248 {}
1249 
1250 /*
1251  * Create a key (index into per thread array)
1252  *	Locks out tsd_create, tsd_destroy, and tsd_exit
1253  *	May allocate memory with lock held
1254  */
1255 void
1256 tsd_create(uint_t *keyp, void (*destructor)(void *))
1257 {
1258 	int	i;
1259 	uint_t	nkeys;
1260 
1261 	/*
1262 	 * if key is allocated, do nothing
1263 	 */
1264 	mutex_enter(&tsd_mutex);
1265 	if (*keyp) {
1266 		mutex_exit(&tsd_mutex);
1267 		return;
1268 	}
1269 	/*
1270 	 * find an unused key
1271 	 */
1272 	if (destructor == NULL)
1273 		destructor = tsd_defaultdestructor;
1274 
1275 	for (i = 0; i < tsd_nkeys; ++i)
1276 		if (tsd_destructor[i] == NULL)
1277 			break;
1278 
1279 	/*
1280 	 * if no unused keys, increase the size of the destructor array
1281 	 */
1282 	if (i == tsd_nkeys) {
1283 		if ((nkeys = (tsd_nkeys << 1)) == 0)
1284 			nkeys = 1;
1285 		tsd_destructor =
1286 		    (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1287 		    (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1288 		    (size_t)(nkeys * sizeof (void (*)(void *))));
1289 		tsd_nkeys = nkeys;
1290 	}
1291 
1292 	/*
1293 	 * allocate the next available unused key
1294 	 */
1295 	tsd_destructor[i] = destructor;
1296 	*keyp = i + 1;
1297 	mutex_exit(&tsd_mutex);
1298 }
1299 
1300 /*
1301  * Destroy a key -- this is for unloadable modules
1302  *
1303  * Assumes that the caller is preventing tsd_set and tsd_get
1304  * Locks out tsd_create, tsd_destroy, and tsd_exit
1305  * May free memory with lock held
1306  */
1307 void
1308 tsd_destroy(uint_t *keyp)
1309 {
1310 	uint_t key;
1311 	struct tsd_thread *tsd;
1312 
1313 	/*
1314 	 * protect the key namespace and our destructor lists
1315 	 */
1316 	mutex_enter(&tsd_mutex);
1317 	key = *keyp;
1318 	*keyp = 0;
1319 
1320 	ASSERT(key <= tsd_nkeys);
1321 
1322 	/*
1323 	 * if the key is valid
1324 	 */
1325 	if (key != 0) {
1326 		uint_t k = key - 1;
1327 		/*
1328 		 * for every thread with TSD, call key's destructor
1329 		 */
1330 		for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1331 			/*
1332 			 * no TSD for key in this thread
1333 			 */
1334 			if (key > tsd->ts_nkeys)
1335 				continue;
1336 			/*
1337 			 * call destructor for key
1338 			 */
1339 			if (tsd->ts_value[k] && tsd_destructor[k])
1340 				(*tsd_destructor[k])(tsd->ts_value[k]);
1341 			/*
1342 			 * reset value for key
1343 			 */
1344 			tsd->ts_value[k] = NULL;
1345 		}
1346 		/*
1347 		 * actually free the key (NULL destructor == unused)
1348 		 */
1349 		tsd_destructor[k] = NULL;
1350 	}
1351 
1352 	mutex_exit(&tsd_mutex);
1353 }
1354 
1355 /*
1356  * Quickly return the per thread value that was stored with the specified key
1357  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1358  */
1359 void *
1360 tsd_get(uint_t key)
1361 {
1362 	return (tsd_agent_get(curthread, key));
1363 }
1364 
1365 /*
1366  * Set a per thread value indexed with the specified key
1367  */
1368 int
1369 tsd_set(uint_t key, void *value)
1370 {
1371 	return (tsd_agent_set(curthread, key, value));
1372 }
1373 
1374 /*
1375  * Like tsd_get(), except that the agent lwp can get the tsd of
1376  * another thread in the same process (the agent thread only runs when the
1377  * process is completely stopped by /proc), or syslwp is creating a new lwp.
1378  */
1379 void *
1380 tsd_agent_get(kthread_t *t, uint_t key)
1381 {
1382 	struct tsd_thread *tsd = t->t_tsd;
1383 
1384 	ASSERT(t == curthread ||
1385 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1386 
1387 	if (key && tsd != NULL && key <= tsd->ts_nkeys)
1388 		return (tsd->ts_value[key - 1]);
1389 	return (NULL);
1390 }
1391 
1392 /*
1393  * Like tsd_set(), except that the agent lwp can set the tsd of
1394  * another thread in the same process, or syslwp can set the tsd
1395  * of a thread it's in the middle of creating.
1396  *
1397  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1398  * May lock out tsd_destroy (and tsd_create), may allocate memory with
1399  * lock held
1400  */
1401 int
1402 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1403 {
1404 	struct tsd_thread *tsd = t->t_tsd;
1405 
1406 	ASSERT(t == curthread ||
1407 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1408 
1409 	if (key == 0)
1410 		return (EINVAL);
1411 	if (tsd == NULL)
1412 		tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1413 	if (key <= tsd->ts_nkeys) {
1414 		tsd->ts_value[key - 1] = value;
1415 		return (0);
1416 	}
1417 
1418 	ASSERT(key <= tsd_nkeys);
1419 
1420 	/*
1421 	 * lock out tsd_destroy()
1422 	 */
1423 	mutex_enter(&tsd_mutex);
1424 	if (tsd->ts_nkeys == 0) {
1425 		/*
1426 		 * Link onto list of threads with TSD
1427 		 */
1428 		if ((tsd->ts_next = tsd_list) != NULL)
1429 			tsd_list->ts_prev = tsd;
1430 		tsd_list = tsd;
1431 	}
1432 
1433 	/*
1434 	 * Allocate thread local storage and set the value for key
1435 	 */
1436 	tsd->ts_value = tsd_realloc(tsd->ts_value,
1437 	    tsd->ts_nkeys * sizeof (void *),
1438 	    key * sizeof (void *));
1439 	tsd->ts_nkeys = key;
1440 	tsd->ts_value[key - 1] = value;
1441 	mutex_exit(&tsd_mutex);
1442 
1443 	return (0);
1444 }
1445 
1446 
1447 /*
1448  * Return the per thread value that was stored with the specified key
1449  *	If necessary, create the key and the value
1450  *	Assumes the caller is protecting *keyp from tsd_destroy
1451  */
1452 void *
1453 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1454 {
1455 	void *value;
1456 	uint_t key = *keyp;
1457 	struct tsd_thread *tsd = curthread->t_tsd;
1458 
1459 	if (tsd == NULL)
1460 		tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1461 	if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1462 		return (value);
1463 	if (key == 0)
1464 		tsd_create(keyp, destroy);
1465 	(void) tsd_set(*keyp, value = (*allocate)());
1466 
1467 	return (value);
1468 }
1469 
1470 /*
1471  * Called from thread_exit() to run the destructor function for each tsd
1472  *	Locks out tsd_create and tsd_destroy
1473  *	Assumes that the destructor *DOES NOT* use tsd
1474  */
1475 void
1476 tsd_exit(void)
1477 {
1478 	int i;
1479 	struct tsd_thread *tsd = curthread->t_tsd;
1480 
1481 	if (tsd == NULL)
1482 		return;
1483 
1484 	if (tsd->ts_nkeys == 0) {
1485 		kmem_free(tsd, sizeof (*tsd));
1486 		curthread->t_tsd = NULL;
1487 		return;
1488 	}
1489 
1490 	/*
1491 	 * lock out tsd_create and tsd_destroy, call
1492 	 * the destructor, and mark the value as destroyed.
1493 	 */
1494 	mutex_enter(&tsd_mutex);
1495 
1496 	for (i = 0; i < tsd->ts_nkeys; i++) {
1497 		if (tsd->ts_value[i] && tsd_destructor[i])
1498 			(*tsd_destructor[i])(tsd->ts_value[i]);
1499 		tsd->ts_value[i] = NULL;
1500 	}
1501 
1502 	/*
1503 	 * remove from linked list of threads with TSD
1504 	 */
1505 	if (tsd->ts_next)
1506 		tsd->ts_next->ts_prev = tsd->ts_prev;
1507 	if (tsd->ts_prev)
1508 		tsd->ts_prev->ts_next = tsd->ts_next;
1509 	if (tsd_list == tsd)
1510 		tsd_list = tsd->ts_next;
1511 
1512 	mutex_exit(&tsd_mutex);
1513 
1514 	/*
1515 	 * free up the TSD
1516 	 */
1517 	kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1518 	kmem_free(tsd, sizeof (struct tsd_thread));
1519 	curthread->t_tsd = NULL;
1520 }
1521 
1522 /*
1523  * realloc
1524  */
1525 static void *
1526 tsd_realloc(void *old, size_t osize, size_t nsize)
1527 {
1528 	void *new;
1529 
1530 	new = kmem_zalloc(nsize, KM_SLEEP);
1531 	if (old) {
1532 		bcopy(old, new, osize);
1533 		kmem_free(old, osize);
1534 	}
1535 	return (new);
1536 }
1537 
1538 /*
1539  * Check to see if an interrupt thread might be active at a given ipl.
1540  * If so return true.
1541  * We must be conservative--it is ok to give a false yes, but a false no
1542  * will cause disaster.  (But if the situation changes after we check it is
1543  * ok--the caller is trying to ensure that an interrupt routine has been
1544  * exited).
1545  * This is used when trying to remove an interrupt handler from an autovector
1546  * list in avintr.c.
1547  */
1548 int
1549 intr_active(struct cpu *cp, int level)
1550 {
1551 	if (level <= LOCK_LEVEL)
1552 		return (cp->cpu_thread != cp->cpu_dispthread);
1553 	else
1554 		return (CPU_ON_INTR(cp));
1555 }
1556 
1557 /*
1558  * Return non-zero if an interrupt is being serviced.
1559  */
1560 int
1561 servicing_interrupt()
1562 {
1563 	int onintr = 0;
1564 
1565 	/* Are we an interrupt thread */
1566 	if (curthread->t_flag & T_INTR_THREAD)
1567 		return (1);
1568 	/* Are we servicing a high level interrupt? */
1569 	if (CPU_ON_INTR(CPU)) {
1570 		kpreempt_disable();
1571 		onintr = CPU_ON_INTR(CPU);
1572 		kpreempt_enable();
1573 	}
1574 	return (onintr);
1575 }
1576 
1577 
1578 /*
1579  * Change the dispatch priority of a thread in the system.
1580  * Used when raising or lowering a thread's priority.
1581  * (E.g., priority inheritance)
1582  *
1583  * Since threads are queued according to their priority, we
1584  * we must check the thread's state to determine whether it
1585  * is on a queue somewhere. If it is, we've got to:
1586  *
1587  *	o Dequeue the thread.
1588  *	o Change its effective priority.
1589  *	o Enqueue the thread.
1590  *
1591  * Assumptions: The thread whose priority we wish to change
1592  * must be locked before we call thread_change_(e)pri().
1593  * The thread_change(e)pri() function doesn't drop the thread
1594  * lock--that must be done by its caller.
1595  */
1596 void
1597 thread_change_epri(kthread_t *t, pri_t disp_pri)
1598 {
1599 	uint_t	state;
1600 
1601 	ASSERT(THREAD_LOCK_HELD(t));
1602 
1603 	/*
1604 	 * If the inherited priority hasn't actually changed,
1605 	 * just return.
1606 	 */
1607 	if (t->t_epri == disp_pri)
1608 		return;
1609 
1610 	state = t->t_state;
1611 
1612 	/*
1613 	 * If it's not on a queue, change the priority with
1614 	 * impunity.
1615 	 */
1616 	if ((state & (TS_SLEEP | TS_RUN)) == 0) {
1617 		t->t_epri = disp_pri;
1618 
1619 		if (state == TS_ONPROC) {
1620 			cpu_t *cp = t->t_disp_queue->disp_cpu;
1621 
1622 			if (t == cp->cpu_dispthread)
1623 				cp->cpu_dispatch_pri = DISP_PRIO(t);
1624 		}
1625 		return;
1626 	}
1627 
1628 	/*
1629 	 * It's either on a sleep queue or a run queue.
1630 	 */
1631 	if (state == TS_SLEEP) {
1632 
1633 		/*
1634 		 * Take the thread out of its sleep queue.
1635 		 * Change the inherited priority.
1636 		 * Re-enqueue the thread.
1637 		 * Each synchronization object exports a function
1638 		 * to do this in an appropriate manner.
1639 		 */
1640 		SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1641 	} else {
1642 		/*
1643 		 * The thread is on a run queue.
1644 		 * Note: setbackdq() may not put the thread
1645 		 * back on the same run queue where it originally
1646 		 * resided.
1647 		 */
1648 		(void) dispdeq(t);
1649 		t->t_epri = disp_pri;
1650 		setbackdq(t);
1651 	}
1652 }	/* end of thread_change_epri */
1653 
1654 /*
1655  * Function: Change the t_pri field of a thread.
1656  * Side Effects: Adjust the thread ordering on a run queue
1657  *		 or sleep queue, if necessary.
1658  * Returns: 1 if the thread was on a run queue, else 0.
1659  */
1660 int
1661 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1662 {
1663 	uint_t	state;
1664 	int	on_rq = 0;
1665 
1666 	ASSERT(THREAD_LOCK_HELD(t));
1667 
1668 	state = t->t_state;
1669 	THREAD_WILLCHANGE_PRI(t, disp_pri);
1670 
1671 	/*
1672 	 * If it's not on a queue, change the priority with
1673 	 * impunity.
1674 	 */
1675 	if ((state & (TS_SLEEP | TS_RUN)) == 0) {
1676 		t->t_pri = disp_pri;
1677 
1678 		if (state == TS_ONPROC) {
1679 			cpu_t *cp = t->t_disp_queue->disp_cpu;
1680 
1681 			if (t == cp->cpu_dispthread)
1682 				cp->cpu_dispatch_pri = DISP_PRIO(t);
1683 		}
1684 		return (0);
1685 	}
1686 
1687 	/*
1688 	 * It's either on a sleep queue or a run queue.
1689 	 */
1690 	if (state == TS_SLEEP) {
1691 		/*
1692 		 * If the priority has changed, take the thread out of
1693 		 * its sleep queue and change the priority.
1694 		 * Re-enqueue the thread.
1695 		 * Each synchronization object exports a function
1696 		 * to do this in an appropriate manner.
1697 		 */
1698 		if (disp_pri != t->t_pri)
1699 			SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1700 	} else {
1701 		/*
1702 		 * The thread is on a run queue.
1703 		 * Note: setbackdq() may not put the thread
1704 		 * back on the same run queue where it originally
1705 		 * resided.
1706 		 *
1707 		 * We still requeue the thread even if the priority
1708 		 * is unchanged to preserve round-robin (and other)
1709 		 * effects between threads of the same priority.
1710 		 */
1711 		on_rq = dispdeq(t);
1712 		ASSERT(on_rq);
1713 		t->t_pri = disp_pri;
1714 		if (front) {
1715 			setfrontdq(t);
1716 		} else {
1717 			setbackdq(t);
1718 		}
1719 	}
1720 	return (on_rq);
1721 }
1722