xref: /titanic_51/usr/src/uts/common/disp/thread.c (revision 58c0eeecf3809d173057fc5a445f5de64d590d9f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/sysmacros.h>
31 #include <sys/signal.h>
32 #include <sys/stack.h>
33 #include <sys/pcb.h>
34 #include <sys/user.h>
35 #include <sys/systm.h>
36 #include <sys/sysinfo.h>
37 #include <sys/var.h>
38 #include <sys/errno.h>
39 #include <sys/cmn_err.h>
40 #include <sys/cred.h>
41 #include <sys/resource.h>
42 #include <sys/task.h>
43 #include <sys/project.h>
44 #include <sys/proc.h>
45 #include <sys/debug.h>
46 #include <sys/inline.h>
47 #include <sys/disp.h>
48 #include <sys/class.h>
49 #include <vm/seg_kmem.h>
50 #include <vm/seg_kp.h>
51 #include <sys/machlock.h>
52 #include <sys/kmem.h>
53 #include <sys/varargs.h>
54 #include <sys/turnstile.h>
55 #include <sys/poll.h>
56 #include <sys/vtrace.h>
57 #include <sys/callb.h>
58 #include <c2/audit.h>
59 #include <sys/tnf.h>
60 #include <sys/sobject.h>
61 #include <sys/cpupart.h>
62 #include <sys/pset.h>
63 #include <sys/door.h>
64 #include <sys/spl.h>
65 #include <sys/copyops.h>
66 #include <sys/rctl.h>
67 #include <sys/brand.h>
68 #include <sys/pool.h>
69 #include <sys/zone.h>
70 #include <sys/tsol/label.h>
71 #include <sys/tsol/tndb.h>
72 #include <sys/cpc_impl.h>
73 #include <sys/sdt.h>
74 #include <sys/reboot.h>
75 #include <sys/kdi.h>
76 #include <sys/waitq.h>
77 #include <sys/cpucaps.h>
78 #include <inet/ip.h>
79 #include <inet/ip_if.h>
80 
81 struct kmem_cache *thread_cache;	/* cache of free threads */
82 struct kmem_cache *lwp_cache;		/* cache of free lwps */
83 struct kmem_cache *turnstile_cache;	/* cache of free turnstiles */
84 
85 /*
86  * allthreads is only for use by kmem_readers.  All kernel loops can use
87  * the current thread as a start/end point.
88  */
89 static kthread_t *allthreads = &t0;	/* circular list of all threads */
90 
91 static kcondvar_t reaper_cv;		/* synchronization var */
92 kthread_t	*thread_deathrow;	/* circular list of reapable threads */
93 kthread_t	*lwp_deathrow;		/* circular list of reapable threads */
94 kmutex_t	reaplock;		/* protects lwp and thread deathrows */
95 kmutex_t	thread_free_lock;	/* protects clock from reaper */
96 int	thread_reapcnt = 0;		/* number of threads on deathrow */
97 int	lwp_reapcnt = 0;		/* number of lwps on deathrow */
98 int	reaplimit = 16;			/* delay reaping until reaplimit */
99 
100 extern int nthread;
101 
102 id_t	syscid;				/* system scheduling class ID */
103 void	*segkp_thread;			/* cookie for segkp pool */
104 
105 int lwp_cache_sz = 32;
106 int t_cache_sz = 8;
107 static kt_did_t next_t_id = 1;
108 
109 /*
110  * Min/Max stack sizes for stack size parameters
111  */
112 #define	MAX_STKSIZE	(32 * DEFAULTSTKSZ)
113 #define	MIN_STKSIZE	DEFAULTSTKSZ
114 
115 /*
116  * default_stksize overrides lwp_default_stksize if it is set.
117  */
118 int	default_stksize;
119 int	lwp_default_stksize;
120 
121 static zone_key_t zone_thread_key;
122 
123 /*
124  * forward declarations for internal thread specific data (tsd)
125  */
126 static void *tsd_realloc(void *, size_t, size_t);
127 
128 void thread_reaper(void);
129 
130 /*ARGSUSED*/
131 static int
132 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
133 {
134 	bzero(buf, sizeof (turnstile_t));
135 	return (0);
136 }
137 
138 /*ARGSUSED*/
139 static void
140 turnstile_destructor(void *buf, void *cdrarg)
141 {
142 	turnstile_t *ts = buf;
143 
144 	ASSERT(ts->ts_free == NULL);
145 	ASSERT(ts->ts_waiters == 0);
146 	ASSERT(ts->ts_inheritor == NULL);
147 	ASSERT(ts->ts_sleepq[0].sq_first == NULL);
148 	ASSERT(ts->ts_sleepq[1].sq_first == NULL);
149 }
150 
151 void
152 thread_init(void)
153 {
154 	kthread_t *tp;
155 	extern char sys_name[];
156 	extern void idle();
157 	struct cpu *cpu = CPU;
158 
159 	mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
160 
161 #if defined(__i386) || defined(__amd64)
162 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
163 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
164 
165 	/*
166 	 * "struct _klwp" includes a "struct pcb", which includes a
167 	 * "struct fpu", which needs to be 16-byte aligned on amd64
168 	 * (and even on i386 for fxsave/fxrstor).
169 	 */
170 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
171 	    16, NULL, NULL, NULL, NULL, NULL, 0);
172 #else
173 	/*
174 	 * Allocate thread structures from static_arena.  This prevents
175 	 * issues where a thread tries to relocate its own thread
176 	 * structure and touches it after the mapping has been suspended.
177 	 */
178 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
179 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
180 
181 	lwp_stk_cache_init();
182 
183 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
184 	    0, NULL, NULL, NULL, NULL, NULL, 0);
185 #endif
186 
187 	turnstile_cache = kmem_cache_create("turnstile_cache",
188 	    sizeof (turnstile_t), 0,
189 	    turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
190 
191 	label_init();
192 	cred_init();
193 
194 	/*
195 	 * Initialize various resource management facilities.
196 	 */
197 	rctl_init();
198 	cpucaps_init();
199 	/*
200 	 * Zone_init() should be called before project_init() so that project ID
201 	 * for the first project is initialized correctly.
202 	 */
203 	zone_init();
204 	project_init();
205 	brand_init();
206 	task_init();
207 	tcache_init();
208 	pool_init();
209 
210 	curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
211 
212 	/*
213 	 * Originally, we had two parameters to set default stack
214 	 * size: one for lwp's (lwp_default_stksize), and one for
215 	 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
216 	 * Now we have a third parameter that overrides both if it is
217 	 * set to a legal stack size, called default_stksize.
218 	 */
219 
220 	if (default_stksize == 0) {
221 		default_stksize = DEFAULTSTKSZ;
222 	} else if (default_stksize % PAGESIZE != 0 ||
223 	    default_stksize > MAX_STKSIZE ||
224 	    default_stksize < MIN_STKSIZE) {
225 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
226 		    (int)DEFAULTSTKSZ);
227 		default_stksize = DEFAULTSTKSZ;
228 	} else {
229 		lwp_default_stksize = default_stksize;
230 	}
231 
232 	if (lwp_default_stksize == 0) {
233 		lwp_default_stksize = default_stksize;
234 	} else if (lwp_default_stksize % PAGESIZE != 0 ||
235 	    lwp_default_stksize > MAX_STKSIZE ||
236 	    lwp_default_stksize < MIN_STKSIZE) {
237 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
238 		    default_stksize);
239 		lwp_default_stksize = default_stksize;
240 	}
241 
242 	segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
243 	    lwp_default_stksize,
244 	    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
245 
246 	segkp_thread = segkp_cache_init(segkp, t_cache_sz,
247 	    default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
248 
249 	(void) getcid(sys_name, &syscid);
250 	curthread->t_cid = syscid;	/* current thread is t0 */
251 
252 	/*
253 	 * Set up the first CPU's idle thread.
254 	 * It runs whenever the CPU has nothing worthwhile to do.
255 	 */
256 	tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
257 	cpu->cpu_idle_thread = tp;
258 	tp->t_preempt = 1;
259 	tp->t_disp_queue = cpu->cpu_disp;
260 	ASSERT(tp->t_disp_queue != NULL);
261 	tp->t_bound_cpu = cpu;
262 	tp->t_affinitycnt = 1;
263 
264 	/*
265 	 * Registering a thread in the callback table is usually
266 	 * done in the initialization code of the thread. In this
267 	 * case, we do it right after thread creation to avoid
268 	 * blocking idle thread while registering itself. It also
269 	 * avoids the possibility of reregistration in case a CPU
270 	 * restarts its idle thread.
271 	 */
272 	CALLB_CPR_INIT_SAFE(tp, "idle");
273 
274 	/*
275 	 * Create the thread_reaper daemon. From this point on, exited
276 	 * threads will get reaped.
277 	 */
278 	(void) thread_create(NULL, 0, (void (*)())thread_reaper,
279 	    NULL, 0, &p0, TS_RUN, minclsyspri);
280 
281 	/*
282 	 * Finish initializing the kernel memory allocator now that
283 	 * thread_create() is available.
284 	 */
285 	kmem_thread_init();
286 
287 	if (boothowto & RB_DEBUG)
288 		kdi_dvec_thravail();
289 }
290 
291 /*
292  * Create a thread.
293  *
294  * thread_create() blocks for memory if necessary.  It never fails.
295  *
296  * If stk is NULL, the thread is created at the base of the stack
297  * and cannot be swapped.
298  */
299 kthread_t *
300 thread_create(
301 	caddr_t	stk,
302 	size_t	stksize,
303 	void	(*proc)(),
304 	void	*arg,
305 	size_t	len,
306 	proc_t	 *pp,
307 	int	state,
308 	pri_t	pri)
309 {
310 	kthread_t *t;
311 	extern struct classfuncs sys_classfuncs;
312 	turnstile_t *ts;
313 
314 	/*
315 	 * Every thread keeps a turnstile around in case it needs to block.
316 	 * The only reason the turnstile is not simply part of the thread
317 	 * structure is that we may have to break the association whenever
318 	 * more than one thread blocks on a given synchronization object.
319 	 * From a memory-management standpoint, turnstiles are like the
320 	 * "attached mblks" that hang off dblks in the streams allocator.
321 	 */
322 	ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
323 
324 	if (stk == NULL) {
325 		/*
326 		 * alloc both thread and stack in segkp chunk
327 		 */
328 
329 		if (stksize < default_stksize)
330 			stksize = default_stksize;
331 
332 		if (stksize == default_stksize) {
333 			stk = (caddr_t)segkp_cache_get(segkp_thread);
334 		} else {
335 			stksize = roundup(stksize, PAGESIZE);
336 			stk = (caddr_t)segkp_get(segkp, stksize,
337 			    (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
338 		}
339 
340 		ASSERT(stk != NULL);
341 
342 		/*
343 		 * The machine-dependent mutex code may require that
344 		 * thread pointers (since they may be used for mutex owner
345 		 * fields) have certain alignment requirements.
346 		 * PTR24_ALIGN is the size of the alignment quanta.
347 		 * XXX - assumes stack grows toward low addresses.
348 		 */
349 		if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
350 			cmn_err(CE_PANIC, "thread_create: proposed stack size"
351 			    " too small to hold thread.");
352 #ifdef STACK_GROWTH_DOWN
353 		stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
354 		stksize &= -PTR24_ALIGN;	/* make thread aligned */
355 		t = (kthread_t *)(stk + stksize);
356 		bzero(t, sizeof (kthread_t));
357 #ifdef	C2_AUDIT
358 		if (audit_active)
359 			audit_thread_create(t);
360 #endif
361 		t->t_stk = stk + stksize;
362 		t->t_stkbase = stk;
363 #else	/* stack grows to larger addresses */
364 		stksize -= SA(sizeof (kthread_t));
365 		t = (kthread_t *)(stk);
366 		bzero(t, sizeof (kthread_t));
367 		t->t_stk = stk + sizeof (kthread_t);
368 		t->t_stkbase = stk + stksize + sizeof (kthread_t);
369 #endif	/* STACK_GROWTH_DOWN */
370 		t->t_flag |= T_TALLOCSTK;
371 		t->t_swap = stk;
372 	} else {
373 		t = kmem_cache_alloc(thread_cache, KM_SLEEP);
374 		bzero(t, sizeof (kthread_t));
375 		ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
376 #ifdef	C2_AUDIT
377 		if (audit_active)
378 			audit_thread_create(t);
379 #endif
380 		/*
381 		 * Initialize t_stk to the kernel stack pointer to use
382 		 * upon entry to the kernel
383 		 */
384 #ifdef STACK_GROWTH_DOWN
385 		t->t_stk = stk + stksize;
386 		t->t_stkbase = stk;
387 #else
388 		t->t_stk = stk;			/* 3b2-like */
389 		t->t_stkbase = stk + stksize;
390 #endif /* STACK_GROWTH_DOWN */
391 	}
392 
393 	/* set default stack flag */
394 	if (stksize == lwp_default_stksize)
395 		t->t_flag |= T_DFLTSTK;
396 
397 	t->t_ts = ts;
398 
399 	/*
400 	 * p_cred could be NULL if it thread_create is called before cred_init
401 	 * is called in main.
402 	 */
403 	mutex_enter(&pp->p_crlock);
404 	if (pp->p_cred)
405 		crhold(t->t_cred = pp->p_cred);
406 	mutex_exit(&pp->p_crlock);
407 	t->t_start = gethrestime_sec();
408 	t->t_startpc = proc;
409 	t->t_procp = pp;
410 	t->t_clfuncs = &sys_classfuncs.thread;
411 	t->t_cid = syscid;
412 	t->t_pri = pri;
413 	t->t_stime = lbolt;
414 	t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
415 	t->t_bind_cpu = PBIND_NONE;
416 	t->t_bind_pset = PS_NONE;
417 	t->t_plockp = &pp->p_lock;
418 	t->t_copyops = NULL;
419 	t->t_taskq = NULL;
420 	t->t_anttime = 0;
421 	t->t_hatdepth = 0;
422 
423 	t->t_dtrace_vtime = 1;	/* assure vtimestamp is always non-zero */
424 
425 	CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
426 #ifndef NPROBE
427 	/* Kernel probe */
428 	tnf_thread_create(t);
429 #endif /* NPROBE */
430 	LOCK_INIT_CLEAR(&t->t_lock);
431 
432 	/*
433 	 * Callers who give us a NULL proc must do their own
434 	 * stack initialization.  e.g. lwp_create()
435 	 */
436 	if (proc != NULL) {
437 		t->t_stk = thread_stk_init(t->t_stk);
438 		thread_load(t, proc, arg, len);
439 	}
440 
441 	/*
442 	 * Put a hold on project0. If this thread is actually in a
443 	 * different project, then t_proj will be changed later in
444 	 * lwp_create().  All kernel-only threads must be in project 0.
445 	 */
446 	t->t_proj = project_hold(proj0p);
447 
448 	lgrp_affinity_init(&t->t_lgrp_affinity);
449 
450 	mutex_enter(&pidlock);
451 	nthread++;
452 	t->t_did = next_t_id++;
453 	t->t_prev = curthread->t_prev;
454 	t->t_next = curthread;
455 
456 	/*
457 	 * Add the thread to the list of all threads, and initialize
458 	 * its t_cpu pointer.  We need to block preemption since
459 	 * cpu_offline walks the thread list looking for threads
460 	 * with t_cpu pointing to the CPU being offlined.  We want
461 	 * to make sure that the list is consistent and that if t_cpu
462 	 * is set, the thread is on the list.
463 	 */
464 	kpreempt_disable();
465 	curthread->t_prev->t_next = t;
466 	curthread->t_prev = t;
467 
468 	/*
469 	 * Threads should never have a NULL t_cpu pointer so assign it
470 	 * here.  If the thread is being created with state TS_RUN a
471 	 * better CPU may be chosen when it is placed on the run queue.
472 	 *
473 	 * We need to keep kernel preemption disabled when setting all
474 	 * three fields to keep them in sync.  Also, always create in
475 	 * the default partition since that's where kernel threads go
476 	 * (if this isn't a kernel thread, t_cpupart will be changed
477 	 * in lwp_create before setting the thread runnable).
478 	 */
479 	t->t_cpupart = &cp_default;
480 
481 	/*
482 	 * For now, affiliate this thread with the root lgroup.
483 	 * Since the kernel does not (presently) allocate its memory
484 	 * in a locality aware fashion, the root is an appropriate home.
485 	 * If this thread is later associated with an lwp, it will have
486 	 * it's lgroup re-assigned at that time.
487 	 */
488 	lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
489 
490 	/*
491 	 * Inherit the current cpu.  If this cpu isn't part of the chosen
492 	 * lgroup, a new cpu will be chosen by cpu_choose when the thread
493 	 * is ready to run.
494 	 */
495 	if (CPU->cpu_part == &cp_default)
496 		t->t_cpu = CPU;
497 	else
498 		t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t->t_lpl,
499 		    t->t_pri, NULL);
500 
501 	t->t_disp_queue = t->t_cpu->cpu_disp;
502 	kpreempt_enable();
503 
504 	/*
505 	 * Initialize thread state and the dispatcher lock pointer.
506 	 * Need to hold onto pidlock to block allthreads walkers until
507 	 * the state is set.
508 	 */
509 	switch (state) {
510 	case TS_RUN:
511 		curthread->t_oldspl = splhigh();	/* get dispatcher spl */
512 		THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
513 		CL_SETRUN(t);
514 		thread_unlock(t);
515 		break;
516 
517 	case TS_ONPROC:
518 		THREAD_ONPROC(t, t->t_cpu);
519 		break;
520 
521 	case TS_FREE:
522 		/*
523 		 * Free state will be used for intr threads.
524 		 * The interrupt routine must set the thread dispatcher
525 		 * lock pointer (t_lockp) if starting on a CPU
526 		 * other than the current one.
527 		 */
528 		THREAD_FREEINTR(t, CPU);
529 		break;
530 
531 	case TS_STOPPED:
532 		THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
533 		break;
534 
535 	default:			/* TS_SLEEP, TS_ZOMB or TS_TRANS */
536 		cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
537 	}
538 	mutex_exit(&pidlock);
539 	return (t);
540 }
541 
542 /*
543  * Move thread to project0 and take care of project reference counters.
544  */
545 void
546 thread_rele(kthread_t *t)
547 {
548 	kproject_t *kpj;
549 
550 	thread_lock(t);
551 
552 	ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
553 	kpj = ttoproj(t);
554 	t->t_proj = proj0p;
555 
556 	thread_unlock(t);
557 
558 	if (kpj != proj0p) {
559 		project_rele(kpj);
560 		(void) project_hold(proj0p);
561 	}
562 }
563 
564 /*
565  * This is a function which is called from thread_exit
566  * that can be used to debug reference count issues in IP.
567  */
568 void (*ip_cleanup_func)(void);
569 
570 void
571 thread_exit()
572 {
573 	kthread_t *t = curthread;
574 
575 	if ((t->t_proc_flag & TP_ZTHREAD) != 0)
576 		cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
577 
578 	if (ip_cleanup_func != NULL)
579 		(*ip_cleanup_func)();
580 
581 	tsd_exit();		/* Clean up this thread's TSD */
582 
583 	kcpc_passivate();	/* clean up performance counter state */
584 
585 	/*
586 	 * No kernel thread should have called poll() without arranging
587 	 * calling pollcleanup() here.
588 	 */
589 	ASSERT(t->t_pollstate == NULL);
590 	ASSERT(t->t_schedctl == NULL);
591 	if (t->t_door)
592 		door_slam();	/* in case thread did an upcall */
593 
594 #ifndef NPROBE
595 	/* Kernel probe */
596 	if (t->t_tnf_tpdp)
597 		tnf_thread_exit();
598 #endif /* NPROBE */
599 
600 	thread_rele(t);
601 	t->t_preempt++;
602 
603 	/*
604 	 * remove thread from the all threads list so that
605 	 * death-row can use the same pointers.
606 	 */
607 	mutex_enter(&pidlock);
608 	t->t_next->t_prev = t->t_prev;
609 	t->t_prev->t_next = t->t_next;
610 	ASSERT(allthreads != t);	/* t0 never exits */
611 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
612 	mutex_exit(&pidlock);
613 
614 	if (t->t_ctx != NULL)
615 		exitctx(t);
616 	if (t->t_procp->p_pctx != NULL)
617 		exitpctx(t->t_procp);
618 
619 	t->t_state = TS_ZOMB;	/* set zombie thread */
620 
621 	swtch_from_zombie();	/* give up the CPU */
622 	/* NOTREACHED */
623 }
624 
625 /*
626  * Check to see if the specified thread is active (defined as being on
627  * the thread list).  This is certainly a slow way to do this; if there's
628  * ever a reason to speed it up, we could maintain a hash table of active
629  * threads indexed by their t_did.
630  */
631 static kthread_t *
632 did_to_thread(kt_did_t tid)
633 {
634 	kthread_t *t;
635 
636 	ASSERT(MUTEX_HELD(&pidlock));
637 	for (t = curthread->t_next; t != curthread; t = t->t_next) {
638 		if (t->t_did == tid)
639 			break;
640 	}
641 	if (t->t_did == tid)
642 		return (t);
643 	else
644 		return (NULL);
645 }
646 
647 /*
648  * Wait for specified thread to exit.  Returns immediately if the thread
649  * could not be found, meaning that it has either already exited or never
650  * existed.
651  */
652 void
653 thread_join(kt_did_t tid)
654 {
655 	kthread_t *t;
656 
657 	ASSERT(tid != curthread->t_did);
658 	ASSERT(tid != t0.t_did);
659 
660 	mutex_enter(&pidlock);
661 	/*
662 	 * Make sure we check that the thread is on the thread list
663 	 * before blocking on it; otherwise we could end up blocking on
664 	 * a cv that's already been freed.  In other words, don't cache
665 	 * the thread pointer across calls to cv_wait.
666 	 *
667 	 * The choice of loop invariant means that whenever a thread
668 	 * is taken off the allthreads list, a cv_broadcast must be
669 	 * performed on that thread's t_joincv to wake up any waiters.
670 	 * The broadcast doesn't have to happen right away, but it
671 	 * shouldn't be postponed indefinitely (e.g., by doing it in
672 	 * thread_free which may only be executed when the deathrow
673 	 * queue is processed.
674 	 */
675 	while (t = did_to_thread(tid))
676 		cv_wait(&t->t_joincv, &pidlock);
677 	mutex_exit(&pidlock);
678 }
679 
680 void
681 thread_free(kthread_t *t)
682 {
683 	ASSERT(t != &t0 && t->t_state == TS_FREE);
684 	ASSERT(t->t_door == NULL);
685 	ASSERT(t->t_schedctl == NULL);
686 	ASSERT(t->t_pollstate == NULL);
687 
688 	t->t_pri = 0;
689 	t->t_pc = 0;
690 	t->t_sp = 0;
691 	t->t_wchan0 = NULL;
692 	t->t_wchan = NULL;
693 	if (t->t_cred != NULL) {
694 		crfree(t->t_cred);
695 		t->t_cred = 0;
696 	}
697 	if (t->t_pdmsg) {
698 		kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
699 		t->t_pdmsg = NULL;
700 	}
701 #ifdef	C2_AUDIT
702 	if (audit_active)
703 		audit_thread_free(t);
704 #endif
705 #ifndef NPROBE
706 	if (t->t_tnf_tpdp)
707 		tnf_thread_free(t);
708 #endif /* NPROBE */
709 	if (t->t_cldata) {
710 		CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
711 	}
712 	if (t->t_rprof != NULL) {
713 		kmem_free(t->t_rprof, sizeof (*t->t_rprof));
714 		t->t_rprof = NULL;
715 	}
716 	t->t_lockp = NULL;	/* nothing should try to lock this thread now */
717 	if (t->t_lwp)
718 		lwp_freeregs(t->t_lwp, 0);
719 	if (t->t_ctx)
720 		freectx(t, 0);
721 	t->t_stk = NULL;
722 	if (t->t_lwp)
723 		lwp_stk_fini(t->t_lwp);
724 	lock_clear(&t->t_lock);
725 
726 	if (t->t_ts->ts_waiters > 0)
727 		panic("thread_free: turnstile still active");
728 
729 	kmem_cache_free(turnstile_cache, t->t_ts);
730 
731 	free_afd(&t->t_activefd);
732 
733 	/*
734 	 * Barrier for clock thread.  The clock holds this lock to
735 	 * keep the thread from going away while it's looking at it.
736 	 */
737 	mutex_enter(&thread_free_lock);
738 	mutex_exit(&thread_free_lock);
739 
740 	ASSERT(ttoproj(t) == proj0p);
741 	project_rele(ttoproj(t));
742 
743 	lgrp_affinity_free(&t->t_lgrp_affinity);
744 
745 	/*
746 	 * Free thread struct and its stack.
747 	 */
748 	if (t->t_flag & T_TALLOCSTK) {
749 		/* thread struct is embedded in stack */
750 		segkp_release(segkp, t->t_swap);
751 		mutex_enter(&pidlock);
752 		nthread--;
753 		mutex_exit(&pidlock);
754 	} else {
755 		if (t->t_swap) {
756 			segkp_release(segkp, t->t_swap);
757 			t->t_swap = NULL;
758 		}
759 		if (t->t_lwp) {
760 			kmem_cache_free(lwp_cache, t->t_lwp);
761 			t->t_lwp = NULL;
762 		}
763 		mutex_enter(&pidlock);
764 		nthread--;
765 		mutex_exit(&pidlock);
766 		kmem_cache_free(thread_cache, t);
767 	}
768 }
769 
770 /*
771  * Removes threads associated with the given zone from a deathrow queue.
772  * tp is a pointer to the head of the deathrow queue, and countp is a
773  * pointer to the current deathrow count.  Returns a linked list of
774  * threads removed from the list.
775  */
776 static kthread_t *
777 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
778 {
779 	kthread_t *tmp, *list = NULL;
780 	cred_t *cr;
781 
782 	ASSERT(MUTEX_HELD(&reaplock));
783 	while (*tp != NULL) {
784 		if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
785 			tmp = *tp;
786 			*tp = tmp->t_forw;
787 			tmp->t_forw = list;
788 			list = tmp;
789 			(*countp)--;
790 		} else {
791 			tp = &(*tp)->t_forw;
792 		}
793 	}
794 	return (list);
795 }
796 
797 static void
798 thread_reap_list(kthread_t *t)
799 {
800 	kthread_t *next;
801 
802 	while (t != NULL) {
803 		next = t->t_forw;
804 		thread_free(t);
805 		t = next;
806 	}
807 }
808 
809 /* ARGSUSED */
810 static void
811 thread_zone_destroy(zoneid_t zoneid, void *unused)
812 {
813 	kthread_t *t, *l;
814 
815 	mutex_enter(&reaplock);
816 	/*
817 	 * Pull threads and lwps associated with zone off deathrow lists.
818 	 */
819 	t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
820 	l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
821 	mutex_exit(&reaplock);
822 
823 	/*
824 	 * Reap threads
825 	 */
826 	thread_reap_list(t);
827 
828 	/*
829 	 * Reap lwps
830 	 */
831 	thread_reap_list(l);
832 }
833 
834 /*
835  * cleanup zombie threads that are on deathrow.
836  */
837 void
838 thread_reaper()
839 {
840 	kthread_t *t, *l;
841 	callb_cpr_t cprinfo;
842 
843 	/*
844 	 * Register callback to clean up threads when zone is destroyed.
845 	 */
846 	zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
847 
848 	CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
849 	for (;;) {
850 		mutex_enter(&reaplock);
851 		while (thread_deathrow == NULL && lwp_deathrow == NULL) {
852 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
853 			cv_wait(&reaper_cv, &reaplock);
854 			CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
855 		}
856 		t = thread_deathrow;
857 		l = lwp_deathrow;
858 		thread_deathrow = NULL;
859 		lwp_deathrow = NULL;
860 		thread_reapcnt = 0;
861 		lwp_reapcnt = 0;
862 		mutex_exit(&reaplock);
863 
864 		/*
865 		 * Reap threads
866 		 */
867 		thread_reap_list(t);
868 
869 		/*
870 		 * Reap lwps
871 		 */
872 		thread_reap_list(l);
873 	}
874 }
875 
876 /*
877  * This is called by resume() to put a zombie thread onto deathrow.
878  * The thread's state is changed to TS_FREE to indicate that is reapable.
879  * This is called from the idle thread so it must not block (just spin).
880  */
881 void
882 reapq_add(kthread_t *t)
883 {
884 	mutex_enter(&reaplock);
885 
886 	/*
887 	 * lwp_deathrow contains only threads with lwp linkage
888 	 * that are of the default stacksize. Anything else goes
889 	 * on thread_deathrow.
890 	 */
891 	if (ttolwp(t) && (t->t_flag & T_DFLTSTK)) {
892 		t->t_forw = lwp_deathrow;
893 		lwp_deathrow = t;
894 		lwp_reapcnt++;
895 	} else {
896 		t->t_forw = thread_deathrow;
897 		thread_deathrow = t;
898 		thread_reapcnt++;
899 	}
900 	if (lwp_reapcnt + thread_reapcnt > reaplimit)
901 		cv_signal(&reaper_cv);	/* wake the reaper */
902 	t->t_state = TS_FREE;
903 	lock_clear(&t->t_lock);
904 	mutex_exit(&reaplock);
905 }
906 
907 /*
908  * Install thread context ops for the current thread.
909  */
910 void
911 installctx(
912 	kthread_t *t,
913 	void	*arg,
914 	void	(*save)(void *),
915 	void	(*restore)(void *),
916 	void	(*fork)(void *, void *),
917 	void	(*lwp_create)(void *, void *),
918 	void	(*exit)(void *),
919 	void	(*free)(void *, int))
920 {
921 	struct ctxop *ctx;
922 
923 	ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
924 	ctx->save_op = save;
925 	ctx->restore_op = restore;
926 	ctx->fork_op = fork;
927 	ctx->lwp_create_op = lwp_create;
928 	ctx->exit_op = exit;
929 	ctx->free_op = free;
930 	ctx->arg = arg;
931 	ctx->next = t->t_ctx;
932 	t->t_ctx = ctx;
933 }
934 
935 /*
936  * Remove the thread context ops from a thread.
937  */
938 int
939 removectx(
940 	kthread_t *t,
941 	void	*arg,
942 	void	(*save)(void *),
943 	void	(*restore)(void *),
944 	void	(*fork)(void *, void *),
945 	void	(*lwp_create)(void *, void *),
946 	void	(*exit)(void *),
947 	void	(*free)(void *, int))
948 {
949 	struct ctxop *ctx, *prev_ctx;
950 
951 	/*
952 	 * The incoming kthread_t (which is the thread for which the
953 	 * context ops will be removed) should be one of the following:
954 	 *
955 	 * a) the current thread,
956 	 *
957 	 * b) a thread of a process that's being forked (SIDL),
958 	 *
959 	 * c) a thread that belongs to the same process as the current
960 	 *    thread and for which the current thread is the agent thread,
961 	 *
962 	 * d) a thread that is TS_STOPPED which is indicative of it
963 	 *    being (if curthread is not an agent) a thread being created
964 	 *    as part of an lwp creation.
965 	 */
966 	ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
967 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
968 
969 	/*
970 	 * Serialize modifications to t->t_ctx to prevent the agent thread
971 	 * and the target thread from racing with each other during lwp exit.
972 	 */
973 	mutex_enter(&t->t_ctx_lock);
974 	prev_ctx = NULL;
975 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) {
976 		if (ctx->save_op == save && ctx->restore_op == restore &&
977 		    ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
978 		    ctx->exit_op == exit && ctx->free_op == free &&
979 		    ctx->arg == arg) {
980 			if (prev_ctx)
981 				prev_ctx->next = ctx->next;
982 			else
983 				t->t_ctx = ctx->next;
984 			mutex_exit(&t->t_ctx_lock);
985 			if (ctx->free_op != NULL)
986 				(ctx->free_op)(ctx->arg, 0);
987 			kmem_free(ctx, sizeof (struct ctxop));
988 			return (1);
989 		}
990 		prev_ctx = ctx;
991 	}
992 	mutex_exit(&t->t_ctx_lock);
993 
994 	return (0);
995 }
996 
997 void
998 savectx(kthread_t *t)
999 {
1000 	struct ctxop *ctx;
1001 
1002 	ASSERT(t == curthread);
1003 	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1004 		if (ctx->save_op != NULL)
1005 			(ctx->save_op)(ctx->arg);
1006 }
1007 
1008 void
1009 restorectx(kthread_t *t)
1010 {
1011 	struct ctxop *ctx;
1012 
1013 	ASSERT(t == curthread);
1014 	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1015 		if (ctx->restore_op != NULL)
1016 			(ctx->restore_op)(ctx->arg);
1017 }
1018 
1019 void
1020 forkctx(kthread_t *t, kthread_t *ct)
1021 {
1022 	struct ctxop *ctx;
1023 
1024 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1025 		if (ctx->fork_op != NULL)
1026 			(ctx->fork_op)(t, ct);
1027 }
1028 
1029 /*
1030  * Note that this operator is only invoked via the _lwp_create
1031  * system call.  The system may have other reasons to create lwps
1032  * e.g. the agent lwp or the doors unreferenced lwp.
1033  */
1034 void
1035 lwp_createctx(kthread_t *t, kthread_t *ct)
1036 {
1037 	struct ctxop *ctx;
1038 
1039 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1040 		if (ctx->lwp_create_op != NULL)
1041 			(ctx->lwp_create_op)(t, ct);
1042 }
1043 
1044 /*
1045  * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1046  * needed when the thread/LWP leaves the processor for the last time. This
1047  * routine is not intended to deal with freeing memory; freectx() is used for
1048  * that purpose during thread_free(). This routine is provided to allow for
1049  * clean-up that can't wait until thread_free().
1050  */
1051 void
1052 exitctx(kthread_t *t)
1053 {
1054 	struct ctxop *ctx;
1055 
1056 	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1057 		if (ctx->exit_op != NULL)
1058 			(ctx->exit_op)(t);
1059 }
1060 
1061 /*
1062  * freectx is called from thread_free() and exec() to get
1063  * rid of old thread context ops.
1064  */
1065 void
1066 freectx(kthread_t *t, int isexec)
1067 {
1068 	struct ctxop *ctx;
1069 
1070 	while ((ctx = t->t_ctx) != NULL) {
1071 		t->t_ctx = ctx->next;
1072 		if (ctx->free_op != NULL)
1073 			(ctx->free_op)(ctx->arg, isexec);
1074 		kmem_free(ctx, sizeof (struct ctxop));
1075 	}
1076 }
1077 
1078 /*
1079  * Set the thread running; arrange for it to be swapped in if necessary.
1080  */
1081 void
1082 setrun_locked(kthread_t *t)
1083 {
1084 	ASSERT(THREAD_LOCK_HELD(t));
1085 	if (t->t_state == TS_SLEEP) {
1086 		/*
1087 		 * Take off sleep queue.
1088 		 */
1089 		SOBJ_UNSLEEP(t->t_sobj_ops, t);
1090 	} else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1091 		/*
1092 		 * Already on dispatcher queue.
1093 		 */
1094 		return;
1095 	} else if (t->t_state == TS_WAIT) {
1096 		waitq_setrun(t);
1097 	} else if (t->t_state == TS_STOPPED) {
1098 		/*
1099 		 * All of the sending of SIGCONT (TC_XSTART) and /proc
1100 		 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1101 		 * requested that the thread be run.
1102 		 * Just calling setrun() is not sufficient to set a stopped
1103 		 * thread running.  TP_TXSTART is always set if the thread
1104 		 * is not stopped by a jobcontrol stop signal.
1105 		 * TP_TPSTART is always set if /proc is not controlling it.
1106 		 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1107 		 * The thread won't be stopped unless one of these
1108 		 * three mechanisms did it.
1109 		 *
1110 		 * These flags must be set before calling setrun_locked(t).
1111 		 * They can't be passed as arguments because the streams
1112 		 * code calls setrun() indirectly and the mechanism for
1113 		 * doing so admits only one argument.  Note that the
1114 		 * thread must be locked in order to change t_schedflags.
1115 		 */
1116 		if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1117 			return;
1118 		/*
1119 		 * Process is no longer stopped (a thread is running).
1120 		 */
1121 		t->t_whystop = 0;
1122 		t->t_whatstop = 0;
1123 		/*
1124 		 * Strictly speaking, we do not have to clear these
1125 		 * flags here; they are cleared on entry to stop().
1126 		 * However, they are confusing when doing kernel
1127 		 * debugging or when they are revealed by ps(1).
1128 		 */
1129 		t->t_schedflag &= ~TS_ALLSTART;
1130 		THREAD_TRANSITION(t);	/* drop stopped-thread lock */
1131 		ASSERT(t->t_lockp == &transition_lock);
1132 		ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1133 		/*
1134 		 * Let the class put the process on the dispatcher queue.
1135 		 */
1136 		CL_SETRUN(t);
1137 	}
1138 }
1139 
1140 void
1141 setrun(kthread_t *t)
1142 {
1143 	thread_lock(t);
1144 	setrun_locked(t);
1145 	thread_unlock(t);
1146 }
1147 
1148 /*
1149  * Unpin an interrupted thread.
1150  *	When an interrupt occurs, the interrupt is handled on the stack
1151  *	of an interrupt thread, taken from a pool linked to the CPU structure.
1152  *
1153  *	When swtch() is switching away from an interrupt thread because it
1154  *	blocked or was preempted, this routine is called to complete the
1155  *	saving of the interrupted thread state, and returns the interrupted
1156  *	thread pointer so it may be resumed.
1157  *
1158  *	Called by swtch() only at high spl.
1159  */
1160 kthread_t *
1161 thread_unpin()
1162 {
1163 	kthread_t	*t = curthread;	/* current thread */
1164 	kthread_t	*itp;		/* interrupted thread */
1165 	int		i;		/* interrupt level */
1166 	extern int	intr_passivate();
1167 
1168 	ASSERT(t->t_intr != NULL);
1169 
1170 	itp = t->t_intr;		/* interrupted thread */
1171 	t->t_intr = NULL;		/* clear interrupt ptr */
1172 
1173 	/*
1174 	 * Get state from interrupt thread for the one
1175 	 * it interrupted.
1176 	 */
1177 
1178 	i = intr_passivate(t, itp);
1179 
1180 	TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1181 		"intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1182 		i, t, t, itp, itp);
1183 
1184 	/*
1185 	 * Dissociate the current thread from the interrupted thread's LWP.
1186 	 */
1187 	t->t_lwp = NULL;
1188 
1189 	/*
1190 	 * Interrupt handlers above the level that spinlocks block must
1191 	 * not block.
1192 	 */
1193 #if DEBUG
1194 	if (i < 0 || i > LOCK_LEVEL)
1195 		cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1196 #endif
1197 
1198 	/*
1199 	 * Compute the CPU's base interrupt level based on the active
1200 	 * interrupts.
1201 	 */
1202 	ASSERT(CPU->cpu_intr_actv & (1 << i));
1203 	set_base_spl();
1204 
1205 	return (itp);
1206 }
1207 
1208 /*
1209  * Create and initialize an interrupt thread.
1210  *	Returns non-zero on error.
1211  *	Called at spl7() or better.
1212  */
1213 void
1214 thread_create_intr(struct cpu *cp)
1215 {
1216 	kthread_t *tp;
1217 
1218 	tp = thread_create(NULL, 0,
1219 	    (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1220 
1221 	/*
1222 	 * Set the thread in the TS_FREE state.  The state will change
1223 	 * to TS_ONPROC only while the interrupt is active.  Think of these
1224 	 * as being on a private free list for the CPU.  Being TS_FREE keeps
1225 	 * inactive interrupt threads out of debugger thread lists.
1226 	 *
1227 	 * We cannot call thread_create with TS_FREE because of the current
1228 	 * checks there for ONPROC.  Fix this when thread_create takes flags.
1229 	 */
1230 	THREAD_FREEINTR(tp, cp);
1231 
1232 	/*
1233 	 * Nobody should ever reference the credentials of an interrupt
1234 	 * thread so make it NULL to catch any such references.
1235 	 */
1236 	tp->t_cred = NULL;
1237 	tp->t_flag |= T_INTR_THREAD;
1238 	tp->t_cpu = cp;
1239 	tp->t_bound_cpu = cp;
1240 	tp->t_disp_queue = cp->cpu_disp;
1241 	tp->t_affinitycnt = 1;
1242 	tp->t_preempt = 1;
1243 
1244 	/*
1245 	 * Don't make a user-requested binding on this thread so that
1246 	 * the processor can be offlined.
1247 	 */
1248 	tp->t_bind_cpu = PBIND_NONE;	/* no USER-requested binding */
1249 	tp->t_bind_pset = PS_NONE;
1250 
1251 #if defined(__i386) || defined(__amd64)
1252 	tp->t_stk -= STACK_ALIGN;
1253 	*(tp->t_stk) = 0;		/* terminate intr thread stack */
1254 #endif
1255 
1256 	/*
1257 	 * Link onto CPU's interrupt pool.
1258 	 */
1259 	tp->t_link = cp->cpu_intr_thread;
1260 	cp->cpu_intr_thread = tp;
1261 }
1262 
1263 /*
1264  * TSD -- THREAD SPECIFIC DATA
1265  */
1266 static kmutex_t		tsd_mutex;	 /* linked list spin lock */
1267 static uint_t		tsd_nkeys;	 /* size of destructor array */
1268 /* per-key destructor funcs */
1269 static void 		(**tsd_destructor)(void *);
1270 /* list of tsd_thread's */
1271 static struct tsd_thread	*tsd_list;
1272 
1273 /*
1274  * Default destructor
1275  *	Needed because NULL destructor means that the key is unused
1276  */
1277 /* ARGSUSED */
1278 void
1279 tsd_defaultdestructor(void *value)
1280 {}
1281 
1282 /*
1283  * Create a key (index into per thread array)
1284  *	Locks out tsd_create, tsd_destroy, and tsd_exit
1285  *	May allocate memory with lock held
1286  */
1287 void
1288 tsd_create(uint_t *keyp, void (*destructor)(void *))
1289 {
1290 	int	i;
1291 	uint_t	nkeys;
1292 
1293 	/*
1294 	 * if key is allocated, do nothing
1295 	 */
1296 	mutex_enter(&tsd_mutex);
1297 	if (*keyp) {
1298 		mutex_exit(&tsd_mutex);
1299 		return;
1300 	}
1301 	/*
1302 	 * find an unused key
1303 	 */
1304 	if (destructor == NULL)
1305 		destructor = tsd_defaultdestructor;
1306 
1307 	for (i = 0; i < tsd_nkeys; ++i)
1308 		if (tsd_destructor[i] == NULL)
1309 			break;
1310 
1311 	/*
1312 	 * if no unused keys, increase the size of the destructor array
1313 	 */
1314 	if (i == tsd_nkeys) {
1315 		if ((nkeys = (tsd_nkeys << 1)) == 0)
1316 			nkeys = 1;
1317 		tsd_destructor =
1318 		    (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1319 		    (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1320 		    (size_t)(nkeys * sizeof (void (*)(void *))));
1321 		tsd_nkeys = nkeys;
1322 	}
1323 
1324 	/*
1325 	 * allocate the next available unused key
1326 	 */
1327 	tsd_destructor[i] = destructor;
1328 	*keyp = i + 1;
1329 	mutex_exit(&tsd_mutex);
1330 }
1331 
1332 /*
1333  * Destroy a key -- this is for unloadable modules
1334  *
1335  * Assumes that the caller is preventing tsd_set and tsd_get
1336  * Locks out tsd_create, tsd_destroy, and tsd_exit
1337  * May free memory with lock held
1338  */
1339 void
1340 tsd_destroy(uint_t *keyp)
1341 {
1342 	uint_t key;
1343 	struct tsd_thread *tsd;
1344 
1345 	/*
1346 	 * protect the key namespace and our destructor lists
1347 	 */
1348 	mutex_enter(&tsd_mutex);
1349 	key = *keyp;
1350 	*keyp = 0;
1351 
1352 	ASSERT(key <= tsd_nkeys);
1353 
1354 	/*
1355 	 * if the key is valid
1356 	 */
1357 	if (key != 0) {
1358 		uint_t k = key - 1;
1359 		/*
1360 		 * for every thread with TSD, call key's destructor
1361 		 */
1362 		for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1363 			/*
1364 			 * no TSD for key in this thread
1365 			 */
1366 			if (key > tsd->ts_nkeys)
1367 				continue;
1368 			/*
1369 			 * call destructor for key
1370 			 */
1371 			if (tsd->ts_value[k] && tsd_destructor[k])
1372 				(*tsd_destructor[k])(tsd->ts_value[k]);
1373 			/*
1374 			 * reset value for key
1375 			 */
1376 			tsd->ts_value[k] = NULL;
1377 		}
1378 		/*
1379 		 * actually free the key (NULL destructor == unused)
1380 		 */
1381 		tsd_destructor[k] = NULL;
1382 	}
1383 
1384 	mutex_exit(&tsd_mutex);
1385 }
1386 
1387 /*
1388  * Quickly return the per thread value that was stored with the specified key
1389  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1390  */
1391 void *
1392 tsd_get(uint_t key)
1393 {
1394 	return (tsd_agent_get(curthread, key));
1395 }
1396 
1397 /*
1398  * Set a per thread value indexed with the specified key
1399  */
1400 int
1401 tsd_set(uint_t key, void *value)
1402 {
1403 	return (tsd_agent_set(curthread, key, value));
1404 }
1405 
1406 /*
1407  * Like tsd_get(), except that the agent lwp can get the tsd of
1408  * another thread in the same process (the agent thread only runs when the
1409  * process is completely stopped by /proc), or syslwp is creating a new lwp.
1410  */
1411 void *
1412 tsd_agent_get(kthread_t *t, uint_t key)
1413 {
1414 	struct tsd_thread *tsd = t->t_tsd;
1415 
1416 	ASSERT(t == curthread ||
1417 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1418 
1419 	if (key && tsd != NULL && key <= tsd->ts_nkeys)
1420 		return (tsd->ts_value[key - 1]);
1421 	return (NULL);
1422 }
1423 
1424 /*
1425  * Like tsd_set(), except that the agent lwp can set the tsd of
1426  * another thread in the same process, or syslwp can set the tsd
1427  * of a thread it's in the middle of creating.
1428  *
1429  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1430  * May lock out tsd_destroy (and tsd_create), may allocate memory with
1431  * lock held
1432  */
1433 int
1434 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1435 {
1436 	struct tsd_thread *tsd = t->t_tsd;
1437 
1438 	ASSERT(t == curthread ||
1439 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1440 
1441 	if (key == 0)
1442 		return (EINVAL);
1443 	if (tsd == NULL)
1444 		tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1445 	if (key <= tsd->ts_nkeys) {
1446 		tsd->ts_value[key - 1] = value;
1447 		return (0);
1448 	}
1449 
1450 	ASSERT(key <= tsd_nkeys);
1451 
1452 	/*
1453 	 * lock out tsd_destroy()
1454 	 */
1455 	mutex_enter(&tsd_mutex);
1456 	if (tsd->ts_nkeys == 0) {
1457 		/*
1458 		 * Link onto list of threads with TSD
1459 		 */
1460 		if ((tsd->ts_next = tsd_list) != NULL)
1461 			tsd_list->ts_prev = tsd;
1462 		tsd_list = tsd;
1463 	}
1464 
1465 	/*
1466 	 * Allocate thread local storage and set the value for key
1467 	 */
1468 	tsd->ts_value = tsd_realloc(tsd->ts_value,
1469 	    tsd->ts_nkeys * sizeof (void *),
1470 	    key * sizeof (void *));
1471 	tsd->ts_nkeys = key;
1472 	tsd->ts_value[key - 1] = value;
1473 	mutex_exit(&tsd_mutex);
1474 
1475 	return (0);
1476 }
1477 
1478 
1479 /*
1480  * Return the per thread value that was stored with the specified key
1481  *	If necessary, create the key and the value
1482  *	Assumes the caller is protecting *keyp from tsd_destroy
1483  */
1484 void *
1485 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1486 {
1487 	void *value;
1488 	uint_t key = *keyp;
1489 	struct tsd_thread *tsd = curthread->t_tsd;
1490 
1491 	if (tsd == NULL)
1492 		tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1493 	if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1494 		return (value);
1495 	if (key == 0)
1496 		tsd_create(keyp, destroy);
1497 	(void) tsd_set(*keyp, value = (*allocate)());
1498 
1499 	return (value);
1500 }
1501 
1502 /*
1503  * Called from thread_exit() to run the destructor function for each tsd
1504  *	Locks out tsd_create and tsd_destroy
1505  *	Assumes that the destructor *DOES NOT* use tsd
1506  */
1507 void
1508 tsd_exit(void)
1509 {
1510 	int i;
1511 	struct tsd_thread *tsd = curthread->t_tsd;
1512 
1513 	if (tsd == NULL)
1514 		return;
1515 
1516 	if (tsd->ts_nkeys == 0) {
1517 		kmem_free(tsd, sizeof (*tsd));
1518 		curthread->t_tsd = NULL;
1519 		return;
1520 	}
1521 
1522 	/*
1523 	 * lock out tsd_create and tsd_destroy, call
1524 	 * the destructor, and mark the value as destroyed.
1525 	 */
1526 	mutex_enter(&tsd_mutex);
1527 
1528 	for (i = 0; i < tsd->ts_nkeys; i++) {
1529 		if (tsd->ts_value[i] && tsd_destructor[i])
1530 			(*tsd_destructor[i])(tsd->ts_value[i]);
1531 		tsd->ts_value[i] = NULL;
1532 	}
1533 
1534 	/*
1535 	 * remove from linked list of threads with TSD
1536 	 */
1537 	if (tsd->ts_next)
1538 		tsd->ts_next->ts_prev = tsd->ts_prev;
1539 	if (tsd->ts_prev)
1540 		tsd->ts_prev->ts_next = tsd->ts_next;
1541 	if (tsd_list == tsd)
1542 		tsd_list = tsd->ts_next;
1543 
1544 	mutex_exit(&tsd_mutex);
1545 
1546 	/*
1547 	 * free up the TSD
1548 	 */
1549 	kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1550 	kmem_free(tsd, sizeof (struct tsd_thread));
1551 	curthread->t_tsd = NULL;
1552 }
1553 
1554 /*
1555  * realloc
1556  */
1557 static void *
1558 tsd_realloc(void *old, size_t osize, size_t nsize)
1559 {
1560 	void *new;
1561 
1562 	new = kmem_zalloc(nsize, KM_SLEEP);
1563 	if (old) {
1564 		bcopy(old, new, osize);
1565 		kmem_free(old, osize);
1566 	}
1567 	return (new);
1568 }
1569 
1570 /*
1571  * Check to see if an interrupt thread might be active at a given ipl.
1572  * If so return true.
1573  * We must be conservative--it is ok to give a false yes, but a false no
1574  * will cause disaster.  (But if the situation changes after we check it is
1575  * ok--the caller is trying to ensure that an interrupt routine has been
1576  * exited).
1577  * This is used when trying to remove an interrupt handler from an autovector
1578  * list in avintr.c.
1579  */
1580 int
1581 intr_active(struct cpu *cp, int level)
1582 {
1583 	if (level <= LOCK_LEVEL)
1584 		return (cp->cpu_thread != cp->cpu_dispthread);
1585 	else
1586 		return (CPU_ON_INTR(cp));
1587 }
1588 
1589 /*
1590  * Return non-zero if an interrupt is being serviced.
1591  */
1592 int
1593 servicing_interrupt()
1594 {
1595 	int onintr = 0;
1596 
1597 	/* Are we an interrupt thread */
1598 	if (curthread->t_flag & T_INTR_THREAD)
1599 		return (1);
1600 	/* Are we servicing a high level interrupt? */
1601 	if (CPU_ON_INTR(CPU)) {
1602 		kpreempt_disable();
1603 		onintr = CPU_ON_INTR(CPU);
1604 		kpreempt_enable();
1605 	}
1606 	return (onintr);
1607 }
1608 
1609 
1610 /*
1611  * Change the dispatch priority of a thread in the system.
1612  * Used when raising or lowering a thread's priority.
1613  * (E.g., priority inheritance)
1614  *
1615  * Since threads are queued according to their priority, we
1616  * we must check the thread's state to determine whether it
1617  * is on a queue somewhere. If it is, we've got to:
1618  *
1619  *	o Dequeue the thread.
1620  *	o Change its effective priority.
1621  *	o Enqueue the thread.
1622  *
1623  * Assumptions: The thread whose priority we wish to change
1624  * must be locked before we call thread_change_(e)pri().
1625  * The thread_change(e)pri() function doesn't drop the thread
1626  * lock--that must be done by its caller.
1627  */
1628 void
1629 thread_change_epri(kthread_t *t, pri_t disp_pri)
1630 {
1631 	uint_t	state;
1632 
1633 	ASSERT(THREAD_LOCK_HELD(t));
1634 
1635 	/*
1636 	 * If the inherited priority hasn't actually changed,
1637 	 * just return.
1638 	 */
1639 	if (t->t_epri == disp_pri)
1640 		return;
1641 
1642 	state = t->t_state;
1643 
1644 	/*
1645 	 * If it's not on a queue, change the priority with
1646 	 * impunity.
1647 	 */
1648 	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1649 		t->t_epri = disp_pri;
1650 
1651 		if (state == TS_ONPROC) {
1652 			cpu_t *cp = t->t_disp_queue->disp_cpu;
1653 
1654 			if (t == cp->cpu_dispthread)
1655 				cp->cpu_dispatch_pri = DISP_PRIO(t);
1656 		}
1657 		return;
1658 	}
1659 
1660 	/*
1661 	 * It's either on a sleep queue or a run queue.
1662 	 */
1663 	if (state == TS_SLEEP) {
1664 		/*
1665 		 * Take the thread out of its sleep queue.
1666 		 * Change the inherited priority.
1667 		 * Re-enqueue the thread.
1668 		 * Each synchronization object exports a function
1669 		 * to do this in an appropriate manner.
1670 		 */
1671 		SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1672 	} else if (state == TS_WAIT) {
1673 		/*
1674 		 * Re-enqueue a thread on the wait queue if its
1675 		 * effective priority needs to change.
1676 		 */
1677 		if (disp_pri != t->t_epri)
1678 			waitq_change_pri(t, disp_pri);
1679 	} else {
1680 		/*
1681 		 * The thread is on a run queue.
1682 		 * Note: setbackdq() may not put the thread
1683 		 * back on the same run queue where it originally
1684 		 * resided.
1685 		 */
1686 		(void) dispdeq(t);
1687 		t->t_epri = disp_pri;
1688 		setbackdq(t);
1689 	}
1690 }	/* end of thread_change_epri */
1691 
1692 /*
1693  * Function: Change the t_pri field of a thread.
1694  * Side Effects: Adjust the thread ordering on a run queue
1695  *		 or sleep queue, if necessary.
1696  * Returns: 1 if the thread was on a run queue, else 0.
1697  */
1698 int
1699 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1700 {
1701 	uint_t	state;
1702 	int	on_rq = 0;
1703 
1704 	ASSERT(THREAD_LOCK_HELD(t));
1705 
1706 	state = t->t_state;
1707 	THREAD_WILLCHANGE_PRI(t, disp_pri);
1708 
1709 	/*
1710 	 * If it's not on a queue, change the priority with
1711 	 * impunity.
1712 	 */
1713 	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1714 		t->t_pri = disp_pri;
1715 
1716 		if (state == TS_ONPROC) {
1717 			cpu_t *cp = t->t_disp_queue->disp_cpu;
1718 
1719 			if (t == cp->cpu_dispthread)
1720 				cp->cpu_dispatch_pri = DISP_PRIO(t);
1721 		}
1722 		return (0);
1723 	}
1724 
1725 	/*
1726 	 * It's either on a sleep queue or a run queue.
1727 	 */
1728 	if (state == TS_SLEEP) {
1729 		/*
1730 		 * If the priority has changed, take the thread out of
1731 		 * its sleep queue and change the priority.
1732 		 * Re-enqueue the thread.
1733 		 * Each synchronization object exports a function
1734 		 * to do this in an appropriate manner.
1735 		 */
1736 		if (disp_pri != t->t_pri)
1737 			SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1738 	} else if (state == TS_WAIT) {
1739 		/*
1740 		 * Re-enqueue a thread on the wait queue if its
1741 		 * priority needs to change.
1742 		 */
1743 		if (disp_pri != t->t_pri)
1744 			waitq_change_pri(t, disp_pri);
1745 	} else {
1746 		/*
1747 		 * The thread is on a run queue.
1748 		 * Note: setbackdq() may not put the thread
1749 		 * back on the same run queue where it originally
1750 		 * resided.
1751 		 *
1752 		 * We still requeue the thread even if the priority
1753 		 * is unchanged to preserve round-robin (and other)
1754 		 * effects between threads of the same priority.
1755 		 */
1756 		on_rq = dispdeq(t);
1757 		ASSERT(on_rq);
1758 		t->t_pri = disp_pri;
1759 		if (front) {
1760 			setfrontdq(t);
1761 		} else {
1762 			setbackdq(t);
1763 		}
1764 	}
1765 	return (on_rq);
1766 }
1767