xref: /titanic_44/usr/src/uts/common/os/cpu.c (revision 30f5cf21f0e4186919b67ac48223d09ca110f8fe)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Architecture-independent CPU control functions.
31  */
32 
33 #include <sys/types.h>
34 #include <sys/param.h>
35 #include <sys/var.h>
36 #include <sys/thread.h>
37 #include <sys/cpuvar.h>
38 #include <sys/kstat.h>
39 #include <sys/uadmin.h>
40 #include <sys/systm.h>
41 #include <sys/errno.h>
42 #include <sys/cmn_err.h>
43 #include <sys/procset.h>
44 #include <sys/processor.h>
45 #include <sys/debug.h>
46 #include <sys/cpupart.h>
47 #include <sys/lgrp.h>
48 #include <sys/pset.h>
49 #include <sys/chip.h>
50 #include <sys/kmem.h>
51 #include <sys/kmem_impl.h>	/* to set per-cpu kmem_cache offset */
52 #include <sys/atomic.h>
53 #include <sys/callb.h>
54 #include <sys/vtrace.h>
55 #include <sys/cyclic.h>
56 #include <sys/bitmap.h>
57 #include <sys/nvpair.h>
58 #include <sys/pool_pset.h>
59 #include <sys/msacct.h>
60 #include <sys/time.h>
61 #include <sys/archsystm.h>
62 
63 extern int	mp_cpu_start(cpu_t *);
64 extern int	mp_cpu_stop(cpu_t *);
65 extern int	mp_cpu_poweron(cpu_t *);
66 extern int	mp_cpu_poweroff(cpu_t *);
67 extern int	mp_cpu_configure(int);
68 extern int	mp_cpu_unconfigure(int);
69 extern void	mp_cpu_faulted_enter(cpu_t *);
70 extern void	mp_cpu_faulted_exit(cpu_t *);
71 
72 extern int cmp_cpu_to_chip(processorid_t cpuid);
73 #ifdef __sparcv9
74 extern char *cpu_fru_fmri(cpu_t *cp);
75 #endif
76 
77 static void cpu_add_active_internal(cpu_t *cp);
78 static void cpu_remove_active(cpu_t *cp);
79 static void cpu_info_kstat_create(cpu_t *cp);
80 static void cpu_info_kstat_destroy(cpu_t *cp);
81 static void cpu_stats_kstat_create(cpu_t *cp);
82 static void cpu_stats_kstat_destroy(cpu_t *cp);
83 
84 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
85 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
86 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
87 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
88 
89 /*
90  * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
91  * and dispatch queue reallocations.  The lock ordering with respect to
92  * related locks is:
93  *
94  *	cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
95  *
96  * Warning:  Certain sections of code do not use the cpu_lock when
97  * traversing the cpu_list (e.g. mutex_vector_enter(), clock()).  Since
98  * all cpus are paused during modifications to this list, a solution
99  * to protect the list is too either disable kernel preemption while
100  * walking the list, *or* recheck the cpu_next pointer at each
101  * iteration in the loop.  Note that in no cases can any cached
102  * copies of the cpu pointers be kept as they may become invalid.
103  */
104 kmutex_t	cpu_lock;
105 cpu_t		*cpu_list;		/* list of all CPUs */
106 cpu_t		*cpu_active;		/* list of active CPUs */
107 static cpuset_t	cpu_available;		/* set of available CPUs */
108 cpuset_t	cpu_seqid_inuse;	/* which cpu_seqids are in use */
109 
110 /*
111  * max_ncpus keeps the max cpus the system can have. Initially
112  * it's NCPU, but since most archs scan the devtree for cpus
113  * fairly early on during boot, the real max can be known before
114  * ncpus is set (useful for early NCPU based allocations).
115  */
116 int max_ncpus = NCPU;
117 /*
118  * platforms that set max_ncpus to maxiumum number of cpus that can be
119  * dynamically added will set boot_max_ncpus to the number of cpus found
120  * at device tree scan time during boot.
121  */
122 int boot_max_ncpus = -1;
123 /*
124  * Maximum possible CPU id.  This can never be >= NCPU since NCPU is
125  * used to size arrays that are indexed by CPU id.
126  */
127 processorid_t max_cpuid = NCPU - 1;
128 
129 int ncpus = 1;
130 int ncpus_online = 1;
131 
132 /*
133  * CPU that we're trying to offline.  Protected by cpu_lock.
134  */
135 cpu_t *cpu_inmotion;
136 
137 /*
138  * Can be raised to suppress further weakbinding, which are instead
139  * satisfied by disabling preemption.  Must be raised/lowered under cpu_lock,
140  * while individual thread weakbinding synchronisation is done under thread
141  * lock.
142  */
143 int weakbindingbarrier;
144 
145 /*
146  * values for safe_list.  Pause state that CPUs are in.
147  */
148 #define	PAUSE_IDLE	0		/* normal state */
149 #define	PAUSE_READY	1		/* paused thread ready to spl */
150 #define	PAUSE_WAIT	2		/* paused thread is spl-ed high */
151 #define	PAUSE_DIE	3		/* tell pause thread to leave */
152 #define	PAUSE_DEAD	4		/* pause thread has left */
153 
154 /*
155  * Variables used in pause_cpus().
156  */
157 static volatile char safe_list[NCPU];
158 
159 static struct _cpu_pause_info {
160 	int		cp_spl;		/* spl saved in pause_cpus() */
161 	volatile int	cp_go;		/* Go signal sent after all ready */
162 	int		cp_count;	/* # of CPUs to pause */
163 	ksema_t		cp_sem;		/* synch pause_cpus & cpu_pause */
164 	kthread_id_t	cp_paused;
165 } cpu_pause_info;
166 
167 static kmutex_t pause_free_mutex;
168 static kcondvar_t pause_free_cv;
169 
170 static struct cpu_sys_stats_ks_data {
171 	kstat_named_t cpu_ticks_idle;
172 	kstat_named_t cpu_ticks_user;
173 	kstat_named_t cpu_ticks_kernel;
174 	kstat_named_t cpu_ticks_wait;
175 	kstat_named_t cpu_nsec_idle;
176 	kstat_named_t cpu_nsec_user;
177 	kstat_named_t cpu_nsec_kernel;
178 	kstat_named_t wait_ticks_io;
179 	kstat_named_t bread;
180 	kstat_named_t bwrite;
181 	kstat_named_t lread;
182 	kstat_named_t lwrite;
183 	kstat_named_t phread;
184 	kstat_named_t phwrite;
185 	kstat_named_t pswitch;
186 	kstat_named_t trap;
187 	kstat_named_t intr;
188 	kstat_named_t syscall;
189 	kstat_named_t sysread;
190 	kstat_named_t syswrite;
191 	kstat_named_t sysfork;
192 	kstat_named_t sysvfork;
193 	kstat_named_t sysexec;
194 	kstat_named_t readch;
195 	kstat_named_t writech;
196 	kstat_named_t rcvint;
197 	kstat_named_t xmtint;
198 	kstat_named_t mdmint;
199 	kstat_named_t rawch;
200 	kstat_named_t canch;
201 	kstat_named_t outch;
202 	kstat_named_t msg;
203 	kstat_named_t sema;
204 	kstat_named_t namei;
205 	kstat_named_t ufsiget;
206 	kstat_named_t ufsdirblk;
207 	kstat_named_t ufsipage;
208 	kstat_named_t ufsinopage;
209 	kstat_named_t procovf;
210 	kstat_named_t intrthread;
211 	kstat_named_t intrblk;
212 	kstat_named_t intrunpin;
213 	kstat_named_t idlethread;
214 	kstat_named_t inv_swtch;
215 	kstat_named_t nthreads;
216 	kstat_named_t cpumigrate;
217 	kstat_named_t xcalls;
218 	kstat_named_t mutex_adenters;
219 	kstat_named_t rw_rdfails;
220 	kstat_named_t rw_wrfails;
221 	kstat_named_t modload;
222 	kstat_named_t modunload;
223 	kstat_named_t bawrite;
224 	kstat_named_t iowait;
225 } cpu_sys_stats_ks_data_template = {
226 	{ "cpu_ticks_idle", 	KSTAT_DATA_UINT64 },
227 	{ "cpu_ticks_user", 	KSTAT_DATA_UINT64 },
228 	{ "cpu_ticks_kernel", 	KSTAT_DATA_UINT64 },
229 	{ "cpu_ticks_wait", 	KSTAT_DATA_UINT64 },
230 	{ "cpu_nsec_idle",	KSTAT_DATA_UINT64 },
231 	{ "cpu_nsec_user",	KSTAT_DATA_UINT64 },
232 	{ "cpu_nsec_kernel",	KSTAT_DATA_UINT64 },
233 	{ "wait_ticks_io", 	KSTAT_DATA_UINT64 },
234 	{ "bread", 		KSTAT_DATA_UINT64 },
235 	{ "bwrite", 		KSTAT_DATA_UINT64 },
236 	{ "lread", 		KSTAT_DATA_UINT64 },
237 	{ "lwrite", 		KSTAT_DATA_UINT64 },
238 	{ "phread", 		KSTAT_DATA_UINT64 },
239 	{ "phwrite", 		KSTAT_DATA_UINT64 },
240 	{ "pswitch", 		KSTAT_DATA_UINT64 },
241 	{ "trap", 		KSTAT_DATA_UINT64 },
242 	{ "intr", 		KSTAT_DATA_UINT64 },
243 	{ "syscall", 		KSTAT_DATA_UINT64 },
244 	{ "sysread", 		KSTAT_DATA_UINT64 },
245 	{ "syswrite", 		KSTAT_DATA_UINT64 },
246 	{ "sysfork", 		KSTAT_DATA_UINT64 },
247 	{ "sysvfork", 		KSTAT_DATA_UINT64 },
248 	{ "sysexec", 		KSTAT_DATA_UINT64 },
249 	{ "readch", 		KSTAT_DATA_UINT64 },
250 	{ "writech", 		KSTAT_DATA_UINT64 },
251 	{ "rcvint", 		KSTAT_DATA_UINT64 },
252 	{ "xmtint", 		KSTAT_DATA_UINT64 },
253 	{ "mdmint", 		KSTAT_DATA_UINT64 },
254 	{ "rawch", 		KSTAT_DATA_UINT64 },
255 	{ "canch", 		KSTAT_DATA_UINT64 },
256 	{ "outch", 		KSTAT_DATA_UINT64 },
257 	{ "msg", 		KSTAT_DATA_UINT64 },
258 	{ "sema", 		KSTAT_DATA_UINT64 },
259 	{ "namei", 		KSTAT_DATA_UINT64 },
260 	{ "ufsiget", 		KSTAT_DATA_UINT64 },
261 	{ "ufsdirblk", 		KSTAT_DATA_UINT64 },
262 	{ "ufsipage", 		KSTAT_DATA_UINT64 },
263 	{ "ufsinopage", 	KSTAT_DATA_UINT64 },
264 	{ "procovf", 		KSTAT_DATA_UINT64 },
265 	{ "intrthread", 	KSTAT_DATA_UINT64 },
266 	{ "intrblk", 		KSTAT_DATA_UINT64 },
267 	{ "intrunpin",		KSTAT_DATA_UINT64 },
268 	{ "idlethread", 	KSTAT_DATA_UINT64 },
269 	{ "inv_swtch", 		KSTAT_DATA_UINT64 },
270 	{ "nthreads", 		KSTAT_DATA_UINT64 },
271 	{ "cpumigrate", 	KSTAT_DATA_UINT64 },
272 	{ "xcalls", 		KSTAT_DATA_UINT64 },
273 	{ "mutex_adenters", 	KSTAT_DATA_UINT64 },
274 	{ "rw_rdfails", 	KSTAT_DATA_UINT64 },
275 	{ "rw_wrfails", 	KSTAT_DATA_UINT64 },
276 	{ "modload", 		KSTAT_DATA_UINT64 },
277 	{ "modunload", 		KSTAT_DATA_UINT64 },
278 	{ "bawrite", 		KSTAT_DATA_UINT64 },
279 	{ "iowait",		KSTAT_DATA_UINT64 },
280 };
281 
282 static struct cpu_vm_stats_ks_data {
283 	kstat_named_t pgrec;
284 	kstat_named_t pgfrec;
285 	kstat_named_t pgin;
286 	kstat_named_t pgpgin;
287 	kstat_named_t pgout;
288 	kstat_named_t pgpgout;
289 	kstat_named_t swapin;
290 	kstat_named_t pgswapin;
291 	kstat_named_t swapout;
292 	kstat_named_t pgswapout;
293 	kstat_named_t zfod;
294 	kstat_named_t dfree;
295 	kstat_named_t scan;
296 	kstat_named_t rev;
297 	kstat_named_t hat_fault;
298 	kstat_named_t as_fault;
299 	kstat_named_t maj_fault;
300 	kstat_named_t cow_fault;
301 	kstat_named_t prot_fault;
302 	kstat_named_t softlock;
303 	kstat_named_t kernel_asflt;
304 	kstat_named_t pgrrun;
305 	kstat_named_t execpgin;
306 	kstat_named_t execpgout;
307 	kstat_named_t execfree;
308 	kstat_named_t anonpgin;
309 	kstat_named_t anonpgout;
310 	kstat_named_t anonfree;
311 	kstat_named_t fspgin;
312 	kstat_named_t fspgout;
313 	kstat_named_t fsfree;
314 } cpu_vm_stats_ks_data_template = {
315 	{ "pgrec",		KSTAT_DATA_UINT64 },
316 	{ "pgfrec",		KSTAT_DATA_UINT64 },
317 	{ "pgin",		KSTAT_DATA_UINT64 },
318 	{ "pgpgin",		KSTAT_DATA_UINT64 },
319 	{ "pgout",		KSTAT_DATA_UINT64 },
320 	{ "pgpgout",		KSTAT_DATA_UINT64 },
321 	{ "swapin",		KSTAT_DATA_UINT64 },
322 	{ "pgswapin",		KSTAT_DATA_UINT64 },
323 	{ "swapout",		KSTAT_DATA_UINT64 },
324 	{ "pgswapout",		KSTAT_DATA_UINT64 },
325 	{ "zfod",		KSTAT_DATA_UINT64 },
326 	{ "dfree",		KSTAT_DATA_UINT64 },
327 	{ "scan",		KSTAT_DATA_UINT64 },
328 	{ "rev",		KSTAT_DATA_UINT64 },
329 	{ "hat_fault",		KSTAT_DATA_UINT64 },
330 	{ "as_fault",		KSTAT_DATA_UINT64 },
331 	{ "maj_fault",		KSTAT_DATA_UINT64 },
332 	{ "cow_fault",		KSTAT_DATA_UINT64 },
333 	{ "prot_fault",		KSTAT_DATA_UINT64 },
334 	{ "softlock",		KSTAT_DATA_UINT64 },
335 	{ "kernel_asflt",	KSTAT_DATA_UINT64 },
336 	{ "pgrrun",		KSTAT_DATA_UINT64 },
337 	{ "execpgin",		KSTAT_DATA_UINT64 },
338 	{ "execpgout",		KSTAT_DATA_UINT64 },
339 	{ "execfree",		KSTAT_DATA_UINT64 },
340 	{ "anonpgin",		KSTAT_DATA_UINT64 },
341 	{ "anonpgout",		KSTAT_DATA_UINT64 },
342 	{ "anonfree",		KSTAT_DATA_UINT64 },
343 	{ "fspgin",		KSTAT_DATA_UINT64 },
344 	{ "fspgout",		KSTAT_DATA_UINT64 },
345 	{ "fsfree",		KSTAT_DATA_UINT64 },
346 };
347 
348 /*
349  * Force the specified thread to migrate to the appropriate processor.
350  * Called with thread lock held, returns with it dropped.
351  */
352 static void
353 force_thread_migrate(kthread_id_t tp)
354 {
355 	ASSERT(THREAD_LOCK_HELD(tp));
356 	if (tp == curthread) {
357 		THREAD_TRANSITION(tp);
358 		CL_SETRUN(tp);
359 		thread_unlock_nopreempt(tp);
360 		swtch();
361 	} else {
362 		if (tp->t_state == TS_ONPROC) {
363 			cpu_surrender(tp);
364 		} else if (tp->t_state == TS_RUN) {
365 			(void) dispdeq(tp);
366 			setbackdq(tp);
367 		}
368 		thread_unlock(tp);
369 	}
370 }
371 
372 /*
373  * Set affinity for a specified CPU.
374  * A reference count is incremented and the affinity is held until the
375  * reference count is decremented to zero by thread_affinity_clear().
376  * This is so regions of code requiring affinity can be nested.
377  * Caller needs to ensure that cpu_id remains valid, which can be
378  * done by holding cpu_lock across this call, unless the caller
379  * specifies CPU_CURRENT in which case the cpu_lock will be acquired
380  * by thread_affinity_set and CPU->cpu_id will be the target CPU.
381  */
382 void
383 thread_affinity_set(kthread_id_t t, int cpu_id)
384 {
385 	cpu_t		*cp;
386 	int		c;
387 
388 	ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
389 
390 	if ((c = cpu_id) == CPU_CURRENT) {
391 		mutex_enter(&cpu_lock);
392 		cpu_id = CPU->cpu_id;
393 	}
394 	/*
395 	 * We should be asserting that cpu_lock is held here, but
396 	 * the NCA code doesn't acquire it.  The following assert
397 	 * should be uncommented when the NCA code is fixed.
398 	 *
399 	 * ASSERT(MUTEX_HELD(&cpu_lock));
400 	 */
401 	ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
402 	cp = cpu[cpu_id];
403 	ASSERT(cp != NULL);		/* user must provide a good cpu_id */
404 	/*
405 	 * If there is already a hard affinity requested, and this affinity
406 	 * conflicts with that, panic.
407 	 */
408 	thread_lock(t);
409 	if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
410 		panic("affinity_set: setting %p but already bound to %p",
411 		    (void *)cp, (void *)t->t_bound_cpu);
412 	}
413 	t->t_affinitycnt++;
414 	t->t_bound_cpu = cp;
415 
416 	/*
417 	 * Make sure we're running on the right CPU.
418 	 */
419 	if (cp != t->t_cpu || t != curthread) {
420 		force_thread_migrate(t);	/* drops thread lock */
421 	} else {
422 		thread_unlock(t);
423 	}
424 
425 	if (c == CPU_CURRENT)
426 		mutex_exit(&cpu_lock);
427 }
428 
429 /*
430  *	Wrapper for backward compatibility.
431  */
432 void
433 affinity_set(int cpu_id)
434 {
435 	thread_affinity_set(curthread, cpu_id);
436 }
437 
438 /*
439  * Decrement the affinity reservation count and if it becomes zero,
440  * clear the CPU affinity for the current thread, or set it to the user's
441  * software binding request.
442  */
443 void
444 thread_affinity_clear(kthread_id_t t)
445 {
446 	register processorid_t binding;
447 
448 	thread_lock(t);
449 	if (--t->t_affinitycnt == 0) {
450 		if ((binding = t->t_bind_cpu) == PBIND_NONE) {
451 			/*
452 			 * Adjust disp_max_unbound_pri if necessary.
453 			 */
454 			disp_adjust_unbound_pri(t);
455 			t->t_bound_cpu = NULL;
456 			if (t->t_cpu->cpu_part != t->t_cpupart) {
457 				force_thread_migrate(t);
458 				return;
459 			}
460 		} else {
461 			t->t_bound_cpu = cpu[binding];
462 			/*
463 			 * Make sure the thread is running on the bound CPU.
464 			 */
465 			if (t->t_cpu != t->t_bound_cpu) {
466 				force_thread_migrate(t);
467 				return;		/* already dropped lock */
468 			}
469 		}
470 	}
471 	thread_unlock(t);
472 }
473 
474 /*
475  * Wrapper for backward compatibility.
476  */
477 void
478 affinity_clear(void)
479 {
480 	thread_affinity_clear(curthread);
481 }
482 
483 /*
484  * Weak cpu affinity.  Bind to the "current" cpu for short periods
485  * of time during which the thread must not block (but may be preempted).
486  * Use this instead of kpreempt_disable() when it is only "no migration"
487  * rather than "no preemption" semantics that are required - disabling
488  * preemption holds higher priority threads off of cpu and if the
489  * operation that is protected is more than momentary this is not good
490  * for realtime etc.
491  *
492  * Weakly bound threads will not prevent a cpu from being offlined -
493  * we'll only run them on the cpu to which they are weakly bound but
494  * (because they do not block) we'll always be able to move them on to
495  * another cpu at offline time if we give them just a short moment to
496  * run during which they will unbind.  To give a cpu a chance of offlining,
497  * however, we require a barrier to weak bindings that may be raised for a
498  * given cpu (offline/move code may set this and then wait a short time for
499  * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
500  *
501  * There are few restrictions on the calling context of thread_nomigrate.
502  * The caller must not hold the thread lock.  Calls may be nested.
503  *
504  * After weakbinding a thread must not perform actions that may block.
505  * In particular it must not call thread_affinity_set; calling that when
506  * already weakbound is nonsensical anyway.
507  *
508  * If curthread is prevented from migrating for other reasons
509  * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
510  * then the weak binding will succeed even if this cpu is the target of an
511  * offline/move request.
512  */
513 void
514 thread_nomigrate(void)
515 {
516 	cpu_t *cp;
517 	kthread_id_t t = curthread;
518 
519 again:
520 	kpreempt_disable();
521 	cp = CPU;
522 
523 	/*
524 	 * A highlevel interrupt must not modify t_nomigrate or
525 	 * t_weakbound_cpu of the thread it has interrupted.  A lowlevel
526 	 * interrupt thread cannot migrate and we can avoid the
527 	 * thread_lock call below by short-circuiting here.  In either
528 	 * case we can just return since no migration is possible and
529 	 * the condition will persist (ie, when we test for these again
530 	 * in thread_allowmigrate they can't have changed).   Migration
531 	 * is also impossible if we're at or above DISP_LEVEL pil.
532 	 */
533 	if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
534 	    getpil() >= DISP_LEVEL) {
535 		kpreempt_enable();
536 		return;
537 	}
538 
539 	/*
540 	 * We must be consistent with existing weak bindings.  Since we
541 	 * may be interrupted between the increment of t_nomigrate and
542 	 * the store to t_weakbound_cpu below we cannot assume that
543 	 * t_weakbound_cpu will be set if t_nomigrate is.  Note that we
544 	 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
545 	 * always the case.
546 	 */
547 	if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
548 		if (!panicstr)
549 			panic("thread_nomigrate: binding to %p but already "
550 			    "bound to %p", (void *)cp,
551 			    (void *)t->t_weakbound_cpu);
552 	}
553 
554 	/*
555 	 * At this point we have preemption disabled and we don't yet hold
556 	 * the thread lock.  So it's possible that somebody else could
557 	 * set t_bind_cpu here and not be able to force us across to the
558 	 * new cpu (since we have preemption disabled).
559 	 */
560 	thread_lock(curthread);
561 
562 	/*
563 	 * If further weak bindings are being (temporarily) suppressed then
564 	 * we'll settle for disabling kernel preemption (which assures
565 	 * no migration provided the thread does not block which it is
566 	 * not allowed to if using thread_nomigrate).  We must remember
567 	 * this disposition so we can take appropriate action in
568 	 * thread_allowmigrate.  If this is a nested call and the
569 	 * thread is already weakbound then fall through as normal.
570 	 * We remember the decision to settle for kpreempt_disable through
571 	 * negative nesting counting in t_nomigrate.  Once a thread has had one
572 	 * weakbinding request satisfied in this way any further (nested)
573 	 * requests will continue to be satisfied in the same way,
574 	 * even if weak bindings have recommenced.
575 	 */
576 	if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
577 		--t->t_nomigrate;
578 		thread_unlock(curthread);
579 		return;		/* with kpreempt_disable still active */
580 	}
581 
582 	/*
583 	 * We hold thread_lock so t_bind_cpu cannot change.  We could,
584 	 * however, be running on a different cpu to which we are t_bound_cpu
585 	 * to (as explained above).  If we grant the weak binding request
586 	 * in that case then the dispatcher must favour our weak binding
587 	 * over our strong (in which case, just as when preemption is
588 	 * disabled, we can continue to run on a cpu other than the one to
589 	 * which we are strongbound; the difference in this case is that
590 	 * this thread can be preempted and so can appear on the dispatch
591 	 * queues of a cpu other than the one it is strongbound to).
592 	 *
593 	 * If the cpu we are running on does not appear to be a current
594 	 * offline target (we check cpu_inmotion to determine this - since
595 	 * we don't hold cpu_lock we may not see a recent store to that,
596 	 * so it's possible that we at times can grant a weak binding to a
597 	 * cpu that is an offline target, but that one request will not
598 	 * prevent the offline from succeeding) then we will always grant
599 	 * the weak binding request.  This includes the case above where
600 	 * we grant a weakbinding not commensurate with our strong binding.
601 	 *
602 	 * If our cpu does appear to be an offline target then we're inclined
603 	 * not to grant the weakbinding request just yet - we'd prefer to
604 	 * migrate to another cpu and grant the request there.  The
605 	 * exceptions are those cases where going through preemption code
606 	 * will not result in us changing cpu:
607 	 *
608 	 *	. interrupts have already bypassed this case (see above)
609 	 *	. we are already weakbound to this cpu (dispatcher code will
610 	 *	  always return us to the weakbound cpu)
611 	 *	. preemption was disabled even before we disabled it above
612 	 *	. we are strongbound to this cpu (if we're strongbound to
613 	 *	another and not yet running there the trip through the
614 	 *	dispatcher will move us to the strongbound cpu and we
615 	 *	will grant the weak binding there)
616 	 */
617 	if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
618 	    t->t_bound_cpu == cp) {
619 		/*
620 		 * Don't be tempted to store to t_weakbound_cpu only on
621 		 * the first nested bind request - if we're interrupted
622 		 * after the increment of t_nomigrate and before the
623 		 * store to t_weakbound_cpu and the interrupt calls
624 		 * thread_nomigrate then the assertion in thread_allowmigrate
625 		 * would fail.
626 		 */
627 		t->t_nomigrate++;
628 		t->t_weakbound_cpu = cp;
629 		membar_producer();
630 		thread_unlock(curthread);
631 		/*
632 		 * Now that we have dropped the thread_lock another thread
633 		 * can set our t_weakbound_cpu, and will try to migrate us
634 		 * to the strongbound cpu (which will not be prevented by
635 		 * preemption being disabled since we're about to enable
636 		 * preemption).  We have granted the weakbinding to the current
637 		 * cpu, so again we are in the position that is is is possible
638 		 * that our weak and strong bindings differ.  Again this
639 		 * is catered for by dispatcher code which will favour our
640 		 * weak binding.
641 		 */
642 		kpreempt_enable();
643 	} else {
644 		/*
645 		 * Move to another cpu before granting the request by
646 		 * forcing this thread through preemption code.  When we
647 		 * get to set{front,back}dq called from CL_PREEMPT()
648 		 * cpu_choose() will be used to select a cpu to queue
649 		 * us on - that will see cpu_inmotion and take
650 		 * steps to avoid returning us to this cpu.
651 		 */
652 		cp->cpu_kprunrun = 1;
653 		thread_unlock(curthread);
654 		kpreempt_enable();	/* will call preempt() */
655 		goto again;
656 	}
657 }
658 
659 void
660 thread_allowmigrate(void)
661 {
662 	kthread_id_t t = curthread;
663 
664 	ASSERT(t->t_weakbound_cpu == CPU ||
665 	    (t->t_nomigrate < 0 && t->t_preempt > 0) ||
666 	    CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
667 	    getpil() >= DISP_LEVEL);
668 
669 	if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
670 	    getpil() >= DISP_LEVEL)
671 		return;
672 
673 	if (t->t_nomigrate < 0) {
674 		/*
675 		 * This thread was granted "weak binding" in the
676 		 * stronger form of kernel preemption disabling.
677 		 * Undo a level of nesting for both t_nomigrate
678 		 * and t_preempt.
679 		 */
680 		++t->t_nomigrate;
681 		kpreempt_enable();
682 	} else if (--t->t_nomigrate == 0) {
683 		/*
684 		 * Time to drop the weak binding.  We need to cater
685 		 * for the case where we're weakbound to a different
686 		 * cpu than that to which we're strongbound (a very
687 		 * temporary arrangement that must only persist until
688 		 * weak binding drops).  We don't acquire thread_lock
689 		 * here so even as this code executes t_bound_cpu
690 		 * may be changing.  So we disable preemption and
691 		 * a) in the case that t_bound_cpu changes while we
692 		 * have preemption disabled kprunrun will be set
693 		 * asynchronously, and b) if before disabling
694 		 * preemption we were already on a different cpu to
695 		 * our t_bound_cpu then we set kprunrun ourselves
696 		 * to force a trip through the dispatcher when
697 		 * preemption is enabled.
698 		 */
699 		kpreempt_disable();
700 		if (t->t_bound_cpu &&
701 		    t->t_weakbound_cpu != t->t_bound_cpu)
702 			CPU->cpu_kprunrun = 1;
703 		t->t_weakbound_cpu = NULL;
704 		membar_producer();
705 		kpreempt_enable();
706 	}
707 }
708 
709 /*
710  * weakbinding_stop can be used to temporarily cause weakbindings made
711  * with thread_nomigrate to be satisfied through the stronger action of
712  * kpreempt_disable.  weakbinding_start recommences normal weakbinding.
713  */
714 
715 void
716 weakbinding_stop(void)
717 {
718 	ASSERT(MUTEX_HELD(&cpu_lock));
719 	weakbindingbarrier = 1;
720 	membar_producer();	/* make visible before subsequent thread_lock */
721 }
722 
723 void
724 weakbinding_start(void)
725 {
726 	ASSERT(MUTEX_HELD(&cpu_lock));
727 	weakbindingbarrier = 0;
728 }
729 
730 /*
731  * This routine is called to place the CPUs in a safe place so that
732  * one of them can be taken off line or placed on line.  What we are
733  * trying to do here is prevent a thread from traversing the list
734  * of active CPUs while we are changing it or from getting placed on
735  * the run queue of a CPU that has just gone off line.  We do this by
736  * creating a thread with the highest possible prio for each CPU and
737  * having it call this routine.  The advantage of this method is that
738  * we can eliminate all checks for CPU_ACTIVE in the disp routines.
739  * This makes disp faster at the expense of making p_online() slower
740  * which is a good trade off.
741  */
742 static void
743 cpu_pause(volatile char *safe)
744 {
745 	int s;
746 	struct _cpu_pause_info *cpi = &cpu_pause_info;
747 
748 	ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
749 
750 	while (*safe != PAUSE_DIE) {
751 		*safe = PAUSE_READY;
752 		membar_enter();		/* make sure stores are flushed */
753 		sema_v(&cpi->cp_sem);	/* signal requesting thread */
754 
755 		/*
756 		 * Wait here until all pause threads are running.  That
757 		 * indicates that it's safe to do the spl.  Until
758 		 * cpu_pause_info.cp_go is set, we don't want to spl
759 		 * because that might block clock interrupts needed
760 		 * to preempt threads on other CPUs.
761 		 */
762 		while (cpi->cp_go == 0)
763 			;
764 		/*
765 		 * Even though we are at the highest disp prio, we need
766 		 * to block out all interrupts below LOCK_LEVEL so that
767 		 * an intr doesn't come in, wake up a thread, and call
768 		 * setbackdq/setfrontdq.
769 		 */
770 		s = splhigh();
771 		/*
772 		 * This cpu is now safe.
773 		 */
774 		*safe = PAUSE_WAIT;
775 		membar_enter();		/* make sure stores are flushed */
776 
777 		/*
778 		 * Now we wait.  When we are allowed to continue, safe will
779 		 * be set to PAUSE_IDLE.
780 		 */
781 		while (*safe != PAUSE_IDLE)
782 			;
783 
784 		splx(s);
785 		/*
786 		 * Waiting is at an end. Switch out of cpu_pause
787 		 * loop and resume useful work.
788 		 */
789 		swtch();
790 	}
791 
792 	mutex_enter(&pause_free_mutex);
793 	*safe = PAUSE_DEAD;
794 	cv_broadcast(&pause_free_cv);
795 	mutex_exit(&pause_free_mutex);
796 }
797 
798 /*
799  * Allow the cpus to start running again.
800  */
801 void
802 start_cpus()
803 {
804 	int i;
805 
806 	ASSERT(MUTEX_HELD(&cpu_lock));
807 	ASSERT(cpu_pause_info.cp_paused);
808 	cpu_pause_info.cp_paused = NULL;
809 	for (i = 0; i < NCPU; i++)
810 		safe_list[i] = PAUSE_IDLE;
811 	membar_enter();			/* make sure stores are flushed */
812 	affinity_clear();
813 	splx(cpu_pause_info.cp_spl);
814 	kpreempt_enable();
815 }
816 
817 /*
818  * Allocate a pause thread for a CPU.
819  */
820 static void
821 cpu_pause_alloc(cpu_t *cp)
822 {
823 	kthread_id_t	t;
824 	int		cpun = cp->cpu_id;
825 
826 	/*
827 	 * Note, v.v_nglobpris will not change value as long as I hold
828 	 * cpu_lock.
829 	 */
830 	t = thread_create(NULL, 0, cpu_pause, (caddr_t)&safe_list[cpun],
831 	    0, &p0, TS_STOPPED, v.v_nglobpris - 1);
832 	thread_lock(t);
833 	t->t_bound_cpu = cp;
834 	t->t_disp_queue = cp->cpu_disp;
835 	t->t_affinitycnt = 1;
836 	t->t_preempt = 1;
837 	thread_unlock(t);
838 	cp->cpu_pause_thread = t;
839 	/*
840 	 * Registering a thread in the callback table is usually done
841 	 * in the initialization code of the thread.  In this
842 	 * case, we do it right after thread creation because the
843 	 * thread itself may never run, and we need to register the
844 	 * fact that it is safe for cpr suspend.
845 	 */
846 	CALLB_CPR_INIT_SAFE(t, "cpu_pause");
847 }
848 
849 /*
850  * Free a pause thread for a CPU.
851  */
852 static void
853 cpu_pause_free(cpu_t *cp)
854 {
855 	kthread_id_t	t;
856 	int		cpun = cp->cpu_id;
857 
858 	ASSERT(MUTEX_HELD(&cpu_lock));
859 	/*
860 	 * We have to get the thread and tell him to die.
861 	 */
862 	if ((t = cp->cpu_pause_thread) == NULL) {
863 		ASSERT(safe_list[cpun] == PAUSE_IDLE);
864 		return;
865 	}
866 	thread_lock(t);
867 	t->t_cpu = CPU;		/* disp gets upset if last cpu is quiesced. */
868 	t->t_bound_cpu = NULL;	/* Must un-bind; cpu may not be running. */
869 	t->t_pri = v.v_nglobpris - 1;
870 	ASSERT(safe_list[cpun] == PAUSE_IDLE);
871 	safe_list[cpun] = PAUSE_DIE;
872 	THREAD_TRANSITION(t);
873 	setbackdq(t);
874 	thread_unlock_nopreempt(t);
875 
876 	/*
877 	 * If we don't wait for the thread to actually die, it may try to
878 	 * run on the wrong cpu as part of an actual call to pause_cpus().
879 	 */
880 	mutex_enter(&pause_free_mutex);
881 	while (safe_list[cpun] != PAUSE_DEAD) {
882 		cv_wait(&pause_free_cv, &pause_free_mutex);
883 	}
884 	mutex_exit(&pause_free_mutex);
885 	safe_list[cpun] = PAUSE_IDLE;
886 
887 	cp->cpu_pause_thread = NULL;
888 }
889 
890 /*
891  * Initialize basic structures for pausing CPUs.
892  */
893 void
894 cpu_pause_init()
895 {
896 	sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
897 	/*
898 	 * Create initial CPU pause thread.
899 	 */
900 	cpu_pause_alloc(CPU);
901 }
902 
903 /*
904  * Start the threads used to pause another CPU.
905  */
906 static int
907 cpu_pause_start(processorid_t cpu_id)
908 {
909 	int	i;
910 	int	cpu_count = 0;
911 
912 	for (i = 0; i < NCPU; i++) {
913 		cpu_t		*cp;
914 		kthread_id_t	t;
915 
916 		cp = cpu[i];
917 		if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
918 			safe_list[i] = PAUSE_WAIT;
919 			continue;
920 		}
921 
922 		/*
923 		 * Skip CPU if it is quiesced or not yet started.
924 		 */
925 		if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
926 			safe_list[i] = PAUSE_WAIT;
927 			continue;
928 		}
929 
930 		/*
931 		 * Start this CPU's pause thread.
932 		 */
933 		t = cp->cpu_pause_thread;
934 		thread_lock(t);
935 		/*
936 		 * Reset the priority, since nglobpris may have
937 		 * changed since the thread was created, if someone
938 		 * has loaded the RT (or some other) scheduling
939 		 * class.
940 		 */
941 		t->t_pri = v.v_nglobpris - 1;
942 		THREAD_TRANSITION(t);
943 		setbackdq(t);
944 		thread_unlock_nopreempt(t);
945 		++cpu_count;
946 	}
947 	return (cpu_count);
948 }
949 
950 
951 /*
952  * Pause all of the CPUs except the one we are on by creating a high
953  * priority thread bound to those CPUs.
954  *
955  * Note that one must be extremely careful regarding code
956  * executed while CPUs are paused.  Since a CPU may be paused
957  * while a thread scheduling on that CPU is holding an adaptive
958  * lock, code executed with CPUs paused must not acquire adaptive
959  * (or low-level spin) locks.  Also, such code must not block,
960  * since the thread that is supposed to initiate the wakeup may
961  * never run.
962  *
963  * With a few exceptions, the restrictions on code executed with CPUs
964  * paused match those for code executed at high-level interrupt
965  * context.
966  */
967 void
968 pause_cpus(cpu_t *off_cp)
969 {
970 	processorid_t	cpu_id;
971 	int		i;
972 	struct _cpu_pause_info	*cpi = &cpu_pause_info;
973 
974 	ASSERT(MUTEX_HELD(&cpu_lock));
975 	ASSERT(cpi->cp_paused == NULL);
976 	cpi->cp_count = 0;
977 	cpi->cp_go = 0;
978 	for (i = 0; i < NCPU; i++)
979 		safe_list[i] = PAUSE_IDLE;
980 	kpreempt_disable();
981 
982 	/*
983 	 * If running on the cpu that is going offline, get off it.
984 	 * This is so that it won't be necessary to rechoose a CPU
985 	 * when done.
986 	 */
987 	if (CPU == off_cp)
988 		cpu_id = off_cp->cpu_next_part->cpu_id;
989 	else
990 		cpu_id = CPU->cpu_id;
991 	affinity_set(cpu_id);
992 
993 	/*
994 	 * Start the pause threads and record how many were started
995 	 */
996 	cpi->cp_count = cpu_pause_start(cpu_id);
997 
998 	/*
999 	 * Now wait for all CPUs to be running the pause thread.
1000 	 */
1001 	while (cpi->cp_count > 0) {
1002 		/*
1003 		 * Spin reading the count without grabbing the disp
1004 		 * lock to make sure we don't prevent the pause
1005 		 * threads from getting the lock.
1006 		 */
1007 		while (sema_held(&cpi->cp_sem))
1008 			;
1009 		if (sema_tryp(&cpi->cp_sem))
1010 			--cpi->cp_count;
1011 	}
1012 	cpi->cp_go = 1;			/* all have reached cpu_pause */
1013 
1014 	/*
1015 	 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1016 	 * to PAUSE_WAIT.)
1017 	 */
1018 	for (i = 0; i < NCPU; i++) {
1019 		while (safe_list[i] != PAUSE_WAIT)
1020 			;
1021 	}
1022 	cpi->cp_spl = splhigh();	/* block dispatcher on this CPU */
1023 	cpi->cp_paused = curthread;
1024 }
1025 
1026 /*
1027  * Check whether the current thread has CPUs paused
1028  */
1029 int
1030 cpus_paused(void)
1031 {
1032 	if (cpu_pause_info.cp_paused != NULL) {
1033 		ASSERT(cpu_pause_info.cp_paused == curthread);
1034 		return (1);
1035 	}
1036 	return (0);
1037 }
1038 
1039 static cpu_t *
1040 cpu_get_all(processorid_t cpun)
1041 {
1042 	ASSERT(MUTEX_HELD(&cpu_lock));
1043 
1044 	if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1045 		return (NULL);
1046 	return (cpu[cpun]);
1047 }
1048 
1049 /*
1050  * Check whether cpun is a valid processor id and whether it should be
1051  * visible from the current zone. If it is, return a pointer to the
1052  * associated CPU structure.
1053  */
1054 cpu_t *
1055 cpu_get(processorid_t cpun)
1056 {
1057 	cpu_t *c;
1058 
1059 	ASSERT(MUTEX_HELD(&cpu_lock));
1060 	c = cpu_get_all(cpun);
1061 	if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1062 	    zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1063 		return (NULL);
1064 	return (c);
1065 }
1066 
1067 /*
1068  * The following functions should be used to check CPU states in the kernel.
1069  * They should be invoked with cpu_lock held.  Kernel subsystems interested
1070  * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1071  * states.  Those are for user-land (and system call) use only.
1072  */
1073 
1074 /*
1075  * Determine whether the CPU is online and handling interrupts.
1076  */
1077 int
1078 cpu_is_online(cpu_t *cpu)
1079 {
1080 	ASSERT(MUTEX_HELD(&cpu_lock));
1081 	return (cpu_flagged_online(cpu->cpu_flags));
1082 }
1083 
1084 /*
1085  * Determine whether the CPU is offline (this includes spare and faulted).
1086  */
1087 int
1088 cpu_is_offline(cpu_t *cpu)
1089 {
1090 	ASSERT(MUTEX_HELD(&cpu_lock));
1091 	return (cpu_flagged_offline(cpu->cpu_flags));
1092 }
1093 
1094 /*
1095  * Determine whether the CPU is powered off.
1096  */
1097 int
1098 cpu_is_poweredoff(cpu_t *cpu)
1099 {
1100 	ASSERT(MUTEX_HELD(&cpu_lock));
1101 	return (cpu_flagged_poweredoff(cpu->cpu_flags));
1102 }
1103 
1104 /*
1105  * Determine whether the CPU is handling interrupts.
1106  */
1107 int
1108 cpu_is_nointr(cpu_t *cpu)
1109 {
1110 	ASSERT(MUTEX_HELD(&cpu_lock));
1111 	return (cpu_flagged_nointr(cpu->cpu_flags));
1112 }
1113 
1114 /*
1115  * Determine whether the CPU is active (scheduling threads).
1116  */
1117 int
1118 cpu_is_active(cpu_t *cpu)
1119 {
1120 	ASSERT(MUTEX_HELD(&cpu_lock));
1121 	return (cpu_flagged_active(cpu->cpu_flags));
1122 }
1123 
1124 /*
1125  * Same as above, but these require cpu_flags instead of cpu_t pointers.
1126  */
1127 int
1128 cpu_flagged_online(cpu_flag_t cpu_flags)
1129 {
1130 	return (cpu_flagged_active(cpu_flags) &&
1131 	    (cpu_flags & CPU_ENABLE));
1132 }
1133 
1134 int
1135 cpu_flagged_offline(cpu_flag_t cpu_flags)
1136 {
1137 	return (((cpu_flags & CPU_POWEROFF) == 0) &&
1138 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1139 }
1140 
1141 int
1142 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1143 {
1144 	return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1145 }
1146 
1147 int
1148 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1149 {
1150 	return (cpu_flagged_active(cpu_flags) &&
1151 	    (cpu_flags & CPU_ENABLE) == 0);
1152 }
1153 
1154 int
1155 cpu_flagged_active(cpu_flag_t cpu_flags)
1156 {
1157 	return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1158 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1159 }
1160 
1161 /*
1162  * Bring the indicated CPU online.
1163  */
1164 int
1165 cpu_online(cpu_t *cp)
1166 {
1167 	int	error = 0;
1168 
1169 	/*
1170 	 * Handle on-line request.
1171 	 *	This code must put the new CPU on the active list before
1172 	 *	starting it because it will not be paused, and will start
1173 	 * 	using the active list immediately.  The real start occurs
1174 	 *	when the CPU_QUIESCED flag is turned off.
1175 	 */
1176 
1177 	ASSERT(MUTEX_HELD(&cpu_lock));
1178 
1179 	/*
1180 	 * Put all the cpus into a known safe place.
1181 	 * No mutexes can be entered while CPUs are paused.
1182 	 */
1183 	error = mp_cpu_start(cp);	/* arch-dep hook */
1184 	if (error == 0) {
1185 		pause_cpus(NULL);
1186 		cpu_add_active_internal(cp);
1187 		if (cp->cpu_flags & CPU_FAULTED) {
1188 			cp->cpu_flags &= ~CPU_FAULTED;
1189 			mp_cpu_faulted_exit(cp);
1190 		}
1191 		cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1192 		    CPU_SPARE);
1193 		start_cpus();
1194 		cpu_stats_kstat_create(cp);
1195 		cpu_create_intrstat(cp);
1196 		lgrp_kstat_create(cp);
1197 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1198 		cpu_intr_enable(cp);	/* arch-dep hook */
1199 		cyclic_online(cp);
1200 		poke_cpu(cp->cpu_id);
1201 	}
1202 
1203 	return (error);
1204 }
1205 
1206 /*
1207  * Take the indicated CPU offline.
1208  */
1209 int
1210 cpu_offline(cpu_t *cp, int flags)
1211 {
1212 	cpupart_t *pp;
1213 	int	error = 0;
1214 	cpu_t	*ncp;
1215 	int	intr_enable;
1216 	int	cyclic_off = 0;
1217 	int	loop_count;
1218 	int	no_quiesce = 0;
1219 	int	(*bound_func)(struct cpu *, int);
1220 	kthread_t *t;
1221 	lpl_t	*cpu_lpl;
1222 	proc_t	*p;
1223 	int	lgrp_diff_lpl;
1224 
1225 	ASSERT(MUTEX_HELD(&cpu_lock));
1226 
1227 	/*
1228 	 * If we're going from faulted or spare to offline, just
1229 	 * clear these flags and update CPU state.
1230 	 */
1231 	if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1232 		if (cp->cpu_flags & CPU_FAULTED) {
1233 			cp->cpu_flags &= ~CPU_FAULTED;
1234 			mp_cpu_faulted_exit(cp);
1235 		}
1236 		cp->cpu_flags &= ~CPU_SPARE;
1237 		cpu_set_state(cp);
1238 		return (0);
1239 	}
1240 
1241 	/*
1242 	 * Handle off-line request.
1243 	 */
1244 	pp = cp->cpu_part;
1245 	/*
1246 	 * Don't offline last online CPU in partition
1247 	 */
1248 	if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1249 		return (EBUSY);
1250 	/*
1251 	 * Unbind all thread bound to our CPU if we were asked to.
1252 	 */
1253 	if (flags & CPU_FORCED && (error = cpu_unbind(cp->cpu_id)) != 0)
1254 		return (error);
1255 	/*
1256 	 * We shouldn't be bound to this CPU ourselves.
1257 	 */
1258 	if (curthread->t_bound_cpu == cp)
1259 		return (EBUSY);
1260 
1261 	/*
1262 	 * Tell interested parties that this CPU is going offline.
1263 	 */
1264 	cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1265 
1266 	/*
1267 	 * Take the CPU out of interrupt participation so we won't find
1268 	 * bound kernel threads.  If the architecture cannot completely
1269 	 * shut off interrupts on the CPU, don't quiesce it, but don't
1270 	 * run anything but interrupt thread... this is indicated by
1271 	 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1272 	 * off.
1273 	 */
1274 	intr_enable = cp->cpu_flags & CPU_ENABLE;
1275 	if (intr_enable)
1276 		no_quiesce = cpu_intr_disable(cp);
1277 
1278 	/*
1279 	 * Record that we are aiming to offline this cpu.  This acts as
1280 	 * a barrier to further weak binding requests in thread_nomigrate
1281 	 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1282 	 * lean away from this cpu.  Further strong bindings are already
1283 	 * avoided since we hold cpu_lock.  Since threads that are set
1284 	 * runnable around now and others coming off the target cpu are
1285 	 * directed away from the target, existing strong and weak bindings
1286 	 * (especially the latter) to the target cpu stand maximum chance of
1287 	 * being able to unbind during the short delay loop below (if other
1288 	 * unbound threads compete they may not see cpu in time to unbind
1289 	 * even if they would do so immediately.
1290 	 */
1291 	cpu_inmotion = cp;
1292 	membar_enter();
1293 
1294 	/*
1295 	 * Check for kernel threads (strong or weak) bound to that CPU.
1296 	 * Strongly bound threads may not unbind, and we'll have to return
1297 	 * EBUSY.  Weakly bound threads should always disappear - we've
1298 	 * stopped more weak binding with cpu_inmotion and existing
1299 	 * bindings will drain imminently (they may not block).  Nonetheless
1300 	 * we will wait for a fixed period for all bound threads to disappear.
1301 	 * Inactive interrupt threads are OK (they'll be in TS_FREE
1302 	 * state).  If test finds some bound threads, wait a few ticks
1303 	 * to give short-lived threads (such as interrupts) chance to
1304 	 * complete.  Note that if no_quiesce is set, i.e. this cpu
1305 	 * is required to service interrupts, then we take the route
1306 	 * that permits interrupt threads to be active (or bypassed).
1307 	 */
1308 	bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1309 
1310 again:	for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1311 		if (loop_count >= 5) {
1312 			error = EBUSY;	/* some threads still bound */
1313 			break;
1314 		}
1315 
1316 		/*
1317 		 * If some threads were assigned, give them
1318 		 * a chance to complete or move.
1319 		 *
1320 		 * This assumes that the clock_thread is not bound
1321 		 * to any CPU, because the clock_thread is needed to
1322 		 * do the delay(hz/100).
1323 		 *
1324 		 * Note: we still hold the cpu_lock while waiting for
1325 		 * the next clock tick.  This is OK since it isn't
1326 		 * needed for anything else except processor_bind(2),
1327 		 * and system initialization.  If we drop the lock,
1328 		 * we would risk another p_online disabling the last
1329 		 * processor.
1330 		 */
1331 		delay(hz/100);
1332 	}
1333 
1334 	if (error == 0 && cyclic_off == 0) {
1335 		if (!cyclic_offline(cp)) {
1336 			/*
1337 			 * We must have bound cyclics...
1338 			 */
1339 			error = EBUSY;
1340 			goto out;
1341 		}
1342 		cyclic_off = 1;
1343 	}
1344 
1345 	/*
1346 	 * Call mp_cpu_stop() to perform any special operations
1347 	 * needed for this machine architecture to offline a CPU.
1348 	 */
1349 	if (error == 0)
1350 		error = mp_cpu_stop(cp);	/* arch-dep hook */
1351 
1352 	/*
1353 	 * If that all worked, take the CPU offline and decrement
1354 	 * ncpus_online.
1355 	 */
1356 	if (error == 0) {
1357 		/*
1358 		 * Put all the cpus into a known safe place.
1359 		 * No mutexes can be entered while CPUs are paused.
1360 		 */
1361 		pause_cpus(cp);
1362 		/*
1363 		 * Repeat the operation, if necessary, to make sure that
1364 		 * all outstanding low-level interrupts run to completion
1365 		 * before we set the CPU_QUIESCED flag.  It's also possible
1366 		 * that a thread has weak bound to the cpu despite our raising
1367 		 * cpu_inmotion above since it may have loaded that
1368 		 * value before the barrier became visible (this would have
1369 		 * to be the thread that was on the target cpu at the time
1370 		 * we raised the barrier).
1371 		 */
1372 		if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1373 		    (*bound_func)(cp, 1)) {
1374 			start_cpus();
1375 			(void) mp_cpu_start(cp);
1376 			goto again;
1377 		}
1378 		ncp = cp->cpu_next_part;
1379 		cpu_lpl = cp->cpu_lpl;
1380 		ASSERT(cpu_lpl != NULL);
1381 
1382 		/*
1383 		 * Remove the CPU from the list of active CPUs.
1384 		 */
1385 		cpu_remove_active(cp);
1386 
1387 		/*
1388 		 * Walk the active process list and look for threads
1389 		 * whose home lgroup needs to be updated, or
1390 		 * the last CPU they run on is the one being offlined now.
1391 		 */
1392 
1393 		ASSERT(curthread->t_cpu != cp);
1394 		for (p = practive; p != NULL; p = p->p_next) {
1395 
1396 			t = p->p_tlist;
1397 
1398 			if (t == NULL)
1399 				continue;
1400 
1401 			lgrp_diff_lpl = 0;
1402 
1403 			do {
1404 				ASSERT(t->t_lpl != NULL);
1405 				/*
1406 				 * Taking last CPU in lpl offline
1407 				 * Rehome thread if it is in this lpl
1408 				 * Otherwise, update the count of how many
1409 				 * threads are in this CPU's lgroup but have
1410 				 * a different lpl.
1411 				 */
1412 
1413 				if (cpu_lpl->lpl_ncpu == 0) {
1414 					if (t->t_lpl == cpu_lpl)
1415 						lgrp_move_thread(t,
1416 						    lgrp_choose(t,
1417 						    t->t_cpupart), 0);
1418 					else if (t->t_lpl->lpl_lgrpid ==
1419 					    cpu_lpl->lpl_lgrpid)
1420 						lgrp_diff_lpl++;
1421 				}
1422 				ASSERT(t->t_lpl->lpl_ncpu > 0);
1423 
1424 				/*
1425 				 * Update CPU last ran on if it was this CPU
1426 				 */
1427 				if (t->t_cpu == cp && t->t_bound_cpu != cp)
1428 				    t->t_cpu = disp_lowpri_cpu(ncp, t->t_lpl,
1429 					t->t_pri, NULL);
1430 				ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1431 				    t->t_weakbound_cpu == cp);
1432 
1433 				t = t->t_forw;
1434 			} while (t != p->p_tlist);
1435 
1436 			/*
1437 			 * Didn't find any threads in the same lgroup as this
1438 			 * CPU with a different lpl, so remove the lgroup from
1439 			 * the process lgroup bitmask.
1440 			 */
1441 
1442 			if (lgrp_diff_lpl == 0)
1443 				klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1444 		}
1445 
1446 		/*
1447 		 * Walk thread list looking for threads that need to be
1448 		 * rehomed, since there are some threads that are not in
1449 		 * their process's p_tlist.
1450 		 */
1451 
1452 		t = curthread;
1453 		do {
1454 			ASSERT(t != NULL && t->t_lpl != NULL);
1455 
1456 			/*
1457 			 * Rehome threads with same lpl as this CPU when this
1458 			 * is the last CPU in the lpl.
1459 			 */
1460 
1461 			if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1462 				lgrp_move_thread(t,
1463 				    lgrp_choose(t, t->t_cpupart), 1);
1464 
1465 			ASSERT(t->t_lpl->lpl_ncpu > 0);
1466 
1467 			/*
1468 			 * Update CPU last ran on if it was this CPU
1469 			 */
1470 
1471 			if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1472 				t->t_cpu = disp_lowpri_cpu(ncp,
1473 				    t->t_lpl, t->t_pri, NULL);
1474 			}
1475 			ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1476 			    t->t_weakbound_cpu == cp);
1477 			t = t->t_next;
1478 
1479 		} while (t != curthread);
1480 		ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1481 		cp->cpu_flags |= CPU_OFFLINE;
1482 		disp_cpu_inactive(cp);
1483 		if (!no_quiesce)
1484 			cp->cpu_flags |= CPU_QUIESCED;
1485 		ncpus_online--;
1486 		cpu_set_state(cp);
1487 		cpu_inmotion = NULL;
1488 		start_cpus();
1489 		cpu_stats_kstat_destroy(cp);
1490 		cpu_delete_intrstat(cp);
1491 		lgrp_kstat_destroy(cp);
1492 	}
1493 
1494 out:
1495 	cpu_inmotion = NULL;
1496 
1497 	/*
1498 	 * If we failed, re-enable interrupts.
1499 	 * Do this even if cpu_intr_disable returned an error, because
1500 	 * it may have partially disabled interrupts.
1501 	 */
1502 	if (error && intr_enable)
1503 		cpu_intr_enable(cp);
1504 
1505 	/*
1506 	 * If we failed, but managed to offline the cyclic subsystem on this
1507 	 * CPU, bring it back online.
1508 	 */
1509 	if (error && cyclic_off)
1510 		cyclic_online(cp);
1511 
1512 	/*
1513 	 * If we failed, we need to notify everyone that this CPU is back on.
1514 	 */
1515 	if (error != 0)
1516 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1517 
1518 	return (error);
1519 }
1520 
1521 /*
1522  * Mark the indicated CPU as faulted, taking it offline.
1523  */
1524 int
1525 cpu_faulted(cpu_t *cp, int flags)
1526 {
1527 	int	error = 0;
1528 
1529 	ASSERT(MUTEX_HELD(&cpu_lock));
1530 	ASSERT(!cpu_is_poweredoff(cp));
1531 
1532 	if (cpu_is_offline(cp)) {
1533 		cp->cpu_flags &= ~CPU_SPARE;
1534 		cp->cpu_flags |= CPU_FAULTED;
1535 		mp_cpu_faulted_enter(cp);
1536 		cpu_set_state(cp);
1537 		return (0);
1538 	}
1539 
1540 	if ((error = cpu_offline(cp, flags)) == 0) {
1541 		cp->cpu_flags |= CPU_FAULTED;
1542 		mp_cpu_faulted_enter(cp);
1543 		cpu_set_state(cp);
1544 	}
1545 
1546 	return (error);
1547 }
1548 
1549 /*
1550  * Mark the indicated CPU as a spare, taking it offline.
1551  */
1552 int
1553 cpu_spare(cpu_t *cp, int flags)
1554 {
1555 	int	error = 0;
1556 
1557 	ASSERT(MUTEX_HELD(&cpu_lock));
1558 	ASSERT(!cpu_is_poweredoff(cp));
1559 
1560 	if (cpu_is_offline(cp)) {
1561 		if (cp->cpu_flags & CPU_FAULTED) {
1562 			cp->cpu_flags &= ~CPU_FAULTED;
1563 			mp_cpu_faulted_exit(cp);
1564 		}
1565 		cp->cpu_flags |= CPU_SPARE;
1566 		cpu_set_state(cp);
1567 		return (0);
1568 	}
1569 
1570 	if ((error = cpu_offline(cp, flags)) == 0) {
1571 		cp->cpu_flags |= CPU_SPARE;
1572 		cpu_set_state(cp);
1573 	}
1574 
1575 	return (error);
1576 }
1577 
1578 /*
1579  * Take the indicated CPU from poweroff to offline.
1580  */
1581 int
1582 cpu_poweron(cpu_t *cp)
1583 {
1584 	int	error = ENOTSUP;
1585 
1586 	ASSERT(MUTEX_HELD(&cpu_lock));
1587 	ASSERT(cpu_is_poweredoff(cp));
1588 
1589 	error = mp_cpu_poweron(cp);	/* arch-dep hook */
1590 	if (error == 0)
1591 		cpu_set_state(cp);
1592 
1593 	return (error);
1594 }
1595 
1596 /*
1597  * Take the indicated CPU from any inactive state to powered off.
1598  */
1599 int
1600 cpu_poweroff(cpu_t *cp)
1601 {
1602 	int	error = ENOTSUP;
1603 
1604 	ASSERT(MUTEX_HELD(&cpu_lock));
1605 	ASSERT(cpu_is_offline(cp));
1606 
1607 	if (!(cp->cpu_flags & CPU_QUIESCED))
1608 		return (EBUSY);		/* not completely idle */
1609 
1610 	error = mp_cpu_poweroff(cp);	/* arch-dep hook */
1611 	if (error == 0)
1612 		cpu_set_state(cp);
1613 
1614 	return (error);
1615 }
1616 
1617 /*
1618  * Initialize the CPU lists for the first CPU.
1619  */
1620 void
1621 cpu_list_init(cpu_t *cp)
1622 {
1623 	cp->cpu_next = cp;
1624 	cp->cpu_prev = cp;
1625 	cpu_list = cp;
1626 
1627 	cp->cpu_next_onln = cp;
1628 	cp->cpu_prev_onln = cp;
1629 	cpu_active = cp;
1630 
1631 	cp->cpu_seqid = 0;
1632 	CPUSET_ADD(cpu_seqid_inuse, 0);
1633 	cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid);
1634 	cp_default.cp_cpulist = cp;
1635 	cp_default.cp_ncpus = 1;
1636 	cp->cpu_next_part = cp;
1637 	cp->cpu_prev_part = cp;
1638 	cp->cpu_part = &cp_default;
1639 
1640 	CPUSET_ADD(cpu_available, cp->cpu_id);
1641 }
1642 
1643 /*
1644  * Insert a CPU into the list of available CPUs.
1645  */
1646 void
1647 cpu_add_unit(cpu_t *cp)
1648 {
1649 	int seqid;
1650 
1651 	ASSERT(MUTEX_HELD(&cpu_lock));
1652 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1653 
1654 	lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1655 
1656 	/*
1657 	 * Note: most users of the cpu_list will grab the
1658 	 * cpu_lock to insure that it isn't modified.  However,
1659 	 * certain users can't or won't do that.  To allow this
1660 	 * we pause the other cpus.  Users who walk the list
1661 	 * without cpu_lock, must disable kernel preemption
1662 	 * to insure that the list isn't modified underneath
1663 	 * them.  Also, any cached pointers to cpu structures
1664 	 * must be revalidated by checking to see if the
1665 	 * cpu_next pointer points to itself.  This check must
1666 	 * be done with the cpu_lock held or kernel preemption
1667 	 * disabled.  This check relies upon the fact that
1668 	 * old cpu structures are not free'ed or cleared after
1669 	 * then are removed from the cpu_list.
1670 	 *
1671 	 * Note that the clock code walks the cpu list dereferencing
1672 	 * the cpu_part pointer, so we need to initialize it before
1673 	 * adding the cpu to the list.
1674 	 */
1675 	cp->cpu_part = &cp_default;
1676 	(void) pause_cpus(NULL);
1677 	cp->cpu_next = cpu_list;
1678 	cp->cpu_prev = cpu_list->cpu_prev;
1679 	cpu_list->cpu_prev->cpu_next = cp;
1680 	cpu_list->cpu_prev = cp;
1681 	start_cpus();
1682 
1683 	for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1684 		continue;
1685 	CPUSET_ADD(cpu_seqid_inuse, seqid);
1686 	cp->cpu_seqid = seqid;
1687 	ASSERT(ncpus < max_ncpus);
1688 	ncpus++;
1689 	cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid);
1690 	cpu[cp->cpu_id] = cp;
1691 	CPUSET_ADD(cpu_available, cp->cpu_id);
1692 
1693 	/*
1694 	 * allocate a pause thread for this CPU.
1695 	 */
1696 	cpu_pause_alloc(cp);
1697 
1698 	/*
1699 	 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1700 	 * link them into a list of just that CPU.
1701 	 * This is so that disp_lowpri_cpu will work for thread_create in
1702 	 * pause_cpus() when called from the startup thread in a new CPU.
1703 	 */
1704 	cp->cpu_next_onln = cp;
1705 	cp->cpu_prev_onln = cp;
1706 	cpu_info_kstat_create(cp);
1707 	cp->cpu_next_part = cp;
1708 	cp->cpu_prev_part = cp;
1709 
1710 	init_cpu_mstate(cp, CMS_SYSTEM);
1711 
1712 	pool_pset_mod = gethrtime();
1713 }
1714 
1715 /*
1716  * Do the opposite of cpu_add_unit().
1717  */
1718 void
1719 cpu_del_unit(int cpuid)
1720 {
1721 	struct cpu	*cp, *cpnext;
1722 
1723 	ASSERT(MUTEX_HELD(&cpu_lock));
1724 	cp = cpu[cpuid];
1725 	ASSERT(cp != NULL);
1726 
1727 	ASSERT(cp->cpu_next_onln == cp);
1728 	ASSERT(cp->cpu_prev_onln == cp);
1729 	ASSERT(cp->cpu_next_part == cp);
1730 	ASSERT(cp->cpu_prev_part == cp);
1731 
1732 	chip_cpu_fini(cp);
1733 
1734 	/*
1735 	 * Destroy kstat stuff.
1736 	 */
1737 	cpu_info_kstat_destroy(cp);
1738 	term_cpu_mstate(cp);
1739 	/*
1740 	 * Free up pause thread.
1741 	 */
1742 	cpu_pause_free(cp);
1743 	CPUSET_DEL(cpu_available, cp->cpu_id);
1744 	cpu[cp->cpu_id] = NULL;
1745 	/*
1746 	 * The clock thread and mutex_vector_enter cannot hold the
1747 	 * cpu_lock while traversing the cpu list, therefore we pause
1748 	 * all other threads by pausing the other cpus. These, and any
1749 	 * other routines holding cpu pointers while possibly sleeping
1750 	 * must be sure to call kpreempt_disable before processing the
1751 	 * list and be sure to check that the cpu has not been deleted
1752 	 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1753 	 * to keep the deleted cpu structure around.
1754 	 *
1755 	 * Note that this MUST be done AFTER cpu_available
1756 	 * has been updated so that we don't waste time
1757 	 * trying to pause the cpu we're trying to delete.
1758 	 */
1759 	(void) pause_cpus(NULL);
1760 
1761 	cpnext = cp->cpu_next;
1762 	cp->cpu_prev->cpu_next = cp->cpu_next;
1763 	cp->cpu_next->cpu_prev = cp->cpu_prev;
1764 	if (cp == cpu_list)
1765 	    cpu_list = cpnext;
1766 
1767 	/*
1768 	 * Signals that the cpu has been deleted (see above).
1769 	 */
1770 	cp->cpu_next = NULL;
1771 	cp->cpu_prev = NULL;
1772 
1773 	start_cpus();
1774 
1775 	CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1776 	ncpus--;
1777 	lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1778 
1779 	pool_pset_mod = gethrtime();
1780 }
1781 
1782 /*
1783  * Add a CPU to the list of active CPUs.
1784  *	This routine must not get any locks, because other CPUs are paused.
1785  */
1786 static void
1787 cpu_add_active_internal(cpu_t *cp)
1788 {
1789 	cpupart_t	*pp = cp->cpu_part;
1790 
1791 	ASSERT(MUTEX_HELD(&cpu_lock));
1792 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1793 
1794 	ncpus_online++;
1795 	cpu_set_state(cp);
1796 	cp->cpu_next_onln = cpu_active;
1797 	cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1798 	cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1799 	cpu_active->cpu_prev_onln = cp;
1800 
1801 	if (pp->cp_cpulist) {
1802 		cp->cpu_next_part = pp->cp_cpulist;
1803 		cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1804 		pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1805 		pp->cp_cpulist->cpu_prev_part = cp;
1806 	} else {
1807 		ASSERT(pp->cp_ncpus == 0);
1808 		pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1809 	}
1810 	pp->cp_ncpus++;
1811 	if (pp->cp_ncpus == 1) {
1812 		cp_numparts_nonempty++;
1813 		ASSERT(cp_numparts_nonempty != 0);
1814 	}
1815 
1816 	chip_cpu_assign(cp);
1817 
1818 	lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1819 
1820 	bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1821 }
1822 
1823 /*
1824  * Add a CPU to the list of active CPUs.
1825  *	This is called from machine-dependent layers when a new CPU is started.
1826  */
1827 void
1828 cpu_add_active(cpu_t *cp)
1829 {
1830 	pause_cpus(NULL);
1831 	cpu_add_active_internal(cp);
1832 	start_cpus();
1833 	cpu_stats_kstat_create(cp);
1834 	cpu_create_intrstat(cp);
1835 	lgrp_kstat_create(cp);
1836 	cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1837 }
1838 
1839 
1840 /*
1841  * Remove a CPU from the list of active CPUs.
1842  *	This routine must not get any locks, because other CPUs are paused.
1843  */
1844 /* ARGSUSED */
1845 static void
1846 cpu_remove_active(cpu_t *cp)
1847 {
1848 	cpupart_t	*pp = cp->cpu_part;
1849 
1850 	ASSERT(MUTEX_HELD(&cpu_lock));
1851 	ASSERT(cp->cpu_next_onln != cp);	/* not the last one */
1852 	ASSERT(cp->cpu_prev_onln != cp);	/* not the last one */
1853 
1854 	chip_cpu_unassign(cp);
1855 
1856 	lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1857 
1858 	cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1859 	cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1860 	if (cpu_active == cp) {
1861 		cpu_active = cp->cpu_next_onln;
1862 	}
1863 	cp->cpu_next_onln = cp;
1864 	cp->cpu_prev_onln = cp;
1865 
1866 	cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1867 	cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1868 	if (pp->cp_cpulist == cp) {
1869 		pp->cp_cpulist = cp->cpu_next_part;
1870 		ASSERT(pp->cp_cpulist != cp);
1871 	}
1872 	cp->cpu_next_part = cp;
1873 	cp->cpu_prev_part = cp;
1874 	pp->cp_ncpus--;
1875 	if (pp->cp_ncpus == 0) {
1876 		cp_numparts_nonempty--;
1877 		ASSERT(cp_numparts_nonempty != 0);
1878 	}
1879 }
1880 
1881 /*
1882  * Routine used to setup a newly inserted CPU in preparation for starting
1883  * it running code.
1884  */
1885 int
1886 cpu_configure(int cpuid)
1887 {
1888 	int retval = 0;
1889 
1890 	ASSERT(MUTEX_HELD(&cpu_lock));
1891 
1892 	/*
1893 	 * Some structures are statically allocated based upon
1894 	 * the maximum number of cpus the system supports.  Do not
1895 	 * try to add anything beyond this limit.
1896 	 */
1897 	if (cpuid < 0 || cpuid >= NCPU) {
1898 		return (EINVAL);
1899 	}
1900 
1901 	if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
1902 		return (EALREADY);
1903 	}
1904 
1905 	if ((retval = mp_cpu_configure(cpuid)) != 0) {
1906 		return (retval);
1907 	}
1908 
1909 	cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
1910 	cpu_set_state(cpu[cpuid]);
1911 	retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
1912 	if (retval != 0)
1913 		(void) mp_cpu_unconfigure(cpuid);
1914 
1915 	return (retval);
1916 }
1917 
1918 /*
1919  * Routine used to cleanup a CPU that has been powered off.  This will
1920  * destroy all per-cpu information related to this cpu.
1921  */
1922 int
1923 cpu_unconfigure(int cpuid)
1924 {
1925 	int error;
1926 
1927 	ASSERT(MUTEX_HELD(&cpu_lock));
1928 
1929 	if (cpu[cpuid] == NULL) {
1930 		return (ENODEV);
1931 	}
1932 
1933 	if (cpu[cpuid]->cpu_flags == 0) {
1934 		return (EALREADY);
1935 	}
1936 
1937 	if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
1938 		return (EBUSY);
1939 	}
1940 
1941 	if (cpu[cpuid]->cpu_props != NULL) {
1942 		(void) nvlist_free(cpu[cpuid]->cpu_props);
1943 		cpu[cpuid]->cpu_props = NULL;
1944 	}
1945 
1946 	error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
1947 
1948 	if (error != 0)
1949 		return (error);
1950 
1951 	return (mp_cpu_unconfigure(cpuid));
1952 }
1953 
1954 /*
1955  * Routines for registering and de-registering cpu_setup callback functions.
1956  *
1957  * Caller's context
1958  *	These routines must not be called from a driver's attach(9E) or
1959  *	detach(9E) entry point.
1960  *
1961  * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
1962  */
1963 
1964 /*
1965  * Ideally, these would be dynamically allocated and put into a linked
1966  * list; however that is not feasible because the registration routine
1967  * has to be available before the kmem allocator is working (in fact,
1968  * it is called by the kmem allocator init code).  In any case, there
1969  * are quite a few extra entries for future users.
1970  */
1971 #define	NCPU_SETUPS	10
1972 
1973 struct cpu_setup {
1974 	cpu_setup_func_t *func;
1975 	void *arg;
1976 } cpu_setups[NCPU_SETUPS];
1977 
1978 void
1979 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
1980 {
1981 	int i;
1982 
1983 	ASSERT(MUTEX_HELD(&cpu_lock));
1984 
1985 	for (i = 0; i < NCPU_SETUPS; i++)
1986 		if (cpu_setups[i].func == NULL)
1987 		    break;
1988 	if (i >= NCPU_SETUPS)
1989 		cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
1990 
1991 	cpu_setups[i].func = func;
1992 	cpu_setups[i].arg = arg;
1993 }
1994 
1995 void
1996 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
1997 {
1998 	int i;
1999 
2000 	ASSERT(MUTEX_HELD(&cpu_lock));
2001 
2002 	for (i = 0; i < NCPU_SETUPS; i++)
2003 		if ((cpu_setups[i].func == func) &&
2004 		    (cpu_setups[i].arg == arg))
2005 		    break;
2006 	if (i >= NCPU_SETUPS)
2007 		cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2008 		    "deregister");
2009 
2010 	cpu_setups[i].func = NULL;
2011 	cpu_setups[i].arg = 0;
2012 }
2013 
2014 /*
2015  * Call any state change hooks for this CPU, ignore any errors.
2016  */
2017 void
2018 cpu_state_change_notify(int id, cpu_setup_t what)
2019 {
2020 	int i;
2021 
2022 	ASSERT(MUTEX_HELD(&cpu_lock));
2023 
2024 	for (i = 0; i < NCPU_SETUPS; i++) {
2025 		if (cpu_setups[i].func != NULL) {
2026 			cpu_setups[i].func(what, id, cpu_setups[i].arg);
2027 		}
2028 	}
2029 }
2030 
2031 /*
2032  * Call any state change hooks for this CPU, undo it if error found.
2033  */
2034 static int
2035 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2036 {
2037 	int i;
2038 	int retval = 0;
2039 
2040 	ASSERT(MUTEX_HELD(&cpu_lock));
2041 
2042 	for (i = 0; i < NCPU_SETUPS; i++) {
2043 		if (cpu_setups[i].func != NULL) {
2044 			retval = cpu_setups[i].func(what, id,
2045 			    cpu_setups[i].arg);
2046 			if (retval) {
2047 				for (i--; i >= 0; i--) {
2048 					if (cpu_setups[i].func != NULL)
2049 						cpu_setups[i].func(undo,
2050 						    id, cpu_setups[i].arg);
2051 				}
2052 				break;
2053 			}
2054 		}
2055 	}
2056 	return (retval);
2057 }
2058 
2059 /*
2060  * Export information about this CPU via the kstat mechanism.
2061  */
2062 static struct {
2063 	kstat_named_t ci_state;
2064 	kstat_named_t ci_state_begin;
2065 	kstat_named_t ci_cpu_type;
2066 	kstat_named_t ci_fpu_type;
2067 	kstat_named_t ci_clock_MHz;
2068 	kstat_named_t ci_chip_id;
2069 	kstat_named_t ci_implementation;
2070 #ifdef __sparcv9
2071 	kstat_named_t ci_device_ID;
2072 	kstat_named_t ci_cpu_fru;
2073 #endif
2074 	kstat_named_t ci_brandstr;
2075 } cpu_info_template = {
2076 	{ "state",		KSTAT_DATA_CHAR },
2077 	{ "state_begin",	KSTAT_DATA_LONG },
2078 	{ "cpu_type",		KSTAT_DATA_CHAR },
2079 	{ "fpu_type",		KSTAT_DATA_CHAR },
2080 	{ "clock_MHz",		KSTAT_DATA_LONG },
2081 	{ "chip_id",		KSTAT_DATA_LONG },
2082 	{ "implementation",	KSTAT_DATA_STRING },
2083 #ifdef __sparcv9
2084 	{ "device_ID",		KSTAT_DATA_UINT64 },
2085 	{ "cpu_fru",		KSTAT_DATA_STRING },
2086 #endif
2087 	{ "brand",		KSTAT_DATA_STRING },
2088 };
2089 
2090 static kmutex_t cpu_info_template_lock;
2091 
2092 static int
2093 cpu_info_kstat_update(kstat_t *ksp, int rw)
2094 {
2095 	cpu_t	*cp = ksp->ks_private;
2096 	const char *pi_state;
2097 
2098 	if (rw == KSTAT_WRITE)
2099 		return (EACCES);
2100 
2101 	switch (cp->cpu_type_info.pi_state) {
2102 	case P_ONLINE:
2103 		pi_state = PS_ONLINE;
2104 		break;
2105 	case P_POWEROFF:
2106 		pi_state = PS_POWEROFF;
2107 		break;
2108 	case P_NOINTR:
2109 		pi_state = PS_NOINTR;
2110 		break;
2111 	case P_FAULTED:
2112 		pi_state = PS_FAULTED;
2113 		break;
2114 	case P_SPARE:
2115 		pi_state = PS_SPARE;
2116 		break;
2117 	case P_OFFLINE:
2118 		pi_state = PS_OFFLINE;
2119 		break;
2120 	default:
2121 		pi_state = "unknown";
2122 	}
2123 	(void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2124 	cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2125 	(void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2126 	    cp->cpu_type_info.pi_processor_type, 15);
2127 	(void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2128 	    cp->cpu_type_info.pi_fputypes, 15);
2129 	cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2130 	cpu_info_template.ci_chip_id.value.l = chip_plat_get_chipid(cp);
2131 	kstat_named_setstr(&cpu_info_template.ci_implementation,
2132 	    cp->cpu_idstr);
2133 #ifdef __sparcv9
2134 	cpu_info_template.ci_device_ID.value.ui64 =
2135 	    cpunodes[cp->cpu_id].device_id;
2136 	kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2137 #endif
2138 	kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2139 	return (0);
2140 }
2141 
2142 static void
2143 cpu_info_kstat_create(cpu_t *cp)
2144 {
2145 	zoneid_t zoneid;
2146 
2147 	ASSERT(MUTEX_HELD(&cpu_lock));
2148 
2149 	if (pool_pset_enabled())
2150 		zoneid = GLOBAL_ZONEID;
2151 	else
2152 		zoneid = ALL_ZONES;
2153 	if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2154 	    NULL, "misc", KSTAT_TYPE_NAMED,
2155 	    sizeof (cpu_info_template) / sizeof (kstat_named_t),
2156 	    KSTAT_FLAG_VIRTUAL, zoneid)) != NULL) {
2157 		cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2158 #ifdef __sparcv9
2159 		cp->cpu_info_kstat->ks_data_size +=
2160 		    strlen(cpu_fru_fmri(cp)) + 1;
2161 #endif
2162 		cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2163 		cp->cpu_info_kstat->ks_data = &cpu_info_template;
2164 		cp->cpu_info_kstat->ks_private = cp;
2165 		cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2166 		kstat_install(cp->cpu_info_kstat);
2167 	}
2168 }
2169 
2170 static void
2171 cpu_info_kstat_destroy(cpu_t *cp)
2172 {
2173 	ASSERT(MUTEX_HELD(&cpu_lock));
2174 
2175 	kstat_delete(cp->cpu_info_kstat);
2176 	cp->cpu_info_kstat = NULL;
2177 }
2178 
2179 /*
2180  * Create and install kstats for the boot CPU.
2181  */
2182 void
2183 cpu_kstat_init(cpu_t *cp)
2184 {
2185 	mutex_enter(&cpu_lock);
2186 	cpu_info_kstat_create(cp);
2187 	cpu_stats_kstat_create(cp);
2188 	cpu_create_intrstat(cp);
2189 	chip_kstat_create(cp->cpu_chip);
2190 	cpu_set_state(cp);
2191 	mutex_exit(&cpu_lock);
2192 }
2193 
2194 /*
2195  * Make visible to the zone that subset of the cpu information that would be
2196  * initialized when a cpu is configured (but still offline).
2197  */
2198 void
2199 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2200 {
2201 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2202 
2203 	ASSERT(MUTEX_HELD(&cpu_lock));
2204 	ASSERT(pool_pset_enabled());
2205 	ASSERT(cp != NULL);
2206 
2207 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2208 		zone->zone_ncpus++;
2209 		ASSERT(zone->zone_ncpus <= ncpus);
2210 	}
2211 	if (cp->cpu_info_kstat != NULL)
2212 		kstat_zone_add(cp->cpu_info_kstat, zoneid);
2213 }
2214 
2215 /*
2216  * Make visible to the zone that subset of the cpu information that would be
2217  * initialized when a previously configured cpu is onlined.
2218  */
2219 void
2220 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2221 {
2222 	kstat_t *ksp;
2223 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2224 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2225 	processorid_t cpun;
2226 
2227 	ASSERT(MUTEX_HELD(&cpu_lock));
2228 	ASSERT(pool_pset_enabled());
2229 	ASSERT(cp != NULL);
2230 	ASSERT(cpu_is_active(cp));
2231 
2232 	cpun = cp->cpu_id;
2233 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2234 		zone->zone_ncpus_online++;
2235 		ASSERT(zone->zone_ncpus_online <= ncpus_online);
2236 	}
2237 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2238 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2239 	    != NULL) {
2240 		kstat_zone_add(ksp, zoneid);
2241 		kstat_rele(ksp);
2242 	}
2243 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2244 		kstat_zone_add(ksp, zoneid);
2245 		kstat_rele(ksp);
2246 	}
2247 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2248 		kstat_zone_add(ksp, zoneid);
2249 		kstat_rele(ksp);
2250 	}
2251 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2252 	    NULL) {
2253 		kstat_zone_add(ksp, zoneid);
2254 		kstat_rele(ksp);
2255 	}
2256 }
2257 
2258 /*
2259  * Update relevant kstats such that cpu is now visible to processes
2260  * executing in specified zone.
2261  */
2262 void
2263 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2264 {
2265 	cpu_visibility_configure(cp, zone);
2266 	if (cpu_is_active(cp))
2267 		cpu_visibility_online(cp, zone);
2268 }
2269 
2270 /*
2271  * Make invisible to the zone that subset of the cpu information that would be
2272  * torn down when a previously offlined cpu is unconfigured.
2273  */
2274 void
2275 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2276 {
2277 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2278 
2279 	ASSERT(MUTEX_HELD(&cpu_lock));
2280 	ASSERT(pool_pset_enabled());
2281 	ASSERT(cp != NULL);
2282 
2283 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2284 		ASSERT(zone->zone_ncpus != 0);
2285 		zone->zone_ncpus--;
2286 	}
2287 	if (cp->cpu_info_kstat)
2288 		kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2289 }
2290 
2291 /*
2292  * Make invisible to the zone that subset of the cpu information that would be
2293  * torn down when a cpu is offlined (but still configured).
2294  */
2295 void
2296 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2297 {
2298 	kstat_t *ksp;
2299 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2300 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2301 	processorid_t cpun;
2302 
2303 	ASSERT(MUTEX_HELD(&cpu_lock));
2304 	ASSERT(pool_pset_enabled());
2305 	ASSERT(cp != NULL);
2306 	ASSERT(cpu_is_active(cp));
2307 
2308 	cpun = cp->cpu_id;
2309 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2310 		ASSERT(zone->zone_ncpus_online != 0);
2311 		zone->zone_ncpus_online--;
2312 	}
2313 
2314 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2315 	    NULL) {
2316 		kstat_zone_remove(ksp, zoneid);
2317 		kstat_rele(ksp);
2318 	}
2319 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2320 		kstat_zone_remove(ksp, zoneid);
2321 		kstat_rele(ksp);
2322 	}
2323 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2324 		kstat_zone_remove(ksp, zoneid);
2325 		kstat_rele(ksp);
2326 	}
2327 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2328 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2329 	    != NULL) {
2330 		kstat_zone_remove(ksp, zoneid);
2331 		kstat_rele(ksp);
2332 	}
2333 }
2334 
2335 /*
2336  * Update relevant kstats such that cpu is no longer visible to processes
2337  * executing in specified zone.
2338  */
2339 void
2340 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2341 {
2342 	if (cpu_is_active(cp))
2343 		cpu_visibility_offline(cp, zone);
2344 	cpu_visibility_unconfigure(cp, zone);
2345 }
2346 
2347 /*
2348  * Bind a thread to a CPU as requested.
2349  */
2350 int
2351 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2352     int *error)
2353 {
2354 	processorid_t	binding;
2355 	cpu_t		*cp;
2356 
2357 	ASSERT(MUTEX_HELD(&cpu_lock));
2358 	ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2359 
2360 	thread_lock(tp);
2361 
2362 	/*
2363 	 * Record old binding, but change the obind, which was initialized
2364 	 * to PBIND_NONE, only if this thread has a binding.  This avoids
2365 	 * reporting PBIND_NONE for a process when some LWPs are bound.
2366 	 */
2367 	binding = tp->t_bind_cpu;
2368 	if (binding != PBIND_NONE)
2369 		*obind = binding;	/* record old binding */
2370 
2371 	if (bind == PBIND_QUERY) {
2372 		thread_unlock(tp);
2373 		return (0);
2374 	}
2375 
2376 	/*
2377 	 * If this thread/LWP cannot be bound because of permission
2378 	 * problems, just note that and return success so that the
2379 	 * other threads/LWPs will be bound.  This is the way
2380 	 * processor_bind() is defined to work.
2381 	 *
2382 	 * Binding will get EPERM if the thread is of system class
2383 	 * or hasprocperm() fails.
2384 	 */
2385 	if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2386 		*error = EPERM;
2387 		thread_unlock(tp);
2388 		return (0);
2389 	}
2390 
2391 	binding = bind;
2392 	if (binding != PBIND_NONE) {
2393 		cp = cpu[binding];
2394 		/*
2395 		 * Make sure binding is in right partition.
2396 		 */
2397 		if (tp->t_cpupart != cp->cpu_part) {
2398 			*error = EINVAL;
2399 			thread_unlock(tp);
2400 			return (0);
2401 		}
2402 	}
2403 	tp->t_bind_cpu = binding;	/* set new binding */
2404 
2405 	/*
2406 	 * If there is no system-set reason for affinity, set
2407 	 * the t_bound_cpu field to reflect the binding.
2408 	 */
2409 	if (tp->t_affinitycnt == 0) {
2410 		if (binding == PBIND_NONE) {
2411 			/*
2412 			 * We may need to adjust disp_max_unbound_pri
2413 			 * since we're becoming unbound.
2414 			 */
2415 			disp_adjust_unbound_pri(tp);
2416 
2417 			tp->t_bound_cpu = NULL;	/* set new binding */
2418 
2419 			/*
2420 			 * Move thread to lgroup with strongest affinity
2421 			 * after unbinding
2422 			 */
2423 			if (tp->t_lgrp_affinity)
2424 				lgrp_move_thread(tp,
2425 				    lgrp_choose(tp, tp->t_cpupart), 1);
2426 
2427 			if (tp->t_state == TS_ONPROC &&
2428 			    tp->t_cpu->cpu_part != tp->t_cpupart)
2429 				cpu_surrender(tp);
2430 		} else {
2431 			lpl_t	*lpl;
2432 
2433 			tp->t_bound_cpu = cp;
2434 			ASSERT(cp->cpu_lpl != NULL);
2435 
2436 			/*
2437 			 * Set home to lgroup with most affinity containing CPU
2438 			 * that thread is being bound or minimum bounding
2439 			 * lgroup if no affinities set
2440 			 */
2441 			if (tp->t_lgrp_affinity)
2442 				lpl = lgrp_affinity_best(tp, tp->t_cpupart, 0);
2443 			else
2444 				lpl = cp->cpu_lpl;
2445 
2446 			if (tp->t_lpl != lpl) {
2447 				/* can't grab cpu_lock */
2448 				lgrp_move_thread(tp, lpl, 1);
2449 			}
2450 
2451 			/*
2452 			 * Make the thread switch to the bound CPU.
2453 			 * If the thread is runnable, we need to
2454 			 * requeue it even if t_cpu is already set
2455 			 * to the right CPU, since it may be on a
2456 			 * kpreempt queue and need to move to a local
2457 			 * queue.  We could check t_disp_queue to
2458 			 * avoid unnecessary overhead if it's already
2459 			 * on the right queue, but since this isn't
2460 			 * a performance-critical operation it doesn't
2461 			 * seem worth the extra code and complexity.
2462 			 *
2463 			 * If the thread is weakbound to the cpu then it will
2464 			 * resist the new binding request until the weak
2465 			 * binding drops.  The cpu_surrender or requeueing
2466 			 * below could be skipped in such cases (since it
2467 			 * will have no effect), but that would require
2468 			 * thread_allowmigrate to acquire thread_lock so
2469 			 * we'll take the very occasional hit here instead.
2470 			 */
2471 			if (tp->t_state == TS_ONPROC) {
2472 				cpu_surrender(tp);
2473 			} else if (tp->t_state == TS_RUN) {
2474 				cpu_t *ocp = tp->t_cpu;
2475 
2476 				(void) dispdeq(tp);
2477 				setbackdq(tp);
2478 				/*
2479 				 * Either on the bound CPU's disp queue now,
2480 				 * or swapped out or on the swap queue.
2481 				 */
2482 				ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2483 				    tp->t_weakbound_cpu == ocp ||
2484 				    (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2485 				    != TS_LOAD);
2486 			}
2487 		}
2488 	}
2489 
2490 	/*
2491 	 * Our binding has changed; set TP_CHANGEBIND.
2492 	 */
2493 	tp->t_proc_flag |= TP_CHANGEBIND;
2494 	aston(tp);
2495 
2496 	thread_unlock(tp);
2497 
2498 	return (0);
2499 }
2500 
2501 #if CPUSET_WORDS > 1
2502 
2503 /*
2504  * Functions for implementing cpuset operations when a cpuset is more
2505  * than one word.  On platforms where a cpuset is a single word these
2506  * are implemented as macros in cpuvar.h.
2507  */
2508 
2509 void
2510 cpuset_all(cpuset_t *s)
2511 {
2512 	int i;
2513 
2514 	for (i = 0; i < CPUSET_WORDS; i++)
2515 		s->cpub[i] = ~0UL;
2516 }
2517 
2518 void
2519 cpuset_all_but(cpuset_t *s, uint_t cpu)
2520 {
2521 	cpuset_all(s);
2522 	CPUSET_DEL(*s, cpu);
2523 }
2524 
2525 void
2526 cpuset_only(cpuset_t *s, uint_t cpu)
2527 {
2528 	CPUSET_ZERO(*s);
2529 	CPUSET_ADD(*s, cpu);
2530 }
2531 
2532 int
2533 cpuset_isnull(cpuset_t *s)
2534 {
2535 	int i;
2536 
2537 	for (i = 0; i < CPUSET_WORDS; i++)
2538 		if (s->cpub[i] != 0)
2539 			return (0);
2540 	return (1);
2541 }
2542 
2543 int
2544 cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2545 {
2546 	int i;
2547 
2548 	for (i = 0; i < CPUSET_WORDS; i++)
2549 		if (s1->cpub[i] != s2->cpub[i])
2550 			return (0);
2551 	return (1);
2552 }
2553 
2554 uint_t
2555 cpuset_find(cpuset_t *s)
2556 {
2557 
2558 	uint_t	i;
2559 	uint_t	cpu = (uint_t)-1;
2560 
2561 	/*
2562 	 * Find a cpu in the cpuset
2563 	 */
2564 	for (i = 0; (i < CPUSET_WORDS && cpu == (uint_t)-1); i++)
2565 		cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2566 	return (cpu);
2567 }
2568 
2569 #endif	/* CPUSET_WORDS */
2570 
2571 /*
2572  * Unbind all user threads bound to a given CPU.
2573  */
2574 int
2575 cpu_unbind(processorid_t cpu)
2576 {
2577 	processorid_t obind;
2578 	kthread_t *tp;
2579 	int ret = 0;
2580 	proc_t *pp;
2581 	int err, berr = 0;
2582 
2583 	ASSERT(MUTEX_HELD(&cpu_lock));
2584 
2585 	mutex_enter(&pidlock);
2586 	for (pp = practive; pp != NULL; pp = pp->p_next) {
2587 		mutex_enter(&pp->p_lock);
2588 		tp = pp->p_tlist;
2589 		/*
2590 		 * Skip zombies, kernel processes, and processes in
2591 		 * other zones, if called from a non-global zone.
2592 		 */
2593 		if (tp == NULL || (pp->p_flag & SSYS) ||
2594 		    !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2595 			mutex_exit(&pp->p_lock);
2596 			continue;
2597 		}
2598 		do {
2599 			if (tp->t_bind_cpu != cpu)
2600 				continue;
2601 			err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2602 			if (ret == 0)
2603 				ret = err;
2604 		} while ((tp = tp->t_forw) != pp->p_tlist);
2605 		mutex_exit(&pp->p_lock);
2606 	}
2607 	mutex_exit(&pidlock);
2608 	if (ret == 0)
2609 		ret = berr;
2610 	return (ret);
2611 }
2612 
2613 
2614 /*
2615  * Destroy all remaining bound threads on a cpu.
2616  */
2617 void
2618 cpu_destroy_bound_threads(cpu_t *cp)
2619 {
2620 	extern id_t syscid;
2621 	register kthread_id_t	t, tlist, tnext;
2622 
2623 	/*
2624 	 * Destroy all remaining bound threads on the cpu.  This
2625 	 * should include both the interrupt threads and the idle thread.
2626 	 * This requires some care, since we need to traverse the
2627 	 * thread list with the pidlock mutex locked, but thread_free
2628 	 * also locks the pidlock mutex.  So, we collect the threads
2629 	 * we're going to reap in a list headed by "tlist", then we
2630 	 * unlock the pidlock mutex and traverse the tlist list,
2631 	 * doing thread_free's on the thread's.	 Simple, n'est pas?
2632 	 * Also, this depends on thread_free not mucking with the
2633 	 * t_next and t_prev links of the thread.
2634 	 */
2635 
2636 	if ((t = curthread) != NULL) {
2637 
2638 		tlist = NULL;
2639 		mutex_enter(&pidlock);
2640 		do {
2641 			tnext = t->t_next;
2642 			if (t->t_bound_cpu == cp) {
2643 
2644 				/*
2645 				 * We've found a bound thread, carefully unlink
2646 				 * it out of the thread list, and add it to
2647 				 * our "tlist".	 We "know" we don't have to
2648 				 * worry about unlinking curthread (the thread
2649 				 * that is executing this code).
2650 				 */
2651 				t->t_next->t_prev = t->t_prev;
2652 				t->t_prev->t_next = t->t_next;
2653 				t->t_next = tlist;
2654 				tlist = t;
2655 				ASSERT(t->t_cid == syscid);
2656 				/* wake up anyone blocked in thread_join */
2657 				cv_broadcast(&t->t_joincv);
2658 				/*
2659 				 * t_lwp set by interrupt threads and not
2660 				 * cleared.
2661 				 */
2662 				t->t_lwp = NULL;
2663 				/*
2664 				 * Pause and idle threads always have
2665 				 * t_state set to TS_ONPROC.
2666 				 */
2667 				t->t_state = TS_FREE;
2668 				t->t_prev = NULL;	/* Just in case */
2669 			}
2670 
2671 		} while ((t = tnext) != curthread);
2672 
2673 		mutex_exit(&pidlock);
2674 
2675 
2676 		for (t = tlist; t != NULL; t = tnext) {
2677 			tnext = t->t_next;
2678 			thread_free(t);
2679 		}
2680 	}
2681 }
2682 
2683 /*
2684  * processor_info(2) and p_online(2) status support functions
2685  *   The constants returned by the cpu_get_state() and cpu_get_state_str() are
2686  *   for use in communicating processor state information to userland.  Kernel
2687  *   subsystems should only be using the cpu_flags value directly.  Subsystems
2688  *   modifying cpu_flags should record the state change via a call to the
2689  *   cpu_set_state().
2690  */
2691 
2692 /*
2693  * Update the pi_state of this CPU.  This function provides the CPU status for
2694  * the information returned by processor_info(2).
2695  */
2696 void
2697 cpu_set_state(cpu_t *cpu)
2698 {
2699 	ASSERT(MUTEX_HELD(&cpu_lock));
2700 	cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
2701 	cpu->cpu_state_begin = gethrestime_sec();
2702 	pool_cpu_mod = gethrtime();
2703 }
2704 
2705 /*
2706  * Return offline/online/other status for the indicated CPU.  Use only for
2707  * communication with user applications; cpu_flags provides the in-kernel
2708  * interface.
2709  */
2710 int
2711 cpu_get_state(cpu_t *cpu)
2712 {
2713 	ASSERT(MUTEX_HELD(&cpu_lock));
2714 	if (cpu->cpu_flags & CPU_POWEROFF)
2715 		return (P_POWEROFF);
2716 	else if (cpu->cpu_flags & CPU_FAULTED)
2717 		return (P_FAULTED);
2718 	else if (cpu->cpu_flags & CPU_SPARE)
2719 		return (P_SPARE);
2720 	else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
2721 		return (P_OFFLINE);
2722 	else if (cpu->cpu_flags & CPU_ENABLE)
2723 		return (P_ONLINE);
2724 	else
2725 		return (P_NOINTR);
2726 }
2727 
2728 /*
2729  * Return processor_info(2) state as a string.
2730  */
2731 const char *
2732 cpu_get_state_str(cpu_t *cpu)
2733 {
2734 	const char *string;
2735 
2736 	switch (cpu_get_state(cpu)) {
2737 	case P_ONLINE:
2738 		string = PS_ONLINE;
2739 		break;
2740 	case P_POWEROFF:
2741 		string = PS_POWEROFF;
2742 		break;
2743 	case P_NOINTR:
2744 		string = PS_NOINTR;
2745 		break;
2746 	case P_SPARE:
2747 		string = PS_SPARE;
2748 		break;
2749 	case P_FAULTED:
2750 		string = PS_FAULTED;
2751 		break;
2752 	case P_OFFLINE:
2753 		string = PS_OFFLINE;
2754 		break;
2755 	default:
2756 		string = "unknown";
2757 		break;
2758 	}
2759 	return (string);
2760 }
2761 
2762 /*
2763  * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
2764  * kstats, respectively.  This is done when a CPU is initialized or placed
2765  * online via p_online(2).
2766  */
2767 static void
2768 cpu_stats_kstat_create(cpu_t *cp)
2769 {
2770 	int 	instance = cp->cpu_id;
2771 	char 	*module = "cpu";
2772 	char 	*class = "misc";
2773 	kstat_t	*ksp;
2774 	zoneid_t zoneid;
2775 
2776 	ASSERT(MUTEX_HELD(&cpu_lock));
2777 
2778 	if (pool_pset_enabled())
2779 		zoneid = GLOBAL_ZONEID;
2780 	else
2781 		zoneid = ALL_ZONES;
2782 	/*
2783 	 * Create named kstats
2784 	 */
2785 #define	CPU_STATS_KS_CREATE(name, tsize, update_func)                    \
2786 	ksp = kstat_create_zone(module, instance, (name), class,         \
2787 	    KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0,       \
2788 	    zoneid);                                                     \
2789 	if (ksp != NULL) {                                               \
2790 		ksp->ks_private = cp;                                    \
2791 		ksp->ks_update = (update_func);                          \
2792 		kstat_install(ksp);                                      \
2793 	} else                                                           \
2794 		cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
2795 		    module, instance, (name));
2796 
2797 	CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
2798 	    cpu_sys_stats_ks_update);
2799 	CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
2800 	    cpu_vm_stats_ks_update);
2801 
2802 	/*
2803 	 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
2804 	 */
2805 	ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
2806 	    "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
2807 	if (ksp != NULL) {
2808 		ksp->ks_update = cpu_stat_ks_update;
2809 		ksp->ks_private = cp;
2810 		kstat_install(ksp);
2811 	}
2812 }
2813 
2814 static void
2815 cpu_stats_kstat_destroy(cpu_t *cp)
2816 {
2817 	char ks_name[KSTAT_STRLEN];
2818 
2819 	(void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
2820 	kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
2821 
2822 	kstat_delete_byname("cpu", cp->cpu_id, "sys");
2823 	kstat_delete_byname("cpu", cp->cpu_id, "vm");
2824 }
2825 
2826 static int
2827 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
2828 {
2829 	cpu_t *cp = (cpu_t *)ksp->ks_private;
2830 	struct cpu_sys_stats_ks_data *csskd;
2831 	cpu_sys_stats_t *css;
2832 	hrtime_t *cmstimep;
2833 	hrtime_t newtime;
2834 	int	i;
2835 
2836 	if (rw == KSTAT_WRITE)
2837 		return (EACCES);
2838 
2839 	csskd = ksp->ks_data;
2840 	css = &cp->cpu_stats.sys;
2841 
2842 	bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
2843 	    sizeof (cpu_sys_stats_ks_data_template));
2844 	csskd->cpu_ticks_wait.value.ui64 = 0;
2845 	csskd->wait_ticks_io.value.ui64 = 0;
2846 	csskd->cpu_nsec_idle.value.ui64 = cp->cpu_acct[CMS_IDLE];
2847 	csskd->cpu_nsec_user.value.ui64 = cp->cpu_acct[CMS_USER];
2848 	csskd->cpu_nsec_kernel.value.ui64 = cp->cpu_acct[CMS_SYSTEM];
2849 
2850 	/*
2851 	 * update kstats to include time spent in current state
2852 	 */
2853 	i = 0;	/* set counter to 0 */
2854 	do {
2855 		switch (cp->cpu_mstate) {
2856 		case CMS_USER:
2857 			cmstimep = (hrtime_t *)&csskd->cpu_nsec_user.value.ui64;
2858 			break;
2859 		case CMS_SYSTEM:
2860 			cmstimep =
2861 			    (hrtime_t *)&csskd->cpu_nsec_kernel.value.ui64;
2862 			break;
2863 		case CMS_IDLE:
2864 			cmstimep = (hrtime_t *)&csskd->cpu_nsec_idle.value.ui64;
2865 			break;
2866 		case CMS_DISABLED:
2867 			panic("cpu_sys_stats_ks_update: disabled state"
2868 			    " reported!");
2869 			break;
2870 		default:
2871 			panic("cpu_sys_stats_ks_update: unknown microstate!");
2872 		}
2873 		newtime = gethrtime_unscaled() - cp->cpu_mstate_start;
2874 		if (newtime < 0) {
2875 			newtime = 0;
2876 			i++;
2877 		}
2878 	} while (newtime <= 0 && i < 5);
2879 
2880 	*cmstimep += newtime;
2881 
2882 	scalehrtime((hrtime_t *)&csskd->cpu_nsec_idle.value.ui64);
2883 	scalehrtime((hrtime_t *)&csskd->cpu_nsec_user.value.ui64);
2884 	scalehrtime((hrtime_t *)&csskd->cpu_nsec_kernel.value.ui64);
2885 	csskd->cpu_ticks_idle.value.ui64 =
2886 	    NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
2887 	csskd->cpu_ticks_user.value.ui64 =
2888 	    NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
2889 	csskd->cpu_ticks_kernel.value.ui64 =
2890 	    NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
2891 	csskd->bread.value.ui64 = css->bread;
2892 	csskd->bwrite.value.ui64 = css->bwrite;
2893 	csskd->lread.value.ui64 = css->lread;
2894 	csskd->lwrite.value.ui64 = css->lwrite;
2895 	csskd->phread.value.ui64 = css->phread;
2896 	csskd->phwrite.value.ui64 = css->phwrite;
2897 	csskd->pswitch.value.ui64 = css->pswitch;
2898 	csskd->trap.value.ui64 = css->trap;
2899 	csskd->intr.value.ui64 = 0;
2900 	for (i = 0; i < PIL_MAX; i++)
2901 		csskd->intr.value.ui64 += css->intr[i];
2902 	csskd->syscall.value.ui64 = css->syscall;
2903 	csskd->sysread.value.ui64 = css->sysread;
2904 	csskd->syswrite.value.ui64 = css->syswrite;
2905 	csskd->sysfork.value.ui64 = css->sysfork;
2906 	csskd->sysvfork.value.ui64 = css->sysvfork;
2907 	csskd->sysexec.value.ui64 = css->sysexec;
2908 	csskd->readch.value.ui64 = css->readch;
2909 	csskd->writech.value.ui64 = css->writech;
2910 	csskd->rcvint.value.ui64 = css->rcvint;
2911 	csskd->xmtint.value.ui64 = css->xmtint;
2912 	csskd->mdmint.value.ui64 = css->mdmint;
2913 	csskd->rawch.value.ui64 = css->rawch;
2914 	csskd->canch.value.ui64 = css->canch;
2915 	csskd->outch.value.ui64 = css->outch;
2916 	csskd->msg.value.ui64 = css->msg;
2917 	csskd->sema.value.ui64 = css->sema;
2918 	csskd->namei.value.ui64 = css->namei;
2919 	csskd->ufsiget.value.ui64 = css->ufsiget;
2920 	csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
2921 	csskd->ufsipage.value.ui64 = css->ufsipage;
2922 	csskd->ufsinopage.value.ui64 = css->ufsinopage;
2923 	csskd->procovf.value.ui64 = css->procovf;
2924 	csskd->intrthread.value.ui64 = 0;
2925 	for (i = 0; i < LOCK_LEVEL; i++)
2926 		csskd->intrthread.value.ui64 += css->intr[i];
2927 	csskd->intrblk.value.ui64 = css->intrblk;
2928 	csskd->intrunpin.value.ui64 = css->intrunpin;
2929 	csskd->idlethread.value.ui64 = css->idlethread;
2930 	csskd->inv_swtch.value.ui64 = css->inv_swtch;
2931 	csskd->nthreads.value.ui64 = css->nthreads;
2932 	csskd->cpumigrate.value.ui64 = css->cpumigrate;
2933 	csskd->xcalls.value.ui64 = css->xcalls;
2934 	csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
2935 	csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
2936 	csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
2937 	csskd->modload.value.ui64 = css->modload;
2938 	csskd->modunload.value.ui64 = css->modunload;
2939 	csskd->bawrite.value.ui64 = css->bawrite;
2940 	csskd->iowait.value.ui64 = 0;
2941 
2942 	return (0);
2943 }
2944 
2945 static int
2946 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
2947 {
2948 	cpu_t *cp = (cpu_t *)ksp->ks_private;
2949 	struct cpu_vm_stats_ks_data *cvskd;
2950 	cpu_vm_stats_t *cvs;
2951 
2952 	if (rw == KSTAT_WRITE)
2953 		return (EACCES);
2954 
2955 	cvs = &cp->cpu_stats.vm;
2956 	cvskd = ksp->ks_data;
2957 
2958 	bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
2959 	    sizeof (cpu_vm_stats_ks_data_template));
2960 	cvskd->pgrec.value.ui64 = cvs->pgrec;
2961 	cvskd->pgfrec.value.ui64 = cvs->pgfrec;
2962 	cvskd->pgin.value.ui64 = cvs->pgin;
2963 	cvskd->pgpgin.value.ui64 = cvs->pgpgin;
2964 	cvskd->pgout.value.ui64 = cvs->pgout;
2965 	cvskd->pgpgout.value.ui64 = cvs->pgpgout;
2966 	cvskd->swapin.value.ui64 = cvs->swapin;
2967 	cvskd->pgswapin.value.ui64 = cvs->pgswapin;
2968 	cvskd->swapout.value.ui64 = cvs->swapout;
2969 	cvskd->pgswapout.value.ui64 = cvs->pgswapout;
2970 	cvskd->zfod.value.ui64 = cvs->zfod;
2971 	cvskd->dfree.value.ui64 = cvs->dfree;
2972 	cvskd->scan.value.ui64 = cvs->scan;
2973 	cvskd->rev.value.ui64 = cvs->rev;
2974 	cvskd->hat_fault.value.ui64 = cvs->hat_fault;
2975 	cvskd->as_fault.value.ui64 = cvs->as_fault;
2976 	cvskd->maj_fault.value.ui64 = cvs->maj_fault;
2977 	cvskd->cow_fault.value.ui64 = cvs->cow_fault;
2978 	cvskd->prot_fault.value.ui64 = cvs->prot_fault;
2979 	cvskd->softlock.value.ui64 = cvs->softlock;
2980 	cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
2981 	cvskd->pgrrun.value.ui64 = cvs->pgrrun;
2982 	cvskd->execpgin.value.ui64 = cvs->execpgin;
2983 	cvskd->execpgout.value.ui64 = cvs->execpgout;
2984 	cvskd->execfree.value.ui64 = cvs->execfree;
2985 	cvskd->anonpgin.value.ui64 = cvs->anonpgin;
2986 	cvskd->anonpgout.value.ui64 = cvs->anonpgout;
2987 	cvskd->anonfree.value.ui64 = cvs->anonfree;
2988 	cvskd->fspgin.value.ui64 = cvs->fspgin;
2989 	cvskd->fspgout.value.ui64 = cvs->fspgout;
2990 	cvskd->fsfree.value.ui64 = cvs->fsfree;
2991 
2992 	return (0);
2993 }
2994 
2995 static int
2996 cpu_stat_ks_update(kstat_t *ksp, int rw)
2997 {
2998 	cpu_stat_t *cso;
2999 	cpu_t *cp;
3000 	int i;
3001 	hrtime_t newtime;
3002 	hrtime_t cphrt_i;
3003 	hrtime_t cphrt_u;
3004 	hrtime_t cphrt_s;
3005 	hrtime_t *cmstimep;
3006 
3007 	cso = (cpu_stat_t *)ksp->ks_data;
3008 	cp = (cpu_t *)ksp->ks_private;
3009 
3010 	if (rw == KSTAT_WRITE)
3011 		return (EACCES);
3012 
3013 	cphrt_i = cp->cpu_acct[CMS_IDLE];
3014 	cphrt_u = cp->cpu_acct[CMS_USER];
3015 	cphrt_s = cp->cpu_acct[CMS_SYSTEM];
3016 
3017 	/*
3018 	 * update kstats to include time spent in current state
3019 	 */
3020 	i = 0;	/* set counter to 0 */
3021 	do {
3022 		switch (cp->cpu_mstate) {
3023 		case CMS_USER:
3024 			cmstimep = &cphrt_u;
3025 			break;
3026 		case CMS_SYSTEM:
3027 			cmstimep = &cphrt_s;
3028 			break;
3029 		case CMS_IDLE:
3030 			cmstimep = &cphrt_i;
3031 			break;
3032 		case CMS_DISABLED:
3033 			panic("cpu_stat_ks_update: disabled state reported!");
3034 			break;
3035 		default:
3036 			panic("cpu_stat_ks_update: unknown microstate!");
3037 		}
3038 		newtime = gethrtime_unscaled() - cp->cpu_mstate_start;
3039 		if (newtime < 0) {
3040 			newtime = 0;
3041 			i++;
3042 		}
3043 	} while (newtime <= 0 && i < 5);
3044 
3045 	*cmstimep += newtime;
3046 
3047 	scalehrtime((hrtime_t *)&cphrt_i);
3048 	cso->cpu_sysinfo.cpu[CPU_IDLE] = NSEC_TO_TICK(cphrt_i);
3049 
3050 	scalehrtime((hrtime_t *)&cphrt_u);
3051 	cso->cpu_sysinfo.cpu[CPU_USER] = NSEC_TO_TICK(cphrt_u);
3052 
3053 	scalehrtime((hrtime_t *)&cphrt_s);
3054 	cso->cpu_sysinfo.cpu[CPU_KERNEL] = NSEC_TO_TICK(cphrt_s);
3055 
3056 	cso->cpu_sysinfo.cpu[CPU_WAIT] 	= 0;
3057 	cso->cpu_sysinfo.wait[W_IO] 	= 0;
3058 	cso->cpu_sysinfo.wait[W_SWAP]	= 0;
3059 	cso->cpu_sysinfo.wait[W_PIO]	= 0;
3060 	cso->cpu_sysinfo.bread 		= CPU_STATS(cp, sys.bread);
3061 	cso->cpu_sysinfo.bwrite 	= CPU_STATS(cp, sys.bwrite);
3062 	cso->cpu_sysinfo.lread 		= CPU_STATS(cp, sys.lread);
3063 	cso->cpu_sysinfo.lwrite 	= CPU_STATS(cp, sys.lwrite);
3064 	cso->cpu_sysinfo.phread 	= CPU_STATS(cp, sys.phread);
3065 	cso->cpu_sysinfo.phwrite 	= CPU_STATS(cp, sys.phwrite);
3066 	cso->cpu_sysinfo.pswitch 	= CPU_STATS(cp, sys.pswitch);
3067 	cso->cpu_sysinfo.trap 		= CPU_STATS(cp, sys.trap);
3068 	cso->cpu_sysinfo.intr		= 0;
3069 	for (i = 0; i < PIL_MAX; i++)
3070 		cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3071 	cso->cpu_sysinfo.syscall	= CPU_STATS(cp, sys.syscall);
3072 	cso->cpu_sysinfo.sysread	= CPU_STATS(cp, sys.sysread);
3073 	cso->cpu_sysinfo.syswrite	= CPU_STATS(cp, sys.syswrite);
3074 	cso->cpu_sysinfo.sysfork	= CPU_STATS(cp, sys.sysfork);
3075 	cso->cpu_sysinfo.sysvfork	= CPU_STATS(cp, sys.sysvfork);
3076 	cso->cpu_sysinfo.sysexec	= CPU_STATS(cp, sys.sysexec);
3077 	cso->cpu_sysinfo.readch		= CPU_STATS(cp, sys.readch);
3078 	cso->cpu_sysinfo.writech	= CPU_STATS(cp, sys.writech);
3079 	cso->cpu_sysinfo.rcvint		= CPU_STATS(cp, sys.rcvint);
3080 	cso->cpu_sysinfo.xmtint		= CPU_STATS(cp, sys.xmtint);
3081 	cso->cpu_sysinfo.mdmint		= CPU_STATS(cp, sys.mdmint);
3082 	cso->cpu_sysinfo.rawch		= CPU_STATS(cp, sys.rawch);
3083 	cso->cpu_sysinfo.canch		= CPU_STATS(cp, sys.canch);
3084 	cso->cpu_sysinfo.outch		= CPU_STATS(cp, sys.outch);
3085 	cso->cpu_sysinfo.msg		= CPU_STATS(cp, sys.msg);
3086 	cso->cpu_sysinfo.sema		= CPU_STATS(cp, sys.sema);
3087 	cso->cpu_sysinfo.namei		= CPU_STATS(cp, sys.namei);
3088 	cso->cpu_sysinfo.ufsiget	= CPU_STATS(cp, sys.ufsiget);
3089 	cso->cpu_sysinfo.ufsdirblk	= CPU_STATS(cp, sys.ufsdirblk);
3090 	cso->cpu_sysinfo.ufsipage	= CPU_STATS(cp, sys.ufsipage);
3091 	cso->cpu_sysinfo.ufsinopage	= CPU_STATS(cp, sys.ufsinopage);
3092 	cso->cpu_sysinfo.inodeovf	= 0;
3093 	cso->cpu_sysinfo.fileovf	= 0;
3094 	cso->cpu_sysinfo.procovf	= CPU_STATS(cp, sys.procovf);
3095 	cso->cpu_sysinfo.intrthread	= 0;
3096 	for (i = 0; i < LOCK_LEVEL; i++)
3097 		cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3098 	cso->cpu_sysinfo.intrblk	= CPU_STATS(cp, sys.intrblk);
3099 	cso->cpu_sysinfo.idlethread	= CPU_STATS(cp, sys.idlethread);
3100 	cso->cpu_sysinfo.inv_swtch	= CPU_STATS(cp, sys.inv_swtch);
3101 	cso->cpu_sysinfo.nthreads	= CPU_STATS(cp, sys.nthreads);
3102 	cso->cpu_sysinfo.cpumigrate	= CPU_STATS(cp, sys.cpumigrate);
3103 	cso->cpu_sysinfo.xcalls		= CPU_STATS(cp, sys.xcalls);
3104 	cso->cpu_sysinfo.mutex_adenters	= CPU_STATS(cp, sys.mutex_adenters);
3105 	cso->cpu_sysinfo.rw_rdfails	= CPU_STATS(cp, sys.rw_rdfails);
3106 	cso->cpu_sysinfo.rw_wrfails	= CPU_STATS(cp, sys.rw_wrfails);
3107 	cso->cpu_sysinfo.modload	= CPU_STATS(cp, sys.modload);
3108 	cso->cpu_sysinfo.modunload	= CPU_STATS(cp, sys.modunload);
3109 	cso->cpu_sysinfo.bawrite	= CPU_STATS(cp, sys.bawrite);
3110 	cso->cpu_sysinfo.rw_enters	= 0;
3111 	cso->cpu_sysinfo.win_uo_cnt	= 0;
3112 	cso->cpu_sysinfo.win_uu_cnt	= 0;
3113 	cso->cpu_sysinfo.win_so_cnt	= 0;
3114 	cso->cpu_sysinfo.win_su_cnt	= 0;
3115 	cso->cpu_sysinfo.win_suo_cnt	= 0;
3116 
3117 	cso->cpu_syswait.iowait		= 0;
3118 	cso->cpu_syswait.swap		= 0;
3119 	cso->cpu_syswait.physio		= 0;
3120 
3121 	cso->cpu_vminfo.pgrec		= CPU_STATS(cp, vm.pgrec);
3122 	cso->cpu_vminfo.pgfrec		= CPU_STATS(cp, vm.pgfrec);
3123 	cso->cpu_vminfo.pgin		= CPU_STATS(cp, vm.pgin);
3124 	cso->cpu_vminfo.pgpgin		= CPU_STATS(cp, vm.pgpgin);
3125 	cso->cpu_vminfo.pgout		= CPU_STATS(cp, vm.pgout);
3126 	cso->cpu_vminfo.pgpgout		= CPU_STATS(cp, vm.pgpgout);
3127 	cso->cpu_vminfo.swapin		= CPU_STATS(cp, vm.swapin);
3128 	cso->cpu_vminfo.pgswapin	= CPU_STATS(cp, vm.pgswapin);
3129 	cso->cpu_vminfo.swapout		= CPU_STATS(cp, vm.swapout);
3130 	cso->cpu_vminfo.pgswapout	= CPU_STATS(cp, vm.pgswapout);
3131 	cso->cpu_vminfo.zfod		= CPU_STATS(cp, vm.zfod);
3132 	cso->cpu_vminfo.dfree		= CPU_STATS(cp, vm.dfree);
3133 	cso->cpu_vminfo.scan		= CPU_STATS(cp, vm.scan);
3134 	cso->cpu_vminfo.rev		= CPU_STATS(cp, vm.rev);
3135 	cso->cpu_vminfo.hat_fault	= CPU_STATS(cp, vm.hat_fault);
3136 	cso->cpu_vminfo.as_fault	= CPU_STATS(cp, vm.as_fault);
3137 	cso->cpu_vminfo.maj_fault	= CPU_STATS(cp, vm.maj_fault);
3138 	cso->cpu_vminfo.cow_fault	= CPU_STATS(cp, vm.cow_fault);
3139 	cso->cpu_vminfo.prot_fault	= CPU_STATS(cp, vm.prot_fault);
3140 	cso->cpu_vminfo.softlock	= CPU_STATS(cp, vm.softlock);
3141 	cso->cpu_vminfo.kernel_asflt	= CPU_STATS(cp, vm.kernel_asflt);
3142 	cso->cpu_vminfo.pgrrun		= CPU_STATS(cp, vm.pgrrun);
3143 	cso->cpu_vminfo.execpgin	= CPU_STATS(cp, vm.execpgin);
3144 	cso->cpu_vminfo.execpgout	= CPU_STATS(cp, vm.execpgout);
3145 	cso->cpu_vminfo.execfree	= CPU_STATS(cp, vm.execfree);
3146 	cso->cpu_vminfo.anonpgin	= CPU_STATS(cp, vm.anonpgin);
3147 	cso->cpu_vminfo.anonpgout	= CPU_STATS(cp, vm.anonpgout);
3148 	cso->cpu_vminfo.anonfree	= CPU_STATS(cp, vm.anonfree);
3149 	cso->cpu_vminfo.fspgin		= CPU_STATS(cp, vm.fspgin);
3150 	cso->cpu_vminfo.fspgout		= CPU_STATS(cp, vm.fspgout);
3151 	cso->cpu_vminfo.fsfree		= CPU_STATS(cp, vm.fsfree);
3152 
3153 	return (0);
3154 }
3155