xref: /illumos-gate/usr/src/uts/common/os/cpu.c (revision 8548bf79039833dba8615afdf63258b2cb122121)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * Architecture-independent CPU control functions.
30  */
31 
32 #include <sys/types.h>
33 #include <sys/param.h>
34 #include <sys/var.h>
35 #include <sys/thread.h>
36 #include <sys/cpuvar.h>
37 #include <sys/kstat.h>
38 #include <sys/uadmin.h>
39 #include <sys/systm.h>
40 #include <sys/errno.h>
41 #include <sys/cmn_err.h>
42 #include <sys/procset.h>
43 #include <sys/processor.h>
44 #include <sys/debug.h>
45 #include <sys/cpupart.h>
46 #include <sys/lgrp.h>
47 #include <sys/pset.h>
48 #include <sys/pghw.h>
49 #include <sys/kmem.h>
50 #include <sys/kmem_impl.h>	/* to set per-cpu kmem_cache offset */
51 #include <sys/atomic.h>
52 #include <sys/callb.h>
53 #include <sys/vtrace.h>
54 #include <sys/cyclic.h>
55 #include <sys/bitmap.h>
56 #include <sys/nvpair.h>
57 #include <sys/pool_pset.h>
58 #include <sys/msacct.h>
59 #include <sys/time.h>
60 #include <sys/archsystm.h>
61 #if defined(__x86)
62 #include <sys/x86_archext.h>
63 #endif
64 
65 extern int	mp_cpu_start(cpu_t *);
66 extern int	mp_cpu_stop(cpu_t *);
67 extern int	mp_cpu_poweron(cpu_t *);
68 extern int	mp_cpu_poweroff(cpu_t *);
69 extern int	mp_cpu_configure(int);
70 extern int	mp_cpu_unconfigure(int);
71 extern void	mp_cpu_faulted_enter(cpu_t *);
72 extern void	mp_cpu_faulted_exit(cpu_t *);
73 
74 extern int cmp_cpu_to_chip(processorid_t cpuid);
75 #ifdef __sparcv9
76 extern char *cpu_fru_fmri(cpu_t *cp);
77 #endif
78 
79 static void cpu_add_active_internal(cpu_t *cp);
80 static void cpu_remove_active(cpu_t *cp);
81 static void cpu_info_kstat_create(cpu_t *cp);
82 static void cpu_info_kstat_destroy(cpu_t *cp);
83 static void cpu_stats_kstat_create(cpu_t *cp);
84 static void cpu_stats_kstat_destroy(cpu_t *cp);
85 
86 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
87 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
88 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
89 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
90 
91 /*
92  * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
93  * and dispatch queue reallocations.  The lock ordering with respect to
94  * related locks is:
95  *
96  *	cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
97  *
98  * Warning:  Certain sections of code do not use the cpu_lock when
99  * traversing the cpu_list (e.g. mutex_vector_enter(), clock()).  Since
100  * all cpus are paused during modifications to this list, a solution
101  * to protect the list is too either disable kernel preemption while
102  * walking the list, *or* recheck the cpu_next pointer at each
103  * iteration in the loop.  Note that in no cases can any cached
104  * copies of the cpu pointers be kept as they may become invalid.
105  */
106 kmutex_t	cpu_lock;
107 cpu_t		*cpu_list;		/* list of all CPUs */
108 cpu_t		*clock_cpu_list;	/* used by clock to walk CPUs */
109 cpu_t		*cpu_active;		/* list of active CPUs */
110 static cpuset_t	cpu_available;		/* set of available CPUs */
111 cpuset_t	cpu_seqid_inuse;	/* which cpu_seqids are in use */
112 
113 /*
114  * max_ncpus keeps the max cpus the system can have. Initially
115  * it's NCPU, but since most archs scan the devtree for cpus
116  * fairly early on during boot, the real max can be known before
117  * ncpus is set (useful for early NCPU based allocations).
118  */
119 int max_ncpus = NCPU;
120 /*
121  * platforms that set max_ncpus to maxiumum number of cpus that can be
122  * dynamically added will set boot_max_ncpus to the number of cpus found
123  * at device tree scan time during boot.
124  */
125 int boot_max_ncpus = -1;
126 /*
127  * Maximum possible CPU id.  This can never be >= NCPU since NCPU is
128  * used to size arrays that are indexed by CPU id.
129  */
130 processorid_t max_cpuid = NCPU - 1;
131 
132 int ncpus = 1;
133 int ncpus_online = 1;
134 
135 /*
136  * CPU that we're trying to offline.  Protected by cpu_lock.
137  */
138 cpu_t *cpu_inmotion;
139 
140 /*
141  * Can be raised to suppress further weakbinding, which are instead
142  * satisfied by disabling preemption.  Must be raised/lowered under cpu_lock,
143  * while individual thread weakbinding synchronisation is done under thread
144  * lock.
145  */
146 int weakbindingbarrier;
147 
148 /*
149  * Variables used in pause_cpus().
150  */
151 static volatile char safe_list[NCPU];
152 
153 static struct _cpu_pause_info {
154 	int		cp_spl;		/* spl saved in pause_cpus() */
155 	volatile int	cp_go;		/* Go signal sent after all ready */
156 	int		cp_count;	/* # of CPUs to pause */
157 	ksema_t		cp_sem;		/* synch pause_cpus & cpu_pause */
158 	kthread_id_t	cp_paused;
159 } cpu_pause_info;
160 
161 static kmutex_t pause_free_mutex;
162 static kcondvar_t pause_free_cv;
163 
164 static struct cpu_sys_stats_ks_data {
165 	kstat_named_t cpu_ticks_idle;
166 	kstat_named_t cpu_ticks_user;
167 	kstat_named_t cpu_ticks_kernel;
168 	kstat_named_t cpu_ticks_wait;
169 	kstat_named_t cpu_nsec_idle;
170 	kstat_named_t cpu_nsec_user;
171 	kstat_named_t cpu_nsec_kernel;
172 	kstat_named_t wait_ticks_io;
173 	kstat_named_t bread;
174 	kstat_named_t bwrite;
175 	kstat_named_t lread;
176 	kstat_named_t lwrite;
177 	kstat_named_t phread;
178 	kstat_named_t phwrite;
179 	kstat_named_t pswitch;
180 	kstat_named_t trap;
181 	kstat_named_t intr;
182 	kstat_named_t syscall;
183 	kstat_named_t sysread;
184 	kstat_named_t syswrite;
185 	kstat_named_t sysfork;
186 	kstat_named_t sysvfork;
187 	kstat_named_t sysexec;
188 	kstat_named_t readch;
189 	kstat_named_t writech;
190 	kstat_named_t rcvint;
191 	kstat_named_t xmtint;
192 	kstat_named_t mdmint;
193 	kstat_named_t rawch;
194 	kstat_named_t canch;
195 	kstat_named_t outch;
196 	kstat_named_t msg;
197 	kstat_named_t sema;
198 	kstat_named_t namei;
199 	kstat_named_t ufsiget;
200 	kstat_named_t ufsdirblk;
201 	kstat_named_t ufsipage;
202 	kstat_named_t ufsinopage;
203 	kstat_named_t procovf;
204 	kstat_named_t intrthread;
205 	kstat_named_t intrblk;
206 	kstat_named_t intrunpin;
207 	kstat_named_t idlethread;
208 	kstat_named_t inv_swtch;
209 	kstat_named_t nthreads;
210 	kstat_named_t cpumigrate;
211 	kstat_named_t xcalls;
212 	kstat_named_t mutex_adenters;
213 	kstat_named_t rw_rdfails;
214 	kstat_named_t rw_wrfails;
215 	kstat_named_t modload;
216 	kstat_named_t modunload;
217 	kstat_named_t bawrite;
218 	kstat_named_t iowait;
219 } cpu_sys_stats_ks_data_template = {
220 	{ "cpu_ticks_idle", 	KSTAT_DATA_UINT64 },
221 	{ "cpu_ticks_user", 	KSTAT_DATA_UINT64 },
222 	{ "cpu_ticks_kernel", 	KSTAT_DATA_UINT64 },
223 	{ "cpu_ticks_wait", 	KSTAT_DATA_UINT64 },
224 	{ "cpu_nsec_idle",	KSTAT_DATA_UINT64 },
225 	{ "cpu_nsec_user",	KSTAT_DATA_UINT64 },
226 	{ "cpu_nsec_kernel",	KSTAT_DATA_UINT64 },
227 	{ "wait_ticks_io", 	KSTAT_DATA_UINT64 },
228 	{ "bread", 		KSTAT_DATA_UINT64 },
229 	{ "bwrite", 		KSTAT_DATA_UINT64 },
230 	{ "lread", 		KSTAT_DATA_UINT64 },
231 	{ "lwrite", 		KSTAT_DATA_UINT64 },
232 	{ "phread", 		KSTAT_DATA_UINT64 },
233 	{ "phwrite", 		KSTAT_DATA_UINT64 },
234 	{ "pswitch", 		KSTAT_DATA_UINT64 },
235 	{ "trap", 		KSTAT_DATA_UINT64 },
236 	{ "intr", 		KSTAT_DATA_UINT64 },
237 	{ "syscall", 		KSTAT_DATA_UINT64 },
238 	{ "sysread", 		KSTAT_DATA_UINT64 },
239 	{ "syswrite", 		KSTAT_DATA_UINT64 },
240 	{ "sysfork", 		KSTAT_DATA_UINT64 },
241 	{ "sysvfork", 		KSTAT_DATA_UINT64 },
242 	{ "sysexec", 		KSTAT_DATA_UINT64 },
243 	{ "readch", 		KSTAT_DATA_UINT64 },
244 	{ "writech", 		KSTAT_DATA_UINT64 },
245 	{ "rcvint", 		KSTAT_DATA_UINT64 },
246 	{ "xmtint", 		KSTAT_DATA_UINT64 },
247 	{ "mdmint", 		KSTAT_DATA_UINT64 },
248 	{ "rawch", 		KSTAT_DATA_UINT64 },
249 	{ "canch", 		KSTAT_DATA_UINT64 },
250 	{ "outch", 		KSTAT_DATA_UINT64 },
251 	{ "msg", 		KSTAT_DATA_UINT64 },
252 	{ "sema", 		KSTAT_DATA_UINT64 },
253 	{ "namei", 		KSTAT_DATA_UINT64 },
254 	{ "ufsiget", 		KSTAT_DATA_UINT64 },
255 	{ "ufsdirblk", 		KSTAT_DATA_UINT64 },
256 	{ "ufsipage", 		KSTAT_DATA_UINT64 },
257 	{ "ufsinopage", 	KSTAT_DATA_UINT64 },
258 	{ "procovf", 		KSTAT_DATA_UINT64 },
259 	{ "intrthread", 	KSTAT_DATA_UINT64 },
260 	{ "intrblk", 		KSTAT_DATA_UINT64 },
261 	{ "intrunpin",		KSTAT_DATA_UINT64 },
262 	{ "idlethread", 	KSTAT_DATA_UINT64 },
263 	{ "inv_swtch", 		KSTAT_DATA_UINT64 },
264 	{ "nthreads", 		KSTAT_DATA_UINT64 },
265 	{ "cpumigrate", 	KSTAT_DATA_UINT64 },
266 	{ "xcalls", 		KSTAT_DATA_UINT64 },
267 	{ "mutex_adenters", 	KSTAT_DATA_UINT64 },
268 	{ "rw_rdfails", 	KSTAT_DATA_UINT64 },
269 	{ "rw_wrfails", 	KSTAT_DATA_UINT64 },
270 	{ "modload", 		KSTAT_DATA_UINT64 },
271 	{ "modunload", 		KSTAT_DATA_UINT64 },
272 	{ "bawrite", 		KSTAT_DATA_UINT64 },
273 	{ "iowait",		KSTAT_DATA_UINT64 },
274 };
275 
276 static struct cpu_vm_stats_ks_data {
277 	kstat_named_t pgrec;
278 	kstat_named_t pgfrec;
279 	kstat_named_t pgin;
280 	kstat_named_t pgpgin;
281 	kstat_named_t pgout;
282 	kstat_named_t pgpgout;
283 	kstat_named_t swapin;
284 	kstat_named_t pgswapin;
285 	kstat_named_t swapout;
286 	kstat_named_t pgswapout;
287 	kstat_named_t zfod;
288 	kstat_named_t dfree;
289 	kstat_named_t scan;
290 	kstat_named_t rev;
291 	kstat_named_t hat_fault;
292 	kstat_named_t as_fault;
293 	kstat_named_t maj_fault;
294 	kstat_named_t cow_fault;
295 	kstat_named_t prot_fault;
296 	kstat_named_t softlock;
297 	kstat_named_t kernel_asflt;
298 	kstat_named_t pgrrun;
299 	kstat_named_t execpgin;
300 	kstat_named_t execpgout;
301 	kstat_named_t execfree;
302 	kstat_named_t anonpgin;
303 	kstat_named_t anonpgout;
304 	kstat_named_t anonfree;
305 	kstat_named_t fspgin;
306 	kstat_named_t fspgout;
307 	kstat_named_t fsfree;
308 } cpu_vm_stats_ks_data_template = {
309 	{ "pgrec",		KSTAT_DATA_UINT64 },
310 	{ "pgfrec",		KSTAT_DATA_UINT64 },
311 	{ "pgin",		KSTAT_DATA_UINT64 },
312 	{ "pgpgin",		KSTAT_DATA_UINT64 },
313 	{ "pgout",		KSTAT_DATA_UINT64 },
314 	{ "pgpgout",		KSTAT_DATA_UINT64 },
315 	{ "swapin",		KSTAT_DATA_UINT64 },
316 	{ "pgswapin",		KSTAT_DATA_UINT64 },
317 	{ "swapout",		KSTAT_DATA_UINT64 },
318 	{ "pgswapout",		KSTAT_DATA_UINT64 },
319 	{ "zfod",		KSTAT_DATA_UINT64 },
320 	{ "dfree",		KSTAT_DATA_UINT64 },
321 	{ "scan",		KSTAT_DATA_UINT64 },
322 	{ "rev",		KSTAT_DATA_UINT64 },
323 	{ "hat_fault",		KSTAT_DATA_UINT64 },
324 	{ "as_fault",		KSTAT_DATA_UINT64 },
325 	{ "maj_fault",		KSTAT_DATA_UINT64 },
326 	{ "cow_fault",		KSTAT_DATA_UINT64 },
327 	{ "prot_fault",		KSTAT_DATA_UINT64 },
328 	{ "softlock",		KSTAT_DATA_UINT64 },
329 	{ "kernel_asflt",	KSTAT_DATA_UINT64 },
330 	{ "pgrrun",		KSTAT_DATA_UINT64 },
331 	{ "execpgin",		KSTAT_DATA_UINT64 },
332 	{ "execpgout",		KSTAT_DATA_UINT64 },
333 	{ "execfree",		KSTAT_DATA_UINT64 },
334 	{ "anonpgin",		KSTAT_DATA_UINT64 },
335 	{ "anonpgout",		KSTAT_DATA_UINT64 },
336 	{ "anonfree",		KSTAT_DATA_UINT64 },
337 	{ "fspgin",		KSTAT_DATA_UINT64 },
338 	{ "fspgout",		KSTAT_DATA_UINT64 },
339 	{ "fsfree",		KSTAT_DATA_UINT64 },
340 };
341 
342 /*
343  * Force the specified thread to migrate to the appropriate processor.
344  * Called with thread lock held, returns with it dropped.
345  */
346 static void
347 force_thread_migrate(kthread_id_t tp)
348 {
349 	ASSERT(THREAD_LOCK_HELD(tp));
350 	if (tp == curthread) {
351 		THREAD_TRANSITION(tp);
352 		CL_SETRUN(tp);
353 		thread_unlock_nopreempt(tp);
354 		swtch();
355 	} else {
356 		if (tp->t_state == TS_ONPROC) {
357 			cpu_surrender(tp);
358 		} else if (tp->t_state == TS_RUN) {
359 			(void) dispdeq(tp);
360 			setbackdq(tp);
361 		}
362 		thread_unlock(tp);
363 	}
364 }
365 
366 /*
367  * Set affinity for a specified CPU.
368  * A reference count is incremented and the affinity is held until the
369  * reference count is decremented to zero by thread_affinity_clear().
370  * This is so regions of code requiring affinity can be nested.
371  * Caller needs to ensure that cpu_id remains valid, which can be
372  * done by holding cpu_lock across this call, unless the caller
373  * specifies CPU_CURRENT in which case the cpu_lock will be acquired
374  * by thread_affinity_set and CPU->cpu_id will be the target CPU.
375  */
376 void
377 thread_affinity_set(kthread_id_t t, int cpu_id)
378 {
379 	cpu_t		*cp;
380 	int		c;
381 
382 	ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
383 
384 	if ((c = cpu_id) == CPU_CURRENT) {
385 		mutex_enter(&cpu_lock);
386 		cpu_id = CPU->cpu_id;
387 	}
388 	/*
389 	 * We should be asserting that cpu_lock is held here, but
390 	 * the NCA code doesn't acquire it.  The following assert
391 	 * should be uncommented when the NCA code is fixed.
392 	 *
393 	 * ASSERT(MUTEX_HELD(&cpu_lock));
394 	 */
395 	ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
396 	cp = cpu[cpu_id];
397 	ASSERT(cp != NULL);		/* user must provide a good cpu_id */
398 	/*
399 	 * If there is already a hard affinity requested, and this affinity
400 	 * conflicts with that, panic.
401 	 */
402 	thread_lock(t);
403 	if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
404 		panic("affinity_set: setting %p but already bound to %p",
405 		    (void *)cp, (void *)t->t_bound_cpu);
406 	}
407 	t->t_affinitycnt++;
408 	t->t_bound_cpu = cp;
409 
410 	/*
411 	 * Make sure we're running on the right CPU.
412 	 */
413 	if (cp != t->t_cpu || t != curthread) {
414 		force_thread_migrate(t);	/* drops thread lock */
415 	} else {
416 		thread_unlock(t);
417 	}
418 
419 	if (c == CPU_CURRENT)
420 		mutex_exit(&cpu_lock);
421 }
422 
423 /*
424  *	Wrapper for backward compatibility.
425  */
426 void
427 affinity_set(int cpu_id)
428 {
429 	thread_affinity_set(curthread, cpu_id);
430 }
431 
432 /*
433  * Decrement the affinity reservation count and if it becomes zero,
434  * clear the CPU affinity for the current thread, or set it to the user's
435  * software binding request.
436  */
437 void
438 thread_affinity_clear(kthread_id_t t)
439 {
440 	register processorid_t binding;
441 
442 	thread_lock(t);
443 	if (--t->t_affinitycnt == 0) {
444 		if ((binding = t->t_bind_cpu) == PBIND_NONE) {
445 			/*
446 			 * Adjust disp_max_unbound_pri if necessary.
447 			 */
448 			disp_adjust_unbound_pri(t);
449 			t->t_bound_cpu = NULL;
450 			if (t->t_cpu->cpu_part != t->t_cpupart) {
451 				force_thread_migrate(t);
452 				return;
453 			}
454 		} else {
455 			t->t_bound_cpu = cpu[binding];
456 			/*
457 			 * Make sure the thread is running on the bound CPU.
458 			 */
459 			if (t->t_cpu != t->t_bound_cpu) {
460 				force_thread_migrate(t);
461 				return;		/* already dropped lock */
462 			}
463 		}
464 	}
465 	thread_unlock(t);
466 }
467 
468 /*
469  * Wrapper for backward compatibility.
470  */
471 void
472 affinity_clear(void)
473 {
474 	thread_affinity_clear(curthread);
475 }
476 
477 /*
478  * Weak cpu affinity.  Bind to the "current" cpu for short periods
479  * of time during which the thread must not block (but may be preempted).
480  * Use this instead of kpreempt_disable() when it is only "no migration"
481  * rather than "no preemption" semantics that are required - disabling
482  * preemption holds higher priority threads off of cpu and if the
483  * operation that is protected is more than momentary this is not good
484  * for realtime etc.
485  *
486  * Weakly bound threads will not prevent a cpu from being offlined -
487  * we'll only run them on the cpu to which they are weakly bound but
488  * (because they do not block) we'll always be able to move them on to
489  * another cpu at offline time if we give them just a short moment to
490  * run during which they will unbind.  To give a cpu a chance of offlining,
491  * however, we require a barrier to weak bindings that may be raised for a
492  * given cpu (offline/move code may set this and then wait a short time for
493  * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
494  *
495  * There are few restrictions on the calling context of thread_nomigrate.
496  * The caller must not hold the thread lock.  Calls may be nested.
497  *
498  * After weakbinding a thread must not perform actions that may block.
499  * In particular it must not call thread_affinity_set; calling that when
500  * already weakbound is nonsensical anyway.
501  *
502  * If curthread is prevented from migrating for other reasons
503  * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
504  * then the weak binding will succeed even if this cpu is the target of an
505  * offline/move request.
506  */
507 void
508 thread_nomigrate(void)
509 {
510 	cpu_t *cp;
511 	kthread_id_t t = curthread;
512 
513 again:
514 	kpreempt_disable();
515 	cp = CPU;
516 
517 	/*
518 	 * A highlevel interrupt must not modify t_nomigrate or
519 	 * t_weakbound_cpu of the thread it has interrupted.  A lowlevel
520 	 * interrupt thread cannot migrate and we can avoid the
521 	 * thread_lock call below by short-circuiting here.  In either
522 	 * case we can just return since no migration is possible and
523 	 * the condition will persist (ie, when we test for these again
524 	 * in thread_allowmigrate they can't have changed).   Migration
525 	 * is also impossible if we're at or above DISP_LEVEL pil.
526 	 */
527 	if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
528 	    getpil() >= DISP_LEVEL) {
529 		kpreempt_enable();
530 		return;
531 	}
532 
533 	/*
534 	 * We must be consistent with existing weak bindings.  Since we
535 	 * may be interrupted between the increment of t_nomigrate and
536 	 * the store to t_weakbound_cpu below we cannot assume that
537 	 * t_weakbound_cpu will be set if t_nomigrate is.  Note that we
538 	 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
539 	 * always the case.
540 	 */
541 	if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
542 		if (!panicstr)
543 			panic("thread_nomigrate: binding to %p but already "
544 			    "bound to %p", (void *)cp,
545 			    (void *)t->t_weakbound_cpu);
546 	}
547 
548 	/*
549 	 * At this point we have preemption disabled and we don't yet hold
550 	 * the thread lock.  So it's possible that somebody else could
551 	 * set t_bind_cpu here and not be able to force us across to the
552 	 * new cpu (since we have preemption disabled).
553 	 */
554 	thread_lock(curthread);
555 
556 	/*
557 	 * If further weak bindings are being (temporarily) suppressed then
558 	 * we'll settle for disabling kernel preemption (which assures
559 	 * no migration provided the thread does not block which it is
560 	 * not allowed to if using thread_nomigrate).  We must remember
561 	 * this disposition so we can take appropriate action in
562 	 * thread_allowmigrate.  If this is a nested call and the
563 	 * thread is already weakbound then fall through as normal.
564 	 * We remember the decision to settle for kpreempt_disable through
565 	 * negative nesting counting in t_nomigrate.  Once a thread has had one
566 	 * weakbinding request satisfied in this way any further (nested)
567 	 * requests will continue to be satisfied in the same way,
568 	 * even if weak bindings have recommenced.
569 	 */
570 	if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
571 		--t->t_nomigrate;
572 		thread_unlock(curthread);
573 		return;		/* with kpreempt_disable still active */
574 	}
575 
576 	/*
577 	 * We hold thread_lock so t_bind_cpu cannot change.  We could,
578 	 * however, be running on a different cpu to which we are t_bound_cpu
579 	 * to (as explained above).  If we grant the weak binding request
580 	 * in that case then the dispatcher must favour our weak binding
581 	 * over our strong (in which case, just as when preemption is
582 	 * disabled, we can continue to run on a cpu other than the one to
583 	 * which we are strongbound; the difference in this case is that
584 	 * this thread can be preempted and so can appear on the dispatch
585 	 * queues of a cpu other than the one it is strongbound to).
586 	 *
587 	 * If the cpu we are running on does not appear to be a current
588 	 * offline target (we check cpu_inmotion to determine this - since
589 	 * we don't hold cpu_lock we may not see a recent store to that,
590 	 * so it's possible that we at times can grant a weak binding to a
591 	 * cpu that is an offline target, but that one request will not
592 	 * prevent the offline from succeeding) then we will always grant
593 	 * the weak binding request.  This includes the case above where
594 	 * we grant a weakbinding not commensurate with our strong binding.
595 	 *
596 	 * If our cpu does appear to be an offline target then we're inclined
597 	 * not to grant the weakbinding request just yet - we'd prefer to
598 	 * migrate to another cpu and grant the request there.  The
599 	 * exceptions are those cases where going through preemption code
600 	 * will not result in us changing cpu:
601 	 *
602 	 *	. interrupts have already bypassed this case (see above)
603 	 *	. we are already weakbound to this cpu (dispatcher code will
604 	 *	  always return us to the weakbound cpu)
605 	 *	. preemption was disabled even before we disabled it above
606 	 *	. we are strongbound to this cpu (if we're strongbound to
607 	 *	another and not yet running there the trip through the
608 	 *	dispatcher will move us to the strongbound cpu and we
609 	 *	will grant the weak binding there)
610 	 */
611 	if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
612 	    t->t_bound_cpu == cp) {
613 		/*
614 		 * Don't be tempted to store to t_weakbound_cpu only on
615 		 * the first nested bind request - if we're interrupted
616 		 * after the increment of t_nomigrate and before the
617 		 * store to t_weakbound_cpu and the interrupt calls
618 		 * thread_nomigrate then the assertion in thread_allowmigrate
619 		 * would fail.
620 		 */
621 		t->t_nomigrate++;
622 		t->t_weakbound_cpu = cp;
623 		membar_producer();
624 		thread_unlock(curthread);
625 		/*
626 		 * Now that we have dropped the thread_lock another thread
627 		 * can set our t_weakbound_cpu, and will try to migrate us
628 		 * to the strongbound cpu (which will not be prevented by
629 		 * preemption being disabled since we're about to enable
630 		 * preemption).  We have granted the weakbinding to the current
631 		 * cpu, so again we are in the position that is is is possible
632 		 * that our weak and strong bindings differ.  Again this
633 		 * is catered for by dispatcher code which will favour our
634 		 * weak binding.
635 		 */
636 		kpreempt_enable();
637 	} else {
638 		/*
639 		 * Move to another cpu before granting the request by
640 		 * forcing this thread through preemption code.  When we
641 		 * get to set{front,back}dq called from CL_PREEMPT()
642 		 * cpu_choose() will be used to select a cpu to queue
643 		 * us on - that will see cpu_inmotion and take
644 		 * steps to avoid returning us to this cpu.
645 		 */
646 		cp->cpu_kprunrun = 1;
647 		thread_unlock(curthread);
648 		kpreempt_enable();	/* will call preempt() */
649 		goto again;
650 	}
651 }
652 
653 void
654 thread_allowmigrate(void)
655 {
656 	kthread_id_t t = curthread;
657 
658 	ASSERT(t->t_weakbound_cpu == CPU ||
659 	    (t->t_nomigrate < 0 && t->t_preempt > 0) ||
660 	    CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
661 	    getpil() >= DISP_LEVEL);
662 
663 	if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
664 	    getpil() >= DISP_LEVEL)
665 		return;
666 
667 	if (t->t_nomigrate < 0) {
668 		/*
669 		 * This thread was granted "weak binding" in the
670 		 * stronger form of kernel preemption disabling.
671 		 * Undo a level of nesting for both t_nomigrate
672 		 * and t_preempt.
673 		 */
674 		++t->t_nomigrate;
675 		kpreempt_enable();
676 	} else if (--t->t_nomigrate == 0) {
677 		/*
678 		 * Time to drop the weak binding.  We need to cater
679 		 * for the case where we're weakbound to a different
680 		 * cpu than that to which we're strongbound (a very
681 		 * temporary arrangement that must only persist until
682 		 * weak binding drops).  We don't acquire thread_lock
683 		 * here so even as this code executes t_bound_cpu
684 		 * may be changing.  So we disable preemption and
685 		 * a) in the case that t_bound_cpu changes while we
686 		 * have preemption disabled kprunrun will be set
687 		 * asynchronously, and b) if before disabling
688 		 * preemption we were already on a different cpu to
689 		 * our t_bound_cpu then we set kprunrun ourselves
690 		 * to force a trip through the dispatcher when
691 		 * preemption is enabled.
692 		 */
693 		kpreempt_disable();
694 		if (t->t_bound_cpu &&
695 		    t->t_weakbound_cpu != t->t_bound_cpu)
696 			CPU->cpu_kprunrun = 1;
697 		t->t_weakbound_cpu = NULL;
698 		membar_producer();
699 		kpreempt_enable();
700 	}
701 }
702 
703 /*
704  * weakbinding_stop can be used to temporarily cause weakbindings made
705  * with thread_nomigrate to be satisfied through the stronger action of
706  * kpreempt_disable.  weakbinding_start recommences normal weakbinding.
707  */
708 
709 void
710 weakbinding_stop(void)
711 {
712 	ASSERT(MUTEX_HELD(&cpu_lock));
713 	weakbindingbarrier = 1;
714 	membar_producer();	/* make visible before subsequent thread_lock */
715 }
716 
717 void
718 weakbinding_start(void)
719 {
720 	ASSERT(MUTEX_HELD(&cpu_lock));
721 	weakbindingbarrier = 0;
722 }
723 
724 /*
725  * This routine is called to place the CPUs in a safe place so that
726  * one of them can be taken off line or placed on line.  What we are
727  * trying to do here is prevent a thread from traversing the list
728  * of active CPUs while we are changing it or from getting placed on
729  * the run queue of a CPU that has just gone off line.  We do this by
730  * creating a thread with the highest possible prio for each CPU and
731  * having it call this routine.  The advantage of this method is that
732  * we can eliminate all checks for CPU_ACTIVE in the disp routines.
733  * This makes disp faster at the expense of making p_online() slower
734  * which is a good trade off.
735  */
736 static void
737 cpu_pause(volatile char *safe)
738 {
739 	int s;
740 	struct _cpu_pause_info *cpi = &cpu_pause_info;
741 
742 	ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
743 
744 	while (*safe != PAUSE_DIE) {
745 		*safe = PAUSE_READY;
746 		membar_enter();		/* make sure stores are flushed */
747 		sema_v(&cpi->cp_sem);	/* signal requesting thread */
748 
749 		/*
750 		 * Wait here until all pause threads are running.  That
751 		 * indicates that it's safe to do the spl.  Until
752 		 * cpu_pause_info.cp_go is set, we don't want to spl
753 		 * because that might block clock interrupts needed
754 		 * to preempt threads on other CPUs.
755 		 */
756 		while (cpi->cp_go == 0)
757 			;
758 		/*
759 		 * Even though we are at the highest disp prio, we need
760 		 * to block out all interrupts below LOCK_LEVEL so that
761 		 * an intr doesn't come in, wake up a thread, and call
762 		 * setbackdq/setfrontdq.
763 		 */
764 		s = splhigh();
765 
766 		mach_cpu_pause(safe);
767 
768 		splx(s);
769 		/*
770 		 * Waiting is at an end. Switch out of cpu_pause
771 		 * loop and resume useful work.
772 		 */
773 		swtch();
774 	}
775 
776 	mutex_enter(&pause_free_mutex);
777 	*safe = PAUSE_DEAD;
778 	cv_broadcast(&pause_free_cv);
779 	mutex_exit(&pause_free_mutex);
780 }
781 
782 /*
783  * Allow the cpus to start running again.
784  */
785 void
786 start_cpus()
787 {
788 	int i;
789 
790 	ASSERT(MUTEX_HELD(&cpu_lock));
791 	ASSERT(cpu_pause_info.cp_paused);
792 	cpu_pause_info.cp_paused = NULL;
793 	for (i = 0; i < NCPU; i++)
794 		safe_list[i] = PAUSE_IDLE;
795 	membar_enter();			/* make sure stores are flushed */
796 	affinity_clear();
797 	splx(cpu_pause_info.cp_spl);
798 	kpreempt_enable();
799 }
800 
801 /*
802  * Allocate a pause thread for a CPU.
803  */
804 static void
805 cpu_pause_alloc(cpu_t *cp)
806 {
807 	kthread_id_t	t;
808 	int		cpun = cp->cpu_id;
809 
810 	/*
811 	 * Note, v.v_nglobpris will not change value as long as I hold
812 	 * cpu_lock.
813 	 */
814 	t = thread_create(NULL, 0, cpu_pause, (caddr_t)&safe_list[cpun],
815 	    0, &p0, TS_STOPPED, v.v_nglobpris - 1);
816 	thread_lock(t);
817 	t->t_bound_cpu = cp;
818 	t->t_disp_queue = cp->cpu_disp;
819 	t->t_affinitycnt = 1;
820 	t->t_preempt = 1;
821 	thread_unlock(t);
822 	cp->cpu_pause_thread = t;
823 	/*
824 	 * Registering a thread in the callback table is usually done
825 	 * in the initialization code of the thread.  In this
826 	 * case, we do it right after thread creation because the
827 	 * thread itself may never run, and we need to register the
828 	 * fact that it is safe for cpr suspend.
829 	 */
830 	CALLB_CPR_INIT_SAFE(t, "cpu_pause");
831 }
832 
833 /*
834  * Free a pause thread for a CPU.
835  */
836 static void
837 cpu_pause_free(cpu_t *cp)
838 {
839 	kthread_id_t	t;
840 	int		cpun = cp->cpu_id;
841 
842 	ASSERT(MUTEX_HELD(&cpu_lock));
843 	/*
844 	 * We have to get the thread and tell him to die.
845 	 */
846 	if ((t = cp->cpu_pause_thread) == NULL) {
847 		ASSERT(safe_list[cpun] == PAUSE_IDLE);
848 		return;
849 	}
850 	thread_lock(t);
851 	t->t_cpu = CPU;		/* disp gets upset if last cpu is quiesced. */
852 	t->t_bound_cpu = NULL;	/* Must un-bind; cpu may not be running. */
853 	t->t_pri = v.v_nglobpris - 1;
854 	ASSERT(safe_list[cpun] == PAUSE_IDLE);
855 	safe_list[cpun] = PAUSE_DIE;
856 	THREAD_TRANSITION(t);
857 	setbackdq(t);
858 	thread_unlock_nopreempt(t);
859 
860 	/*
861 	 * If we don't wait for the thread to actually die, it may try to
862 	 * run on the wrong cpu as part of an actual call to pause_cpus().
863 	 */
864 	mutex_enter(&pause_free_mutex);
865 	while (safe_list[cpun] != PAUSE_DEAD) {
866 		cv_wait(&pause_free_cv, &pause_free_mutex);
867 	}
868 	mutex_exit(&pause_free_mutex);
869 	safe_list[cpun] = PAUSE_IDLE;
870 
871 	cp->cpu_pause_thread = NULL;
872 }
873 
874 /*
875  * Initialize basic structures for pausing CPUs.
876  */
877 void
878 cpu_pause_init()
879 {
880 	sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
881 	/*
882 	 * Create initial CPU pause thread.
883 	 */
884 	cpu_pause_alloc(CPU);
885 }
886 
887 /*
888  * Start the threads used to pause another CPU.
889  */
890 static int
891 cpu_pause_start(processorid_t cpu_id)
892 {
893 	int	i;
894 	int	cpu_count = 0;
895 
896 	for (i = 0; i < NCPU; i++) {
897 		cpu_t		*cp;
898 		kthread_id_t	t;
899 
900 		cp = cpu[i];
901 		if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
902 			safe_list[i] = PAUSE_WAIT;
903 			continue;
904 		}
905 
906 		/*
907 		 * Skip CPU if it is quiesced or not yet started.
908 		 */
909 		if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
910 			safe_list[i] = PAUSE_WAIT;
911 			continue;
912 		}
913 
914 		/*
915 		 * Start this CPU's pause thread.
916 		 */
917 		t = cp->cpu_pause_thread;
918 		thread_lock(t);
919 		/*
920 		 * Reset the priority, since nglobpris may have
921 		 * changed since the thread was created, if someone
922 		 * has loaded the RT (or some other) scheduling
923 		 * class.
924 		 */
925 		t->t_pri = v.v_nglobpris - 1;
926 		THREAD_TRANSITION(t);
927 		setbackdq(t);
928 		thread_unlock_nopreempt(t);
929 		++cpu_count;
930 	}
931 	return (cpu_count);
932 }
933 
934 
935 /*
936  * Pause all of the CPUs except the one we are on by creating a high
937  * priority thread bound to those CPUs.
938  *
939  * Note that one must be extremely careful regarding code
940  * executed while CPUs are paused.  Since a CPU may be paused
941  * while a thread scheduling on that CPU is holding an adaptive
942  * lock, code executed with CPUs paused must not acquire adaptive
943  * (or low-level spin) locks.  Also, such code must not block,
944  * since the thread that is supposed to initiate the wakeup may
945  * never run.
946  *
947  * With a few exceptions, the restrictions on code executed with CPUs
948  * paused match those for code executed at high-level interrupt
949  * context.
950  */
951 void
952 pause_cpus(cpu_t *off_cp)
953 {
954 	processorid_t	cpu_id;
955 	int		i;
956 	struct _cpu_pause_info	*cpi = &cpu_pause_info;
957 
958 	ASSERT(MUTEX_HELD(&cpu_lock));
959 	ASSERT(cpi->cp_paused == NULL);
960 	cpi->cp_count = 0;
961 	cpi->cp_go = 0;
962 	for (i = 0; i < NCPU; i++)
963 		safe_list[i] = PAUSE_IDLE;
964 	kpreempt_disable();
965 
966 	/*
967 	 * If running on the cpu that is going offline, get off it.
968 	 * This is so that it won't be necessary to rechoose a CPU
969 	 * when done.
970 	 */
971 	if (CPU == off_cp)
972 		cpu_id = off_cp->cpu_next_part->cpu_id;
973 	else
974 		cpu_id = CPU->cpu_id;
975 	affinity_set(cpu_id);
976 
977 	/*
978 	 * Start the pause threads and record how many were started
979 	 */
980 	cpi->cp_count = cpu_pause_start(cpu_id);
981 
982 	/*
983 	 * Now wait for all CPUs to be running the pause thread.
984 	 */
985 	while (cpi->cp_count > 0) {
986 		/*
987 		 * Spin reading the count without grabbing the disp
988 		 * lock to make sure we don't prevent the pause
989 		 * threads from getting the lock.
990 		 */
991 		while (sema_held(&cpi->cp_sem))
992 			;
993 		if (sema_tryp(&cpi->cp_sem))
994 			--cpi->cp_count;
995 	}
996 	cpi->cp_go = 1;			/* all have reached cpu_pause */
997 
998 	/*
999 	 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1000 	 * to PAUSE_WAIT.)
1001 	 */
1002 	for (i = 0; i < NCPU; i++) {
1003 		while (safe_list[i] != PAUSE_WAIT)
1004 			;
1005 	}
1006 	cpi->cp_spl = splhigh();	/* block dispatcher on this CPU */
1007 	cpi->cp_paused = curthread;
1008 }
1009 
1010 /*
1011  * Check whether the current thread has CPUs paused
1012  */
1013 int
1014 cpus_paused(void)
1015 {
1016 	if (cpu_pause_info.cp_paused != NULL) {
1017 		ASSERT(cpu_pause_info.cp_paused == curthread);
1018 		return (1);
1019 	}
1020 	return (0);
1021 }
1022 
1023 static cpu_t *
1024 cpu_get_all(processorid_t cpun)
1025 {
1026 	ASSERT(MUTEX_HELD(&cpu_lock));
1027 
1028 	if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1029 		return (NULL);
1030 	return (cpu[cpun]);
1031 }
1032 
1033 /*
1034  * Check whether cpun is a valid processor id and whether it should be
1035  * visible from the current zone. If it is, return a pointer to the
1036  * associated CPU structure.
1037  */
1038 cpu_t *
1039 cpu_get(processorid_t cpun)
1040 {
1041 	cpu_t *c;
1042 
1043 	ASSERT(MUTEX_HELD(&cpu_lock));
1044 	c = cpu_get_all(cpun);
1045 	if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1046 	    zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1047 		return (NULL);
1048 	return (c);
1049 }
1050 
1051 /*
1052  * The following functions should be used to check CPU states in the kernel.
1053  * They should be invoked with cpu_lock held.  Kernel subsystems interested
1054  * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1055  * states.  Those are for user-land (and system call) use only.
1056  */
1057 
1058 /*
1059  * Determine whether the CPU is online and handling interrupts.
1060  */
1061 int
1062 cpu_is_online(cpu_t *cpu)
1063 {
1064 	ASSERT(MUTEX_HELD(&cpu_lock));
1065 	return (cpu_flagged_online(cpu->cpu_flags));
1066 }
1067 
1068 /*
1069  * Determine whether the CPU is offline (this includes spare and faulted).
1070  */
1071 int
1072 cpu_is_offline(cpu_t *cpu)
1073 {
1074 	ASSERT(MUTEX_HELD(&cpu_lock));
1075 	return (cpu_flagged_offline(cpu->cpu_flags));
1076 }
1077 
1078 /*
1079  * Determine whether the CPU is powered off.
1080  */
1081 int
1082 cpu_is_poweredoff(cpu_t *cpu)
1083 {
1084 	ASSERT(MUTEX_HELD(&cpu_lock));
1085 	return (cpu_flagged_poweredoff(cpu->cpu_flags));
1086 }
1087 
1088 /*
1089  * Determine whether the CPU is handling interrupts.
1090  */
1091 int
1092 cpu_is_nointr(cpu_t *cpu)
1093 {
1094 	ASSERT(MUTEX_HELD(&cpu_lock));
1095 	return (cpu_flagged_nointr(cpu->cpu_flags));
1096 }
1097 
1098 /*
1099  * Determine whether the CPU is active (scheduling threads).
1100  */
1101 int
1102 cpu_is_active(cpu_t *cpu)
1103 {
1104 	ASSERT(MUTEX_HELD(&cpu_lock));
1105 	return (cpu_flagged_active(cpu->cpu_flags));
1106 }
1107 
1108 /*
1109  * Same as above, but these require cpu_flags instead of cpu_t pointers.
1110  */
1111 int
1112 cpu_flagged_online(cpu_flag_t cpu_flags)
1113 {
1114 	return (cpu_flagged_active(cpu_flags) &&
1115 	    (cpu_flags & CPU_ENABLE));
1116 }
1117 
1118 int
1119 cpu_flagged_offline(cpu_flag_t cpu_flags)
1120 {
1121 	return (((cpu_flags & CPU_POWEROFF) == 0) &&
1122 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1123 }
1124 
1125 int
1126 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1127 {
1128 	return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1129 }
1130 
1131 int
1132 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1133 {
1134 	return (cpu_flagged_active(cpu_flags) &&
1135 	    (cpu_flags & CPU_ENABLE) == 0);
1136 }
1137 
1138 int
1139 cpu_flagged_active(cpu_flag_t cpu_flags)
1140 {
1141 	return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1142 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1143 }
1144 
1145 /*
1146  * Bring the indicated CPU online.
1147  */
1148 int
1149 cpu_online(cpu_t *cp)
1150 {
1151 	int	error = 0;
1152 
1153 	/*
1154 	 * Handle on-line request.
1155 	 *	This code must put the new CPU on the active list before
1156 	 *	starting it because it will not be paused, and will start
1157 	 * 	using the active list immediately.  The real start occurs
1158 	 *	when the CPU_QUIESCED flag is turned off.
1159 	 */
1160 
1161 	ASSERT(MUTEX_HELD(&cpu_lock));
1162 
1163 	/*
1164 	 * Put all the cpus into a known safe place.
1165 	 * No mutexes can be entered while CPUs are paused.
1166 	 */
1167 	error = mp_cpu_start(cp);	/* arch-dep hook */
1168 	if (error == 0) {
1169 		pg_cpupart_in(cp, cp->cpu_part);
1170 		pause_cpus(NULL);
1171 		cpu_add_active_internal(cp);
1172 		if (cp->cpu_flags & CPU_FAULTED) {
1173 			cp->cpu_flags &= ~CPU_FAULTED;
1174 			mp_cpu_faulted_exit(cp);
1175 		}
1176 		cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1177 		    CPU_SPARE);
1178 		start_cpus();
1179 		cpu_stats_kstat_create(cp);
1180 		cpu_create_intrstat(cp);
1181 		lgrp_kstat_create(cp);
1182 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1183 		cpu_intr_enable(cp);	/* arch-dep hook */
1184 		cpu_set_state(cp);
1185 		cyclic_online(cp);
1186 		poke_cpu(cp->cpu_id);
1187 	}
1188 
1189 	return (error);
1190 }
1191 
1192 /*
1193  * Take the indicated CPU offline.
1194  */
1195 int
1196 cpu_offline(cpu_t *cp, int flags)
1197 {
1198 	cpupart_t *pp;
1199 	int	error = 0;
1200 	cpu_t	*ncp;
1201 	int	intr_enable;
1202 	int	cyclic_off = 0;
1203 	int	loop_count;
1204 	int	no_quiesce = 0;
1205 	int	(*bound_func)(struct cpu *, int);
1206 	kthread_t *t;
1207 	lpl_t	*cpu_lpl;
1208 	proc_t	*p;
1209 	int	lgrp_diff_lpl;
1210 
1211 	ASSERT(MUTEX_HELD(&cpu_lock));
1212 
1213 	/*
1214 	 * If we're going from faulted or spare to offline, just
1215 	 * clear these flags and update CPU state.
1216 	 */
1217 	if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1218 		if (cp->cpu_flags & CPU_FAULTED) {
1219 			cp->cpu_flags &= ~CPU_FAULTED;
1220 			mp_cpu_faulted_exit(cp);
1221 		}
1222 		cp->cpu_flags &= ~CPU_SPARE;
1223 		cpu_set_state(cp);
1224 		return (0);
1225 	}
1226 
1227 	/*
1228 	 * Handle off-line request.
1229 	 */
1230 	pp = cp->cpu_part;
1231 	/*
1232 	 * Don't offline last online CPU in partition
1233 	 */
1234 	if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1235 		return (EBUSY);
1236 	/*
1237 	 * Unbind all thread bound to our CPU if we were asked to.
1238 	 */
1239 	if (flags & CPU_FORCED && (error = cpu_unbind(cp->cpu_id)) != 0)
1240 		return (error);
1241 	/*
1242 	 * We shouldn't be bound to this CPU ourselves.
1243 	 */
1244 	if (curthread->t_bound_cpu == cp)
1245 		return (EBUSY);
1246 
1247 	/*
1248 	 * Tell interested parties that this CPU is going offline.
1249 	 */
1250 	cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1251 
1252 	/*
1253 	 * Tell the PG subsystem that the CPU is leaving the partition
1254 	 */
1255 	pg_cpupart_out(cp, pp);
1256 
1257 	/*
1258 	 * Take the CPU out of interrupt participation so we won't find
1259 	 * bound kernel threads.  If the architecture cannot completely
1260 	 * shut off interrupts on the CPU, don't quiesce it, but don't
1261 	 * run anything but interrupt thread... this is indicated by
1262 	 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1263 	 * off.
1264 	 */
1265 	intr_enable = cp->cpu_flags & CPU_ENABLE;
1266 	if (intr_enable)
1267 		no_quiesce = cpu_intr_disable(cp);
1268 
1269 	/*
1270 	 * Record that we are aiming to offline this cpu.  This acts as
1271 	 * a barrier to further weak binding requests in thread_nomigrate
1272 	 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1273 	 * lean away from this cpu.  Further strong bindings are already
1274 	 * avoided since we hold cpu_lock.  Since threads that are set
1275 	 * runnable around now and others coming off the target cpu are
1276 	 * directed away from the target, existing strong and weak bindings
1277 	 * (especially the latter) to the target cpu stand maximum chance of
1278 	 * being able to unbind during the short delay loop below (if other
1279 	 * unbound threads compete they may not see cpu in time to unbind
1280 	 * even if they would do so immediately.
1281 	 */
1282 	cpu_inmotion = cp;
1283 	membar_enter();
1284 
1285 	/*
1286 	 * Check for kernel threads (strong or weak) bound to that CPU.
1287 	 * Strongly bound threads may not unbind, and we'll have to return
1288 	 * EBUSY.  Weakly bound threads should always disappear - we've
1289 	 * stopped more weak binding with cpu_inmotion and existing
1290 	 * bindings will drain imminently (they may not block).  Nonetheless
1291 	 * we will wait for a fixed period for all bound threads to disappear.
1292 	 * Inactive interrupt threads are OK (they'll be in TS_FREE
1293 	 * state).  If test finds some bound threads, wait a few ticks
1294 	 * to give short-lived threads (such as interrupts) chance to
1295 	 * complete.  Note that if no_quiesce is set, i.e. this cpu
1296 	 * is required to service interrupts, then we take the route
1297 	 * that permits interrupt threads to be active (or bypassed).
1298 	 */
1299 	bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1300 
1301 again:	for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1302 		if (loop_count >= 5) {
1303 			error = EBUSY;	/* some threads still bound */
1304 			break;
1305 		}
1306 
1307 		/*
1308 		 * If some threads were assigned, give them
1309 		 * a chance to complete or move.
1310 		 *
1311 		 * This assumes that the clock_thread is not bound
1312 		 * to any CPU, because the clock_thread is needed to
1313 		 * do the delay(hz/100).
1314 		 *
1315 		 * Note: we still hold the cpu_lock while waiting for
1316 		 * the next clock tick.  This is OK since it isn't
1317 		 * needed for anything else except processor_bind(2),
1318 		 * and system initialization.  If we drop the lock,
1319 		 * we would risk another p_online disabling the last
1320 		 * processor.
1321 		 */
1322 		delay(hz/100);
1323 	}
1324 
1325 	if (error == 0 && cyclic_off == 0) {
1326 		if (!cyclic_offline(cp)) {
1327 			/*
1328 			 * We must have bound cyclics...
1329 			 */
1330 			error = EBUSY;
1331 			goto out;
1332 		}
1333 		cyclic_off = 1;
1334 	}
1335 
1336 	/*
1337 	 * Call mp_cpu_stop() to perform any special operations
1338 	 * needed for this machine architecture to offline a CPU.
1339 	 */
1340 	if (error == 0)
1341 		error = mp_cpu_stop(cp);	/* arch-dep hook */
1342 
1343 	/*
1344 	 * If that all worked, take the CPU offline and decrement
1345 	 * ncpus_online.
1346 	 */
1347 	if (error == 0) {
1348 		/*
1349 		 * Put all the cpus into a known safe place.
1350 		 * No mutexes can be entered while CPUs are paused.
1351 		 */
1352 		pause_cpus(cp);
1353 		/*
1354 		 * Repeat the operation, if necessary, to make sure that
1355 		 * all outstanding low-level interrupts run to completion
1356 		 * before we set the CPU_QUIESCED flag.  It's also possible
1357 		 * that a thread has weak bound to the cpu despite our raising
1358 		 * cpu_inmotion above since it may have loaded that
1359 		 * value before the barrier became visible (this would have
1360 		 * to be the thread that was on the target cpu at the time
1361 		 * we raised the barrier).
1362 		 */
1363 		if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1364 		    (*bound_func)(cp, 1)) {
1365 			start_cpus();
1366 			(void) mp_cpu_start(cp);
1367 			goto again;
1368 		}
1369 		ncp = cp->cpu_next_part;
1370 		cpu_lpl = cp->cpu_lpl;
1371 		ASSERT(cpu_lpl != NULL);
1372 
1373 		/*
1374 		 * Remove the CPU from the list of active CPUs.
1375 		 */
1376 		cpu_remove_active(cp);
1377 
1378 		/*
1379 		 * Walk the active process list and look for threads
1380 		 * whose home lgroup needs to be updated, or
1381 		 * the last CPU they run on is the one being offlined now.
1382 		 */
1383 
1384 		ASSERT(curthread->t_cpu != cp);
1385 		for (p = practive; p != NULL; p = p->p_next) {
1386 
1387 			t = p->p_tlist;
1388 
1389 			if (t == NULL)
1390 				continue;
1391 
1392 			lgrp_diff_lpl = 0;
1393 
1394 			do {
1395 				ASSERT(t->t_lpl != NULL);
1396 				/*
1397 				 * Taking last CPU in lpl offline
1398 				 * Rehome thread if it is in this lpl
1399 				 * Otherwise, update the count of how many
1400 				 * threads are in this CPU's lgroup but have
1401 				 * a different lpl.
1402 				 */
1403 
1404 				if (cpu_lpl->lpl_ncpu == 0) {
1405 					if (t->t_lpl == cpu_lpl)
1406 						lgrp_move_thread(t,
1407 						    lgrp_choose(t,
1408 						    t->t_cpupart), 0);
1409 					else if (t->t_lpl->lpl_lgrpid ==
1410 					    cpu_lpl->lpl_lgrpid)
1411 						lgrp_diff_lpl++;
1412 				}
1413 				ASSERT(t->t_lpl->lpl_ncpu > 0);
1414 
1415 				/*
1416 				 * Update CPU last ran on if it was this CPU
1417 				 */
1418 				if (t->t_cpu == cp && t->t_bound_cpu != cp)
1419 				    t->t_cpu = disp_lowpri_cpu(ncp, t->t_lpl,
1420 					t->t_pri, NULL);
1421 				ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1422 				    t->t_weakbound_cpu == cp);
1423 
1424 				t = t->t_forw;
1425 			} while (t != p->p_tlist);
1426 
1427 			/*
1428 			 * Didn't find any threads in the same lgroup as this
1429 			 * CPU with a different lpl, so remove the lgroup from
1430 			 * the process lgroup bitmask.
1431 			 */
1432 
1433 			if (lgrp_diff_lpl == 0)
1434 				klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1435 		}
1436 
1437 		/*
1438 		 * Walk thread list looking for threads that need to be
1439 		 * rehomed, since there are some threads that are not in
1440 		 * their process's p_tlist.
1441 		 */
1442 
1443 		t = curthread;
1444 		do {
1445 			ASSERT(t != NULL && t->t_lpl != NULL);
1446 
1447 			/*
1448 			 * Rehome threads with same lpl as this CPU when this
1449 			 * is the last CPU in the lpl.
1450 			 */
1451 
1452 			if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1453 				lgrp_move_thread(t,
1454 				    lgrp_choose(t, t->t_cpupart), 1);
1455 
1456 			ASSERT(t->t_lpl->lpl_ncpu > 0);
1457 
1458 			/*
1459 			 * Update CPU last ran on if it was this CPU
1460 			 */
1461 
1462 			if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1463 				t->t_cpu = disp_lowpri_cpu(ncp,
1464 				    t->t_lpl, t->t_pri, NULL);
1465 			}
1466 			ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1467 			    t->t_weakbound_cpu == cp);
1468 			t = t->t_next;
1469 
1470 		} while (t != curthread);
1471 		ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1472 		cp->cpu_flags |= CPU_OFFLINE;
1473 		disp_cpu_inactive(cp);
1474 		if (!no_quiesce)
1475 			cp->cpu_flags |= CPU_QUIESCED;
1476 		ncpus_online--;
1477 		cpu_set_state(cp);
1478 		cpu_inmotion = NULL;
1479 		start_cpus();
1480 		cpu_stats_kstat_destroy(cp);
1481 		cpu_delete_intrstat(cp);
1482 		lgrp_kstat_destroy(cp);
1483 	}
1484 
1485 out:
1486 	cpu_inmotion = NULL;
1487 
1488 	/*
1489 	 * If we failed, re-enable interrupts.
1490 	 * Do this even if cpu_intr_disable returned an error, because
1491 	 * it may have partially disabled interrupts.
1492 	 */
1493 	if (error && intr_enable)
1494 		cpu_intr_enable(cp);
1495 
1496 	/*
1497 	 * If we failed, but managed to offline the cyclic subsystem on this
1498 	 * CPU, bring it back online.
1499 	 */
1500 	if (error && cyclic_off)
1501 		cyclic_online(cp);
1502 
1503 	/*
1504 	 * If we failed, tell the PG subsystem that the CPU is back
1505 	 */
1506 	pg_cpupart_in(cp, pp);
1507 
1508 	/*
1509 	 * If we failed, we need to notify everyone that this CPU is back on.
1510 	 */
1511 	if (error != 0)
1512 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1513 
1514 	return (error);
1515 }
1516 
1517 /*
1518  * Mark the indicated CPU as faulted, taking it offline.
1519  */
1520 int
1521 cpu_faulted(cpu_t *cp, int flags)
1522 {
1523 	int	error = 0;
1524 
1525 	ASSERT(MUTEX_HELD(&cpu_lock));
1526 	ASSERT(!cpu_is_poweredoff(cp));
1527 
1528 	if (cpu_is_offline(cp)) {
1529 		cp->cpu_flags &= ~CPU_SPARE;
1530 		cp->cpu_flags |= CPU_FAULTED;
1531 		mp_cpu_faulted_enter(cp);
1532 		cpu_set_state(cp);
1533 		return (0);
1534 	}
1535 
1536 	if ((error = cpu_offline(cp, flags)) == 0) {
1537 		cp->cpu_flags |= CPU_FAULTED;
1538 		mp_cpu_faulted_enter(cp);
1539 		cpu_set_state(cp);
1540 	}
1541 
1542 	return (error);
1543 }
1544 
1545 /*
1546  * Mark the indicated CPU as a spare, taking it offline.
1547  */
1548 int
1549 cpu_spare(cpu_t *cp, int flags)
1550 {
1551 	int	error = 0;
1552 
1553 	ASSERT(MUTEX_HELD(&cpu_lock));
1554 	ASSERT(!cpu_is_poweredoff(cp));
1555 
1556 	if (cpu_is_offline(cp)) {
1557 		if (cp->cpu_flags & CPU_FAULTED) {
1558 			cp->cpu_flags &= ~CPU_FAULTED;
1559 			mp_cpu_faulted_exit(cp);
1560 		}
1561 		cp->cpu_flags |= CPU_SPARE;
1562 		cpu_set_state(cp);
1563 		return (0);
1564 	}
1565 
1566 	if ((error = cpu_offline(cp, flags)) == 0) {
1567 		cp->cpu_flags |= CPU_SPARE;
1568 		cpu_set_state(cp);
1569 	}
1570 
1571 	return (error);
1572 }
1573 
1574 /*
1575  * Take the indicated CPU from poweroff to offline.
1576  */
1577 int
1578 cpu_poweron(cpu_t *cp)
1579 {
1580 	int	error = ENOTSUP;
1581 
1582 	ASSERT(MUTEX_HELD(&cpu_lock));
1583 	ASSERT(cpu_is_poweredoff(cp));
1584 
1585 	error = mp_cpu_poweron(cp);	/* arch-dep hook */
1586 	if (error == 0)
1587 		cpu_set_state(cp);
1588 
1589 	return (error);
1590 }
1591 
1592 /*
1593  * Take the indicated CPU from any inactive state to powered off.
1594  */
1595 int
1596 cpu_poweroff(cpu_t *cp)
1597 {
1598 	int	error = ENOTSUP;
1599 
1600 	ASSERT(MUTEX_HELD(&cpu_lock));
1601 	ASSERT(cpu_is_offline(cp));
1602 
1603 	if (!(cp->cpu_flags & CPU_QUIESCED))
1604 		return (EBUSY);		/* not completely idle */
1605 
1606 	error = mp_cpu_poweroff(cp);	/* arch-dep hook */
1607 	if (error == 0)
1608 		cpu_set_state(cp);
1609 
1610 	return (error);
1611 }
1612 
1613 /*
1614  * Initialize the CPU lists for the first CPU.
1615  */
1616 void
1617 cpu_list_init(cpu_t *cp)
1618 {
1619 	cp->cpu_next = cp;
1620 	cp->cpu_prev = cp;
1621 	cpu_list = cp;
1622 	clock_cpu_list = cp;
1623 
1624 	cp->cpu_next_onln = cp;
1625 	cp->cpu_prev_onln = cp;
1626 	cpu_active = cp;
1627 
1628 	cp->cpu_seqid = 0;
1629 	CPUSET_ADD(cpu_seqid_inuse, 0);
1630 	cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid);
1631 	cp_default.cp_mach = &cp_default_mach;
1632 	cp_default.cp_cpulist = cp;
1633 	cp_default.cp_ncpus = 1;
1634 	cp->cpu_next_part = cp;
1635 	cp->cpu_prev_part = cp;
1636 	cp->cpu_part = &cp_default;
1637 
1638 	CPUSET_ADD(cpu_available, cp->cpu_id);
1639 }
1640 
1641 /*
1642  * Insert a CPU into the list of available CPUs.
1643  */
1644 void
1645 cpu_add_unit(cpu_t *cp)
1646 {
1647 	int seqid;
1648 
1649 	ASSERT(MUTEX_HELD(&cpu_lock));
1650 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1651 
1652 	lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1653 
1654 	/*
1655 	 * Note: most users of the cpu_list will grab the
1656 	 * cpu_lock to insure that it isn't modified.  However,
1657 	 * certain users can't or won't do that.  To allow this
1658 	 * we pause the other cpus.  Users who walk the list
1659 	 * without cpu_lock, must disable kernel preemption
1660 	 * to insure that the list isn't modified underneath
1661 	 * them.  Also, any cached pointers to cpu structures
1662 	 * must be revalidated by checking to see if the
1663 	 * cpu_next pointer points to itself.  This check must
1664 	 * be done with the cpu_lock held or kernel preemption
1665 	 * disabled.  This check relies upon the fact that
1666 	 * old cpu structures are not free'ed or cleared after
1667 	 * then are removed from the cpu_list.
1668 	 *
1669 	 * Note that the clock code walks the cpu list dereferencing
1670 	 * the cpu_part pointer, so we need to initialize it before
1671 	 * adding the cpu to the list.
1672 	 */
1673 	cp->cpu_part = &cp_default;
1674 	(void) pause_cpus(NULL);
1675 	cp->cpu_next = cpu_list;
1676 	cp->cpu_prev = cpu_list->cpu_prev;
1677 	cpu_list->cpu_prev->cpu_next = cp;
1678 	cpu_list->cpu_prev = cp;
1679 	start_cpus();
1680 
1681 	for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1682 		continue;
1683 	CPUSET_ADD(cpu_seqid_inuse, seqid);
1684 	cp->cpu_seqid = seqid;
1685 	ASSERT(ncpus < max_ncpus);
1686 	ncpus++;
1687 	cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid);
1688 	cpu[cp->cpu_id] = cp;
1689 	CPUSET_ADD(cpu_available, cp->cpu_id);
1690 
1691 	/*
1692 	 * allocate a pause thread for this CPU.
1693 	 */
1694 	cpu_pause_alloc(cp);
1695 
1696 	/*
1697 	 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1698 	 * link them into a list of just that CPU.
1699 	 * This is so that disp_lowpri_cpu will work for thread_create in
1700 	 * pause_cpus() when called from the startup thread in a new CPU.
1701 	 */
1702 	cp->cpu_next_onln = cp;
1703 	cp->cpu_prev_onln = cp;
1704 	cpu_info_kstat_create(cp);
1705 	cp->cpu_next_part = cp;
1706 	cp->cpu_prev_part = cp;
1707 
1708 	init_cpu_mstate(cp, CMS_SYSTEM);
1709 
1710 	pool_pset_mod = gethrtime();
1711 }
1712 
1713 /*
1714  * Do the opposite of cpu_add_unit().
1715  */
1716 void
1717 cpu_del_unit(int cpuid)
1718 {
1719 	struct cpu	*cp, *cpnext;
1720 
1721 	ASSERT(MUTEX_HELD(&cpu_lock));
1722 	cp = cpu[cpuid];
1723 	ASSERT(cp != NULL);
1724 
1725 	ASSERT(cp->cpu_next_onln == cp);
1726 	ASSERT(cp->cpu_prev_onln == cp);
1727 	ASSERT(cp->cpu_next_part == cp);
1728 	ASSERT(cp->cpu_prev_part == cp);
1729 
1730 	/*
1731 	 * Tear down the CPU's physical ID cache, and update any
1732 	 * processor groups
1733 	 */
1734 	pg_cpu_fini(cp);
1735 	pghw_physid_destroy(cp);
1736 
1737 	/*
1738 	 * Destroy kstat stuff.
1739 	 */
1740 	cpu_info_kstat_destroy(cp);
1741 	term_cpu_mstate(cp);
1742 	/*
1743 	 * Free up pause thread.
1744 	 */
1745 	cpu_pause_free(cp);
1746 	CPUSET_DEL(cpu_available, cp->cpu_id);
1747 	cpu[cp->cpu_id] = NULL;
1748 	/*
1749 	 * The clock thread and mutex_vector_enter cannot hold the
1750 	 * cpu_lock while traversing the cpu list, therefore we pause
1751 	 * all other threads by pausing the other cpus. These, and any
1752 	 * other routines holding cpu pointers while possibly sleeping
1753 	 * must be sure to call kpreempt_disable before processing the
1754 	 * list and be sure to check that the cpu has not been deleted
1755 	 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1756 	 * to keep the deleted cpu structure around.
1757 	 *
1758 	 * Note that this MUST be done AFTER cpu_available
1759 	 * has been updated so that we don't waste time
1760 	 * trying to pause the cpu we're trying to delete.
1761 	 */
1762 	(void) pause_cpus(NULL);
1763 
1764 	cpnext = cp->cpu_next;
1765 	cp->cpu_prev->cpu_next = cp->cpu_next;
1766 	cp->cpu_next->cpu_prev = cp->cpu_prev;
1767 	if (cp == cpu_list)
1768 		cpu_list = cpnext;
1769 	if (cp == clock_cpu_list)
1770 		clock_cpu_list = cpnext;
1771 
1772 
1773 	/*
1774 	 * Signals that the cpu has been deleted (see above).
1775 	 */
1776 	cp->cpu_next = NULL;
1777 	cp->cpu_prev = NULL;
1778 
1779 	start_cpus();
1780 
1781 	CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1782 	ncpus--;
1783 	lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1784 
1785 	pool_pset_mod = gethrtime();
1786 }
1787 
1788 /*
1789  * Add a CPU to the list of active CPUs.
1790  *	This routine must not get any locks, because other CPUs are paused.
1791  */
1792 static void
1793 cpu_add_active_internal(cpu_t *cp)
1794 {
1795 	cpupart_t	*pp = cp->cpu_part;
1796 
1797 	ASSERT(MUTEX_HELD(&cpu_lock));
1798 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1799 
1800 	ncpus_online++;
1801 	cpu_set_state(cp);
1802 	cp->cpu_next_onln = cpu_active;
1803 	cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1804 	cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1805 	cpu_active->cpu_prev_onln = cp;
1806 
1807 	if (pp->cp_cpulist) {
1808 		cp->cpu_next_part = pp->cp_cpulist;
1809 		cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1810 		pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1811 		pp->cp_cpulist->cpu_prev_part = cp;
1812 	} else {
1813 		ASSERT(pp->cp_ncpus == 0);
1814 		pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1815 	}
1816 	pp->cp_ncpus++;
1817 	if (pp->cp_ncpus == 1) {
1818 		cp_numparts_nonempty++;
1819 		ASSERT(cp_numparts_nonempty != 0);
1820 	}
1821 
1822 	pg_cpu_active(cp);
1823 	lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1824 
1825 	bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1826 }
1827 
1828 /*
1829  * Add a CPU to the list of active CPUs.
1830  *	This is called from machine-dependent layers when a new CPU is started.
1831  */
1832 void
1833 cpu_add_active(cpu_t *cp)
1834 {
1835 	pg_cpupart_in(cp, cp->cpu_part);
1836 
1837 	pause_cpus(NULL);
1838 	cpu_add_active_internal(cp);
1839 	start_cpus();
1840 
1841 	cpu_stats_kstat_create(cp);
1842 	cpu_create_intrstat(cp);
1843 	lgrp_kstat_create(cp);
1844 	cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1845 }
1846 
1847 
1848 /*
1849  * Remove a CPU from the list of active CPUs.
1850  *	This routine must not get any locks, because other CPUs are paused.
1851  */
1852 /* ARGSUSED */
1853 static void
1854 cpu_remove_active(cpu_t *cp)
1855 {
1856 	cpupart_t	*pp = cp->cpu_part;
1857 
1858 	ASSERT(MUTEX_HELD(&cpu_lock));
1859 	ASSERT(cp->cpu_next_onln != cp);	/* not the last one */
1860 	ASSERT(cp->cpu_prev_onln != cp);	/* not the last one */
1861 
1862 	pg_cpu_inactive(cp);
1863 
1864 	lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1865 
1866 	cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1867 	cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1868 	if (cpu_active == cp) {
1869 		cpu_active = cp->cpu_next_onln;
1870 	}
1871 	cp->cpu_next_onln = cp;
1872 	cp->cpu_prev_onln = cp;
1873 
1874 	cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1875 	cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1876 	if (pp->cp_cpulist == cp) {
1877 		pp->cp_cpulist = cp->cpu_next_part;
1878 		ASSERT(pp->cp_cpulist != cp);
1879 	}
1880 	cp->cpu_next_part = cp;
1881 	cp->cpu_prev_part = cp;
1882 	pp->cp_ncpus--;
1883 	if (pp->cp_ncpus == 0) {
1884 		cp_numparts_nonempty--;
1885 		ASSERT(cp_numparts_nonempty != 0);
1886 	}
1887 }
1888 
1889 /*
1890  * Routine used to setup a newly inserted CPU in preparation for starting
1891  * it running code.
1892  */
1893 int
1894 cpu_configure(int cpuid)
1895 {
1896 	int retval = 0;
1897 
1898 	ASSERT(MUTEX_HELD(&cpu_lock));
1899 
1900 	/*
1901 	 * Some structures are statically allocated based upon
1902 	 * the maximum number of cpus the system supports.  Do not
1903 	 * try to add anything beyond this limit.
1904 	 */
1905 	if (cpuid < 0 || cpuid >= NCPU) {
1906 		return (EINVAL);
1907 	}
1908 
1909 	if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
1910 		return (EALREADY);
1911 	}
1912 
1913 	if ((retval = mp_cpu_configure(cpuid)) != 0) {
1914 		return (retval);
1915 	}
1916 
1917 	cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
1918 	cpu_set_state(cpu[cpuid]);
1919 	retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
1920 	if (retval != 0)
1921 		(void) mp_cpu_unconfigure(cpuid);
1922 
1923 	return (retval);
1924 }
1925 
1926 /*
1927  * Routine used to cleanup a CPU that has been powered off.  This will
1928  * destroy all per-cpu information related to this cpu.
1929  */
1930 int
1931 cpu_unconfigure(int cpuid)
1932 {
1933 	int error;
1934 
1935 	ASSERT(MUTEX_HELD(&cpu_lock));
1936 
1937 	if (cpu[cpuid] == NULL) {
1938 		return (ENODEV);
1939 	}
1940 
1941 	if (cpu[cpuid]->cpu_flags == 0) {
1942 		return (EALREADY);
1943 	}
1944 
1945 	if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
1946 		return (EBUSY);
1947 	}
1948 
1949 	if (cpu[cpuid]->cpu_props != NULL) {
1950 		(void) nvlist_free(cpu[cpuid]->cpu_props);
1951 		cpu[cpuid]->cpu_props = NULL;
1952 	}
1953 
1954 	error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
1955 
1956 	if (error != 0)
1957 		return (error);
1958 
1959 	return (mp_cpu_unconfigure(cpuid));
1960 }
1961 
1962 /*
1963  * Routines for registering and de-registering cpu_setup callback functions.
1964  *
1965  * Caller's context
1966  *	These routines must not be called from a driver's attach(9E) or
1967  *	detach(9E) entry point.
1968  *
1969  * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
1970  */
1971 
1972 /*
1973  * Ideally, these would be dynamically allocated and put into a linked
1974  * list; however that is not feasible because the registration routine
1975  * has to be available before the kmem allocator is working (in fact,
1976  * it is called by the kmem allocator init code).  In any case, there
1977  * are quite a few extra entries for future users.
1978  */
1979 #define	NCPU_SETUPS	20
1980 
1981 struct cpu_setup {
1982 	cpu_setup_func_t *func;
1983 	void *arg;
1984 } cpu_setups[NCPU_SETUPS];
1985 
1986 void
1987 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
1988 {
1989 	int i;
1990 
1991 	ASSERT(MUTEX_HELD(&cpu_lock));
1992 
1993 	for (i = 0; i < NCPU_SETUPS; i++)
1994 		if (cpu_setups[i].func == NULL)
1995 		    break;
1996 	if (i >= NCPU_SETUPS)
1997 		cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
1998 
1999 	cpu_setups[i].func = func;
2000 	cpu_setups[i].arg = arg;
2001 }
2002 
2003 void
2004 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2005 {
2006 	int i;
2007 
2008 	ASSERT(MUTEX_HELD(&cpu_lock));
2009 
2010 	for (i = 0; i < NCPU_SETUPS; i++)
2011 		if ((cpu_setups[i].func == func) &&
2012 		    (cpu_setups[i].arg == arg))
2013 		    break;
2014 	if (i >= NCPU_SETUPS)
2015 		cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2016 		    "deregister");
2017 
2018 	cpu_setups[i].func = NULL;
2019 	cpu_setups[i].arg = 0;
2020 }
2021 
2022 /*
2023  * Call any state change hooks for this CPU, ignore any errors.
2024  */
2025 void
2026 cpu_state_change_notify(int id, cpu_setup_t what)
2027 {
2028 	int i;
2029 
2030 	ASSERT(MUTEX_HELD(&cpu_lock));
2031 
2032 	for (i = 0; i < NCPU_SETUPS; i++) {
2033 		if (cpu_setups[i].func != NULL) {
2034 			cpu_setups[i].func(what, id, cpu_setups[i].arg);
2035 		}
2036 	}
2037 }
2038 
2039 /*
2040  * Call any state change hooks for this CPU, undo it if error found.
2041  */
2042 static int
2043 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2044 {
2045 	int i;
2046 	int retval = 0;
2047 
2048 	ASSERT(MUTEX_HELD(&cpu_lock));
2049 
2050 	for (i = 0; i < NCPU_SETUPS; i++) {
2051 		if (cpu_setups[i].func != NULL) {
2052 			retval = cpu_setups[i].func(what, id,
2053 			    cpu_setups[i].arg);
2054 			if (retval) {
2055 				for (i--; i >= 0; i--) {
2056 					if (cpu_setups[i].func != NULL)
2057 						cpu_setups[i].func(undo,
2058 						    id, cpu_setups[i].arg);
2059 				}
2060 				break;
2061 			}
2062 		}
2063 	}
2064 	return (retval);
2065 }
2066 
2067 /*
2068  * Export information about this CPU via the kstat mechanism.
2069  */
2070 static struct {
2071 	kstat_named_t ci_state;
2072 	kstat_named_t ci_state_begin;
2073 	kstat_named_t ci_cpu_type;
2074 	kstat_named_t ci_fpu_type;
2075 	kstat_named_t ci_clock_MHz;
2076 	kstat_named_t ci_chip_id;
2077 	kstat_named_t ci_implementation;
2078 	kstat_named_t ci_brandstr;
2079 	kstat_named_t ci_core_id;
2080 #if defined(__sparcv9)
2081 	kstat_named_t ci_device_ID;
2082 	kstat_named_t ci_cpu_fru;
2083 #endif
2084 #if defined(__x86)
2085 	kstat_named_t ci_vendorstr;
2086 	kstat_named_t ci_family;
2087 	kstat_named_t ci_model;
2088 	kstat_named_t ci_step;
2089 	kstat_named_t ci_clogid;
2090 #endif
2091 } cpu_info_template = {
2092 	{ "state",		KSTAT_DATA_CHAR },
2093 	{ "state_begin",	KSTAT_DATA_LONG },
2094 	{ "cpu_type",		KSTAT_DATA_CHAR },
2095 	{ "fpu_type",		KSTAT_DATA_CHAR },
2096 	{ "clock_MHz",		KSTAT_DATA_LONG },
2097 	{ "chip_id",		KSTAT_DATA_LONG },
2098 	{ "implementation",	KSTAT_DATA_STRING },
2099 	{ "brand",		KSTAT_DATA_STRING },
2100 	{ "core_id",		KSTAT_DATA_LONG },
2101 #if defined(__sparcv9)
2102 	{ "device_ID",		KSTAT_DATA_UINT64 },
2103 	{ "cpu_fru",		KSTAT_DATA_STRING },
2104 #endif
2105 #if defined(__x86)
2106 	{ "vendor_id",		KSTAT_DATA_STRING },
2107 	{ "family",		KSTAT_DATA_INT32 },
2108 	{ "model",		KSTAT_DATA_INT32 },
2109 	{ "stepping",		KSTAT_DATA_INT32 },
2110 	{ "clog_id",		KSTAT_DATA_INT32 },
2111 #endif
2112 };
2113 
2114 static kmutex_t cpu_info_template_lock;
2115 
2116 static int
2117 cpu_info_kstat_update(kstat_t *ksp, int rw)
2118 {
2119 	cpu_t	*cp = ksp->ks_private;
2120 	const char *pi_state;
2121 
2122 	if (rw == KSTAT_WRITE)
2123 		return (EACCES);
2124 
2125 	switch (cp->cpu_type_info.pi_state) {
2126 	case P_ONLINE:
2127 		pi_state = PS_ONLINE;
2128 		break;
2129 	case P_POWEROFF:
2130 		pi_state = PS_POWEROFF;
2131 		break;
2132 	case P_NOINTR:
2133 		pi_state = PS_NOINTR;
2134 		break;
2135 	case P_FAULTED:
2136 		pi_state = PS_FAULTED;
2137 		break;
2138 	case P_SPARE:
2139 		pi_state = PS_SPARE;
2140 		break;
2141 	case P_OFFLINE:
2142 		pi_state = PS_OFFLINE;
2143 		break;
2144 	default:
2145 		pi_state = "unknown";
2146 	}
2147 	(void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2148 	cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2149 	(void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2150 	    cp->cpu_type_info.pi_processor_type, 15);
2151 	(void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2152 	    cp->cpu_type_info.pi_fputypes, 15);
2153 	cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2154 	cpu_info_template.ci_chip_id.value.l =
2155 	    pg_plat_hw_instance_id(cp, PGHW_CHIP);
2156 	kstat_named_setstr(&cpu_info_template.ci_implementation,
2157 	    cp->cpu_idstr);
2158 	kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2159 	cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2160 
2161 #if defined(__sparcv9)
2162 	cpu_info_template.ci_device_ID.value.ui64 =
2163 	    cpunodes[cp->cpu_id].device_id;
2164 	kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2165 #endif
2166 #if defined(__x86)
2167 	kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2168 	    cpuid_getvendorstr(cp));
2169 	cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2170 	cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2171 	cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2172 	cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2173 #endif
2174 
2175 	return (0);
2176 }
2177 
2178 static void
2179 cpu_info_kstat_create(cpu_t *cp)
2180 {
2181 	zoneid_t zoneid;
2182 
2183 	ASSERT(MUTEX_HELD(&cpu_lock));
2184 
2185 	if (pool_pset_enabled())
2186 		zoneid = GLOBAL_ZONEID;
2187 	else
2188 		zoneid = ALL_ZONES;
2189 	if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2190 	    NULL, "misc", KSTAT_TYPE_NAMED,
2191 		    sizeof (cpu_info_template) / sizeof (kstat_named_t),
2192 		    KSTAT_FLAG_VIRTUAL, zoneid)) != NULL) {
2193 		cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2194 #if defined(__sparcv9)
2195 		cp->cpu_info_kstat->ks_data_size +=
2196 		    strlen(cpu_fru_fmri(cp)) + 1;
2197 #endif
2198 #if defined(__x86)
2199 		cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2200 #endif
2201 		cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2202 		cp->cpu_info_kstat->ks_data = &cpu_info_template;
2203 		cp->cpu_info_kstat->ks_private = cp;
2204 		cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2205 		kstat_install(cp->cpu_info_kstat);
2206 	}
2207 }
2208 
2209 static void
2210 cpu_info_kstat_destroy(cpu_t *cp)
2211 {
2212 	ASSERT(MUTEX_HELD(&cpu_lock));
2213 
2214 	kstat_delete(cp->cpu_info_kstat);
2215 	cp->cpu_info_kstat = NULL;
2216 }
2217 
2218 /*
2219  * Create and install kstats for the boot CPU.
2220  */
2221 void
2222 cpu_kstat_init(cpu_t *cp)
2223 {
2224 	mutex_enter(&cpu_lock);
2225 	cpu_info_kstat_create(cp);
2226 	cpu_stats_kstat_create(cp);
2227 	cpu_create_intrstat(cp);
2228 	cpu_set_state(cp);
2229 	mutex_exit(&cpu_lock);
2230 }
2231 
2232 /*
2233  * Make visible to the zone that subset of the cpu information that would be
2234  * initialized when a cpu is configured (but still offline).
2235  */
2236 void
2237 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2238 {
2239 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2240 
2241 	ASSERT(MUTEX_HELD(&cpu_lock));
2242 	ASSERT(pool_pset_enabled());
2243 	ASSERT(cp != NULL);
2244 
2245 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2246 		zone->zone_ncpus++;
2247 		ASSERT(zone->zone_ncpus <= ncpus);
2248 	}
2249 	if (cp->cpu_info_kstat != NULL)
2250 		kstat_zone_add(cp->cpu_info_kstat, zoneid);
2251 }
2252 
2253 /*
2254  * Make visible to the zone that subset of the cpu information that would be
2255  * initialized when a previously configured cpu is onlined.
2256  */
2257 void
2258 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2259 {
2260 	kstat_t *ksp;
2261 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2262 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2263 	processorid_t cpun;
2264 
2265 	ASSERT(MUTEX_HELD(&cpu_lock));
2266 	ASSERT(pool_pset_enabled());
2267 	ASSERT(cp != NULL);
2268 	ASSERT(cpu_is_active(cp));
2269 
2270 	cpun = cp->cpu_id;
2271 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2272 		zone->zone_ncpus_online++;
2273 		ASSERT(zone->zone_ncpus_online <= ncpus_online);
2274 	}
2275 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2276 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2277 	    != NULL) {
2278 		kstat_zone_add(ksp, zoneid);
2279 		kstat_rele(ksp);
2280 	}
2281 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2282 		kstat_zone_add(ksp, zoneid);
2283 		kstat_rele(ksp);
2284 	}
2285 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2286 		kstat_zone_add(ksp, zoneid);
2287 		kstat_rele(ksp);
2288 	}
2289 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2290 	    NULL) {
2291 		kstat_zone_add(ksp, zoneid);
2292 		kstat_rele(ksp);
2293 	}
2294 }
2295 
2296 /*
2297  * Update relevant kstats such that cpu is now visible to processes
2298  * executing in specified zone.
2299  */
2300 void
2301 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2302 {
2303 	cpu_visibility_configure(cp, zone);
2304 	if (cpu_is_active(cp))
2305 		cpu_visibility_online(cp, zone);
2306 }
2307 
2308 /*
2309  * Make invisible to the zone that subset of the cpu information that would be
2310  * torn down when a previously offlined cpu is unconfigured.
2311  */
2312 void
2313 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2314 {
2315 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2316 
2317 	ASSERT(MUTEX_HELD(&cpu_lock));
2318 	ASSERT(pool_pset_enabled());
2319 	ASSERT(cp != NULL);
2320 
2321 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2322 		ASSERT(zone->zone_ncpus != 0);
2323 		zone->zone_ncpus--;
2324 	}
2325 	if (cp->cpu_info_kstat)
2326 		kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2327 }
2328 
2329 /*
2330  * Make invisible to the zone that subset of the cpu information that would be
2331  * torn down when a cpu is offlined (but still configured).
2332  */
2333 void
2334 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2335 {
2336 	kstat_t *ksp;
2337 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2338 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2339 	processorid_t cpun;
2340 
2341 	ASSERT(MUTEX_HELD(&cpu_lock));
2342 	ASSERT(pool_pset_enabled());
2343 	ASSERT(cp != NULL);
2344 	ASSERT(cpu_is_active(cp));
2345 
2346 	cpun = cp->cpu_id;
2347 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2348 		ASSERT(zone->zone_ncpus_online != 0);
2349 		zone->zone_ncpus_online--;
2350 	}
2351 
2352 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2353 	    NULL) {
2354 		kstat_zone_remove(ksp, zoneid);
2355 		kstat_rele(ksp);
2356 	}
2357 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2358 		kstat_zone_remove(ksp, zoneid);
2359 		kstat_rele(ksp);
2360 	}
2361 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2362 		kstat_zone_remove(ksp, zoneid);
2363 		kstat_rele(ksp);
2364 	}
2365 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2366 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2367 	    != NULL) {
2368 		kstat_zone_remove(ksp, zoneid);
2369 		kstat_rele(ksp);
2370 	}
2371 }
2372 
2373 /*
2374  * Update relevant kstats such that cpu is no longer visible to processes
2375  * executing in specified zone.
2376  */
2377 void
2378 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2379 {
2380 	if (cpu_is_active(cp))
2381 		cpu_visibility_offline(cp, zone);
2382 	cpu_visibility_unconfigure(cp, zone);
2383 }
2384 
2385 /*
2386  * Bind a thread to a CPU as requested.
2387  */
2388 int
2389 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2390     int *error)
2391 {
2392 	processorid_t	binding;
2393 	cpu_t		*cp;
2394 
2395 	ASSERT(MUTEX_HELD(&cpu_lock));
2396 	ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2397 
2398 	thread_lock(tp);
2399 
2400 	/*
2401 	 * Record old binding, but change the obind, which was initialized
2402 	 * to PBIND_NONE, only if this thread has a binding.  This avoids
2403 	 * reporting PBIND_NONE for a process when some LWPs are bound.
2404 	 */
2405 	binding = tp->t_bind_cpu;
2406 	if (binding != PBIND_NONE)
2407 		*obind = binding;	/* record old binding */
2408 
2409 	if (bind == PBIND_QUERY) {
2410 		thread_unlock(tp);
2411 		return (0);
2412 	}
2413 
2414 	/*
2415 	 * If this thread/LWP cannot be bound because of permission
2416 	 * problems, just note that and return success so that the
2417 	 * other threads/LWPs will be bound.  This is the way
2418 	 * processor_bind() is defined to work.
2419 	 *
2420 	 * Binding will get EPERM if the thread is of system class
2421 	 * or hasprocperm() fails.
2422 	 */
2423 	if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2424 		*error = EPERM;
2425 		thread_unlock(tp);
2426 		return (0);
2427 	}
2428 
2429 	binding = bind;
2430 	if (binding != PBIND_NONE) {
2431 		cp = cpu[binding];
2432 		/*
2433 		 * Make sure binding is in right partition.
2434 		 */
2435 		if (tp->t_cpupart != cp->cpu_part) {
2436 			*error = EINVAL;
2437 			thread_unlock(tp);
2438 			return (0);
2439 		}
2440 	}
2441 	tp->t_bind_cpu = binding;	/* set new binding */
2442 
2443 	/*
2444 	 * If there is no system-set reason for affinity, set
2445 	 * the t_bound_cpu field to reflect the binding.
2446 	 */
2447 	if (tp->t_affinitycnt == 0) {
2448 		if (binding == PBIND_NONE) {
2449 			/*
2450 			 * We may need to adjust disp_max_unbound_pri
2451 			 * since we're becoming unbound.
2452 			 */
2453 			disp_adjust_unbound_pri(tp);
2454 
2455 			tp->t_bound_cpu = NULL;	/* set new binding */
2456 
2457 			/*
2458 			 * Move thread to lgroup with strongest affinity
2459 			 * after unbinding
2460 			 */
2461 			if (tp->t_lgrp_affinity)
2462 				lgrp_move_thread(tp,
2463 				    lgrp_choose(tp, tp->t_cpupart), 1);
2464 
2465 			if (tp->t_state == TS_ONPROC &&
2466 			    tp->t_cpu->cpu_part != tp->t_cpupart)
2467 				cpu_surrender(tp);
2468 		} else {
2469 			lpl_t	*lpl;
2470 
2471 			tp->t_bound_cpu = cp;
2472 			ASSERT(cp->cpu_lpl != NULL);
2473 
2474 			/*
2475 			 * Set home to lgroup with most affinity containing CPU
2476 			 * that thread is being bound or minimum bounding
2477 			 * lgroup if no affinities set
2478 			 */
2479 			if (tp->t_lgrp_affinity)
2480 				lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2481 				    LGRP_NONE, B_FALSE);
2482 			else
2483 				lpl = cp->cpu_lpl;
2484 
2485 			if (tp->t_lpl != lpl) {
2486 				/* can't grab cpu_lock */
2487 				lgrp_move_thread(tp, lpl, 1);
2488 			}
2489 
2490 			/*
2491 			 * Make the thread switch to the bound CPU.
2492 			 * If the thread is runnable, we need to
2493 			 * requeue it even if t_cpu is already set
2494 			 * to the right CPU, since it may be on a
2495 			 * kpreempt queue and need to move to a local
2496 			 * queue.  We could check t_disp_queue to
2497 			 * avoid unnecessary overhead if it's already
2498 			 * on the right queue, but since this isn't
2499 			 * a performance-critical operation it doesn't
2500 			 * seem worth the extra code and complexity.
2501 			 *
2502 			 * If the thread is weakbound to the cpu then it will
2503 			 * resist the new binding request until the weak
2504 			 * binding drops.  The cpu_surrender or requeueing
2505 			 * below could be skipped in such cases (since it
2506 			 * will have no effect), but that would require
2507 			 * thread_allowmigrate to acquire thread_lock so
2508 			 * we'll take the very occasional hit here instead.
2509 			 */
2510 			if (tp->t_state == TS_ONPROC) {
2511 				cpu_surrender(tp);
2512 			} else if (tp->t_state == TS_RUN) {
2513 				cpu_t *ocp = tp->t_cpu;
2514 
2515 				(void) dispdeq(tp);
2516 				setbackdq(tp);
2517 				/*
2518 				 * Either on the bound CPU's disp queue now,
2519 				 * or swapped out or on the swap queue.
2520 				 */
2521 				ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2522 				    tp->t_weakbound_cpu == ocp ||
2523 				    (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2524 				    != TS_LOAD);
2525 			}
2526 		}
2527 	}
2528 
2529 	/*
2530 	 * Our binding has changed; set TP_CHANGEBIND.
2531 	 */
2532 	tp->t_proc_flag |= TP_CHANGEBIND;
2533 	aston(tp);
2534 
2535 	thread_unlock(tp);
2536 
2537 	return (0);
2538 }
2539 
2540 #if CPUSET_WORDS > 1
2541 
2542 /*
2543  * Functions for implementing cpuset operations when a cpuset is more
2544  * than one word.  On platforms where a cpuset is a single word these
2545  * are implemented as macros in cpuvar.h.
2546  */
2547 
2548 void
2549 cpuset_all(cpuset_t *s)
2550 {
2551 	int i;
2552 
2553 	for (i = 0; i < CPUSET_WORDS; i++)
2554 		s->cpub[i] = ~0UL;
2555 }
2556 
2557 void
2558 cpuset_all_but(cpuset_t *s, uint_t cpu)
2559 {
2560 	cpuset_all(s);
2561 	CPUSET_DEL(*s, cpu);
2562 }
2563 
2564 void
2565 cpuset_only(cpuset_t *s, uint_t cpu)
2566 {
2567 	CPUSET_ZERO(*s);
2568 	CPUSET_ADD(*s, cpu);
2569 }
2570 
2571 int
2572 cpuset_isnull(cpuset_t *s)
2573 {
2574 	int i;
2575 
2576 	for (i = 0; i < CPUSET_WORDS; i++)
2577 		if (s->cpub[i] != 0)
2578 			return (0);
2579 	return (1);
2580 }
2581 
2582 int
2583 cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2584 {
2585 	int i;
2586 
2587 	for (i = 0; i < CPUSET_WORDS; i++)
2588 		if (s1->cpub[i] != s2->cpub[i])
2589 			return (0);
2590 	return (1);
2591 }
2592 
2593 uint_t
2594 cpuset_find(cpuset_t *s)
2595 {
2596 
2597 	uint_t	i;
2598 	uint_t	cpu = (uint_t)-1;
2599 
2600 	/*
2601 	 * Find a cpu in the cpuset
2602 	 */
2603 	for (i = 0; i < CPUSET_WORDS; i++) {
2604 		cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2605 		if (cpu != (uint_t)-1) {
2606 			cpu += i * BT_NBIPUL;
2607 			break;
2608 		}
2609 	}
2610 	return (cpu);
2611 }
2612 
2613 void
2614 cpuset_bounds(cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2615 {
2616 	int	i, j;
2617 	uint_t	bit;
2618 
2619 	/*
2620 	 * First, find the smallest cpu id in the set.
2621 	 */
2622 	for (i = 0; i < CPUSET_WORDS; i++) {
2623 		if (s->cpub[i] != 0) {
2624 			bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2625 			ASSERT(bit != (uint_t)-1);
2626 			*smallestid = bit + (i * BT_NBIPUL);
2627 
2628 			/*
2629 			 * Now find the largest cpu id in
2630 			 * the set and return immediately.
2631 			 * Done in an inner loop to avoid
2632 			 * having to break out of the first
2633 			 * loop.
2634 			 */
2635 			for (j = CPUSET_WORDS - 1; j >= i; j--) {
2636 				if (s->cpub[j] != 0) {
2637 					bit = (uint_t)(highbit(s->cpub[j]) - 1);
2638 					ASSERT(bit != (uint_t)-1);
2639 					*largestid = bit + (j * BT_NBIPUL);
2640 					ASSERT(*largestid >= *smallestid);
2641 					return;
2642 				}
2643 			}
2644 
2645 			/*
2646 			 * If this code is reached, a
2647 			 * smallestid was found, but not a
2648 			 * largestid. The cpuset must have
2649 			 * been changed during the course
2650 			 * of this function call.
2651 			 */
2652 			ASSERT(0);
2653 		}
2654 	}
2655 	*smallestid = *largestid = CPUSET_NOTINSET;
2656 }
2657 
2658 #endif	/* CPUSET_WORDS */
2659 
2660 /*
2661  * Unbind all user threads bound to a given CPU.
2662  */
2663 int
2664 cpu_unbind(processorid_t cpu)
2665 {
2666 	processorid_t obind;
2667 	kthread_t *tp;
2668 	int ret = 0;
2669 	proc_t *pp;
2670 	int err, berr = 0;
2671 
2672 	ASSERT(MUTEX_HELD(&cpu_lock));
2673 
2674 	mutex_enter(&pidlock);
2675 	for (pp = practive; pp != NULL; pp = pp->p_next) {
2676 		mutex_enter(&pp->p_lock);
2677 		tp = pp->p_tlist;
2678 		/*
2679 		 * Skip zombies, kernel processes, and processes in
2680 		 * other zones, if called from a non-global zone.
2681 		 */
2682 		if (tp == NULL || (pp->p_flag & SSYS) ||
2683 		    !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2684 			mutex_exit(&pp->p_lock);
2685 			continue;
2686 		}
2687 		do {
2688 			if (tp->t_bind_cpu != cpu)
2689 				continue;
2690 			err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2691 			if (ret == 0)
2692 				ret = err;
2693 		} while ((tp = tp->t_forw) != pp->p_tlist);
2694 		mutex_exit(&pp->p_lock);
2695 	}
2696 	mutex_exit(&pidlock);
2697 	if (ret == 0)
2698 		ret = berr;
2699 	return (ret);
2700 }
2701 
2702 
2703 /*
2704  * Destroy all remaining bound threads on a cpu.
2705  */
2706 void
2707 cpu_destroy_bound_threads(cpu_t *cp)
2708 {
2709 	extern id_t syscid;
2710 	register kthread_id_t	t, tlist, tnext;
2711 
2712 	/*
2713 	 * Destroy all remaining bound threads on the cpu.  This
2714 	 * should include both the interrupt threads and the idle thread.
2715 	 * This requires some care, since we need to traverse the
2716 	 * thread list with the pidlock mutex locked, but thread_free
2717 	 * also locks the pidlock mutex.  So, we collect the threads
2718 	 * we're going to reap in a list headed by "tlist", then we
2719 	 * unlock the pidlock mutex and traverse the tlist list,
2720 	 * doing thread_free's on the thread's.	 Simple, n'est pas?
2721 	 * Also, this depends on thread_free not mucking with the
2722 	 * t_next and t_prev links of the thread.
2723 	 */
2724 
2725 	if ((t = curthread) != NULL) {
2726 
2727 		tlist = NULL;
2728 		mutex_enter(&pidlock);
2729 		do {
2730 			tnext = t->t_next;
2731 			if (t->t_bound_cpu == cp) {
2732 
2733 				/*
2734 				 * We've found a bound thread, carefully unlink
2735 				 * it out of the thread list, and add it to
2736 				 * our "tlist".	 We "know" we don't have to
2737 				 * worry about unlinking curthread (the thread
2738 				 * that is executing this code).
2739 				 */
2740 				t->t_next->t_prev = t->t_prev;
2741 				t->t_prev->t_next = t->t_next;
2742 				t->t_next = tlist;
2743 				tlist = t;
2744 				ASSERT(t->t_cid == syscid);
2745 				/* wake up anyone blocked in thread_join */
2746 				cv_broadcast(&t->t_joincv);
2747 				/*
2748 				 * t_lwp set by interrupt threads and not
2749 				 * cleared.
2750 				 */
2751 				t->t_lwp = NULL;
2752 				/*
2753 				 * Pause and idle threads always have
2754 				 * t_state set to TS_ONPROC.
2755 				 */
2756 				t->t_state = TS_FREE;
2757 				t->t_prev = NULL;	/* Just in case */
2758 			}
2759 
2760 		} while ((t = tnext) != curthread);
2761 
2762 		mutex_exit(&pidlock);
2763 
2764 
2765 		for (t = tlist; t != NULL; t = tnext) {
2766 			tnext = t->t_next;
2767 			thread_free(t);
2768 		}
2769 	}
2770 }
2771 
2772 /*
2773  * processor_info(2) and p_online(2) status support functions
2774  *   The constants returned by the cpu_get_state() and cpu_get_state_str() are
2775  *   for use in communicating processor state information to userland.  Kernel
2776  *   subsystems should only be using the cpu_flags value directly.  Subsystems
2777  *   modifying cpu_flags should record the state change via a call to the
2778  *   cpu_set_state().
2779  */
2780 
2781 /*
2782  * Update the pi_state of this CPU.  This function provides the CPU status for
2783  * the information returned by processor_info(2).
2784  */
2785 void
2786 cpu_set_state(cpu_t *cpu)
2787 {
2788 	ASSERT(MUTEX_HELD(&cpu_lock));
2789 	cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
2790 	cpu->cpu_state_begin = gethrestime_sec();
2791 	pool_cpu_mod = gethrtime();
2792 }
2793 
2794 /*
2795  * Return offline/online/other status for the indicated CPU.  Use only for
2796  * communication with user applications; cpu_flags provides the in-kernel
2797  * interface.
2798  */
2799 int
2800 cpu_get_state(cpu_t *cpu)
2801 {
2802 	ASSERT(MUTEX_HELD(&cpu_lock));
2803 	if (cpu->cpu_flags & CPU_POWEROFF)
2804 		return (P_POWEROFF);
2805 	else if (cpu->cpu_flags & CPU_FAULTED)
2806 		return (P_FAULTED);
2807 	else if (cpu->cpu_flags & CPU_SPARE)
2808 		return (P_SPARE);
2809 	else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
2810 		return (P_OFFLINE);
2811 	else if (cpu->cpu_flags & CPU_ENABLE)
2812 		return (P_ONLINE);
2813 	else
2814 		return (P_NOINTR);
2815 }
2816 
2817 /*
2818  * Return processor_info(2) state as a string.
2819  */
2820 const char *
2821 cpu_get_state_str(cpu_t *cpu)
2822 {
2823 	const char *string;
2824 
2825 	switch (cpu_get_state(cpu)) {
2826 	case P_ONLINE:
2827 		string = PS_ONLINE;
2828 		break;
2829 	case P_POWEROFF:
2830 		string = PS_POWEROFF;
2831 		break;
2832 	case P_NOINTR:
2833 		string = PS_NOINTR;
2834 		break;
2835 	case P_SPARE:
2836 		string = PS_SPARE;
2837 		break;
2838 	case P_FAULTED:
2839 		string = PS_FAULTED;
2840 		break;
2841 	case P_OFFLINE:
2842 		string = PS_OFFLINE;
2843 		break;
2844 	default:
2845 		string = "unknown";
2846 		break;
2847 	}
2848 	return (string);
2849 }
2850 
2851 /*
2852  * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
2853  * kstats, respectively.  This is done when a CPU is initialized or placed
2854  * online via p_online(2).
2855  */
2856 static void
2857 cpu_stats_kstat_create(cpu_t *cp)
2858 {
2859 	int 	instance = cp->cpu_id;
2860 	char 	*module = "cpu";
2861 	char 	*class = "misc";
2862 	kstat_t	*ksp;
2863 	zoneid_t zoneid;
2864 
2865 	ASSERT(MUTEX_HELD(&cpu_lock));
2866 
2867 	if (pool_pset_enabled())
2868 		zoneid = GLOBAL_ZONEID;
2869 	else
2870 		zoneid = ALL_ZONES;
2871 	/*
2872 	 * Create named kstats
2873 	 */
2874 #define	CPU_STATS_KS_CREATE(name, tsize, update_func)                    \
2875 	ksp = kstat_create_zone(module, instance, (name), class,         \
2876 	    KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0,       \
2877 	    zoneid);                                                     \
2878 	if (ksp != NULL) {                                               \
2879 		ksp->ks_private = cp;                                    \
2880 		ksp->ks_update = (update_func);                          \
2881 		kstat_install(ksp);                                      \
2882 	} else                                                           \
2883 		cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
2884 		    module, instance, (name));
2885 
2886 	CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
2887 	    cpu_sys_stats_ks_update);
2888 	CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
2889 	    cpu_vm_stats_ks_update);
2890 
2891 	/*
2892 	 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
2893 	 */
2894 	ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
2895 	    "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
2896 	if (ksp != NULL) {
2897 		ksp->ks_update = cpu_stat_ks_update;
2898 		ksp->ks_private = cp;
2899 		kstat_install(ksp);
2900 	}
2901 }
2902 
2903 static void
2904 cpu_stats_kstat_destroy(cpu_t *cp)
2905 {
2906 	char ks_name[KSTAT_STRLEN];
2907 
2908 	(void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
2909 	kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
2910 
2911 	kstat_delete_byname("cpu", cp->cpu_id, "sys");
2912 	kstat_delete_byname("cpu", cp->cpu_id, "vm");
2913 }
2914 
2915 static int
2916 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
2917 {
2918 	cpu_t *cp = (cpu_t *)ksp->ks_private;
2919 	struct cpu_sys_stats_ks_data *csskd;
2920 	cpu_sys_stats_t *css;
2921 	hrtime_t msnsecs[NCMSTATES];
2922 	int	i;
2923 
2924 	if (rw == KSTAT_WRITE)
2925 		return (EACCES);
2926 
2927 	csskd = ksp->ks_data;
2928 	css = &cp->cpu_stats.sys;
2929 
2930 	/*
2931 	 * Read CPU mstate, but compare with the last values we
2932 	 * received to make sure that the returned kstats never
2933 	 * decrease.
2934 	 */
2935 
2936 	get_cpu_mstate(cp, msnsecs);
2937 	if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
2938 		msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
2939 	if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
2940 		msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
2941 	if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
2942 		msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
2943 
2944 	bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
2945 	    sizeof (cpu_sys_stats_ks_data_template));
2946 
2947 	csskd->cpu_ticks_wait.value.ui64 = 0;
2948 	csskd->wait_ticks_io.value.ui64 = 0;
2949 
2950 	csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
2951 	csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
2952 	csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
2953 	csskd->cpu_ticks_idle.value.ui64 =
2954 	    NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
2955 	csskd->cpu_ticks_user.value.ui64 =
2956 	    NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
2957 	csskd->cpu_ticks_kernel.value.ui64 =
2958 	    NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
2959 	csskd->bread.value.ui64 = css->bread;
2960 	csskd->bwrite.value.ui64 = css->bwrite;
2961 	csskd->lread.value.ui64 = css->lread;
2962 	csskd->lwrite.value.ui64 = css->lwrite;
2963 	csskd->phread.value.ui64 = css->phread;
2964 	csskd->phwrite.value.ui64 = css->phwrite;
2965 	csskd->pswitch.value.ui64 = css->pswitch;
2966 	csskd->trap.value.ui64 = css->trap;
2967 	csskd->intr.value.ui64 = 0;
2968 	for (i = 0; i < PIL_MAX; i++)
2969 		csskd->intr.value.ui64 += css->intr[i];
2970 	csskd->syscall.value.ui64 = css->syscall;
2971 	csskd->sysread.value.ui64 = css->sysread;
2972 	csskd->syswrite.value.ui64 = css->syswrite;
2973 	csskd->sysfork.value.ui64 = css->sysfork;
2974 	csskd->sysvfork.value.ui64 = css->sysvfork;
2975 	csskd->sysexec.value.ui64 = css->sysexec;
2976 	csskd->readch.value.ui64 = css->readch;
2977 	csskd->writech.value.ui64 = css->writech;
2978 	csskd->rcvint.value.ui64 = css->rcvint;
2979 	csskd->xmtint.value.ui64 = css->xmtint;
2980 	csskd->mdmint.value.ui64 = css->mdmint;
2981 	csskd->rawch.value.ui64 = css->rawch;
2982 	csskd->canch.value.ui64 = css->canch;
2983 	csskd->outch.value.ui64 = css->outch;
2984 	csskd->msg.value.ui64 = css->msg;
2985 	csskd->sema.value.ui64 = css->sema;
2986 	csskd->namei.value.ui64 = css->namei;
2987 	csskd->ufsiget.value.ui64 = css->ufsiget;
2988 	csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
2989 	csskd->ufsipage.value.ui64 = css->ufsipage;
2990 	csskd->ufsinopage.value.ui64 = css->ufsinopage;
2991 	csskd->procovf.value.ui64 = css->procovf;
2992 	csskd->intrthread.value.ui64 = 0;
2993 	for (i = 0; i < LOCK_LEVEL; i++)
2994 		csskd->intrthread.value.ui64 += css->intr[i];
2995 	csskd->intrblk.value.ui64 = css->intrblk;
2996 	csskd->intrunpin.value.ui64 = css->intrunpin;
2997 	csskd->idlethread.value.ui64 = css->idlethread;
2998 	csskd->inv_swtch.value.ui64 = css->inv_swtch;
2999 	csskd->nthreads.value.ui64 = css->nthreads;
3000 	csskd->cpumigrate.value.ui64 = css->cpumigrate;
3001 	csskd->xcalls.value.ui64 = css->xcalls;
3002 	csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3003 	csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3004 	csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3005 	csskd->modload.value.ui64 = css->modload;
3006 	csskd->modunload.value.ui64 = css->modunload;
3007 	csskd->bawrite.value.ui64 = css->bawrite;
3008 	csskd->iowait.value.ui64 = css->iowait;
3009 
3010 	return (0);
3011 }
3012 
3013 static int
3014 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3015 {
3016 	cpu_t *cp = (cpu_t *)ksp->ks_private;
3017 	struct cpu_vm_stats_ks_data *cvskd;
3018 	cpu_vm_stats_t *cvs;
3019 
3020 	if (rw == KSTAT_WRITE)
3021 		return (EACCES);
3022 
3023 	cvs = &cp->cpu_stats.vm;
3024 	cvskd = ksp->ks_data;
3025 
3026 	bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3027 	    sizeof (cpu_vm_stats_ks_data_template));
3028 	cvskd->pgrec.value.ui64 = cvs->pgrec;
3029 	cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3030 	cvskd->pgin.value.ui64 = cvs->pgin;
3031 	cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3032 	cvskd->pgout.value.ui64 = cvs->pgout;
3033 	cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3034 	cvskd->swapin.value.ui64 = cvs->swapin;
3035 	cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3036 	cvskd->swapout.value.ui64 = cvs->swapout;
3037 	cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3038 	cvskd->zfod.value.ui64 = cvs->zfod;
3039 	cvskd->dfree.value.ui64 = cvs->dfree;
3040 	cvskd->scan.value.ui64 = cvs->scan;
3041 	cvskd->rev.value.ui64 = cvs->rev;
3042 	cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3043 	cvskd->as_fault.value.ui64 = cvs->as_fault;
3044 	cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3045 	cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3046 	cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3047 	cvskd->softlock.value.ui64 = cvs->softlock;
3048 	cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3049 	cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3050 	cvskd->execpgin.value.ui64 = cvs->execpgin;
3051 	cvskd->execpgout.value.ui64 = cvs->execpgout;
3052 	cvskd->execfree.value.ui64 = cvs->execfree;
3053 	cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3054 	cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3055 	cvskd->anonfree.value.ui64 = cvs->anonfree;
3056 	cvskd->fspgin.value.ui64 = cvs->fspgin;
3057 	cvskd->fspgout.value.ui64 = cvs->fspgout;
3058 	cvskd->fsfree.value.ui64 = cvs->fsfree;
3059 
3060 	return (0);
3061 }
3062 
3063 static int
3064 cpu_stat_ks_update(kstat_t *ksp, int rw)
3065 {
3066 	cpu_stat_t *cso;
3067 	cpu_t *cp;
3068 	int i;
3069 	hrtime_t msnsecs[NCMSTATES];
3070 
3071 	cso = (cpu_stat_t *)ksp->ks_data;
3072 	cp = (cpu_t *)ksp->ks_private;
3073 
3074 	if (rw == KSTAT_WRITE)
3075 		return (EACCES);
3076 
3077 	/*
3078 	 * Read CPU mstate, but compare with the last values we
3079 	 * received to make sure that the returned kstats never
3080 	 * decrease.
3081 	 */
3082 
3083 	get_cpu_mstate(cp, msnsecs);
3084 	msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3085 	msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3086 	msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3087 	if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3088 		cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3089 	if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3090 		cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3091 	if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3092 		cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3093 	cso->cpu_sysinfo.cpu[CPU_WAIT] 	= 0;
3094 	cso->cpu_sysinfo.wait[W_IO] 	= 0;
3095 	cso->cpu_sysinfo.wait[W_SWAP]	= 0;
3096 	cso->cpu_sysinfo.wait[W_PIO]	= 0;
3097 	cso->cpu_sysinfo.bread 		= CPU_STATS(cp, sys.bread);
3098 	cso->cpu_sysinfo.bwrite 	= CPU_STATS(cp, sys.bwrite);
3099 	cso->cpu_sysinfo.lread 		= CPU_STATS(cp, sys.lread);
3100 	cso->cpu_sysinfo.lwrite 	= CPU_STATS(cp, sys.lwrite);
3101 	cso->cpu_sysinfo.phread 	= CPU_STATS(cp, sys.phread);
3102 	cso->cpu_sysinfo.phwrite 	= CPU_STATS(cp, sys.phwrite);
3103 	cso->cpu_sysinfo.pswitch 	= CPU_STATS(cp, sys.pswitch);
3104 	cso->cpu_sysinfo.trap 		= CPU_STATS(cp, sys.trap);
3105 	cso->cpu_sysinfo.intr		= 0;
3106 	for (i = 0; i < PIL_MAX; i++)
3107 		cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3108 	cso->cpu_sysinfo.syscall	= CPU_STATS(cp, sys.syscall);
3109 	cso->cpu_sysinfo.sysread	= CPU_STATS(cp, sys.sysread);
3110 	cso->cpu_sysinfo.syswrite	= CPU_STATS(cp, sys.syswrite);
3111 	cso->cpu_sysinfo.sysfork	= CPU_STATS(cp, sys.sysfork);
3112 	cso->cpu_sysinfo.sysvfork	= CPU_STATS(cp, sys.sysvfork);
3113 	cso->cpu_sysinfo.sysexec	= CPU_STATS(cp, sys.sysexec);
3114 	cso->cpu_sysinfo.readch		= CPU_STATS(cp, sys.readch);
3115 	cso->cpu_sysinfo.writech	= CPU_STATS(cp, sys.writech);
3116 	cso->cpu_sysinfo.rcvint		= CPU_STATS(cp, sys.rcvint);
3117 	cso->cpu_sysinfo.xmtint		= CPU_STATS(cp, sys.xmtint);
3118 	cso->cpu_sysinfo.mdmint		= CPU_STATS(cp, sys.mdmint);
3119 	cso->cpu_sysinfo.rawch		= CPU_STATS(cp, sys.rawch);
3120 	cso->cpu_sysinfo.canch		= CPU_STATS(cp, sys.canch);
3121 	cso->cpu_sysinfo.outch		= CPU_STATS(cp, sys.outch);
3122 	cso->cpu_sysinfo.msg		= CPU_STATS(cp, sys.msg);
3123 	cso->cpu_sysinfo.sema		= CPU_STATS(cp, sys.sema);
3124 	cso->cpu_sysinfo.namei		= CPU_STATS(cp, sys.namei);
3125 	cso->cpu_sysinfo.ufsiget	= CPU_STATS(cp, sys.ufsiget);
3126 	cso->cpu_sysinfo.ufsdirblk	= CPU_STATS(cp, sys.ufsdirblk);
3127 	cso->cpu_sysinfo.ufsipage	= CPU_STATS(cp, sys.ufsipage);
3128 	cso->cpu_sysinfo.ufsinopage	= CPU_STATS(cp, sys.ufsinopage);
3129 	cso->cpu_sysinfo.inodeovf	= 0;
3130 	cso->cpu_sysinfo.fileovf	= 0;
3131 	cso->cpu_sysinfo.procovf	= CPU_STATS(cp, sys.procovf);
3132 	cso->cpu_sysinfo.intrthread	= 0;
3133 	for (i = 0; i < LOCK_LEVEL; i++)
3134 		cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3135 	cso->cpu_sysinfo.intrblk	= CPU_STATS(cp, sys.intrblk);
3136 	cso->cpu_sysinfo.idlethread	= CPU_STATS(cp, sys.idlethread);
3137 	cso->cpu_sysinfo.inv_swtch	= CPU_STATS(cp, sys.inv_swtch);
3138 	cso->cpu_sysinfo.nthreads	= CPU_STATS(cp, sys.nthreads);
3139 	cso->cpu_sysinfo.cpumigrate	= CPU_STATS(cp, sys.cpumigrate);
3140 	cso->cpu_sysinfo.xcalls		= CPU_STATS(cp, sys.xcalls);
3141 	cso->cpu_sysinfo.mutex_adenters	= CPU_STATS(cp, sys.mutex_adenters);
3142 	cso->cpu_sysinfo.rw_rdfails	= CPU_STATS(cp, sys.rw_rdfails);
3143 	cso->cpu_sysinfo.rw_wrfails	= CPU_STATS(cp, sys.rw_wrfails);
3144 	cso->cpu_sysinfo.modload	= CPU_STATS(cp, sys.modload);
3145 	cso->cpu_sysinfo.modunload	= CPU_STATS(cp, sys.modunload);
3146 	cso->cpu_sysinfo.bawrite	= CPU_STATS(cp, sys.bawrite);
3147 	cso->cpu_sysinfo.rw_enters	= 0;
3148 	cso->cpu_sysinfo.win_uo_cnt	= 0;
3149 	cso->cpu_sysinfo.win_uu_cnt	= 0;
3150 	cso->cpu_sysinfo.win_so_cnt	= 0;
3151 	cso->cpu_sysinfo.win_su_cnt	= 0;
3152 	cso->cpu_sysinfo.win_suo_cnt	= 0;
3153 
3154 	cso->cpu_syswait.iowait		= CPU_STATS(cp, sys.iowait);
3155 	cso->cpu_syswait.swap		= 0;
3156 	cso->cpu_syswait.physio		= 0;
3157 
3158 	cso->cpu_vminfo.pgrec		= CPU_STATS(cp, vm.pgrec);
3159 	cso->cpu_vminfo.pgfrec		= CPU_STATS(cp, vm.pgfrec);
3160 	cso->cpu_vminfo.pgin		= CPU_STATS(cp, vm.pgin);
3161 	cso->cpu_vminfo.pgpgin		= CPU_STATS(cp, vm.pgpgin);
3162 	cso->cpu_vminfo.pgout		= CPU_STATS(cp, vm.pgout);
3163 	cso->cpu_vminfo.pgpgout		= CPU_STATS(cp, vm.pgpgout);
3164 	cso->cpu_vminfo.swapin		= CPU_STATS(cp, vm.swapin);
3165 	cso->cpu_vminfo.pgswapin	= CPU_STATS(cp, vm.pgswapin);
3166 	cso->cpu_vminfo.swapout		= CPU_STATS(cp, vm.swapout);
3167 	cso->cpu_vminfo.pgswapout	= CPU_STATS(cp, vm.pgswapout);
3168 	cso->cpu_vminfo.zfod		= CPU_STATS(cp, vm.zfod);
3169 	cso->cpu_vminfo.dfree		= CPU_STATS(cp, vm.dfree);
3170 	cso->cpu_vminfo.scan		= CPU_STATS(cp, vm.scan);
3171 	cso->cpu_vminfo.rev		= CPU_STATS(cp, vm.rev);
3172 	cso->cpu_vminfo.hat_fault	= CPU_STATS(cp, vm.hat_fault);
3173 	cso->cpu_vminfo.as_fault	= CPU_STATS(cp, vm.as_fault);
3174 	cso->cpu_vminfo.maj_fault	= CPU_STATS(cp, vm.maj_fault);
3175 	cso->cpu_vminfo.cow_fault	= CPU_STATS(cp, vm.cow_fault);
3176 	cso->cpu_vminfo.prot_fault	= CPU_STATS(cp, vm.prot_fault);
3177 	cso->cpu_vminfo.softlock	= CPU_STATS(cp, vm.softlock);
3178 	cso->cpu_vminfo.kernel_asflt	= CPU_STATS(cp, vm.kernel_asflt);
3179 	cso->cpu_vminfo.pgrrun		= CPU_STATS(cp, vm.pgrrun);
3180 	cso->cpu_vminfo.execpgin	= CPU_STATS(cp, vm.execpgin);
3181 	cso->cpu_vminfo.execpgout	= CPU_STATS(cp, vm.execpgout);
3182 	cso->cpu_vminfo.execfree	= CPU_STATS(cp, vm.execfree);
3183 	cso->cpu_vminfo.anonpgin	= CPU_STATS(cp, vm.anonpgin);
3184 	cso->cpu_vminfo.anonpgout	= CPU_STATS(cp, vm.anonpgout);
3185 	cso->cpu_vminfo.anonfree	= CPU_STATS(cp, vm.anonfree);
3186 	cso->cpu_vminfo.fspgin		= CPU_STATS(cp, vm.fspgin);
3187 	cso->cpu_vminfo.fspgout		= CPU_STATS(cp, vm.fspgout);
3188 	cso->cpu_vminfo.fsfree		= CPU_STATS(cp, vm.fsfree);
3189 
3190 	return (0);
3191 }
3192