xref: /titanic_41/usr/src/uts/common/os/cpu.c (revision bce835f2166e1510cc776830775859ba3e49c1ce)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2012 by Delphix. All rights reserved.
24  */
25 
26 /*
27  * Architecture-independent CPU control functions.
28  */
29 
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/var.h>
33 #include <sys/thread.h>
34 #include <sys/cpuvar.h>
35 #include <sys/cpu_event.h>
36 #include <sys/kstat.h>
37 #include <sys/uadmin.h>
38 #include <sys/systm.h>
39 #include <sys/errno.h>
40 #include <sys/cmn_err.h>
41 #include <sys/procset.h>
42 #include <sys/processor.h>
43 #include <sys/debug.h>
44 #include <sys/cpupart.h>
45 #include <sys/lgrp.h>
46 #include <sys/pset.h>
47 #include <sys/pghw.h>
48 #include <sys/kmem.h>
49 #include <sys/kmem_impl.h>	/* to set per-cpu kmem_cache offset */
50 #include <sys/atomic.h>
51 #include <sys/callb.h>
52 #include <sys/vtrace.h>
53 #include <sys/cyclic.h>
54 #include <sys/bitmap.h>
55 #include <sys/nvpair.h>
56 #include <sys/pool_pset.h>
57 #include <sys/msacct.h>
58 #include <sys/time.h>
59 #include <sys/archsystm.h>
60 #include <sys/sdt.h>
61 #if defined(__x86) || defined(__amd64)
62 #include <sys/x86_archext.h>
63 #endif
64 #include <sys/callo.h>
65 
66 extern int	mp_cpu_start(cpu_t *);
67 extern int	mp_cpu_stop(cpu_t *);
68 extern int	mp_cpu_poweron(cpu_t *);
69 extern int	mp_cpu_poweroff(cpu_t *);
70 extern int	mp_cpu_configure(int);
71 extern int	mp_cpu_unconfigure(int);
72 extern void	mp_cpu_faulted_enter(cpu_t *);
73 extern void	mp_cpu_faulted_exit(cpu_t *);
74 
75 extern int cmp_cpu_to_chip(processorid_t cpuid);
76 #ifdef __sparcv9
77 extern char *cpu_fru_fmri(cpu_t *cp);
78 #endif
79 
80 static void cpu_add_active_internal(cpu_t *cp);
81 static void cpu_remove_active(cpu_t *cp);
82 static void cpu_info_kstat_create(cpu_t *cp);
83 static void cpu_info_kstat_destroy(cpu_t *cp);
84 static void cpu_stats_kstat_create(cpu_t *cp);
85 static void cpu_stats_kstat_destroy(cpu_t *cp);
86 
87 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
88 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
89 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
90 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
91 
92 /*
93  * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
94  * max_cpu_seqid_ever, and dispatch queue reallocations.  The lock ordering with
95  * respect to related locks is:
96  *
97  *	cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
98  *
99  * Warning:  Certain sections of code do not use the cpu_lock when
100  * traversing the cpu_list (e.g. mutex_vector_enter(), clock()).  Since
101  * all cpus are paused during modifications to this list, a solution
102  * to protect the list is too either disable kernel preemption while
103  * walking the list, *or* recheck the cpu_next pointer at each
104  * iteration in the loop.  Note that in no cases can any cached
105  * copies of the cpu pointers be kept as they may become invalid.
106  */
107 kmutex_t	cpu_lock;
108 cpu_t		*cpu_list;		/* list of all CPUs */
109 cpu_t		*clock_cpu_list;	/* used by clock to walk CPUs */
110 cpu_t		*cpu_active;		/* list of active CPUs */
111 static cpuset_t	cpu_available;		/* set of available CPUs */
112 cpuset_t	cpu_seqid_inuse;	/* which cpu_seqids are in use */
113 
114 cpu_t		**cpu_seq;		/* ptrs to CPUs, indexed by seq_id */
115 
116 /*
117  * max_ncpus keeps the max cpus the system can have. Initially
118  * it's NCPU, but since most archs scan the devtree for cpus
119  * fairly early on during boot, the real max can be known before
120  * ncpus is set (useful for early NCPU based allocations).
121  */
122 int max_ncpus = NCPU;
123 /*
124  * platforms that set max_ncpus to maxiumum number of cpus that can be
125  * dynamically added will set boot_max_ncpus to the number of cpus found
126  * at device tree scan time during boot.
127  */
128 int boot_max_ncpus = -1;
129 int boot_ncpus = -1;
130 /*
131  * Maximum possible CPU id.  This can never be >= NCPU since NCPU is
132  * used to size arrays that are indexed by CPU id.
133  */
134 processorid_t max_cpuid = NCPU - 1;
135 
136 /*
137  * Maximum cpu_seqid was given. This number can only grow and never shrink. It
138  * can be used to optimize NCPU loops to avoid going through CPUs which were
139  * never on-line.
140  */
141 processorid_t max_cpu_seqid_ever = 0;
142 
143 int ncpus = 1;
144 int ncpus_online = 1;
145 
146 /*
147  * CPU that we're trying to offline.  Protected by cpu_lock.
148  */
149 cpu_t *cpu_inmotion;
150 
151 /*
152  * Can be raised to suppress further weakbinding, which are instead
153  * satisfied by disabling preemption.  Must be raised/lowered under cpu_lock,
154  * while individual thread weakbinding synchronization is done under thread
155  * lock.
156  */
157 int weakbindingbarrier;
158 
159 /*
160  * Variables used in pause_cpus().
161  */
162 static volatile char safe_list[NCPU];
163 
164 static struct _cpu_pause_info {
165 	int		cp_spl;		/* spl saved in pause_cpus() */
166 	volatile int	cp_go;		/* Go signal sent after all ready */
167 	int		cp_count;	/* # of CPUs to pause */
168 	ksema_t		cp_sem;		/* synch pause_cpus & cpu_pause */
169 	kthread_id_t	cp_paused;
170 	void		*(*cp_func)(void *);
171 } cpu_pause_info;
172 
173 static kmutex_t pause_free_mutex;
174 static kcondvar_t pause_free_cv;
175 
176 
177 static struct cpu_sys_stats_ks_data {
178 	kstat_named_t cpu_ticks_idle;
179 	kstat_named_t cpu_ticks_user;
180 	kstat_named_t cpu_ticks_kernel;
181 	kstat_named_t cpu_ticks_wait;
182 	kstat_named_t cpu_nsec_idle;
183 	kstat_named_t cpu_nsec_user;
184 	kstat_named_t cpu_nsec_kernel;
185 	kstat_named_t cpu_nsec_dtrace;
186 	kstat_named_t cpu_nsec_intr;
187 	kstat_named_t cpu_load_intr;
188 	kstat_named_t wait_ticks_io;
189 	kstat_named_t dtrace_probes;
190 	kstat_named_t bread;
191 	kstat_named_t bwrite;
192 	kstat_named_t lread;
193 	kstat_named_t lwrite;
194 	kstat_named_t phread;
195 	kstat_named_t phwrite;
196 	kstat_named_t pswitch;
197 	kstat_named_t trap;
198 	kstat_named_t intr;
199 	kstat_named_t syscall;
200 	kstat_named_t sysread;
201 	kstat_named_t syswrite;
202 	kstat_named_t sysfork;
203 	kstat_named_t sysvfork;
204 	kstat_named_t sysexec;
205 	kstat_named_t readch;
206 	kstat_named_t writech;
207 	kstat_named_t rcvint;
208 	kstat_named_t xmtint;
209 	kstat_named_t mdmint;
210 	kstat_named_t rawch;
211 	kstat_named_t canch;
212 	kstat_named_t outch;
213 	kstat_named_t msg;
214 	kstat_named_t sema;
215 	kstat_named_t namei;
216 	kstat_named_t ufsiget;
217 	kstat_named_t ufsdirblk;
218 	kstat_named_t ufsipage;
219 	kstat_named_t ufsinopage;
220 	kstat_named_t procovf;
221 	kstat_named_t intrthread;
222 	kstat_named_t intrblk;
223 	kstat_named_t intrunpin;
224 	kstat_named_t idlethread;
225 	kstat_named_t inv_swtch;
226 	kstat_named_t nthreads;
227 	kstat_named_t cpumigrate;
228 	kstat_named_t xcalls;
229 	kstat_named_t mutex_adenters;
230 	kstat_named_t rw_rdfails;
231 	kstat_named_t rw_wrfails;
232 	kstat_named_t modload;
233 	kstat_named_t modunload;
234 	kstat_named_t bawrite;
235 	kstat_named_t iowait;
236 } cpu_sys_stats_ks_data_template = {
237 	{ "cpu_ticks_idle",	KSTAT_DATA_UINT64 },
238 	{ "cpu_ticks_user",	KSTAT_DATA_UINT64 },
239 	{ "cpu_ticks_kernel",	KSTAT_DATA_UINT64 },
240 	{ "cpu_ticks_wait",	KSTAT_DATA_UINT64 },
241 	{ "cpu_nsec_idle",	KSTAT_DATA_UINT64 },
242 	{ "cpu_nsec_user",	KSTAT_DATA_UINT64 },
243 	{ "cpu_nsec_kernel",	KSTAT_DATA_UINT64 },
244 	{ "cpu_nsec_dtrace",	KSTAT_DATA_UINT64 },
245 	{ "cpu_nsec_intr",	KSTAT_DATA_UINT64 },
246 	{ "cpu_load_intr",	KSTAT_DATA_UINT64 },
247 	{ "wait_ticks_io",	KSTAT_DATA_UINT64 },
248 	{ "dtrace_probes",	KSTAT_DATA_UINT64 },
249 	{ "bread",		KSTAT_DATA_UINT64 },
250 	{ "bwrite",		KSTAT_DATA_UINT64 },
251 	{ "lread",		KSTAT_DATA_UINT64 },
252 	{ "lwrite",		KSTAT_DATA_UINT64 },
253 	{ "phread",		KSTAT_DATA_UINT64 },
254 	{ "phwrite",		KSTAT_DATA_UINT64 },
255 	{ "pswitch",		KSTAT_DATA_UINT64 },
256 	{ "trap",		KSTAT_DATA_UINT64 },
257 	{ "intr",		KSTAT_DATA_UINT64 },
258 	{ "syscall",		KSTAT_DATA_UINT64 },
259 	{ "sysread",		KSTAT_DATA_UINT64 },
260 	{ "syswrite",		KSTAT_DATA_UINT64 },
261 	{ "sysfork",		KSTAT_DATA_UINT64 },
262 	{ "sysvfork",		KSTAT_DATA_UINT64 },
263 	{ "sysexec",		KSTAT_DATA_UINT64 },
264 	{ "readch",		KSTAT_DATA_UINT64 },
265 	{ "writech",		KSTAT_DATA_UINT64 },
266 	{ "rcvint",		KSTAT_DATA_UINT64 },
267 	{ "xmtint",		KSTAT_DATA_UINT64 },
268 	{ "mdmint",		KSTAT_DATA_UINT64 },
269 	{ "rawch",		KSTAT_DATA_UINT64 },
270 	{ "canch",		KSTAT_DATA_UINT64 },
271 	{ "outch",		KSTAT_DATA_UINT64 },
272 	{ "msg",		KSTAT_DATA_UINT64 },
273 	{ "sema",		KSTAT_DATA_UINT64 },
274 	{ "namei",		KSTAT_DATA_UINT64 },
275 	{ "ufsiget",		KSTAT_DATA_UINT64 },
276 	{ "ufsdirblk",		KSTAT_DATA_UINT64 },
277 	{ "ufsipage",		KSTAT_DATA_UINT64 },
278 	{ "ufsinopage",		KSTAT_DATA_UINT64 },
279 	{ "procovf",		KSTAT_DATA_UINT64 },
280 	{ "intrthread",		KSTAT_DATA_UINT64 },
281 	{ "intrblk",		KSTAT_DATA_UINT64 },
282 	{ "intrunpin",		KSTAT_DATA_UINT64 },
283 	{ "idlethread",		KSTAT_DATA_UINT64 },
284 	{ "inv_swtch",		KSTAT_DATA_UINT64 },
285 	{ "nthreads",		KSTAT_DATA_UINT64 },
286 	{ "cpumigrate",		KSTAT_DATA_UINT64 },
287 	{ "xcalls",		KSTAT_DATA_UINT64 },
288 	{ "mutex_adenters",	KSTAT_DATA_UINT64 },
289 	{ "rw_rdfails",		KSTAT_DATA_UINT64 },
290 	{ "rw_wrfails",		KSTAT_DATA_UINT64 },
291 	{ "modload",		KSTAT_DATA_UINT64 },
292 	{ "modunload",		KSTAT_DATA_UINT64 },
293 	{ "bawrite",		KSTAT_DATA_UINT64 },
294 	{ "iowait",		KSTAT_DATA_UINT64 },
295 };
296 
297 static struct cpu_vm_stats_ks_data {
298 	kstat_named_t pgrec;
299 	kstat_named_t pgfrec;
300 	kstat_named_t pgin;
301 	kstat_named_t pgpgin;
302 	kstat_named_t pgout;
303 	kstat_named_t pgpgout;
304 	kstat_named_t swapin;
305 	kstat_named_t pgswapin;
306 	kstat_named_t swapout;
307 	kstat_named_t pgswapout;
308 	kstat_named_t zfod;
309 	kstat_named_t dfree;
310 	kstat_named_t scan;
311 	kstat_named_t rev;
312 	kstat_named_t hat_fault;
313 	kstat_named_t as_fault;
314 	kstat_named_t maj_fault;
315 	kstat_named_t cow_fault;
316 	kstat_named_t prot_fault;
317 	kstat_named_t softlock;
318 	kstat_named_t kernel_asflt;
319 	kstat_named_t pgrrun;
320 	kstat_named_t execpgin;
321 	kstat_named_t execpgout;
322 	kstat_named_t execfree;
323 	kstat_named_t anonpgin;
324 	kstat_named_t anonpgout;
325 	kstat_named_t anonfree;
326 	kstat_named_t fspgin;
327 	kstat_named_t fspgout;
328 	kstat_named_t fsfree;
329 } cpu_vm_stats_ks_data_template = {
330 	{ "pgrec",		KSTAT_DATA_UINT64 },
331 	{ "pgfrec",		KSTAT_DATA_UINT64 },
332 	{ "pgin",		KSTAT_DATA_UINT64 },
333 	{ "pgpgin",		KSTAT_DATA_UINT64 },
334 	{ "pgout",		KSTAT_DATA_UINT64 },
335 	{ "pgpgout",		KSTAT_DATA_UINT64 },
336 	{ "swapin",		KSTAT_DATA_UINT64 },
337 	{ "pgswapin",		KSTAT_DATA_UINT64 },
338 	{ "swapout",		KSTAT_DATA_UINT64 },
339 	{ "pgswapout",		KSTAT_DATA_UINT64 },
340 	{ "zfod",		KSTAT_DATA_UINT64 },
341 	{ "dfree",		KSTAT_DATA_UINT64 },
342 	{ "scan",		KSTAT_DATA_UINT64 },
343 	{ "rev",		KSTAT_DATA_UINT64 },
344 	{ "hat_fault",		KSTAT_DATA_UINT64 },
345 	{ "as_fault",		KSTAT_DATA_UINT64 },
346 	{ "maj_fault",		KSTAT_DATA_UINT64 },
347 	{ "cow_fault",		KSTAT_DATA_UINT64 },
348 	{ "prot_fault",		KSTAT_DATA_UINT64 },
349 	{ "softlock",		KSTAT_DATA_UINT64 },
350 	{ "kernel_asflt",	KSTAT_DATA_UINT64 },
351 	{ "pgrrun",		KSTAT_DATA_UINT64 },
352 	{ "execpgin",		KSTAT_DATA_UINT64 },
353 	{ "execpgout",		KSTAT_DATA_UINT64 },
354 	{ "execfree",		KSTAT_DATA_UINT64 },
355 	{ "anonpgin",		KSTAT_DATA_UINT64 },
356 	{ "anonpgout",		KSTAT_DATA_UINT64 },
357 	{ "anonfree",		KSTAT_DATA_UINT64 },
358 	{ "fspgin",		KSTAT_DATA_UINT64 },
359 	{ "fspgout",		KSTAT_DATA_UINT64 },
360 	{ "fsfree",		KSTAT_DATA_UINT64 },
361 };
362 
363 /*
364  * Force the specified thread to migrate to the appropriate processor.
365  * Called with thread lock held, returns with it dropped.
366  */
367 static void
force_thread_migrate(kthread_id_t tp)368 force_thread_migrate(kthread_id_t tp)
369 {
370 	ASSERT(THREAD_LOCK_HELD(tp));
371 	if (tp == curthread) {
372 		THREAD_TRANSITION(tp);
373 		CL_SETRUN(tp);
374 		thread_unlock_nopreempt(tp);
375 		swtch();
376 	} else {
377 		if (tp->t_state == TS_ONPROC) {
378 			cpu_surrender(tp);
379 		} else if (tp->t_state == TS_RUN) {
380 			(void) dispdeq(tp);
381 			setbackdq(tp);
382 		}
383 		thread_unlock(tp);
384 	}
385 }
386 
387 /*
388  * Set affinity for a specified CPU.
389  * A reference count is incremented and the affinity is held until the
390  * reference count is decremented to zero by thread_affinity_clear().
391  * This is so regions of code requiring affinity can be nested.
392  * Caller needs to ensure that cpu_id remains valid, which can be
393  * done by holding cpu_lock across this call, unless the caller
394  * specifies CPU_CURRENT in which case the cpu_lock will be acquired
395  * by thread_affinity_set and CPU->cpu_id will be the target CPU.
396  */
397 void
thread_affinity_set(kthread_id_t t,int cpu_id)398 thread_affinity_set(kthread_id_t t, int cpu_id)
399 {
400 	cpu_t		*cp;
401 	int		c;
402 
403 	ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
404 
405 	if ((c = cpu_id) == CPU_CURRENT) {
406 		mutex_enter(&cpu_lock);
407 		cpu_id = CPU->cpu_id;
408 	}
409 	/*
410 	 * We should be asserting that cpu_lock is held here, but
411 	 * the NCA code doesn't acquire it.  The following assert
412 	 * should be uncommented when the NCA code is fixed.
413 	 *
414 	 * ASSERT(MUTEX_HELD(&cpu_lock));
415 	 */
416 	ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
417 	cp = cpu[cpu_id];
418 	ASSERT(cp != NULL);		/* user must provide a good cpu_id */
419 	/*
420 	 * If there is already a hard affinity requested, and this affinity
421 	 * conflicts with that, panic.
422 	 */
423 	thread_lock(t);
424 	if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
425 		panic("affinity_set: setting %p but already bound to %p",
426 		    (void *)cp, (void *)t->t_bound_cpu);
427 	}
428 	t->t_affinitycnt++;
429 	t->t_bound_cpu = cp;
430 
431 	/*
432 	 * Make sure we're running on the right CPU.
433 	 */
434 	if (cp != t->t_cpu || t != curthread) {
435 		force_thread_migrate(t);	/* drops thread lock */
436 	} else {
437 		thread_unlock(t);
438 	}
439 
440 	if (c == CPU_CURRENT)
441 		mutex_exit(&cpu_lock);
442 }
443 
444 /*
445  *	Wrapper for backward compatibility.
446  */
447 void
affinity_set(int cpu_id)448 affinity_set(int cpu_id)
449 {
450 	thread_affinity_set(curthread, cpu_id);
451 }
452 
453 /*
454  * Decrement the affinity reservation count and if it becomes zero,
455  * clear the CPU affinity for the current thread, or set it to the user's
456  * software binding request.
457  */
458 void
thread_affinity_clear(kthread_id_t t)459 thread_affinity_clear(kthread_id_t t)
460 {
461 	register processorid_t binding;
462 
463 	thread_lock(t);
464 	if (--t->t_affinitycnt == 0) {
465 		if ((binding = t->t_bind_cpu) == PBIND_NONE) {
466 			/*
467 			 * Adjust disp_max_unbound_pri if necessary.
468 			 */
469 			disp_adjust_unbound_pri(t);
470 			t->t_bound_cpu = NULL;
471 			if (t->t_cpu->cpu_part != t->t_cpupart) {
472 				force_thread_migrate(t);
473 				return;
474 			}
475 		} else {
476 			t->t_bound_cpu = cpu[binding];
477 			/*
478 			 * Make sure the thread is running on the bound CPU.
479 			 */
480 			if (t->t_cpu != t->t_bound_cpu) {
481 				force_thread_migrate(t);
482 				return;		/* already dropped lock */
483 			}
484 		}
485 	}
486 	thread_unlock(t);
487 }
488 
489 /*
490  * Wrapper for backward compatibility.
491  */
492 void
affinity_clear(void)493 affinity_clear(void)
494 {
495 	thread_affinity_clear(curthread);
496 }
497 
498 /*
499  * Weak cpu affinity.  Bind to the "current" cpu for short periods
500  * of time during which the thread must not block (but may be preempted).
501  * Use this instead of kpreempt_disable() when it is only "no migration"
502  * rather than "no preemption" semantics that are required - disabling
503  * preemption holds higher priority threads off of cpu and if the
504  * operation that is protected is more than momentary this is not good
505  * for realtime etc.
506  *
507  * Weakly bound threads will not prevent a cpu from being offlined -
508  * we'll only run them on the cpu to which they are weakly bound but
509  * (because they do not block) we'll always be able to move them on to
510  * another cpu at offline time if we give them just a short moment to
511  * run during which they will unbind.  To give a cpu a chance of offlining,
512  * however, we require a barrier to weak bindings that may be raised for a
513  * given cpu (offline/move code may set this and then wait a short time for
514  * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
515  *
516  * There are few restrictions on the calling context of thread_nomigrate.
517  * The caller must not hold the thread lock.  Calls may be nested.
518  *
519  * After weakbinding a thread must not perform actions that may block.
520  * In particular it must not call thread_affinity_set; calling that when
521  * already weakbound is nonsensical anyway.
522  *
523  * If curthread is prevented from migrating for other reasons
524  * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
525  * then the weak binding will succeed even if this cpu is the target of an
526  * offline/move request.
527  */
528 void
thread_nomigrate(void)529 thread_nomigrate(void)
530 {
531 	cpu_t *cp;
532 	kthread_id_t t = curthread;
533 
534 again:
535 	kpreempt_disable();
536 	cp = CPU;
537 
538 	/*
539 	 * A highlevel interrupt must not modify t_nomigrate or
540 	 * t_weakbound_cpu of the thread it has interrupted.  A lowlevel
541 	 * interrupt thread cannot migrate and we can avoid the
542 	 * thread_lock call below by short-circuiting here.  In either
543 	 * case we can just return since no migration is possible and
544 	 * the condition will persist (ie, when we test for these again
545 	 * in thread_allowmigrate they can't have changed).   Migration
546 	 * is also impossible if we're at or above DISP_LEVEL pil.
547 	 */
548 	if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
549 	    getpil() >= DISP_LEVEL) {
550 		kpreempt_enable();
551 		return;
552 	}
553 
554 	/*
555 	 * We must be consistent with existing weak bindings.  Since we
556 	 * may be interrupted between the increment of t_nomigrate and
557 	 * the store to t_weakbound_cpu below we cannot assume that
558 	 * t_weakbound_cpu will be set if t_nomigrate is.  Note that we
559 	 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
560 	 * always the case.
561 	 */
562 	if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
563 		if (!panicstr)
564 			panic("thread_nomigrate: binding to %p but already "
565 			    "bound to %p", (void *)cp,
566 			    (void *)t->t_weakbound_cpu);
567 	}
568 
569 	/*
570 	 * At this point we have preemption disabled and we don't yet hold
571 	 * the thread lock.  So it's possible that somebody else could
572 	 * set t_bind_cpu here and not be able to force us across to the
573 	 * new cpu (since we have preemption disabled).
574 	 */
575 	thread_lock(curthread);
576 
577 	/*
578 	 * If further weak bindings are being (temporarily) suppressed then
579 	 * we'll settle for disabling kernel preemption (which assures
580 	 * no migration provided the thread does not block which it is
581 	 * not allowed to if using thread_nomigrate).  We must remember
582 	 * this disposition so we can take appropriate action in
583 	 * thread_allowmigrate.  If this is a nested call and the
584 	 * thread is already weakbound then fall through as normal.
585 	 * We remember the decision to settle for kpreempt_disable through
586 	 * negative nesting counting in t_nomigrate.  Once a thread has had one
587 	 * weakbinding request satisfied in this way any further (nested)
588 	 * requests will continue to be satisfied in the same way,
589 	 * even if weak bindings have recommenced.
590 	 */
591 	if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
592 		--t->t_nomigrate;
593 		thread_unlock(curthread);
594 		return;		/* with kpreempt_disable still active */
595 	}
596 
597 	/*
598 	 * We hold thread_lock so t_bind_cpu cannot change.  We could,
599 	 * however, be running on a different cpu to which we are t_bound_cpu
600 	 * to (as explained above).  If we grant the weak binding request
601 	 * in that case then the dispatcher must favour our weak binding
602 	 * over our strong (in which case, just as when preemption is
603 	 * disabled, we can continue to run on a cpu other than the one to
604 	 * which we are strongbound; the difference in this case is that
605 	 * this thread can be preempted and so can appear on the dispatch
606 	 * queues of a cpu other than the one it is strongbound to).
607 	 *
608 	 * If the cpu we are running on does not appear to be a current
609 	 * offline target (we check cpu_inmotion to determine this - since
610 	 * we don't hold cpu_lock we may not see a recent store to that,
611 	 * so it's possible that we at times can grant a weak binding to a
612 	 * cpu that is an offline target, but that one request will not
613 	 * prevent the offline from succeeding) then we will always grant
614 	 * the weak binding request.  This includes the case above where
615 	 * we grant a weakbinding not commensurate with our strong binding.
616 	 *
617 	 * If our cpu does appear to be an offline target then we're inclined
618 	 * not to grant the weakbinding request just yet - we'd prefer to
619 	 * migrate to another cpu and grant the request there.  The
620 	 * exceptions are those cases where going through preemption code
621 	 * will not result in us changing cpu:
622 	 *
623 	 *	. interrupts have already bypassed this case (see above)
624 	 *	. we are already weakbound to this cpu (dispatcher code will
625 	 *	  always return us to the weakbound cpu)
626 	 *	. preemption was disabled even before we disabled it above
627 	 *	. we are strongbound to this cpu (if we're strongbound to
628 	 *	another and not yet running there the trip through the
629 	 *	dispatcher will move us to the strongbound cpu and we
630 	 *	will grant the weak binding there)
631 	 */
632 	if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
633 	    t->t_bound_cpu == cp) {
634 		/*
635 		 * Don't be tempted to store to t_weakbound_cpu only on
636 		 * the first nested bind request - if we're interrupted
637 		 * after the increment of t_nomigrate and before the
638 		 * store to t_weakbound_cpu and the interrupt calls
639 		 * thread_nomigrate then the assertion in thread_allowmigrate
640 		 * would fail.
641 		 */
642 		t->t_nomigrate++;
643 		t->t_weakbound_cpu = cp;
644 		membar_producer();
645 		thread_unlock(curthread);
646 		/*
647 		 * Now that we have dropped the thread_lock another thread
648 		 * can set our t_weakbound_cpu, and will try to migrate us
649 		 * to the strongbound cpu (which will not be prevented by
650 		 * preemption being disabled since we're about to enable
651 		 * preemption).  We have granted the weakbinding to the current
652 		 * cpu, so again we are in the position that is is is possible
653 		 * that our weak and strong bindings differ.  Again this
654 		 * is catered for by dispatcher code which will favour our
655 		 * weak binding.
656 		 */
657 		kpreempt_enable();
658 	} else {
659 		/*
660 		 * Move to another cpu before granting the request by
661 		 * forcing this thread through preemption code.  When we
662 		 * get to set{front,back}dq called from CL_PREEMPT()
663 		 * cpu_choose() will be used to select a cpu to queue
664 		 * us on - that will see cpu_inmotion and take
665 		 * steps to avoid returning us to this cpu.
666 		 */
667 		cp->cpu_kprunrun = 1;
668 		thread_unlock(curthread);
669 		kpreempt_enable();	/* will call preempt() */
670 		goto again;
671 	}
672 }
673 
674 void
thread_allowmigrate(void)675 thread_allowmigrate(void)
676 {
677 	kthread_id_t t = curthread;
678 
679 	ASSERT(t->t_weakbound_cpu == CPU ||
680 	    (t->t_nomigrate < 0 && t->t_preempt > 0) ||
681 	    CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
682 	    getpil() >= DISP_LEVEL);
683 
684 	if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
685 	    getpil() >= DISP_LEVEL)
686 		return;
687 
688 	if (t->t_nomigrate < 0) {
689 		/*
690 		 * This thread was granted "weak binding" in the
691 		 * stronger form of kernel preemption disabling.
692 		 * Undo a level of nesting for both t_nomigrate
693 		 * and t_preempt.
694 		 */
695 		++t->t_nomigrate;
696 		kpreempt_enable();
697 	} else if (--t->t_nomigrate == 0) {
698 		/*
699 		 * Time to drop the weak binding.  We need to cater
700 		 * for the case where we're weakbound to a different
701 		 * cpu than that to which we're strongbound (a very
702 		 * temporary arrangement that must only persist until
703 		 * weak binding drops).  We don't acquire thread_lock
704 		 * here so even as this code executes t_bound_cpu
705 		 * may be changing.  So we disable preemption and
706 		 * a) in the case that t_bound_cpu changes while we
707 		 * have preemption disabled kprunrun will be set
708 		 * asynchronously, and b) if before disabling
709 		 * preemption we were already on a different cpu to
710 		 * our t_bound_cpu then we set kprunrun ourselves
711 		 * to force a trip through the dispatcher when
712 		 * preemption is enabled.
713 		 */
714 		kpreempt_disable();
715 		if (t->t_bound_cpu &&
716 		    t->t_weakbound_cpu != t->t_bound_cpu)
717 			CPU->cpu_kprunrun = 1;
718 		t->t_weakbound_cpu = NULL;
719 		membar_producer();
720 		kpreempt_enable();
721 	}
722 }
723 
724 /*
725  * weakbinding_stop can be used to temporarily cause weakbindings made
726  * with thread_nomigrate to be satisfied through the stronger action of
727  * kpreempt_disable.  weakbinding_start recommences normal weakbinding.
728  */
729 
730 void
weakbinding_stop(void)731 weakbinding_stop(void)
732 {
733 	ASSERT(MUTEX_HELD(&cpu_lock));
734 	weakbindingbarrier = 1;
735 	membar_producer();	/* make visible before subsequent thread_lock */
736 }
737 
738 void
weakbinding_start(void)739 weakbinding_start(void)
740 {
741 	ASSERT(MUTEX_HELD(&cpu_lock));
742 	weakbindingbarrier = 0;
743 }
744 
745 void
null_xcall(void)746 null_xcall(void)
747 {
748 }
749 
750 /*
751  * This routine is called to place the CPUs in a safe place so that
752  * one of them can be taken off line or placed on line.  What we are
753  * trying to do here is prevent a thread from traversing the list
754  * of active CPUs while we are changing it or from getting placed on
755  * the run queue of a CPU that has just gone off line.  We do this by
756  * creating a thread with the highest possible prio for each CPU and
757  * having it call this routine.  The advantage of this method is that
758  * we can eliminate all checks for CPU_ACTIVE in the disp routines.
759  * This makes disp faster at the expense of making p_online() slower
760  * which is a good trade off.
761  */
762 static void
cpu_pause(int index)763 cpu_pause(int index)
764 {
765 	int s;
766 	struct _cpu_pause_info *cpi = &cpu_pause_info;
767 	volatile char *safe = &safe_list[index];
768 	long    lindex = index;
769 
770 	ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
771 
772 	while (*safe != PAUSE_DIE) {
773 		*safe = PAUSE_READY;
774 		membar_enter();		/* make sure stores are flushed */
775 		sema_v(&cpi->cp_sem);	/* signal requesting thread */
776 
777 		/*
778 		 * Wait here until all pause threads are running.  That
779 		 * indicates that it's safe to do the spl.  Until
780 		 * cpu_pause_info.cp_go is set, we don't want to spl
781 		 * because that might block clock interrupts needed
782 		 * to preempt threads on other CPUs.
783 		 */
784 		while (cpi->cp_go == 0)
785 			;
786 		/*
787 		 * Even though we are at the highest disp prio, we need
788 		 * to block out all interrupts below LOCK_LEVEL so that
789 		 * an intr doesn't come in, wake up a thread, and call
790 		 * setbackdq/setfrontdq.
791 		 */
792 		s = splhigh();
793 		/*
794 		 * if cp_func has been set then call it using index as the
795 		 * argument, currently only used by cpr_suspend_cpus().
796 		 * This function is used as the code to execute on the
797 		 * "paused" cpu's when a machine comes out of a sleep state
798 		 * and CPU's were powered off.  (could also be used for
799 		 * hotplugging CPU's).
800 		 */
801 		if (cpi->cp_func != NULL)
802 			(*cpi->cp_func)((void *)lindex);
803 
804 		mach_cpu_pause(safe);
805 
806 		splx(s);
807 		/*
808 		 * Waiting is at an end. Switch out of cpu_pause
809 		 * loop and resume useful work.
810 		 */
811 		swtch();
812 	}
813 
814 	mutex_enter(&pause_free_mutex);
815 	*safe = PAUSE_DEAD;
816 	cv_broadcast(&pause_free_cv);
817 	mutex_exit(&pause_free_mutex);
818 }
819 
820 /*
821  * Allow the cpus to start running again.
822  */
823 void
start_cpus()824 start_cpus()
825 {
826 	int i;
827 
828 	ASSERT(MUTEX_HELD(&cpu_lock));
829 	ASSERT(cpu_pause_info.cp_paused);
830 	cpu_pause_info.cp_paused = NULL;
831 	for (i = 0; i < NCPU; i++)
832 		safe_list[i] = PAUSE_IDLE;
833 	membar_enter();			/* make sure stores are flushed */
834 	affinity_clear();
835 	splx(cpu_pause_info.cp_spl);
836 	kpreempt_enable();
837 }
838 
839 /*
840  * Allocate a pause thread for a CPU.
841  */
842 static void
cpu_pause_alloc(cpu_t * cp)843 cpu_pause_alloc(cpu_t *cp)
844 {
845 	kthread_id_t	t;
846 	long		cpun = cp->cpu_id;
847 
848 	/*
849 	 * Note, v.v_nglobpris will not change value as long as I hold
850 	 * cpu_lock.
851 	 */
852 	t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
853 	    0, &p0, TS_STOPPED, v.v_nglobpris - 1);
854 	thread_lock(t);
855 	t->t_bound_cpu = cp;
856 	t->t_disp_queue = cp->cpu_disp;
857 	t->t_affinitycnt = 1;
858 	t->t_preempt = 1;
859 	thread_unlock(t);
860 	cp->cpu_pause_thread = t;
861 	/*
862 	 * Registering a thread in the callback table is usually done
863 	 * in the initialization code of the thread.  In this
864 	 * case, we do it right after thread creation because the
865 	 * thread itself may never run, and we need to register the
866 	 * fact that it is safe for cpr suspend.
867 	 */
868 	CALLB_CPR_INIT_SAFE(t, "cpu_pause");
869 }
870 
871 /*
872  * Free a pause thread for a CPU.
873  */
874 static void
cpu_pause_free(cpu_t * cp)875 cpu_pause_free(cpu_t *cp)
876 {
877 	kthread_id_t	t;
878 	int		cpun = cp->cpu_id;
879 
880 	ASSERT(MUTEX_HELD(&cpu_lock));
881 	/*
882 	 * We have to get the thread and tell him to die.
883 	 */
884 	if ((t = cp->cpu_pause_thread) == NULL) {
885 		ASSERT(safe_list[cpun] == PAUSE_IDLE);
886 		return;
887 	}
888 	thread_lock(t);
889 	t->t_cpu = CPU;		/* disp gets upset if last cpu is quiesced. */
890 	t->t_bound_cpu = NULL;	/* Must un-bind; cpu may not be running. */
891 	t->t_pri = v.v_nglobpris - 1;
892 	ASSERT(safe_list[cpun] == PAUSE_IDLE);
893 	safe_list[cpun] = PAUSE_DIE;
894 	THREAD_TRANSITION(t);
895 	setbackdq(t);
896 	thread_unlock_nopreempt(t);
897 
898 	/*
899 	 * If we don't wait for the thread to actually die, it may try to
900 	 * run on the wrong cpu as part of an actual call to pause_cpus().
901 	 */
902 	mutex_enter(&pause_free_mutex);
903 	while (safe_list[cpun] != PAUSE_DEAD) {
904 		cv_wait(&pause_free_cv, &pause_free_mutex);
905 	}
906 	mutex_exit(&pause_free_mutex);
907 	safe_list[cpun] = PAUSE_IDLE;
908 
909 	cp->cpu_pause_thread = NULL;
910 }
911 
912 /*
913  * Initialize basic structures for pausing CPUs.
914  */
915 void
cpu_pause_init()916 cpu_pause_init()
917 {
918 	sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
919 	/*
920 	 * Create initial CPU pause thread.
921 	 */
922 	cpu_pause_alloc(CPU);
923 }
924 
925 /*
926  * Start the threads used to pause another CPU.
927  */
928 static int
cpu_pause_start(processorid_t cpu_id)929 cpu_pause_start(processorid_t cpu_id)
930 {
931 	int	i;
932 	int	cpu_count = 0;
933 
934 	for (i = 0; i < NCPU; i++) {
935 		cpu_t		*cp;
936 		kthread_id_t	t;
937 
938 		cp = cpu[i];
939 		if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
940 			safe_list[i] = PAUSE_WAIT;
941 			continue;
942 		}
943 
944 		/*
945 		 * Skip CPU if it is quiesced or not yet started.
946 		 */
947 		if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
948 			safe_list[i] = PAUSE_WAIT;
949 			continue;
950 		}
951 
952 		/*
953 		 * Start this CPU's pause thread.
954 		 */
955 		t = cp->cpu_pause_thread;
956 		thread_lock(t);
957 		/*
958 		 * Reset the priority, since nglobpris may have
959 		 * changed since the thread was created, if someone
960 		 * has loaded the RT (or some other) scheduling
961 		 * class.
962 		 */
963 		t->t_pri = v.v_nglobpris - 1;
964 		THREAD_TRANSITION(t);
965 		setbackdq(t);
966 		thread_unlock_nopreempt(t);
967 		++cpu_count;
968 	}
969 	return (cpu_count);
970 }
971 
972 
973 /*
974  * Pause all of the CPUs except the one we are on by creating a high
975  * priority thread bound to those CPUs.
976  *
977  * Note that one must be extremely careful regarding code
978  * executed while CPUs are paused.  Since a CPU may be paused
979  * while a thread scheduling on that CPU is holding an adaptive
980  * lock, code executed with CPUs paused must not acquire adaptive
981  * (or low-level spin) locks.  Also, such code must not block,
982  * since the thread that is supposed to initiate the wakeup may
983  * never run.
984  *
985  * With a few exceptions, the restrictions on code executed with CPUs
986  * paused match those for code executed at high-level interrupt
987  * context.
988  */
989 void
pause_cpus(cpu_t * off_cp,void * (* func)(void *))990 pause_cpus(cpu_t *off_cp, void *(*func)(void *))
991 {
992 	processorid_t	cpu_id;
993 	int		i;
994 	struct _cpu_pause_info	*cpi = &cpu_pause_info;
995 
996 	ASSERT(MUTEX_HELD(&cpu_lock));
997 	ASSERT(cpi->cp_paused == NULL);
998 	cpi->cp_count = 0;
999 	cpi->cp_go = 0;
1000 	for (i = 0; i < NCPU; i++)
1001 		safe_list[i] = PAUSE_IDLE;
1002 	kpreempt_disable();
1003 
1004 	cpi->cp_func = func;
1005 
1006 	/*
1007 	 * If running on the cpu that is going offline, get off it.
1008 	 * This is so that it won't be necessary to rechoose a CPU
1009 	 * when done.
1010 	 */
1011 	if (CPU == off_cp)
1012 		cpu_id = off_cp->cpu_next_part->cpu_id;
1013 	else
1014 		cpu_id = CPU->cpu_id;
1015 	affinity_set(cpu_id);
1016 
1017 	/*
1018 	 * Start the pause threads and record how many were started
1019 	 */
1020 	cpi->cp_count = cpu_pause_start(cpu_id);
1021 
1022 	/*
1023 	 * Now wait for all CPUs to be running the pause thread.
1024 	 */
1025 	while (cpi->cp_count > 0) {
1026 		/*
1027 		 * Spin reading the count without grabbing the disp
1028 		 * lock to make sure we don't prevent the pause
1029 		 * threads from getting the lock.
1030 		 */
1031 		while (sema_held(&cpi->cp_sem))
1032 			;
1033 		if (sema_tryp(&cpi->cp_sem))
1034 			--cpi->cp_count;
1035 	}
1036 	cpi->cp_go = 1;			/* all have reached cpu_pause */
1037 
1038 	/*
1039 	 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1040 	 * to PAUSE_WAIT.)
1041 	 */
1042 	for (i = 0; i < NCPU; i++) {
1043 		while (safe_list[i] != PAUSE_WAIT)
1044 			;
1045 	}
1046 	cpi->cp_spl = splhigh();	/* block dispatcher on this CPU */
1047 	cpi->cp_paused = curthread;
1048 }
1049 
1050 /*
1051  * Check whether the current thread has CPUs paused
1052  */
1053 int
cpus_paused(void)1054 cpus_paused(void)
1055 {
1056 	if (cpu_pause_info.cp_paused != NULL) {
1057 		ASSERT(cpu_pause_info.cp_paused == curthread);
1058 		return (1);
1059 	}
1060 	return (0);
1061 }
1062 
1063 static cpu_t *
cpu_get_all(processorid_t cpun)1064 cpu_get_all(processorid_t cpun)
1065 {
1066 	ASSERT(MUTEX_HELD(&cpu_lock));
1067 
1068 	if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1069 		return (NULL);
1070 	return (cpu[cpun]);
1071 }
1072 
1073 /*
1074  * Check whether cpun is a valid processor id and whether it should be
1075  * visible from the current zone. If it is, return a pointer to the
1076  * associated CPU structure.
1077  */
1078 cpu_t *
cpu_get(processorid_t cpun)1079 cpu_get(processorid_t cpun)
1080 {
1081 	cpu_t *c;
1082 
1083 	ASSERT(MUTEX_HELD(&cpu_lock));
1084 	c = cpu_get_all(cpun);
1085 	if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1086 	    zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1087 		return (NULL);
1088 	return (c);
1089 }
1090 
1091 /*
1092  * The following functions should be used to check CPU states in the kernel.
1093  * They should be invoked with cpu_lock held.  Kernel subsystems interested
1094  * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1095  * states.  Those are for user-land (and system call) use only.
1096  */
1097 
1098 /*
1099  * Determine whether the CPU is online and handling interrupts.
1100  */
1101 int
cpu_is_online(cpu_t * cpu)1102 cpu_is_online(cpu_t *cpu)
1103 {
1104 	ASSERT(MUTEX_HELD(&cpu_lock));
1105 	return (cpu_flagged_online(cpu->cpu_flags));
1106 }
1107 
1108 /*
1109  * Determine whether the CPU is offline (this includes spare and faulted).
1110  */
1111 int
cpu_is_offline(cpu_t * cpu)1112 cpu_is_offline(cpu_t *cpu)
1113 {
1114 	ASSERT(MUTEX_HELD(&cpu_lock));
1115 	return (cpu_flagged_offline(cpu->cpu_flags));
1116 }
1117 
1118 /*
1119  * Determine whether the CPU is powered off.
1120  */
1121 int
cpu_is_poweredoff(cpu_t * cpu)1122 cpu_is_poweredoff(cpu_t *cpu)
1123 {
1124 	ASSERT(MUTEX_HELD(&cpu_lock));
1125 	return (cpu_flagged_poweredoff(cpu->cpu_flags));
1126 }
1127 
1128 /*
1129  * Determine whether the CPU is handling interrupts.
1130  */
1131 int
cpu_is_nointr(cpu_t * cpu)1132 cpu_is_nointr(cpu_t *cpu)
1133 {
1134 	ASSERT(MUTEX_HELD(&cpu_lock));
1135 	return (cpu_flagged_nointr(cpu->cpu_flags));
1136 }
1137 
1138 /*
1139  * Determine whether the CPU is active (scheduling threads).
1140  */
1141 int
cpu_is_active(cpu_t * cpu)1142 cpu_is_active(cpu_t *cpu)
1143 {
1144 	ASSERT(MUTEX_HELD(&cpu_lock));
1145 	return (cpu_flagged_active(cpu->cpu_flags));
1146 }
1147 
1148 /*
1149  * Same as above, but these require cpu_flags instead of cpu_t pointers.
1150  */
1151 int
cpu_flagged_online(cpu_flag_t cpu_flags)1152 cpu_flagged_online(cpu_flag_t cpu_flags)
1153 {
1154 	return (cpu_flagged_active(cpu_flags) &&
1155 	    (cpu_flags & CPU_ENABLE));
1156 }
1157 
1158 int
cpu_flagged_offline(cpu_flag_t cpu_flags)1159 cpu_flagged_offline(cpu_flag_t cpu_flags)
1160 {
1161 	return (((cpu_flags & CPU_POWEROFF) == 0) &&
1162 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1163 }
1164 
1165 int
cpu_flagged_poweredoff(cpu_flag_t cpu_flags)1166 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1167 {
1168 	return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1169 }
1170 
1171 int
cpu_flagged_nointr(cpu_flag_t cpu_flags)1172 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1173 {
1174 	return (cpu_flagged_active(cpu_flags) &&
1175 	    (cpu_flags & CPU_ENABLE) == 0);
1176 }
1177 
1178 int
cpu_flagged_active(cpu_flag_t cpu_flags)1179 cpu_flagged_active(cpu_flag_t cpu_flags)
1180 {
1181 	return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1182 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1183 }
1184 
1185 /*
1186  * Bring the indicated CPU online.
1187  */
1188 int
cpu_online(cpu_t * cp)1189 cpu_online(cpu_t *cp)
1190 {
1191 	int	error = 0;
1192 
1193 	/*
1194 	 * Handle on-line request.
1195 	 *	This code must put the new CPU on the active list before
1196 	 *	starting it because it will not be paused, and will start
1197 	 * 	using the active list immediately.  The real start occurs
1198 	 *	when the CPU_QUIESCED flag is turned off.
1199 	 */
1200 
1201 	ASSERT(MUTEX_HELD(&cpu_lock));
1202 
1203 	/*
1204 	 * Put all the cpus into a known safe place.
1205 	 * No mutexes can be entered while CPUs are paused.
1206 	 */
1207 	error = mp_cpu_start(cp);	/* arch-dep hook */
1208 	if (error == 0) {
1209 		pg_cpupart_in(cp, cp->cpu_part);
1210 		pause_cpus(NULL, NULL);
1211 		cpu_add_active_internal(cp);
1212 		if (cp->cpu_flags & CPU_FAULTED) {
1213 			cp->cpu_flags &= ~CPU_FAULTED;
1214 			mp_cpu_faulted_exit(cp);
1215 		}
1216 		cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1217 		    CPU_SPARE);
1218 		CPU_NEW_GENERATION(cp);
1219 		start_cpus();
1220 		cpu_stats_kstat_create(cp);
1221 		cpu_create_intrstat(cp);
1222 		lgrp_kstat_create(cp);
1223 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1224 		cpu_intr_enable(cp);	/* arch-dep hook */
1225 		cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1226 		cpu_set_state(cp);
1227 		cyclic_online(cp);
1228 		/*
1229 		 * This has to be called only after cyclic_online(). This
1230 		 * function uses cyclics.
1231 		 */
1232 		callout_cpu_online(cp);
1233 		poke_cpu(cp->cpu_id);
1234 	}
1235 
1236 	return (error);
1237 }
1238 
1239 /*
1240  * Take the indicated CPU offline.
1241  */
1242 int
cpu_offline(cpu_t * cp,int flags)1243 cpu_offline(cpu_t *cp, int flags)
1244 {
1245 	cpupart_t *pp;
1246 	int	error = 0;
1247 	cpu_t	*ncp;
1248 	int	intr_enable;
1249 	int	cyclic_off = 0;
1250 	int	callout_off = 0;
1251 	int	loop_count;
1252 	int	no_quiesce = 0;
1253 	int	(*bound_func)(struct cpu *, int);
1254 	kthread_t *t;
1255 	lpl_t	*cpu_lpl;
1256 	proc_t	*p;
1257 	int	lgrp_diff_lpl;
1258 	boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
1259 
1260 	ASSERT(MUTEX_HELD(&cpu_lock));
1261 
1262 	/*
1263 	 * If we're going from faulted or spare to offline, just
1264 	 * clear these flags and update CPU state.
1265 	 */
1266 	if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1267 		if (cp->cpu_flags & CPU_FAULTED) {
1268 			cp->cpu_flags &= ~CPU_FAULTED;
1269 			mp_cpu_faulted_exit(cp);
1270 		}
1271 		cp->cpu_flags &= ~CPU_SPARE;
1272 		cpu_set_state(cp);
1273 		return (0);
1274 	}
1275 
1276 	/*
1277 	 * Handle off-line request.
1278 	 */
1279 	pp = cp->cpu_part;
1280 	/*
1281 	 * Don't offline last online CPU in partition
1282 	 */
1283 	if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1284 		return (EBUSY);
1285 	/*
1286 	 * Unbind all soft-bound threads bound to our CPU and hard bound threads
1287 	 * if we were asked to.
1288 	 */
1289 	error = cpu_unbind(cp->cpu_id, unbind_all_threads);
1290 	if (error != 0)
1291 		return (error);
1292 	/*
1293 	 * We shouldn't be bound to this CPU ourselves.
1294 	 */
1295 	if (curthread->t_bound_cpu == cp)
1296 		return (EBUSY);
1297 
1298 	/*
1299 	 * Tell interested parties that this CPU is going offline.
1300 	 */
1301 	CPU_NEW_GENERATION(cp);
1302 	cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1303 
1304 	/*
1305 	 * Tell the PG subsystem that the CPU is leaving the partition
1306 	 */
1307 	pg_cpupart_out(cp, pp);
1308 
1309 	/*
1310 	 * Take the CPU out of interrupt participation so we won't find
1311 	 * bound kernel threads.  If the architecture cannot completely
1312 	 * shut off interrupts on the CPU, don't quiesce it, but don't
1313 	 * run anything but interrupt thread... this is indicated by
1314 	 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1315 	 * off.
1316 	 */
1317 	intr_enable = cp->cpu_flags & CPU_ENABLE;
1318 	if (intr_enable)
1319 		no_quiesce = cpu_intr_disable(cp);
1320 
1321 	/*
1322 	 * Record that we are aiming to offline this cpu.  This acts as
1323 	 * a barrier to further weak binding requests in thread_nomigrate
1324 	 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1325 	 * lean away from this cpu.  Further strong bindings are already
1326 	 * avoided since we hold cpu_lock.  Since threads that are set
1327 	 * runnable around now and others coming off the target cpu are
1328 	 * directed away from the target, existing strong and weak bindings
1329 	 * (especially the latter) to the target cpu stand maximum chance of
1330 	 * being able to unbind during the short delay loop below (if other
1331 	 * unbound threads compete they may not see cpu in time to unbind
1332 	 * even if they would do so immediately.
1333 	 */
1334 	cpu_inmotion = cp;
1335 	membar_enter();
1336 
1337 	/*
1338 	 * Check for kernel threads (strong or weak) bound to that CPU.
1339 	 * Strongly bound threads may not unbind, and we'll have to return
1340 	 * EBUSY.  Weakly bound threads should always disappear - we've
1341 	 * stopped more weak binding with cpu_inmotion and existing
1342 	 * bindings will drain imminently (they may not block).  Nonetheless
1343 	 * we will wait for a fixed period for all bound threads to disappear.
1344 	 * Inactive interrupt threads are OK (they'll be in TS_FREE
1345 	 * state).  If test finds some bound threads, wait a few ticks
1346 	 * to give short-lived threads (such as interrupts) chance to
1347 	 * complete.  Note that if no_quiesce is set, i.e. this cpu
1348 	 * is required to service interrupts, then we take the route
1349 	 * that permits interrupt threads to be active (or bypassed).
1350 	 */
1351 	bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1352 
1353 again:	for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1354 		if (loop_count >= 5) {
1355 			error = EBUSY;	/* some threads still bound */
1356 			break;
1357 		}
1358 
1359 		/*
1360 		 * If some threads were assigned, give them
1361 		 * a chance to complete or move.
1362 		 *
1363 		 * This assumes that the clock_thread is not bound
1364 		 * to any CPU, because the clock_thread is needed to
1365 		 * do the delay(hz/100).
1366 		 *
1367 		 * Note: we still hold the cpu_lock while waiting for
1368 		 * the next clock tick.  This is OK since it isn't
1369 		 * needed for anything else except processor_bind(2),
1370 		 * and system initialization.  If we drop the lock,
1371 		 * we would risk another p_online disabling the last
1372 		 * processor.
1373 		 */
1374 		delay(hz/100);
1375 	}
1376 
1377 	if (error == 0 && callout_off == 0) {
1378 		callout_cpu_offline(cp);
1379 		callout_off = 1;
1380 	}
1381 
1382 	if (error == 0 && cyclic_off == 0) {
1383 		if (!cyclic_offline(cp)) {
1384 			/*
1385 			 * We must have bound cyclics...
1386 			 */
1387 			error = EBUSY;
1388 			goto out;
1389 		}
1390 		cyclic_off = 1;
1391 	}
1392 
1393 	/*
1394 	 * Call mp_cpu_stop() to perform any special operations
1395 	 * needed for this machine architecture to offline a CPU.
1396 	 */
1397 	if (error == 0)
1398 		error = mp_cpu_stop(cp);	/* arch-dep hook */
1399 
1400 	/*
1401 	 * If that all worked, take the CPU offline and decrement
1402 	 * ncpus_online.
1403 	 */
1404 	if (error == 0) {
1405 		/*
1406 		 * Put all the cpus into a known safe place.
1407 		 * No mutexes can be entered while CPUs are paused.
1408 		 */
1409 		pause_cpus(cp, NULL);
1410 		/*
1411 		 * Repeat the operation, if necessary, to make sure that
1412 		 * all outstanding low-level interrupts run to completion
1413 		 * before we set the CPU_QUIESCED flag.  It's also possible
1414 		 * that a thread has weak bound to the cpu despite our raising
1415 		 * cpu_inmotion above since it may have loaded that
1416 		 * value before the barrier became visible (this would have
1417 		 * to be the thread that was on the target cpu at the time
1418 		 * we raised the barrier).
1419 		 */
1420 		if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1421 		    (*bound_func)(cp, 1)) {
1422 			start_cpus();
1423 			(void) mp_cpu_start(cp);
1424 			goto again;
1425 		}
1426 		ncp = cp->cpu_next_part;
1427 		cpu_lpl = cp->cpu_lpl;
1428 		ASSERT(cpu_lpl != NULL);
1429 
1430 		/*
1431 		 * Remove the CPU from the list of active CPUs.
1432 		 */
1433 		cpu_remove_active(cp);
1434 
1435 		/*
1436 		 * Walk the active process list and look for threads
1437 		 * whose home lgroup needs to be updated, or
1438 		 * the last CPU they run on is the one being offlined now.
1439 		 */
1440 
1441 		ASSERT(curthread->t_cpu != cp);
1442 		for (p = practive; p != NULL; p = p->p_next) {
1443 
1444 			t = p->p_tlist;
1445 
1446 			if (t == NULL)
1447 				continue;
1448 
1449 			lgrp_diff_lpl = 0;
1450 
1451 			do {
1452 				ASSERT(t->t_lpl != NULL);
1453 				/*
1454 				 * Taking last CPU in lpl offline
1455 				 * Rehome thread if it is in this lpl
1456 				 * Otherwise, update the count of how many
1457 				 * threads are in this CPU's lgroup but have
1458 				 * a different lpl.
1459 				 */
1460 
1461 				if (cpu_lpl->lpl_ncpu == 0) {
1462 					if (t->t_lpl == cpu_lpl)
1463 						lgrp_move_thread(t,
1464 						    lgrp_choose(t,
1465 						    t->t_cpupart), 0);
1466 					else if (t->t_lpl->lpl_lgrpid ==
1467 					    cpu_lpl->lpl_lgrpid)
1468 						lgrp_diff_lpl++;
1469 				}
1470 				ASSERT(t->t_lpl->lpl_ncpu > 0);
1471 
1472 				/*
1473 				 * Update CPU last ran on if it was this CPU
1474 				 */
1475 				if (t->t_cpu == cp && t->t_bound_cpu != cp)
1476 					t->t_cpu = disp_lowpri_cpu(ncp,
1477 					    t->t_lpl, t->t_pri, NULL);
1478 				ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1479 				    t->t_weakbound_cpu == cp);
1480 
1481 				t = t->t_forw;
1482 			} while (t != p->p_tlist);
1483 
1484 			/*
1485 			 * Didn't find any threads in the same lgroup as this
1486 			 * CPU with a different lpl, so remove the lgroup from
1487 			 * the process lgroup bitmask.
1488 			 */
1489 
1490 			if (lgrp_diff_lpl == 0)
1491 				klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1492 		}
1493 
1494 		/*
1495 		 * Walk thread list looking for threads that need to be
1496 		 * rehomed, since there are some threads that are not in
1497 		 * their process's p_tlist.
1498 		 */
1499 
1500 		t = curthread;
1501 		do {
1502 			ASSERT(t != NULL && t->t_lpl != NULL);
1503 
1504 			/*
1505 			 * Rehome threads with same lpl as this CPU when this
1506 			 * is the last CPU in the lpl.
1507 			 */
1508 
1509 			if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1510 				lgrp_move_thread(t,
1511 				    lgrp_choose(t, t->t_cpupart), 1);
1512 
1513 			ASSERT(t->t_lpl->lpl_ncpu > 0);
1514 
1515 			/*
1516 			 * Update CPU last ran on if it was this CPU
1517 			 */
1518 
1519 			if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1520 				t->t_cpu = disp_lowpri_cpu(ncp,
1521 				    t->t_lpl, t->t_pri, NULL);
1522 			}
1523 			ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1524 			    t->t_weakbound_cpu == cp);
1525 			t = t->t_next;
1526 
1527 		} while (t != curthread);
1528 		ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1529 		cp->cpu_flags |= CPU_OFFLINE;
1530 		disp_cpu_inactive(cp);
1531 		if (!no_quiesce)
1532 			cp->cpu_flags |= CPU_QUIESCED;
1533 		ncpus_online--;
1534 		cpu_set_state(cp);
1535 		cpu_inmotion = NULL;
1536 		start_cpus();
1537 		cpu_stats_kstat_destroy(cp);
1538 		cpu_delete_intrstat(cp);
1539 		lgrp_kstat_destroy(cp);
1540 	}
1541 
1542 out:
1543 	cpu_inmotion = NULL;
1544 
1545 	/*
1546 	 * If we failed, re-enable interrupts.
1547 	 * Do this even if cpu_intr_disable returned an error, because
1548 	 * it may have partially disabled interrupts.
1549 	 */
1550 	if (error && intr_enable)
1551 		cpu_intr_enable(cp);
1552 
1553 	/*
1554 	 * If we failed, but managed to offline the cyclic subsystem on this
1555 	 * CPU, bring it back online.
1556 	 */
1557 	if (error && cyclic_off)
1558 		cyclic_online(cp);
1559 
1560 	/*
1561 	 * If we failed, but managed to offline callouts on this CPU,
1562 	 * bring it back online.
1563 	 */
1564 	if (error && callout_off)
1565 		callout_cpu_online(cp);
1566 
1567 	/*
1568 	 * If we failed, tell the PG subsystem that the CPU is back
1569 	 */
1570 	pg_cpupart_in(cp, pp);
1571 
1572 	/*
1573 	 * If we failed, we need to notify everyone that this CPU is back on.
1574 	 */
1575 	if (error != 0) {
1576 		CPU_NEW_GENERATION(cp);
1577 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1578 		cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1579 	}
1580 
1581 	return (error);
1582 }
1583 
1584 /*
1585  * Mark the indicated CPU as faulted, taking it offline.
1586  */
1587 int
cpu_faulted(cpu_t * cp,int flags)1588 cpu_faulted(cpu_t *cp, int flags)
1589 {
1590 	int	error = 0;
1591 
1592 	ASSERT(MUTEX_HELD(&cpu_lock));
1593 	ASSERT(!cpu_is_poweredoff(cp));
1594 
1595 	if (cpu_is_offline(cp)) {
1596 		cp->cpu_flags &= ~CPU_SPARE;
1597 		cp->cpu_flags |= CPU_FAULTED;
1598 		mp_cpu_faulted_enter(cp);
1599 		cpu_set_state(cp);
1600 		return (0);
1601 	}
1602 
1603 	if ((error = cpu_offline(cp, flags)) == 0) {
1604 		cp->cpu_flags |= CPU_FAULTED;
1605 		mp_cpu_faulted_enter(cp);
1606 		cpu_set_state(cp);
1607 	}
1608 
1609 	return (error);
1610 }
1611 
1612 /*
1613  * Mark the indicated CPU as a spare, taking it offline.
1614  */
1615 int
cpu_spare(cpu_t * cp,int flags)1616 cpu_spare(cpu_t *cp, int flags)
1617 {
1618 	int	error = 0;
1619 
1620 	ASSERT(MUTEX_HELD(&cpu_lock));
1621 	ASSERT(!cpu_is_poweredoff(cp));
1622 
1623 	if (cpu_is_offline(cp)) {
1624 		if (cp->cpu_flags & CPU_FAULTED) {
1625 			cp->cpu_flags &= ~CPU_FAULTED;
1626 			mp_cpu_faulted_exit(cp);
1627 		}
1628 		cp->cpu_flags |= CPU_SPARE;
1629 		cpu_set_state(cp);
1630 		return (0);
1631 	}
1632 
1633 	if ((error = cpu_offline(cp, flags)) == 0) {
1634 		cp->cpu_flags |= CPU_SPARE;
1635 		cpu_set_state(cp);
1636 	}
1637 
1638 	return (error);
1639 }
1640 
1641 /*
1642  * Take the indicated CPU from poweroff to offline.
1643  */
1644 int
cpu_poweron(cpu_t * cp)1645 cpu_poweron(cpu_t *cp)
1646 {
1647 	int	error = ENOTSUP;
1648 
1649 	ASSERT(MUTEX_HELD(&cpu_lock));
1650 	ASSERT(cpu_is_poweredoff(cp));
1651 
1652 	error = mp_cpu_poweron(cp);	/* arch-dep hook */
1653 	if (error == 0)
1654 		cpu_set_state(cp);
1655 
1656 	return (error);
1657 }
1658 
1659 /*
1660  * Take the indicated CPU from any inactive state to powered off.
1661  */
1662 int
cpu_poweroff(cpu_t * cp)1663 cpu_poweroff(cpu_t *cp)
1664 {
1665 	int	error = ENOTSUP;
1666 
1667 	ASSERT(MUTEX_HELD(&cpu_lock));
1668 	ASSERT(cpu_is_offline(cp));
1669 
1670 	if (!(cp->cpu_flags & CPU_QUIESCED))
1671 		return (EBUSY);		/* not completely idle */
1672 
1673 	error = mp_cpu_poweroff(cp);	/* arch-dep hook */
1674 	if (error == 0)
1675 		cpu_set_state(cp);
1676 
1677 	return (error);
1678 }
1679 
1680 /*
1681  * Initialize the Sequential CPU id lookup table
1682  */
1683 void
cpu_seq_tbl_init()1684 cpu_seq_tbl_init()
1685 {
1686 	cpu_t	**tbl;
1687 
1688 	tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1689 	tbl[0] = CPU;
1690 
1691 	cpu_seq = tbl;
1692 }
1693 
1694 /*
1695  * Initialize the CPU lists for the first CPU.
1696  */
1697 void
cpu_list_init(cpu_t * cp)1698 cpu_list_init(cpu_t *cp)
1699 {
1700 	cp->cpu_next = cp;
1701 	cp->cpu_prev = cp;
1702 	cpu_list = cp;
1703 	clock_cpu_list = cp;
1704 
1705 	cp->cpu_next_onln = cp;
1706 	cp->cpu_prev_onln = cp;
1707 	cpu_active = cp;
1708 
1709 	cp->cpu_seqid = 0;
1710 	CPUSET_ADD(cpu_seqid_inuse, 0);
1711 
1712 	/*
1713 	 * Bootstrap cpu_seq using cpu_list
1714 	 * The cpu_seq[] table will be dynamically allocated
1715 	 * when kmem later becomes available (but before going MP)
1716 	 */
1717 	cpu_seq = &cpu_list;
1718 
1719 	cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1720 	cp_default.cp_cpulist = cp;
1721 	cp_default.cp_ncpus = 1;
1722 	cp->cpu_next_part = cp;
1723 	cp->cpu_prev_part = cp;
1724 	cp->cpu_part = &cp_default;
1725 
1726 	CPUSET_ADD(cpu_available, cp->cpu_id);
1727 }
1728 
1729 /*
1730  * Insert a CPU into the list of available CPUs.
1731  */
1732 void
cpu_add_unit(cpu_t * cp)1733 cpu_add_unit(cpu_t *cp)
1734 {
1735 	int seqid;
1736 
1737 	ASSERT(MUTEX_HELD(&cpu_lock));
1738 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1739 
1740 	lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1741 
1742 	/*
1743 	 * Note: most users of the cpu_list will grab the
1744 	 * cpu_lock to insure that it isn't modified.  However,
1745 	 * certain users can't or won't do that.  To allow this
1746 	 * we pause the other cpus.  Users who walk the list
1747 	 * without cpu_lock, must disable kernel preemption
1748 	 * to insure that the list isn't modified underneath
1749 	 * them.  Also, any cached pointers to cpu structures
1750 	 * must be revalidated by checking to see if the
1751 	 * cpu_next pointer points to itself.  This check must
1752 	 * be done with the cpu_lock held or kernel preemption
1753 	 * disabled.  This check relies upon the fact that
1754 	 * old cpu structures are not free'ed or cleared after
1755 	 * then are removed from the cpu_list.
1756 	 *
1757 	 * Note that the clock code walks the cpu list dereferencing
1758 	 * the cpu_part pointer, so we need to initialize it before
1759 	 * adding the cpu to the list.
1760 	 */
1761 	cp->cpu_part = &cp_default;
1762 	pause_cpus(NULL, NULL);
1763 	cp->cpu_next = cpu_list;
1764 	cp->cpu_prev = cpu_list->cpu_prev;
1765 	cpu_list->cpu_prev->cpu_next = cp;
1766 	cpu_list->cpu_prev = cp;
1767 	start_cpus();
1768 
1769 	for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1770 		continue;
1771 	CPUSET_ADD(cpu_seqid_inuse, seqid);
1772 	cp->cpu_seqid = seqid;
1773 
1774 	if (seqid > max_cpu_seqid_ever)
1775 		max_cpu_seqid_ever = seqid;
1776 
1777 	ASSERT(ncpus < max_ncpus);
1778 	ncpus++;
1779 	cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1780 	cpu[cp->cpu_id] = cp;
1781 	CPUSET_ADD(cpu_available, cp->cpu_id);
1782 	cpu_seq[cp->cpu_seqid] = cp;
1783 
1784 	/*
1785 	 * allocate a pause thread for this CPU.
1786 	 */
1787 	cpu_pause_alloc(cp);
1788 
1789 	/*
1790 	 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1791 	 * link them into a list of just that CPU.
1792 	 * This is so that disp_lowpri_cpu will work for thread_create in
1793 	 * pause_cpus() when called from the startup thread in a new CPU.
1794 	 */
1795 	cp->cpu_next_onln = cp;
1796 	cp->cpu_prev_onln = cp;
1797 	cpu_info_kstat_create(cp);
1798 	cp->cpu_next_part = cp;
1799 	cp->cpu_prev_part = cp;
1800 
1801 	init_cpu_mstate(cp, CMS_SYSTEM);
1802 
1803 	pool_pset_mod = gethrtime();
1804 }
1805 
1806 /*
1807  * Do the opposite of cpu_add_unit().
1808  */
1809 void
cpu_del_unit(int cpuid)1810 cpu_del_unit(int cpuid)
1811 {
1812 	struct cpu	*cp, *cpnext;
1813 
1814 	ASSERT(MUTEX_HELD(&cpu_lock));
1815 	cp = cpu[cpuid];
1816 	ASSERT(cp != NULL);
1817 
1818 	ASSERT(cp->cpu_next_onln == cp);
1819 	ASSERT(cp->cpu_prev_onln == cp);
1820 	ASSERT(cp->cpu_next_part == cp);
1821 	ASSERT(cp->cpu_prev_part == cp);
1822 
1823 	/*
1824 	 * Tear down the CPU's physical ID cache, and update any
1825 	 * processor groups
1826 	 */
1827 	pg_cpu_fini(cp, NULL);
1828 	pghw_physid_destroy(cp);
1829 
1830 	/*
1831 	 * Destroy kstat stuff.
1832 	 */
1833 	cpu_info_kstat_destroy(cp);
1834 	term_cpu_mstate(cp);
1835 	/*
1836 	 * Free up pause thread.
1837 	 */
1838 	cpu_pause_free(cp);
1839 	CPUSET_DEL(cpu_available, cp->cpu_id);
1840 	cpu[cp->cpu_id] = NULL;
1841 	cpu_seq[cp->cpu_seqid] = NULL;
1842 
1843 	/*
1844 	 * The clock thread and mutex_vector_enter cannot hold the
1845 	 * cpu_lock while traversing the cpu list, therefore we pause
1846 	 * all other threads by pausing the other cpus. These, and any
1847 	 * other routines holding cpu pointers while possibly sleeping
1848 	 * must be sure to call kpreempt_disable before processing the
1849 	 * list and be sure to check that the cpu has not been deleted
1850 	 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1851 	 * to keep the deleted cpu structure around.
1852 	 *
1853 	 * Note that this MUST be done AFTER cpu_available
1854 	 * has been updated so that we don't waste time
1855 	 * trying to pause the cpu we're trying to delete.
1856 	 */
1857 	pause_cpus(NULL, NULL);
1858 
1859 	cpnext = cp->cpu_next;
1860 	cp->cpu_prev->cpu_next = cp->cpu_next;
1861 	cp->cpu_next->cpu_prev = cp->cpu_prev;
1862 	if (cp == cpu_list)
1863 		cpu_list = cpnext;
1864 
1865 	/*
1866 	 * Signals that the cpu has been deleted (see above).
1867 	 */
1868 	cp->cpu_next = NULL;
1869 	cp->cpu_prev = NULL;
1870 
1871 	start_cpus();
1872 
1873 	CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1874 	ncpus--;
1875 	lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1876 
1877 	pool_pset_mod = gethrtime();
1878 }
1879 
1880 /*
1881  * Add a CPU to the list of active CPUs.
1882  *	This routine must not get any locks, because other CPUs are paused.
1883  */
1884 static void
cpu_add_active_internal(cpu_t * cp)1885 cpu_add_active_internal(cpu_t *cp)
1886 {
1887 	cpupart_t	*pp = cp->cpu_part;
1888 
1889 	ASSERT(MUTEX_HELD(&cpu_lock));
1890 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1891 
1892 	ncpus_online++;
1893 	cpu_set_state(cp);
1894 	cp->cpu_next_onln = cpu_active;
1895 	cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1896 	cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1897 	cpu_active->cpu_prev_onln = cp;
1898 
1899 	if (pp->cp_cpulist) {
1900 		cp->cpu_next_part = pp->cp_cpulist;
1901 		cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1902 		pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1903 		pp->cp_cpulist->cpu_prev_part = cp;
1904 	} else {
1905 		ASSERT(pp->cp_ncpus == 0);
1906 		pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1907 	}
1908 	pp->cp_ncpus++;
1909 	if (pp->cp_ncpus == 1) {
1910 		cp_numparts_nonempty++;
1911 		ASSERT(cp_numparts_nonempty != 0);
1912 	}
1913 
1914 	pg_cpu_active(cp);
1915 	lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1916 
1917 	bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1918 }
1919 
1920 /*
1921  * Add a CPU to the list of active CPUs.
1922  *	This is called from machine-dependent layers when a new CPU is started.
1923  */
1924 void
cpu_add_active(cpu_t * cp)1925 cpu_add_active(cpu_t *cp)
1926 {
1927 	pg_cpupart_in(cp, cp->cpu_part);
1928 
1929 	pause_cpus(NULL, NULL);
1930 	cpu_add_active_internal(cp);
1931 	start_cpus();
1932 
1933 	cpu_stats_kstat_create(cp);
1934 	cpu_create_intrstat(cp);
1935 	lgrp_kstat_create(cp);
1936 	cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1937 }
1938 
1939 
1940 /*
1941  * Remove a CPU from the list of active CPUs.
1942  *	This routine must not get any locks, because other CPUs are paused.
1943  */
1944 /* ARGSUSED */
1945 static void
cpu_remove_active(cpu_t * cp)1946 cpu_remove_active(cpu_t *cp)
1947 {
1948 	cpupart_t	*pp = cp->cpu_part;
1949 
1950 	ASSERT(MUTEX_HELD(&cpu_lock));
1951 	ASSERT(cp->cpu_next_onln != cp);	/* not the last one */
1952 	ASSERT(cp->cpu_prev_onln != cp);	/* not the last one */
1953 
1954 	pg_cpu_inactive(cp);
1955 
1956 	lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1957 
1958 	if (cp == clock_cpu_list)
1959 		clock_cpu_list = cp->cpu_next_onln;
1960 
1961 	cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1962 	cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1963 	if (cpu_active == cp) {
1964 		cpu_active = cp->cpu_next_onln;
1965 	}
1966 	cp->cpu_next_onln = cp;
1967 	cp->cpu_prev_onln = cp;
1968 
1969 	cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1970 	cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1971 	if (pp->cp_cpulist == cp) {
1972 		pp->cp_cpulist = cp->cpu_next_part;
1973 		ASSERT(pp->cp_cpulist != cp);
1974 	}
1975 	cp->cpu_next_part = cp;
1976 	cp->cpu_prev_part = cp;
1977 	pp->cp_ncpus--;
1978 	if (pp->cp_ncpus == 0) {
1979 		cp_numparts_nonempty--;
1980 		ASSERT(cp_numparts_nonempty != 0);
1981 	}
1982 }
1983 
1984 /*
1985  * Routine used to setup a newly inserted CPU in preparation for starting
1986  * it running code.
1987  */
1988 int
cpu_configure(int cpuid)1989 cpu_configure(int cpuid)
1990 {
1991 	int retval = 0;
1992 
1993 	ASSERT(MUTEX_HELD(&cpu_lock));
1994 
1995 	/*
1996 	 * Some structures are statically allocated based upon
1997 	 * the maximum number of cpus the system supports.  Do not
1998 	 * try to add anything beyond this limit.
1999 	 */
2000 	if (cpuid < 0 || cpuid >= NCPU) {
2001 		return (EINVAL);
2002 	}
2003 
2004 	if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
2005 		return (EALREADY);
2006 	}
2007 
2008 	if ((retval = mp_cpu_configure(cpuid)) != 0) {
2009 		return (retval);
2010 	}
2011 
2012 	cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2013 	cpu_set_state(cpu[cpuid]);
2014 	retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2015 	if (retval != 0)
2016 		(void) mp_cpu_unconfigure(cpuid);
2017 
2018 	return (retval);
2019 }
2020 
2021 /*
2022  * Routine used to cleanup a CPU that has been powered off.  This will
2023  * destroy all per-cpu information related to this cpu.
2024  */
2025 int
cpu_unconfigure(int cpuid)2026 cpu_unconfigure(int cpuid)
2027 {
2028 	int error;
2029 
2030 	ASSERT(MUTEX_HELD(&cpu_lock));
2031 
2032 	if (cpu[cpuid] == NULL) {
2033 		return (ENODEV);
2034 	}
2035 
2036 	if (cpu[cpuid]->cpu_flags == 0) {
2037 		return (EALREADY);
2038 	}
2039 
2040 	if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2041 		return (EBUSY);
2042 	}
2043 
2044 	if (cpu[cpuid]->cpu_props != NULL) {
2045 		(void) nvlist_free(cpu[cpuid]->cpu_props);
2046 		cpu[cpuid]->cpu_props = NULL;
2047 	}
2048 
2049 	error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2050 
2051 	if (error != 0)
2052 		return (error);
2053 
2054 	return (mp_cpu_unconfigure(cpuid));
2055 }
2056 
2057 /*
2058  * Routines for registering and de-registering cpu_setup callback functions.
2059  *
2060  * Caller's context
2061  *	These routines must not be called from a driver's attach(9E) or
2062  *	detach(9E) entry point.
2063  *
2064  * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2065  */
2066 
2067 /*
2068  * Ideally, these would be dynamically allocated and put into a linked
2069  * list; however that is not feasible because the registration routine
2070  * has to be available before the kmem allocator is working (in fact,
2071  * it is called by the kmem allocator init code).  In any case, there
2072  * are quite a few extra entries for future users.
2073  */
2074 #define	NCPU_SETUPS	20
2075 
2076 struct cpu_setup {
2077 	cpu_setup_func_t *func;
2078 	void *arg;
2079 } cpu_setups[NCPU_SETUPS];
2080 
2081 void
register_cpu_setup_func(cpu_setup_func_t * func,void * arg)2082 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2083 {
2084 	int i;
2085 
2086 	ASSERT(MUTEX_HELD(&cpu_lock));
2087 
2088 	for (i = 0; i < NCPU_SETUPS; i++)
2089 		if (cpu_setups[i].func == NULL)
2090 			break;
2091 	if (i >= NCPU_SETUPS)
2092 		cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2093 
2094 	cpu_setups[i].func = func;
2095 	cpu_setups[i].arg = arg;
2096 }
2097 
2098 void
unregister_cpu_setup_func(cpu_setup_func_t * func,void * arg)2099 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2100 {
2101 	int i;
2102 
2103 	ASSERT(MUTEX_HELD(&cpu_lock));
2104 
2105 	for (i = 0; i < NCPU_SETUPS; i++)
2106 		if ((cpu_setups[i].func == func) &&
2107 		    (cpu_setups[i].arg == arg))
2108 			break;
2109 	if (i >= NCPU_SETUPS)
2110 		cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2111 		    "deregister");
2112 
2113 	cpu_setups[i].func = NULL;
2114 	cpu_setups[i].arg = 0;
2115 }
2116 
2117 /*
2118  * Call any state change hooks for this CPU, ignore any errors.
2119  */
2120 void
cpu_state_change_notify(int id,cpu_setup_t what)2121 cpu_state_change_notify(int id, cpu_setup_t what)
2122 {
2123 	int i;
2124 
2125 	ASSERT(MUTEX_HELD(&cpu_lock));
2126 
2127 	for (i = 0; i < NCPU_SETUPS; i++) {
2128 		if (cpu_setups[i].func != NULL) {
2129 			cpu_setups[i].func(what, id, cpu_setups[i].arg);
2130 		}
2131 	}
2132 }
2133 
2134 /*
2135  * Call any state change hooks for this CPU, undo it if error found.
2136  */
2137 static int
cpu_state_change_hooks(int id,cpu_setup_t what,cpu_setup_t undo)2138 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2139 {
2140 	int i;
2141 	int retval = 0;
2142 
2143 	ASSERT(MUTEX_HELD(&cpu_lock));
2144 
2145 	for (i = 0; i < NCPU_SETUPS; i++) {
2146 		if (cpu_setups[i].func != NULL) {
2147 			retval = cpu_setups[i].func(what, id,
2148 			    cpu_setups[i].arg);
2149 			if (retval) {
2150 				for (i--; i >= 0; i--) {
2151 					if (cpu_setups[i].func != NULL)
2152 						cpu_setups[i].func(undo,
2153 						    id, cpu_setups[i].arg);
2154 				}
2155 				break;
2156 			}
2157 		}
2158 	}
2159 	return (retval);
2160 }
2161 
2162 /*
2163  * Export information about this CPU via the kstat mechanism.
2164  */
2165 static struct {
2166 	kstat_named_t ci_state;
2167 	kstat_named_t ci_state_begin;
2168 	kstat_named_t ci_cpu_type;
2169 	kstat_named_t ci_fpu_type;
2170 	kstat_named_t ci_clock_MHz;
2171 	kstat_named_t ci_chip_id;
2172 	kstat_named_t ci_implementation;
2173 	kstat_named_t ci_brandstr;
2174 	kstat_named_t ci_core_id;
2175 	kstat_named_t ci_curr_clock_Hz;
2176 	kstat_named_t ci_supp_freq_Hz;
2177 	kstat_named_t ci_pg_id;
2178 #if defined(__sparcv9)
2179 	kstat_named_t ci_device_ID;
2180 	kstat_named_t ci_cpu_fru;
2181 #endif
2182 #if defined(__x86)
2183 	kstat_named_t ci_vendorstr;
2184 	kstat_named_t ci_family;
2185 	kstat_named_t ci_model;
2186 	kstat_named_t ci_step;
2187 	kstat_named_t ci_clogid;
2188 	kstat_named_t ci_pkg_core_id;
2189 	kstat_named_t ci_ncpuperchip;
2190 	kstat_named_t ci_ncoreperchip;
2191 	kstat_named_t ci_max_cstates;
2192 	kstat_named_t ci_curr_cstate;
2193 	kstat_named_t ci_cacheid;
2194 	kstat_named_t ci_sktstr;
2195 #endif
2196 } cpu_info_template = {
2197 	{ "state",			KSTAT_DATA_CHAR },
2198 	{ "state_begin",		KSTAT_DATA_LONG },
2199 	{ "cpu_type",			KSTAT_DATA_CHAR },
2200 	{ "fpu_type",			KSTAT_DATA_CHAR },
2201 	{ "clock_MHz",			KSTAT_DATA_LONG },
2202 	{ "chip_id",			KSTAT_DATA_LONG },
2203 	{ "implementation",		KSTAT_DATA_STRING },
2204 	{ "brand",			KSTAT_DATA_STRING },
2205 	{ "core_id",			KSTAT_DATA_LONG },
2206 	{ "current_clock_Hz",		KSTAT_DATA_UINT64 },
2207 	{ "supported_frequencies_Hz",	KSTAT_DATA_STRING },
2208 	{ "pg_id",			KSTAT_DATA_LONG },
2209 #if defined(__sparcv9)
2210 	{ "device_ID",			KSTAT_DATA_UINT64 },
2211 	{ "cpu_fru",			KSTAT_DATA_STRING },
2212 #endif
2213 #if defined(__x86)
2214 	{ "vendor_id",			KSTAT_DATA_STRING },
2215 	{ "family",			KSTAT_DATA_INT32 },
2216 	{ "model",			KSTAT_DATA_INT32 },
2217 	{ "stepping",			KSTAT_DATA_INT32 },
2218 	{ "clog_id",			KSTAT_DATA_INT32 },
2219 	{ "pkg_core_id",		KSTAT_DATA_LONG },
2220 	{ "ncpu_per_chip",		KSTAT_DATA_INT32 },
2221 	{ "ncore_per_chip",		KSTAT_DATA_INT32 },
2222 	{ "supported_max_cstates",	KSTAT_DATA_INT32 },
2223 	{ "current_cstate",		KSTAT_DATA_INT32 },
2224 	{ "cache_id",			KSTAT_DATA_INT32 },
2225 	{ "socket_type",		KSTAT_DATA_STRING },
2226 #endif
2227 };
2228 
2229 static kmutex_t cpu_info_template_lock;
2230 
2231 static int
cpu_info_kstat_update(kstat_t * ksp,int rw)2232 cpu_info_kstat_update(kstat_t *ksp, int rw)
2233 {
2234 	cpu_t	*cp = ksp->ks_private;
2235 	const char *pi_state;
2236 
2237 	if (rw == KSTAT_WRITE)
2238 		return (EACCES);
2239 
2240 #if defined(__x86)
2241 	/* Is the cpu still initialising itself? */
2242 	if (cpuid_checkpass(cp, 1) == 0)
2243 		return (ENXIO);
2244 #endif
2245 	switch (cp->cpu_type_info.pi_state) {
2246 	case P_ONLINE:
2247 		pi_state = PS_ONLINE;
2248 		break;
2249 	case P_POWEROFF:
2250 		pi_state = PS_POWEROFF;
2251 		break;
2252 	case P_NOINTR:
2253 		pi_state = PS_NOINTR;
2254 		break;
2255 	case P_FAULTED:
2256 		pi_state = PS_FAULTED;
2257 		break;
2258 	case P_SPARE:
2259 		pi_state = PS_SPARE;
2260 		break;
2261 	case P_OFFLINE:
2262 		pi_state = PS_OFFLINE;
2263 		break;
2264 	default:
2265 		pi_state = "unknown";
2266 	}
2267 	(void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2268 	cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2269 	(void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2270 	    cp->cpu_type_info.pi_processor_type, 15);
2271 	(void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2272 	    cp->cpu_type_info.pi_fputypes, 15);
2273 	cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2274 	cpu_info_template.ci_chip_id.value.l =
2275 	    pg_plat_hw_instance_id(cp, PGHW_CHIP);
2276 	kstat_named_setstr(&cpu_info_template.ci_implementation,
2277 	    cp->cpu_idstr);
2278 	kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2279 	cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2280 	cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2281 	    cp->cpu_curr_clock;
2282 	cpu_info_template.ci_pg_id.value.l =
2283 	    cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2284 	    cp->cpu_pg->cmt_lineage->pg_id : -1;
2285 	kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2286 	    cp->cpu_supp_freqs);
2287 #if defined(__sparcv9)
2288 	cpu_info_template.ci_device_ID.value.ui64 =
2289 	    cpunodes[cp->cpu_id].device_id;
2290 	kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2291 #endif
2292 #if defined(__x86)
2293 	kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2294 	    cpuid_getvendorstr(cp));
2295 	cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2296 	cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2297 	cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2298 	cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2299 	cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2300 	cpu_info_template.ci_ncoreperchip.value.l =
2301 	    cpuid_get_ncore_per_chip(cp);
2302 	cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2303 	cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2304 	cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2305 	cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2306 	kstat_named_setstr(&cpu_info_template.ci_sktstr,
2307 	    cpuid_getsocketstr(cp));
2308 #endif
2309 
2310 	return (0);
2311 }
2312 
2313 static void
cpu_info_kstat_create(cpu_t * cp)2314 cpu_info_kstat_create(cpu_t *cp)
2315 {
2316 	zoneid_t zoneid;
2317 
2318 	ASSERT(MUTEX_HELD(&cpu_lock));
2319 
2320 	if (pool_pset_enabled())
2321 		zoneid = GLOBAL_ZONEID;
2322 	else
2323 		zoneid = ALL_ZONES;
2324 	if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2325 	    NULL, "misc", KSTAT_TYPE_NAMED,
2326 	    sizeof (cpu_info_template) / sizeof (kstat_named_t),
2327 	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2328 		cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2329 #if defined(__sparcv9)
2330 		cp->cpu_info_kstat->ks_data_size +=
2331 		    strlen(cpu_fru_fmri(cp)) + 1;
2332 #endif
2333 #if defined(__x86)
2334 		cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2335 #endif
2336 		if (cp->cpu_supp_freqs != NULL)
2337 			cp->cpu_info_kstat->ks_data_size +=
2338 			    strlen(cp->cpu_supp_freqs) + 1;
2339 		cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2340 		cp->cpu_info_kstat->ks_data = &cpu_info_template;
2341 		cp->cpu_info_kstat->ks_private = cp;
2342 		cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2343 		kstat_install(cp->cpu_info_kstat);
2344 	}
2345 }
2346 
2347 static void
cpu_info_kstat_destroy(cpu_t * cp)2348 cpu_info_kstat_destroy(cpu_t *cp)
2349 {
2350 	ASSERT(MUTEX_HELD(&cpu_lock));
2351 
2352 	kstat_delete(cp->cpu_info_kstat);
2353 	cp->cpu_info_kstat = NULL;
2354 }
2355 
2356 /*
2357  * Create and install kstats for the boot CPU.
2358  */
2359 void
cpu_kstat_init(cpu_t * cp)2360 cpu_kstat_init(cpu_t *cp)
2361 {
2362 	mutex_enter(&cpu_lock);
2363 	cpu_info_kstat_create(cp);
2364 	cpu_stats_kstat_create(cp);
2365 	cpu_create_intrstat(cp);
2366 	cpu_set_state(cp);
2367 	mutex_exit(&cpu_lock);
2368 }
2369 
2370 /*
2371  * Make visible to the zone that subset of the cpu information that would be
2372  * initialized when a cpu is configured (but still offline).
2373  */
2374 void
cpu_visibility_configure(cpu_t * cp,zone_t * zone)2375 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2376 {
2377 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2378 
2379 	ASSERT(MUTEX_HELD(&cpu_lock));
2380 	ASSERT(pool_pset_enabled());
2381 	ASSERT(cp != NULL);
2382 
2383 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2384 		zone->zone_ncpus++;
2385 		ASSERT(zone->zone_ncpus <= ncpus);
2386 	}
2387 	if (cp->cpu_info_kstat != NULL)
2388 		kstat_zone_add(cp->cpu_info_kstat, zoneid);
2389 }
2390 
2391 /*
2392  * Make visible to the zone that subset of the cpu information that would be
2393  * initialized when a previously configured cpu is onlined.
2394  */
2395 void
cpu_visibility_online(cpu_t * cp,zone_t * zone)2396 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2397 {
2398 	kstat_t *ksp;
2399 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2400 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2401 	processorid_t cpun;
2402 
2403 	ASSERT(MUTEX_HELD(&cpu_lock));
2404 	ASSERT(pool_pset_enabled());
2405 	ASSERT(cp != NULL);
2406 	ASSERT(cpu_is_active(cp));
2407 
2408 	cpun = cp->cpu_id;
2409 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2410 		zone->zone_ncpus_online++;
2411 		ASSERT(zone->zone_ncpus_online <= ncpus_online);
2412 	}
2413 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2414 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2415 	    != NULL) {
2416 		kstat_zone_add(ksp, zoneid);
2417 		kstat_rele(ksp);
2418 	}
2419 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2420 		kstat_zone_add(ksp, zoneid);
2421 		kstat_rele(ksp);
2422 	}
2423 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2424 		kstat_zone_add(ksp, zoneid);
2425 		kstat_rele(ksp);
2426 	}
2427 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2428 	    NULL) {
2429 		kstat_zone_add(ksp, zoneid);
2430 		kstat_rele(ksp);
2431 	}
2432 }
2433 
2434 /*
2435  * Update relevant kstats such that cpu is now visible to processes
2436  * executing in specified zone.
2437  */
2438 void
cpu_visibility_add(cpu_t * cp,zone_t * zone)2439 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2440 {
2441 	cpu_visibility_configure(cp, zone);
2442 	if (cpu_is_active(cp))
2443 		cpu_visibility_online(cp, zone);
2444 }
2445 
2446 /*
2447  * Make invisible to the zone that subset of the cpu information that would be
2448  * torn down when a previously offlined cpu is unconfigured.
2449  */
2450 void
cpu_visibility_unconfigure(cpu_t * cp,zone_t * zone)2451 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2452 {
2453 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2454 
2455 	ASSERT(MUTEX_HELD(&cpu_lock));
2456 	ASSERT(pool_pset_enabled());
2457 	ASSERT(cp != NULL);
2458 
2459 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2460 		ASSERT(zone->zone_ncpus != 0);
2461 		zone->zone_ncpus--;
2462 	}
2463 	if (cp->cpu_info_kstat)
2464 		kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2465 }
2466 
2467 /*
2468  * Make invisible to the zone that subset of the cpu information that would be
2469  * torn down when a cpu is offlined (but still configured).
2470  */
2471 void
cpu_visibility_offline(cpu_t * cp,zone_t * zone)2472 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2473 {
2474 	kstat_t *ksp;
2475 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2476 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2477 	processorid_t cpun;
2478 
2479 	ASSERT(MUTEX_HELD(&cpu_lock));
2480 	ASSERT(pool_pset_enabled());
2481 	ASSERT(cp != NULL);
2482 	ASSERT(cpu_is_active(cp));
2483 
2484 	cpun = cp->cpu_id;
2485 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2486 		ASSERT(zone->zone_ncpus_online != 0);
2487 		zone->zone_ncpus_online--;
2488 	}
2489 
2490 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2491 	    NULL) {
2492 		kstat_zone_remove(ksp, zoneid);
2493 		kstat_rele(ksp);
2494 	}
2495 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2496 		kstat_zone_remove(ksp, zoneid);
2497 		kstat_rele(ksp);
2498 	}
2499 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2500 		kstat_zone_remove(ksp, zoneid);
2501 		kstat_rele(ksp);
2502 	}
2503 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2504 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2505 	    != NULL) {
2506 		kstat_zone_remove(ksp, zoneid);
2507 		kstat_rele(ksp);
2508 	}
2509 }
2510 
2511 /*
2512  * Update relevant kstats such that cpu is no longer visible to processes
2513  * executing in specified zone.
2514  */
2515 void
cpu_visibility_remove(cpu_t * cp,zone_t * zone)2516 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2517 {
2518 	if (cpu_is_active(cp))
2519 		cpu_visibility_offline(cp, zone);
2520 	cpu_visibility_unconfigure(cp, zone);
2521 }
2522 
2523 /*
2524  * Bind a thread to a CPU as requested.
2525  */
2526 int
cpu_bind_thread(kthread_id_t tp,processorid_t bind,processorid_t * obind,int * error)2527 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2528     int *error)
2529 {
2530 	processorid_t	binding;
2531 	cpu_t		*cp = NULL;
2532 
2533 	ASSERT(MUTEX_HELD(&cpu_lock));
2534 	ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2535 
2536 	thread_lock(tp);
2537 
2538 	/*
2539 	 * Record old binding, but change the obind, which was initialized
2540 	 * to PBIND_NONE, only if this thread has a binding.  This avoids
2541 	 * reporting PBIND_NONE for a process when some LWPs are bound.
2542 	 */
2543 	binding = tp->t_bind_cpu;
2544 	if (binding != PBIND_NONE)
2545 		*obind = binding;	/* record old binding */
2546 
2547 	switch (bind) {
2548 	case PBIND_QUERY:
2549 		/* Just return the old binding */
2550 		thread_unlock(tp);
2551 		return (0);
2552 
2553 	case PBIND_QUERY_TYPE:
2554 		/* Return the binding type */
2555 		*obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2556 		thread_unlock(tp);
2557 		return (0);
2558 
2559 	case PBIND_SOFT:
2560 		/*
2561 		 *  Set soft binding for this thread and return the actual
2562 		 *  binding
2563 		 */
2564 		TB_CPU_SOFT_SET(tp);
2565 		thread_unlock(tp);
2566 		return (0);
2567 
2568 	case PBIND_HARD:
2569 		/*
2570 		 *  Set hard binding for this thread and return the actual
2571 		 *  binding
2572 		 */
2573 		TB_CPU_HARD_SET(tp);
2574 		thread_unlock(tp);
2575 		return (0);
2576 
2577 	default:
2578 		break;
2579 	}
2580 
2581 	/*
2582 	 * If this thread/LWP cannot be bound because of permission
2583 	 * problems, just note that and return success so that the
2584 	 * other threads/LWPs will be bound.  This is the way
2585 	 * processor_bind() is defined to work.
2586 	 *
2587 	 * Binding will get EPERM if the thread is of system class
2588 	 * or hasprocperm() fails.
2589 	 */
2590 	if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2591 		*error = EPERM;
2592 		thread_unlock(tp);
2593 		return (0);
2594 	}
2595 
2596 	binding = bind;
2597 	if (binding != PBIND_NONE) {
2598 		cp = cpu_get((processorid_t)binding);
2599 		/*
2600 		 * Make sure binding is valid and is in right partition.
2601 		 */
2602 		if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2603 			*error = EINVAL;
2604 			thread_unlock(tp);
2605 			return (0);
2606 		}
2607 	}
2608 	tp->t_bind_cpu = binding;	/* set new binding */
2609 
2610 	/*
2611 	 * If there is no system-set reason for affinity, set
2612 	 * the t_bound_cpu field to reflect the binding.
2613 	 */
2614 	if (tp->t_affinitycnt == 0) {
2615 		if (binding == PBIND_NONE) {
2616 			/*
2617 			 * We may need to adjust disp_max_unbound_pri
2618 			 * since we're becoming unbound.
2619 			 */
2620 			disp_adjust_unbound_pri(tp);
2621 
2622 			tp->t_bound_cpu = NULL;	/* set new binding */
2623 
2624 			/*
2625 			 * Move thread to lgroup with strongest affinity
2626 			 * after unbinding
2627 			 */
2628 			if (tp->t_lgrp_affinity)
2629 				lgrp_move_thread(tp,
2630 				    lgrp_choose(tp, tp->t_cpupart), 1);
2631 
2632 			if (tp->t_state == TS_ONPROC &&
2633 			    tp->t_cpu->cpu_part != tp->t_cpupart)
2634 				cpu_surrender(tp);
2635 		} else {
2636 			lpl_t	*lpl;
2637 
2638 			tp->t_bound_cpu = cp;
2639 			ASSERT(cp->cpu_lpl != NULL);
2640 
2641 			/*
2642 			 * Set home to lgroup with most affinity containing CPU
2643 			 * that thread is being bound or minimum bounding
2644 			 * lgroup if no affinities set
2645 			 */
2646 			if (tp->t_lgrp_affinity)
2647 				lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2648 				    LGRP_NONE, B_FALSE);
2649 			else
2650 				lpl = cp->cpu_lpl;
2651 
2652 			if (tp->t_lpl != lpl) {
2653 				/* can't grab cpu_lock */
2654 				lgrp_move_thread(tp, lpl, 1);
2655 			}
2656 
2657 			/*
2658 			 * Make the thread switch to the bound CPU.
2659 			 * If the thread is runnable, we need to
2660 			 * requeue it even if t_cpu is already set
2661 			 * to the right CPU, since it may be on a
2662 			 * kpreempt queue and need to move to a local
2663 			 * queue.  We could check t_disp_queue to
2664 			 * avoid unnecessary overhead if it's already
2665 			 * on the right queue, but since this isn't
2666 			 * a performance-critical operation it doesn't
2667 			 * seem worth the extra code and complexity.
2668 			 *
2669 			 * If the thread is weakbound to the cpu then it will
2670 			 * resist the new binding request until the weak
2671 			 * binding drops.  The cpu_surrender or requeueing
2672 			 * below could be skipped in such cases (since it
2673 			 * will have no effect), but that would require
2674 			 * thread_allowmigrate to acquire thread_lock so
2675 			 * we'll take the very occasional hit here instead.
2676 			 */
2677 			if (tp->t_state == TS_ONPROC) {
2678 				cpu_surrender(tp);
2679 			} else if (tp->t_state == TS_RUN) {
2680 				cpu_t *ocp = tp->t_cpu;
2681 
2682 				(void) dispdeq(tp);
2683 				setbackdq(tp);
2684 				/*
2685 				 * Either on the bound CPU's disp queue now,
2686 				 * or swapped out or on the swap queue.
2687 				 */
2688 				ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2689 				    tp->t_weakbound_cpu == ocp ||
2690 				    (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2691 				    != TS_LOAD);
2692 			}
2693 		}
2694 	}
2695 
2696 	/*
2697 	 * Our binding has changed; set TP_CHANGEBIND.
2698 	 */
2699 	tp->t_proc_flag |= TP_CHANGEBIND;
2700 	aston(tp);
2701 
2702 	thread_unlock(tp);
2703 
2704 	return (0);
2705 }
2706 
2707 #if CPUSET_WORDS > 1
2708 
2709 /*
2710  * Functions for implementing cpuset operations when a cpuset is more
2711  * than one word.  On platforms where a cpuset is a single word these
2712  * are implemented as macros in cpuvar.h.
2713  */
2714 
2715 void
cpuset_all(cpuset_t * s)2716 cpuset_all(cpuset_t *s)
2717 {
2718 	int i;
2719 
2720 	for (i = 0; i < CPUSET_WORDS; i++)
2721 		s->cpub[i] = ~0UL;
2722 }
2723 
2724 void
cpuset_all_but(cpuset_t * s,uint_t cpu)2725 cpuset_all_but(cpuset_t *s, uint_t cpu)
2726 {
2727 	cpuset_all(s);
2728 	CPUSET_DEL(*s, cpu);
2729 }
2730 
2731 void
cpuset_only(cpuset_t * s,uint_t cpu)2732 cpuset_only(cpuset_t *s, uint_t cpu)
2733 {
2734 	CPUSET_ZERO(*s);
2735 	CPUSET_ADD(*s, cpu);
2736 }
2737 
2738 int
cpuset_isnull(cpuset_t * s)2739 cpuset_isnull(cpuset_t *s)
2740 {
2741 	int i;
2742 
2743 	for (i = 0; i < CPUSET_WORDS; i++)
2744 		if (s->cpub[i] != 0)
2745 			return (0);
2746 	return (1);
2747 }
2748 
2749 int
cpuset_cmp(cpuset_t * s1,cpuset_t * s2)2750 cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2751 {
2752 	int i;
2753 
2754 	for (i = 0; i < CPUSET_WORDS; i++)
2755 		if (s1->cpub[i] != s2->cpub[i])
2756 			return (0);
2757 	return (1);
2758 }
2759 
2760 uint_t
cpuset_find(cpuset_t * s)2761 cpuset_find(cpuset_t *s)
2762 {
2763 
2764 	uint_t	i;
2765 	uint_t	cpu = (uint_t)-1;
2766 
2767 	/*
2768 	 * Find a cpu in the cpuset
2769 	 */
2770 	for (i = 0; i < CPUSET_WORDS; i++) {
2771 		cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2772 		if (cpu != (uint_t)-1) {
2773 			cpu += i * BT_NBIPUL;
2774 			break;
2775 		}
2776 	}
2777 	return (cpu);
2778 }
2779 
2780 void
cpuset_bounds(cpuset_t * s,uint_t * smallestid,uint_t * largestid)2781 cpuset_bounds(cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2782 {
2783 	int	i, j;
2784 	uint_t	bit;
2785 
2786 	/*
2787 	 * First, find the smallest cpu id in the set.
2788 	 */
2789 	for (i = 0; i < CPUSET_WORDS; i++) {
2790 		if (s->cpub[i] != 0) {
2791 			bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2792 			ASSERT(bit != (uint_t)-1);
2793 			*smallestid = bit + (i * BT_NBIPUL);
2794 
2795 			/*
2796 			 * Now find the largest cpu id in
2797 			 * the set and return immediately.
2798 			 * Done in an inner loop to avoid
2799 			 * having to break out of the first
2800 			 * loop.
2801 			 */
2802 			for (j = CPUSET_WORDS - 1; j >= i; j--) {
2803 				if (s->cpub[j] != 0) {
2804 					bit = (uint_t)(highbit(s->cpub[j]) - 1);
2805 					ASSERT(bit != (uint_t)-1);
2806 					*largestid = bit + (j * BT_NBIPUL);
2807 					ASSERT(*largestid >= *smallestid);
2808 					return;
2809 				}
2810 			}
2811 
2812 			/*
2813 			 * If this code is reached, a
2814 			 * smallestid was found, but not a
2815 			 * largestid. The cpuset must have
2816 			 * been changed during the course
2817 			 * of this function call.
2818 			 */
2819 			ASSERT(0);
2820 		}
2821 	}
2822 	*smallestid = *largestid = CPUSET_NOTINSET;
2823 }
2824 
2825 #endif	/* CPUSET_WORDS */
2826 
2827 /*
2828  * Unbind threads bound to specified CPU.
2829  *
2830  * If `unbind_all_threads' is true, unbind all user threads bound to a given
2831  * CPU. Otherwise unbind all soft-bound user threads.
2832  */
2833 int
cpu_unbind(processorid_t cpu,boolean_t unbind_all_threads)2834 cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2835 {
2836 	processorid_t obind;
2837 	kthread_t *tp;
2838 	int ret = 0;
2839 	proc_t *pp;
2840 	int err, berr = 0;
2841 
2842 	ASSERT(MUTEX_HELD(&cpu_lock));
2843 
2844 	mutex_enter(&pidlock);
2845 	for (pp = practive; pp != NULL; pp = pp->p_next) {
2846 		mutex_enter(&pp->p_lock);
2847 		tp = pp->p_tlist;
2848 		/*
2849 		 * Skip zombies, kernel processes, and processes in
2850 		 * other zones, if called from a non-global zone.
2851 		 */
2852 		if (tp == NULL || (pp->p_flag & SSYS) ||
2853 		    !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2854 			mutex_exit(&pp->p_lock);
2855 			continue;
2856 		}
2857 		do {
2858 			if (tp->t_bind_cpu != cpu)
2859 				continue;
2860 			/*
2861 			 * Skip threads with hard binding when
2862 			 * `unbind_all_threads' is not specified.
2863 			 */
2864 			if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2865 				continue;
2866 			err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2867 			if (ret == 0)
2868 				ret = err;
2869 		} while ((tp = tp->t_forw) != pp->p_tlist);
2870 		mutex_exit(&pp->p_lock);
2871 	}
2872 	mutex_exit(&pidlock);
2873 	if (ret == 0)
2874 		ret = berr;
2875 	return (ret);
2876 }
2877 
2878 
2879 /*
2880  * Destroy all remaining bound threads on a cpu.
2881  */
2882 void
cpu_destroy_bound_threads(cpu_t * cp)2883 cpu_destroy_bound_threads(cpu_t *cp)
2884 {
2885 	extern id_t syscid;
2886 	register kthread_id_t	t, tlist, tnext;
2887 
2888 	/*
2889 	 * Destroy all remaining bound threads on the cpu.  This
2890 	 * should include both the interrupt threads and the idle thread.
2891 	 * This requires some care, since we need to traverse the
2892 	 * thread list with the pidlock mutex locked, but thread_free
2893 	 * also locks the pidlock mutex.  So, we collect the threads
2894 	 * we're going to reap in a list headed by "tlist", then we
2895 	 * unlock the pidlock mutex and traverse the tlist list,
2896 	 * doing thread_free's on the thread's.	 Simple, n'est pas?
2897 	 * Also, this depends on thread_free not mucking with the
2898 	 * t_next and t_prev links of the thread.
2899 	 */
2900 
2901 	if ((t = curthread) != NULL) {
2902 
2903 		tlist = NULL;
2904 		mutex_enter(&pidlock);
2905 		do {
2906 			tnext = t->t_next;
2907 			if (t->t_bound_cpu == cp) {
2908 
2909 				/*
2910 				 * We've found a bound thread, carefully unlink
2911 				 * it out of the thread list, and add it to
2912 				 * our "tlist".	 We "know" we don't have to
2913 				 * worry about unlinking curthread (the thread
2914 				 * that is executing this code).
2915 				 */
2916 				t->t_next->t_prev = t->t_prev;
2917 				t->t_prev->t_next = t->t_next;
2918 				t->t_next = tlist;
2919 				tlist = t;
2920 				ASSERT(t->t_cid == syscid);
2921 				/* wake up anyone blocked in thread_join */
2922 				cv_broadcast(&t->t_joincv);
2923 				/*
2924 				 * t_lwp set by interrupt threads and not
2925 				 * cleared.
2926 				 */
2927 				t->t_lwp = NULL;
2928 				/*
2929 				 * Pause and idle threads always have
2930 				 * t_state set to TS_ONPROC.
2931 				 */
2932 				t->t_state = TS_FREE;
2933 				t->t_prev = NULL;	/* Just in case */
2934 			}
2935 
2936 		} while ((t = tnext) != curthread);
2937 
2938 		mutex_exit(&pidlock);
2939 
2940 		mutex_sync();
2941 		for (t = tlist; t != NULL; t = tnext) {
2942 			tnext = t->t_next;
2943 			thread_free(t);
2944 		}
2945 	}
2946 }
2947 
2948 /*
2949  * Update the cpu_supp_freqs of this cpu. This information is returned
2950  * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
2951  * maintain the kstat data size.
2952  */
2953 void
cpu_set_supp_freqs(cpu_t * cp,const char * freqs)2954 cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
2955 {
2956 	char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
2957 	const char *lfreqs = clkstr;
2958 	boolean_t kstat_exists = B_FALSE;
2959 	kstat_t *ksp;
2960 	size_t len;
2961 
2962 	/*
2963 	 * A NULL pointer means we only support one speed.
2964 	 */
2965 	if (freqs == NULL)
2966 		(void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
2967 		    cp->cpu_curr_clock);
2968 	else
2969 		lfreqs = freqs;
2970 
2971 	/*
2972 	 * Make sure the frequency doesn't change while a snapshot is
2973 	 * going on. Of course, we only need to worry about this if
2974 	 * the kstat exists.
2975 	 */
2976 	if ((ksp = cp->cpu_info_kstat) != NULL) {
2977 		mutex_enter(ksp->ks_lock);
2978 		kstat_exists = B_TRUE;
2979 	}
2980 
2981 	/*
2982 	 * Free any previously allocated string and if the kstat
2983 	 * already exists, then update its data size.
2984 	 */
2985 	if (cp->cpu_supp_freqs != NULL) {
2986 		len = strlen(cp->cpu_supp_freqs) + 1;
2987 		kmem_free(cp->cpu_supp_freqs, len);
2988 		if (kstat_exists)
2989 			ksp->ks_data_size -= len;
2990 	}
2991 
2992 	/*
2993 	 * Allocate the new string and set the pointer.
2994 	 */
2995 	len = strlen(lfreqs) + 1;
2996 	cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
2997 	(void) strcpy(cp->cpu_supp_freqs, lfreqs);
2998 
2999 	/*
3000 	 * If the kstat already exists then update the data size and
3001 	 * free the lock.
3002 	 */
3003 	if (kstat_exists) {
3004 		ksp->ks_data_size += len;
3005 		mutex_exit(ksp->ks_lock);
3006 	}
3007 }
3008 
3009 /*
3010  * Indicate the current CPU's clock freqency (in Hz).
3011  * The calling context must be such that CPU references are safe.
3012  */
3013 void
cpu_set_curr_clock(uint64_t new_clk)3014 cpu_set_curr_clock(uint64_t new_clk)
3015 {
3016 	uint64_t old_clk;
3017 
3018 	old_clk = CPU->cpu_curr_clock;
3019 	CPU->cpu_curr_clock = new_clk;
3020 
3021 	/*
3022 	 * The cpu-change-speed DTrace probe exports the frequency in Hz
3023 	 */
3024 	DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3025 	    uint64_t, old_clk, uint64_t, new_clk);
3026 }
3027 
3028 /*
3029  * processor_info(2) and p_online(2) status support functions
3030  *   The constants returned by the cpu_get_state() and cpu_get_state_str() are
3031  *   for use in communicating processor state information to userland.  Kernel
3032  *   subsystems should only be using the cpu_flags value directly.  Subsystems
3033  *   modifying cpu_flags should record the state change via a call to the
3034  *   cpu_set_state().
3035  */
3036 
3037 /*
3038  * Update the pi_state of this CPU.  This function provides the CPU status for
3039  * the information returned by processor_info(2).
3040  */
3041 void
cpu_set_state(cpu_t * cpu)3042 cpu_set_state(cpu_t *cpu)
3043 {
3044 	ASSERT(MUTEX_HELD(&cpu_lock));
3045 	cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3046 	cpu->cpu_state_begin = gethrestime_sec();
3047 	pool_cpu_mod = gethrtime();
3048 }
3049 
3050 /*
3051  * Return offline/online/other status for the indicated CPU.  Use only for
3052  * communication with user applications; cpu_flags provides the in-kernel
3053  * interface.
3054  */
3055 int
cpu_get_state(cpu_t * cpu)3056 cpu_get_state(cpu_t *cpu)
3057 {
3058 	ASSERT(MUTEX_HELD(&cpu_lock));
3059 	if (cpu->cpu_flags & CPU_POWEROFF)
3060 		return (P_POWEROFF);
3061 	else if (cpu->cpu_flags & CPU_FAULTED)
3062 		return (P_FAULTED);
3063 	else if (cpu->cpu_flags & CPU_SPARE)
3064 		return (P_SPARE);
3065 	else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3066 		return (P_OFFLINE);
3067 	else if (cpu->cpu_flags & CPU_ENABLE)
3068 		return (P_ONLINE);
3069 	else
3070 		return (P_NOINTR);
3071 }
3072 
3073 /*
3074  * Return processor_info(2) state as a string.
3075  */
3076 const char *
cpu_get_state_str(cpu_t * cpu)3077 cpu_get_state_str(cpu_t *cpu)
3078 {
3079 	const char *string;
3080 
3081 	switch (cpu_get_state(cpu)) {
3082 	case P_ONLINE:
3083 		string = PS_ONLINE;
3084 		break;
3085 	case P_POWEROFF:
3086 		string = PS_POWEROFF;
3087 		break;
3088 	case P_NOINTR:
3089 		string = PS_NOINTR;
3090 		break;
3091 	case P_SPARE:
3092 		string = PS_SPARE;
3093 		break;
3094 	case P_FAULTED:
3095 		string = PS_FAULTED;
3096 		break;
3097 	case P_OFFLINE:
3098 		string = PS_OFFLINE;
3099 		break;
3100 	default:
3101 		string = "unknown";
3102 		break;
3103 	}
3104 	return (string);
3105 }
3106 
3107 /*
3108  * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3109  * kstats, respectively.  This is done when a CPU is initialized or placed
3110  * online via p_online(2).
3111  */
3112 static void
cpu_stats_kstat_create(cpu_t * cp)3113 cpu_stats_kstat_create(cpu_t *cp)
3114 {
3115 	int 	instance = cp->cpu_id;
3116 	char 	*module = "cpu";
3117 	char 	*class = "misc";
3118 	kstat_t	*ksp;
3119 	zoneid_t zoneid;
3120 
3121 	ASSERT(MUTEX_HELD(&cpu_lock));
3122 
3123 	if (pool_pset_enabled())
3124 		zoneid = GLOBAL_ZONEID;
3125 	else
3126 		zoneid = ALL_ZONES;
3127 	/*
3128 	 * Create named kstats
3129 	 */
3130 #define	CPU_STATS_KS_CREATE(name, tsize, update_func)                    \
3131 	ksp = kstat_create_zone(module, instance, (name), class,         \
3132 	    KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0,       \
3133 	    zoneid);                                                     \
3134 	if (ksp != NULL) {                                               \
3135 		ksp->ks_private = cp;                                    \
3136 		ksp->ks_update = (update_func);                          \
3137 		kstat_install(ksp);                                      \
3138 	} else                                                           \
3139 		cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3140 		    module, instance, (name));
3141 
3142 	CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
3143 	    cpu_sys_stats_ks_update);
3144 	CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
3145 	    cpu_vm_stats_ks_update);
3146 
3147 	/*
3148 	 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3149 	 */
3150 	ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3151 	    "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3152 	if (ksp != NULL) {
3153 		ksp->ks_update = cpu_stat_ks_update;
3154 		ksp->ks_private = cp;
3155 		kstat_install(ksp);
3156 	}
3157 }
3158 
3159 static void
cpu_stats_kstat_destroy(cpu_t * cp)3160 cpu_stats_kstat_destroy(cpu_t *cp)
3161 {
3162 	char ks_name[KSTAT_STRLEN];
3163 
3164 	(void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3165 	kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3166 
3167 	kstat_delete_byname("cpu", cp->cpu_id, "sys");
3168 	kstat_delete_byname("cpu", cp->cpu_id, "vm");
3169 }
3170 
3171 static int
cpu_sys_stats_ks_update(kstat_t * ksp,int rw)3172 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3173 {
3174 	cpu_t *cp = (cpu_t *)ksp->ks_private;
3175 	struct cpu_sys_stats_ks_data *csskd;
3176 	cpu_sys_stats_t *css;
3177 	hrtime_t msnsecs[NCMSTATES];
3178 	int	i;
3179 
3180 	if (rw == KSTAT_WRITE)
3181 		return (EACCES);
3182 
3183 	csskd = ksp->ks_data;
3184 	css = &cp->cpu_stats.sys;
3185 
3186 	/*
3187 	 * Read CPU mstate, but compare with the last values we
3188 	 * received to make sure that the returned kstats never
3189 	 * decrease.
3190 	 */
3191 
3192 	get_cpu_mstate(cp, msnsecs);
3193 	if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3194 		msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3195 	if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3196 		msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3197 	if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3198 		msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3199 
3200 	bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3201 	    sizeof (cpu_sys_stats_ks_data_template));
3202 
3203 	csskd->cpu_ticks_wait.value.ui64 = 0;
3204 	csskd->wait_ticks_io.value.ui64 = 0;
3205 
3206 	csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3207 	csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3208 	csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3209 	csskd->cpu_ticks_idle.value.ui64 =
3210 	    NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3211 	csskd->cpu_ticks_user.value.ui64 =
3212 	    NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3213 	csskd->cpu_ticks_kernel.value.ui64 =
3214 	    NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3215 	csskd->cpu_nsec_dtrace.value.ui64 = cp->cpu_dtrace_nsec;
3216 	csskd->dtrace_probes.value.ui64 = cp->cpu_dtrace_probes;
3217 	csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3218 	csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3219 	csskd->bread.value.ui64 = css->bread;
3220 	csskd->bwrite.value.ui64 = css->bwrite;
3221 	csskd->lread.value.ui64 = css->lread;
3222 	csskd->lwrite.value.ui64 = css->lwrite;
3223 	csskd->phread.value.ui64 = css->phread;
3224 	csskd->phwrite.value.ui64 = css->phwrite;
3225 	csskd->pswitch.value.ui64 = css->pswitch;
3226 	csskd->trap.value.ui64 = css->trap;
3227 	csskd->intr.value.ui64 = 0;
3228 	for (i = 0; i < PIL_MAX; i++)
3229 		csskd->intr.value.ui64 += css->intr[i];
3230 	csskd->syscall.value.ui64 = css->syscall;
3231 	csskd->sysread.value.ui64 = css->sysread;
3232 	csskd->syswrite.value.ui64 = css->syswrite;
3233 	csskd->sysfork.value.ui64 = css->sysfork;
3234 	csskd->sysvfork.value.ui64 = css->sysvfork;
3235 	csskd->sysexec.value.ui64 = css->sysexec;
3236 	csskd->readch.value.ui64 = css->readch;
3237 	csskd->writech.value.ui64 = css->writech;
3238 	csskd->rcvint.value.ui64 = css->rcvint;
3239 	csskd->xmtint.value.ui64 = css->xmtint;
3240 	csskd->mdmint.value.ui64 = css->mdmint;
3241 	csskd->rawch.value.ui64 = css->rawch;
3242 	csskd->canch.value.ui64 = css->canch;
3243 	csskd->outch.value.ui64 = css->outch;
3244 	csskd->msg.value.ui64 = css->msg;
3245 	csskd->sema.value.ui64 = css->sema;
3246 	csskd->namei.value.ui64 = css->namei;
3247 	csskd->ufsiget.value.ui64 = css->ufsiget;
3248 	csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3249 	csskd->ufsipage.value.ui64 = css->ufsipage;
3250 	csskd->ufsinopage.value.ui64 = css->ufsinopage;
3251 	csskd->procovf.value.ui64 = css->procovf;
3252 	csskd->intrthread.value.ui64 = 0;
3253 	for (i = 0; i < LOCK_LEVEL - 1; i++)
3254 		csskd->intrthread.value.ui64 += css->intr[i];
3255 	csskd->intrblk.value.ui64 = css->intrblk;
3256 	csskd->intrunpin.value.ui64 = css->intrunpin;
3257 	csskd->idlethread.value.ui64 = css->idlethread;
3258 	csskd->inv_swtch.value.ui64 = css->inv_swtch;
3259 	csskd->nthreads.value.ui64 = css->nthreads;
3260 	csskd->cpumigrate.value.ui64 = css->cpumigrate;
3261 	csskd->xcalls.value.ui64 = css->xcalls;
3262 	csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3263 	csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3264 	csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3265 	csskd->modload.value.ui64 = css->modload;
3266 	csskd->modunload.value.ui64 = css->modunload;
3267 	csskd->bawrite.value.ui64 = css->bawrite;
3268 	csskd->iowait.value.ui64 = css->iowait;
3269 
3270 	return (0);
3271 }
3272 
3273 static int
cpu_vm_stats_ks_update(kstat_t * ksp,int rw)3274 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3275 {
3276 	cpu_t *cp = (cpu_t *)ksp->ks_private;
3277 	struct cpu_vm_stats_ks_data *cvskd;
3278 	cpu_vm_stats_t *cvs;
3279 
3280 	if (rw == KSTAT_WRITE)
3281 		return (EACCES);
3282 
3283 	cvs = &cp->cpu_stats.vm;
3284 	cvskd = ksp->ks_data;
3285 
3286 	bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3287 	    sizeof (cpu_vm_stats_ks_data_template));
3288 	cvskd->pgrec.value.ui64 = cvs->pgrec;
3289 	cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3290 	cvskd->pgin.value.ui64 = cvs->pgin;
3291 	cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3292 	cvskd->pgout.value.ui64 = cvs->pgout;
3293 	cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3294 	cvskd->swapin.value.ui64 = cvs->swapin;
3295 	cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3296 	cvskd->swapout.value.ui64 = cvs->swapout;
3297 	cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3298 	cvskd->zfod.value.ui64 = cvs->zfod;
3299 	cvskd->dfree.value.ui64 = cvs->dfree;
3300 	cvskd->scan.value.ui64 = cvs->scan;
3301 	cvskd->rev.value.ui64 = cvs->rev;
3302 	cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3303 	cvskd->as_fault.value.ui64 = cvs->as_fault;
3304 	cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3305 	cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3306 	cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3307 	cvskd->softlock.value.ui64 = cvs->softlock;
3308 	cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3309 	cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3310 	cvskd->execpgin.value.ui64 = cvs->execpgin;
3311 	cvskd->execpgout.value.ui64 = cvs->execpgout;
3312 	cvskd->execfree.value.ui64 = cvs->execfree;
3313 	cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3314 	cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3315 	cvskd->anonfree.value.ui64 = cvs->anonfree;
3316 	cvskd->fspgin.value.ui64 = cvs->fspgin;
3317 	cvskd->fspgout.value.ui64 = cvs->fspgout;
3318 	cvskd->fsfree.value.ui64 = cvs->fsfree;
3319 
3320 	return (0);
3321 }
3322 
3323 static int
cpu_stat_ks_update(kstat_t * ksp,int rw)3324 cpu_stat_ks_update(kstat_t *ksp, int rw)
3325 {
3326 	cpu_stat_t *cso;
3327 	cpu_t *cp;
3328 	int i;
3329 	hrtime_t msnsecs[NCMSTATES];
3330 
3331 	cso = (cpu_stat_t *)ksp->ks_data;
3332 	cp = (cpu_t *)ksp->ks_private;
3333 
3334 	if (rw == KSTAT_WRITE)
3335 		return (EACCES);
3336 
3337 	/*
3338 	 * Read CPU mstate, but compare with the last values we
3339 	 * received to make sure that the returned kstats never
3340 	 * decrease.
3341 	 */
3342 
3343 	get_cpu_mstate(cp, msnsecs);
3344 	msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3345 	msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3346 	msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3347 	if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3348 		cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3349 	if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3350 		cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3351 	if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3352 		cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3353 	cso->cpu_sysinfo.cpu[CPU_WAIT] 	= 0;
3354 	cso->cpu_sysinfo.wait[W_IO] 	= 0;
3355 	cso->cpu_sysinfo.wait[W_SWAP]	= 0;
3356 	cso->cpu_sysinfo.wait[W_PIO]	= 0;
3357 	cso->cpu_sysinfo.bread 		= CPU_STATS(cp, sys.bread);
3358 	cso->cpu_sysinfo.bwrite 	= CPU_STATS(cp, sys.bwrite);
3359 	cso->cpu_sysinfo.lread 		= CPU_STATS(cp, sys.lread);
3360 	cso->cpu_sysinfo.lwrite 	= CPU_STATS(cp, sys.lwrite);
3361 	cso->cpu_sysinfo.phread 	= CPU_STATS(cp, sys.phread);
3362 	cso->cpu_sysinfo.phwrite 	= CPU_STATS(cp, sys.phwrite);
3363 	cso->cpu_sysinfo.pswitch 	= CPU_STATS(cp, sys.pswitch);
3364 	cso->cpu_sysinfo.trap 		= CPU_STATS(cp, sys.trap);
3365 	cso->cpu_sysinfo.intr		= 0;
3366 	for (i = 0; i < PIL_MAX; i++)
3367 		cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3368 	cso->cpu_sysinfo.syscall	= CPU_STATS(cp, sys.syscall);
3369 	cso->cpu_sysinfo.sysread	= CPU_STATS(cp, sys.sysread);
3370 	cso->cpu_sysinfo.syswrite	= CPU_STATS(cp, sys.syswrite);
3371 	cso->cpu_sysinfo.sysfork	= CPU_STATS(cp, sys.sysfork);
3372 	cso->cpu_sysinfo.sysvfork	= CPU_STATS(cp, sys.sysvfork);
3373 	cso->cpu_sysinfo.sysexec	= CPU_STATS(cp, sys.sysexec);
3374 	cso->cpu_sysinfo.readch		= CPU_STATS(cp, sys.readch);
3375 	cso->cpu_sysinfo.writech	= CPU_STATS(cp, sys.writech);
3376 	cso->cpu_sysinfo.rcvint		= CPU_STATS(cp, sys.rcvint);
3377 	cso->cpu_sysinfo.xmtint		= CPU_STATS(cp, sys.xmtint);
3378 	cso->cpu_sysinfo.mdmint		= CPU_STATS(cp, sys.mdmint);
3379 	cso->cpu_sysinfo.rawch		= CPU_STATS(cp, sys.rawch);
3380 	cso->cpu_sysinfo.canch		= CPU_STATS(cp, sys.canch);
3381 	cso->cpu_sysinfo.outch		= CPU_STATS(cp, sys.outch);
3382 	cso->cpu_sysinfo.msg		= CPU_STATS(cp, sys.msg);
3383 	cso->cpu_sysinfo.sema		= CPU_STATS(cp, sys.sema);
3384 	cso->cpu_sysinfo.namei		= CPU_STATS(cp, sys.namei);
3385 	cso->cpu_sysinfo.ufsiget	= CPU_STATS(cp, sys.ufsiget);
3386 	cso->cpu_sysinfo.ufsdirblk	= CPU_STATS(cp, sys.ufsdirblk);
3387 	cso->cpu_sysinfo.ufsipage	= CPU_STATS(cp, sys.ufsipage);
3388 	cso->cpu_sysinfo.ufsinopage	= CPU_STATS(cp, sys.ufsinopage);
3389 	cso->cpu_sysinfo.inodeovf	= 0;
3390 	cso->cpu_sysinfo.fileovf	= 0;
3391 	cso->cpu_sysinfo.procovf	= CPU_STATS(cp, sys.procovf);
3392 	cso->cpu_sysinfo.intrthread	= 0;
3393 	for (i = 0; i < LOCK_LEVEL - 1; i++)
3394 		cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3395 	cso->cpu_sysinfo.intrblk	= CPU_STATS(cp, sys.intrblk);
3396 	cso->cpu_sysinfo.idlethread	= CPU_STATS(cp, sys.idlethread);
3397 	cso->cpu_sysinfo.inv_swtch	= CPU_STATS(cp, sys.inv_swtch);
3398 	cso->cpu_sysinfo.nthreads	= CPU_STATS(cp, sys.nthreads);
3399 	cso->cpu_sysinfo.cpumigrate	= CPU_STATS(cp, sys.cpumigrate);
3400 	cso->cpu_sysinfo.xcalls		= CPU_STATS(cp, sys.xcalls);
3401 	cso->cpu_sysinfo.mutex_adenters	= CPU_STATS(cp, sys.mutex_adenters);
3402 	cso->cpu_sysinfo.rw_rdfails	= CPU_STATS(cp, sys.rw_rdfails);
3403 	cso->cpu_sysinfo.rw_wrfails	= CPU_STATS(cp, sys.rw_wrfails);
3404 	cso->cpu_sysinfo.modload	= CPU_STATS(cp, sys.modload);
3405 	cso->cpu_sysinfo.modunload	= CPU_STATS(cp, sys.modunload);
3406 	cso->cpu_sysinfo.bawrite	= CPU_STATS(cp, sys.bawrite);
3407 	cso->cpu_sysinfo.rw_enters	= 0;
3408 	cso->cpu_sysinfo.win_uo_cnt	= 0;
3409 	cso->cpu_sysinfo.win_uu_cnt	= 0;
3410 	cso->cpu_sysinfo.win_so_cnt	= 0;
3411 	cso->cpu_sysinfo.win_su_cnt	= 0;
3412 	cso->cpu_sysinfo.win_suo_cnt	= 0;
3413 
3414 	cso->cpu_syswait.iowait		= CPU_STATS(cp, sys.iowait);
3415 	cso->cpu_syswait.swap		= 0;
3416 	cso->cpu_syswait.physio		= 0;
3417 
3418 	cso->cpu_vminfo.pgrec		= CPU_STATS(cp, vm.pgrec);
3419 	cso->cpu_vminfo.pgfrec		= CPU_STATS(cp, vm.pgfrec);
3420 	cso->cpu_vminfo.pgin		= CPU_STATS(cp, vm.pgin);
3421 	cso->cpu_vminfo.pgpgin		= CPU_STATS(cp, vm.pgpgin);
3422 	cso->cpu_vminfo.pgout		= CPU_STATS(cp, vm.pgout);
3423 	cso->cpu_vminfo.pgpgout		= CPU_STATS(cp, vm.pgpgout);
3424 	cso->cpu_vminfo.swapin		= CPU_STATS(cp, vm.swapin);
3425 	cso->cpu_vminfo.pgswapin	= CPU_STATS(cp, vm.pgswapin);
3426 	cso->cpu_vminfo.swapout		= CPU_STATS(cp, vm.swapout);
3427 	cso->cpu_vminfo.pgswapout	= CPU_STATS(cp, vm.pgswapout);
3428 	cso->cpu_vminfo.zfod		= CPU_STATS(cp, vm.zfod);
3429 	cso->cpu_vminfo.dfree		= CPU_STATS(cp, vm.dfree);
3430 	cso->cpu_vminfo.scan		= CPU_STATS(cp, vm.scan);
3431 	cso->cpu_vminfo.rev		= CPU_STATS(cp, vm.rev);
3432 	cso->cpu_vminfo.hat_fault	= CPU_STATS(cp, vm.hat_fault);
3433 	cso->cpu_vminfo.as_fault	= CPU_STATS(cp, vm.as_fault);
3434 	cso->cpu_vminfo.maj_fault	= CPU_STATS(cp, vm.maj_fault);
3435 	cso->cpu_vminfo.cow_fault	= CPU_STATS(cp, vm.cow_fault);
3436 	cso->cpu_vminfo.prot_fault	= CPU_STATS(cp, vm.prot_fault);
3437 	cso->cpu_vminfo.softlock	= CPU_STATS(cp, vm.softlock);
3438 	cso->cpu_vminfo.kernel_asflt	= CPU_STATS(cp, vm.kernel_asflt);
3439 	cso->cpu_vminfo.pgrrun		= CPU_STATS(cp, vm.pgrrun);
3440 	cso->cpu_vminfo.execpgin	= CPU_STATS(cp, vm.execpgin);
3441 	cso->cpu_vminfo.execpgout	= CPU_STATS(cp, vm.execpgout);
3442 	cso->cpu_vminfo.execfree	= CPU_STATS(cp, vm.execfree);
3443 	cso->cpu_vminfo.anonpgin	= CPU_STATS(cp, vm.anonpgin);
3444 	cso->cpu_vminfo.anonpgout	= CPU_STATS(cp, vm.anonpgout);
3445 	cso->cpu_vminfo.anonfree	= CPU_STATS(cp, vm.anonfree);
3446 	cso->cpu_vminfo.fspgin		= CPU_STATS(cp, vm.fspgin);
3447 	cso->cpu_vminfo.fspgout		= CPU_STATS(cp, vm.fspgout);
3448 	cso->cpu_vminfo.fsfree		= CPU_STATS(cp, vm.fsfree);
3449 
3450 	return (0);
3451 }
3452