1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * Architecture-independent CPU control functions. 31 */ 32 33 #include <sys/types.h> 34 #include <sys/param.h> 35 #include <sys/var.h> 36 #include <sys/thread.h> 37 #include <sys/cpuvar.h> 38 #include <sys/kstat.h> 39 #include <sys/uadmin.h> 40 #include <sys/systm.h> 41 #include <sys/errno.h> 42 #include <sys/cmn_err.h> 43 #include <sys/procset.h> 44 #include <sys/processor.h> 45 #include <sys/debug.h> 46 #include <sys/cpupart.h> 47 #include <sys/lgrp.h> 48 #include <sys/pset.h> 49 #include <sys/chip.h> 50 #include <sys/kmem.h> 51 #include <sys/kmem_impl.h> /* to set per-cpu kmem_cache offset */ 52 #include <sys/atomic.h> 53 #include <sys/callb.h> 54 #include <sys/vtrace.h> 55 #include <sys/cyclic.h> 56 #include <sys/bitmap.h> 57 #include <sys/nvpair.h> 58 #include <sys/pool_pset.h> 59 #include <sys/msacct.h> 60 #include <sys/time.h> 61 #include <sys/archsystm.h> 62 #if defined(__i386) || defined(__amd64) 63 #include <sys/x86_archext.h> 64 #endif 65 66 extern int mp_cpu_start(cpu_t *); 67 extern int mp_cpu_stop(cpu_t *); 68 extern int mp_cpu_poweron(cpu_t *); 69 extern int mp_cpu_poweroff(cpu_t *); 70 extern int mp_cpu_configure(int); 71 extern int mp_cpu_unconfigure(int); 72 extern void mp_cpu_faulted_enter(cpu_t *); 73 extern void mp_cpu_faulted_exit(cpu_t *); 74 75 extern int cmp_cpu_to_chip(processorid_t cpuid); 76 #ifdef __sparcv9 77 extern char *cpu_fru_fmri(cpu_t *cp); 78 #endif 79 80 static void cpu_add_active_internal(cpu_t *cp); 81 static void cpu_remove_active(cpu_t *cp); 82 static void cpu_info_kstat_create(cpu_t *cp); 83 static void cpu_info_kstat_destroy(cpu_t *cp); 84 static void cpu_stats_kstat_create(cpu_t *cp); 85 static void cpu_stats_kstat_destroy(cpu_t *cp); 86 87 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw); 88 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw); 89 static int cpu_stat_ks_update(kstat_t *ksp, int rw); 90 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t); 91 92 /* 93 * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active, 94 * and dispatch queue reallocations. The lock ordering with respect to 95 * related locks is: 96 * 97 * cpu_lock --> thread_free_lock ---> p_lock ---> thread_lock() 98 * 99 * Warning: Certain sections of code do not use the cpu_lock when 100 * traversing the cpu_list (e.g. mutex_vector_enter(), clock()). Since 101 * all cpus are paused during modifications to this list, a solution 102 * to protect the list is too either disable kernel preemption while 103 * walking the list, *or* recheck the cpu_next pointer at each 104 * iteration in the loop. Note that in no cases can any cached 105 * copies of the cpu pointers be kept as they may become invalid. 106 */ 107 kmutex_t cpu_lock; 108 cpu_t *cpu_list; /* list of all CPUs */ 109 cpu_t *cpu_active; /* list of active CPUs */ 110 static cpuset_t cpu_available; /* set of available CPUs */ 111 cpuset_t cpu_seqid_inuse; /* which cpu_seqids are in use */ 112 113 /* 114 * max_ncpus keeps the max cpus the system can have. Initially 115 * it's NCPU, but since most archs scan the devtree for cpus 116 * fairly early on during boot, the real max can be known before 117 * ncpus is set (useful for early NCPU based allocations). 118 */ 119 int max_ncpus = NCPU; 120 /* 121 * platforms that set max_ncpus to maxiumum number of cpus that can be 122 * dynamically added will set boot_max_ncpus to the number of cpus found 123 * at device tree scan time during boot. 124 */ 125 int boot_max_ncpus = -1; 126 /* 127 * Maximum possible CPU id. This can never be >= NCPU since NCPU is 128 * used to size arrays that are indexed by CPU id. 129 */ 130 processorid_t max_cpuid = NCPU - 1; 131 132 int ncpus = 1; 133 int ncpus_online = 1; 134 135 /* 136 * CPU that we're trying to offline. Protected by cpu_lock. 137 */ 138 cpu_t *cpu_inmotion; 139 140 /* 141 * Can be raised to suppress further weakbinding, which are instead 142 * satisfied by disabling preemption. Must be raised/lowered under cpu_lock, 143 * while individual thread weakbinding synchronisation is done under thread 144 * lock. 145 */ 146 int weakbindingbarrier; 147 148 /* 149 * values for safe_list. Pause state that CPUs are in. 150 */ 151 #define PAUSE_IDLE 0 /* normal state */ 152 #define PAUSE_READY 1 /* paused thread ready to spl */ 153 #define PAUSE_WAIT 2 /* paused thread is spl-ed high */ 154 #define PAUSE_DIE 3 /* tell pause thread to leave */ 155 #define PAUSE_DEAD 4 /* pause thread has left */ 156 157 /* 158 * Variables used in pause_cpus(). 159 */ 160 static volatile char safe_list[NCPU]; 161 162 static struct _cpu_pause_info { 163 int cp_spl; /* spl saved in pause_cpus() */ 164 volatile int cp_go; /* Go signal sent after all ready */ 165 int cp_count; /* # of CPUs to pause */ 166 ksema_t cp_sem; /* synch pause_cpus & cpu_pause */ 167 kthread_id_t cp_paused; 168 } cpu_pause_info; 169 170 static kmutex_t pause_free_mutex; 171 static kcondvar_t pause_free_cv; 172 173 static struct cpu_sys_stats_ks_data { 174 kstat_named_t cpu_ticks_idle; 175 kstat_named_t cpu_ticks_user; 176 kstat_named_t cpu_ticks_kernel; 177 kstat_named_t cpu_ticks_wait; 178 kstat_named_t cpu_nsec_idle; 179 kstat_named_t cpu_nsec_user; 180 kstat_named_t cpu_nsec_kernel; 181 kstat_named_t wait_ticks_io; 182 kstat_named_t bread; 183 kstat_named_t bwrite; 184 kstat_named_t lread; 185 kstat_named_t lwrite; 186 kstat_named_t phread; 187 kstat_named_t phwrite; 188 kstat_named_t pswitch; 189 kstat_named_t trap; 190 kstat_named_t intr; 191 kstat_named_t syscall; 192 kstat_named_t sysread; 193 kstat_named_t syswrite; 194 kstat_named_t sysfork; 195 kstat_named_t sysvfork; 196 kstat_named_t sysexec; 197 kstat_named_t readch; 198 kstat_named_t writech; 199 kstat_named_t rcvint; 200 kstat_named_t xmtint; 201 kstat_named_t mdmint; 202 kstat_named_t rawch; 203 kstat_named_t canch; 204 kstat_named_t outch; 205 kstat_named_t msg; 206 kstat_named_t sema; 207 kstat_named_t namei; 208 kstat_named_t ufsiget; 209 kstat_named_t ufsdirblk; 210 kstat_named_t ufsipage; 211 kstat_named_t ufsinopage; 212 kstat_named_t procovf; 213 kstat_named_t intrthread; 214 kstat_named_t intrblk; 215 kstat_named_t intrunpin; 216 kstat_named_t idlethread; 217 kstat_named_t inv_swtch; 218 kstat_named_t nthreads; 219 kstat_named_t cpumigrate; 220 kstat_named_t xcalls; 221 kstat_named_t mutex_adenters; 222 kstat_named_t rw_rdfails; 223 kstat_named_t rw_wrfails; 224 kstat_named_t modload; 225 kstat_named_t modunload; 226 kstat_named_t bawrite; 227 kstat_named_t iowait; 228 } cpu_sys_stats_ks_data_template = { 229 { "cpu_ticks_idle", KSTAT_DATA_UINT64 }, 230 { "cpu_ticks_user", KSTAT_DATA_UINT64 }, 231 { "cpu_ticks_kernel", KSTAT_DATA_UINT64 }, 232 { "cpu_ticks_wait", KSTAT_DATA_UINT64 }, 233 { "cpu_nsec_idle", KSTAT_DATA_UINT64 }, 234 { "cpu_nsec_user", KSTAT_DATA_UINT64 }, 235 { "cpu_nsec_kernel", KSTAT_DATA_UINT64 }, 236 { "wait_ticks_io", KSTAT_DATA_UINT64 }, 237 { "bread", KSTAT_DATA_UINT64 }, 238 { "bwrite", KSTAT_DATA_UINT64 }, 239 { "lread", KSTAT_DATA_UINT64 }, 240 { "lwrite", KSTAT_DATA_UINT64 }, 241 { "phread", KSTAT_DATA_UINT64 }, 242 { "phwrite", KSTAT_DATA_UINT64 }, 243 { "pswitch", KSTAT_DATA_UINT64 }, 244 { "trap", KSTAT_DATA_UINT64 }, 245 { "intr", KSTAT_DATA_UINT64 }, 246 { "syscall", KSTAT_DATA_UINT64 }, 247 { "sysread", KSTAT_DATA_UINT64 }, 248 { "syswrite", KSTAT_DATA_UINT64 }, 249 { "sysfork", KSTAT_DATA_UINT64 }, 250 { "sysvfork", KSTAT_DATA_UINT64 }, 251 { "sysexec", KSTAT_DATA_UINT64 }, 252 { "readch", KSTAT_DATA_UINT64 }, 253 { "writech", KSTAT_DATA_UINT64 }, 254 { "rcvint", KSTAT_DATA_UINT64 }, 255 { "xmtint", KSTAT_DATA_UINT64 }, 256 { "mdmint", KSTAT_DATA_UINT64 }, 257 { "rawch", KSTAT_DATA_UINT64 }, 258 { "canch", KSTAT_DATA_UINT64 }, 259 { "outch", KSTAT_DATA_UINT64 }, 260 { "msg", KSTAT_DATA_UINT64 }, 261 { "sema", KSTAT_DATA_UINT64 }, 262 { "namei", KSTAT_DATA_UINT64 }, 263 { "ufsiget", KSTAT_DATA_UINT64 }, 264 { "ufsdirblk", KSTAT_DATA_UINT64 }, 265 { "ufsipage", KSTAT_DATA_UINT64 }, 266 { "ufsinopage", KSTAT_DATA_UINT64 }, 267 { "procovf", KSTAT_DATA_UINT64 }, 268 { "intrthread", KSTAT_DATA_UINT64 }, 269 { "intrblk", KSTAT_DATA_UINT64 }, 270 { "intrunpin", KSTAT_DATA_UINT64 }, 271 { "idlethread", KSTAT_DATA_UINT64 }, 272 { "inv_swtch", KSTAT_DATA_UINT64 }, 273 { "nthreads", KSTAT_DATA_UINT64 }, 274 { "cpumigrate", KSTAT_DATA_UINT64 }, 275 { "xcalls", KSTAT_DATA_UINT64 }, 276 { "mutex_adenters", KSTAT_DATA_UINT64 }, 277 { "rw_rdfails", KSTAT_DATA_UINT64 }, 278 { "rw_wrfails", KSTAT_DATA_UINT64 }, 279 { "modload", KSTAT_DATA_UINT64 }, 280 { "modunload", KSTAT_DATA_UINT64 }, 281 { "bawrite", KSTAT_DATA_UINT64 }, 282 { "iowait", KSTAT_DATA_UINT64 }, 283 }; 284 285 static struct cpu_vm_stats_ks_data { 286 kstat_named_t pgrec; 287 kstat_named_t pgfrec; 288 kstat_named_t pgin; 289 kstat_named_t pgpgin; 290 kstat_named_t pgout; 291 kstat_named_t pgpgout; 292 kstat_named_t swapin; 293 kstat_named_t pgswapin; 294 kstat_named_t swapout; 295 kstat_named_t pgswapout; 296 kstat_named_t zfod; 297 kstat_named_t dfree; 298 kstat_named_t scan; 299 kstat_named_t rev; 300 kstat_named_t hat_fault; 301 kstat_named_t as_fault; 302 kstat_named_t maj_fault; 303 kstat_named_t cow_fault; 304 kstat_named_t prot_fault; 305 kstat_named_t softlock; 306 kstat_named_t kernel_asflt; 307 kstat_named_t pgrrun; 308 kstat_named_t execpgin; 309 kstat_named_t execpgout; 310 kstat_named_t execfree; 311 kstat_named_t anonpgin; 312 kstat_named_t anonpgout; 313 kstat_named_t anonfree; 314 kstat_named_t fspgin; 315 kstat_named_t fspgout; 316 kstat_named_t fsfree; 317 } cpu_vm_stats_ks_data_template = { 318 { "pgrec", KSTAT_DATA_UINT64 }, 319 { "pgfrec", KSTAT_DATA_UINT64 }, 320 { "pgin", KSTAT_DATA_UINT64 }, 321 { "pgpgin", KSTAT_DATA_UINT64 }, 322 { "pgout", KSTAT_DATA_UINT64 }, 323 { "pgpgout", KSTAT_DATA_UINT64 }, 324 { "swapin", KSTAT_DATA_UINT64 }, 325 { "pgswapin", KSTAT_DATA_UINT64 }, 326 { "swapout", KSTAT_DATA_UINT64 }, 327 { "pgswapout", KSTAT_DATA_UINT64 }, 328 { "zfod", KSTAT_DATA_UINT64 }, 329 { "dfree", KSTAT_DATA_UINT64 }, 330 { "scan", KSTAT_DATA_UINT64 }, 331 { "rev", KSTAT_DATA_UINT64 }, 332 { "hat_fault", KSTAT_DATA_UINT64 }, 333 { "as_fault", KSTAT_DATA_UINT64 }, 334 { "maj_fault", KSTAT_DATA_UINT64 }, 335 { "cow_fault", KSTAT_DATA_UINT64 }, 336 { "prot_fault", KSTAT_DATA_UINT64 }, 337 { "softlock", KSTAT_DATA_UINT64 }, 338 { "kernel_asflt", KSTAT_DATA_UINT64 }, 339 { "pgrrun", KSTAT_DATA_UINT64 }, 340 { "execpgin", KSTAT_DATA_UINT64 }, 341 { "execpgout", KSTAT_DATA_UINT64 }, 342 { "execfree", KSTAT_DATA_UINT64 }, 343 { "anonpgin", KSTAT_DATA_UINT64 }, 344 { "anonpgout", KSTAT_DATA_UINT64 }, 345 { "anonfree", KSTAT_DATA_UINT64 }, 346 { "fspgin", KSTAT_DATA_UINT64 }, 347 { "fspgout", KSTAT_DATA_UINT64 }, 348 { "fsfree", KSTAT_DATA_UINT64 }, 349 }; 350 351 /* 352 * Force the specified thread to migrate to the appropriate processor. 353 * Called with thread lock held, returns with it dropped. 354 */ 355 static void 356 force_thread_migrate(kthread_id_t tp) 357 { 358 ASSERT(THREAD_LOCK_HELD(tp)); 359 if (tp == curthread) { 360 THREAD_TRANSITION(tp); 361 CL_SETRUN(tp); 362 thread_unlock_nopreempt(tp); 363 swtch(); 364 } else { 365 if (tp->t_state == TS_ONPROC) { 366 cpu_surrender(tp); 367 } else if (tp->t_state == TS_RUN) { 368 (void) dispdeq(tp); 369 setbackdq(tp); 370 } 371 thread_unlock(tp); 372 } 373 } 374 375 /* 376 * Set affinity for a specified CPU. 377 * A reference count is incremented and the affinity is held until the 378 * reference count is decremented to zero by thread_affinity_clear(). 379 * This is so regions of code requiring affinity can be nested. 380 * Caller needs to ensure that cpu_id remains valid, which can be 381 * done by holding cpu_lock across this call, unless the caller 382 * specifies CPU_CURRENT in which case the cpu_lock will be acquired 383 * by thread_affinity_set and CPU->cpu_id will be the target CPU. 384 */ 385 void 386 thread_affinity_set(kthread_id_t t, int cpu_id) 387 { 388 cpu_t *cp; 389 int c; 390 391 ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL)); 392 393 if ((c = cpu_id) == CPU_CURRENT) { 394 mutex_enter(&cpu_lock); 395 cpu_id = CPU->cpu_id; 396 } 397 /* 398 * We should be asserting that cpu_lock is held here, but 399 * the NCA code doesn't acquire it. The following assert 400 * should be uncommented when the NCA code is fixed. 401 * 402 * ASSERT(MUTEX_HELD(&cpu_lock)); 403 */ 404 ASSERT((cpu_id >= 0) && (cpu_id < NCPU)); 405 cp = cpu[cpu_id]; 406 ASSERT(cp != NULL); /* user must provide a good cpu_id */ 407 /* 408 * If there is already a hard affinity requested, and this affinity 409 * conflicts with that, panic. 410 */ 411 thread_lock(t); 412 if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) { 413 panic("affinity_set: setting %p but already bound to %p", 414 (void *)cp, (void *)t->t_bound_cpu); 415 } 416 t->t_affinitycnt++; 417 t->t_bound_cpu = cp; 418 419 /* 420 * Make sure we're running on the right CPU. 421 */ 422 if (cp != t->t_cpu || t != curthread) { 423 force_thread_migrate(t); /* drops thread lock */ 424 } else { 425 thread_unlock(t); 426 } 427 428 if (c == CPU_CURRENT) 429 mutex_exit(&cpu_lock); 430 } 431 432 /* 433 * Wrapper for backward compatibility. 434 */ 435 void 436 affinity_set(int cpu_id) 437 { 438 thread_affinity_set(curthread, cpu_id); 439 } 440 441 /* 442 * Decrement the affinity reservation count and if it becomes zero, 443 * clear the CPU affinity for the current thread, or set it to the user's 444 * software binding request. 445 */ 446 void 447 thread_affinity_clear(kthread_id_t t) 448 { 449 register processorid_t binding; 450 451 thread_lock(t); 452 if (--t->t_affinitycnt == 0) { 453 if ((binding = t->t_bind_cpu) == PBIND_NONE) { 454 /* 455 * Adjust disp_max_unbound_pri if necessary. 456 */ 457 disp_adjust_unbound_pri(t); 458 t->t_bound_cpu = NULL; 459 if (t->t_cpu->cpu_part != t->t_cpupart) { 460 force_thread_migrate(t); 461 return; 462 } 463 } else { 464 t->t_bound_cpu = cpu[binding]; 465 /* 466 * Make sure the thread is running on the bound CPU. 467 */ 468 if (t->t_cpu != t->t_bound_cpu) { 469 force_thread_migrate(t); 470 return; /* already dropped lock */ 471 } 472 } 473 } 474 thread_unlock(t); 475 } 476 477 /* 478 * Wrapper for backward compatibility. 479 */ 480 void 481 affinity_clear(void) 482 { 483 thread_affinity_clear(curthread); 484 } 485 486 /* 487 * Weak cpu affinity. Bind to the "current" cpu for short periods 488 * of time during which the thread must not block (but may be preempted). 489 * Use this instead of kpreempt_disable() when it is only "no migration" 490 * rather than "no preemption" semantics that are required - disabling 491 * preemption holds higher priority threads off of cpu and if the 492 * operation that is protected is more than momentary this is not good 493 * for realtime etc. 494 * 495 * Weakly bound threads will not prevent a cpu from being offlined - 496 * we'll only run them on the cpu to which they are weakly bound but 497 * (because they do not block) we'll always be able to move them on to 498 * another cpu at offline time if we give them just a short moment to 499 * run during which they will unbind. To give a cpu a chance of offlining, 500 * however, we require a barrier to weak bindings that may be raised for a 501 * given cpu (offline/move code may set this and then wait a short time for 502 * existing weak bindings to drop); the cpu_inmotion pointer is that barrier. 503 * 504 * There are few restrictions on the calling context of thread_nomigrate. 505 * The caller must not hold the thread lock. Calls may be nested. 506 * 507 * After weakbinding a thread must not perform actions that may block. 508 * In particular it must not call thread_affinity_set; calling that when 509 * already weakbound is nonsensical anyway. 510 * 511 * If curthread is prevented from migrating for other reasons 512 * (kernel preemption disabled; high pil; strongly bound; interrupt thread) 513 * then the weak binding will succeed even if this cpu is the target of an 514 * offline/move request. 515 */ 516 void 517 thread_nomigrate(void) 518 { 519 cpu_t *cp; 520 kthread_id_t t = curthread; 521 522 again: 523 kpreempt_disable(); 524 cp = CPU; 525 526 /* 527 * A highlevel interrupt must not modify t_nomigrate or 528 * t_weakbound_cpu of the thread it has interrupted. A lowlevel 529 * interrupt thread cannot migrate and we can avoid the 530 * thread_lock call below by short-circuiting here. In either 531 * case we can just return since no migration is possible and 532 * the condition will persist (ie, when we test for these again 533 * in thread_allowmigrate they can't have changed). Migration 534 * is also impossible if we're at or above DISP_LEVEL pil. 535 */ 536 if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD || 537 getpil() >= DISP_LEVEL) { 538 kpreempt_enable(); 539 return; 540 } 541 542 /* 543 * We must be consistent with existing weak bindings. Since we 544 * may be interrupted between the increment of t_nomigrate and 545 * the store to t_weakbound_cpu below we cannot assume that 546 * t_weakbound_cpu will be set if t_nomigrate is. Note that we 547 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not 548 * always the case. 549 */ 550 if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) { 551 if (!panicstr) 552 panic("thread_nomigrate: binding to %p but already " 553 "bound to %p", (void *)cp, 554 (void *)t->t_weakbound_cpu); 555 } 556 557 /* 558 * At this point we have preemption disabled and we don't yet hold 559 * the thread lock. So it's possible that somebody else could 560 * set t_bind_cpu here and not be able to force us across to the 561 * new cpu (since we have preemption disabled). 562 */ 563 thread_lock(curthread); 564 565 /* 566 * If further weak bindings are being (temporarily) suppressed then 567 * we'll settle for disabling kernel preemption (which assures 568 * no migration provided the thread does not block which it is 569 * not allowed to if using thread_nomigrate). We must remember 570 * this disposition so we can take appropriate action in 571 * thread_allowmigrate. If this is a nested call and the 572 * thread is already weakbound then fall through as normal. 573 * We remember the decision to settle for kpreempt_disable through 574 * negative nesting counting in t_nomigrate. Once a thread has had one 575 * weakbinding request satisfied in this way any further (nested) 576 * requests will continue to be satisfied in the same way, 577 * even if weak bindings have recommenced. 578 */ 579 if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) { 580 --t->t_nomigrate; 581 thread_unlock(curthread); 582 return; /* with kpreempt_disable still active */ 583 } 584 585 /* 586 * We hold thread_lock so t_bind_cpu cannot change. We could, 587 * however, be running on a different cpu to which we are t_bound_cpu 588 * to (as explained above). If we grant the weak binding request 589 * in that case then the dispatcher must favour our weak binding 590 * over our strong (in which case, just as when preemption is 591 * disabled, we can continue to run on a cpu other than the one to 592 * which we are strongbound; the difference in this case is that 593 * this thread can be preempted and so can appear on the dispatch 594 * queues of a cpu other than the one it is strongbound to). 595 * 596 * If the cpu we are running on does not appear to be a current 597 * offline target (we check cpu_inmotion to determine this - since 598 * we don't hold cpu_lock we may not see a recent store to that, 599 * so it's possible that we at times can grant a weak binding to a 600 * cpu that is an offline target, but that one request will not 601 * prevent the offline from succeeding) then we will always grant 602 * the weak binding request. This includes the case above where 603 * we grant a weakbinding not commensurate with our strong binding. 604 * 605 * If our cpu does appear to be an offline target then we're inclined 606 * not to grant the weakbinding request just yet - we'd prefer to 607 * migrate to another cpu and grant the request there. The 608 * exceptions are those cases where going through preemption code 609 * will not result in us changing cpu: 610 * 611 * . interrupts have already bypassed this case (see above) 612 * . we are already weakbound to this cpu (dispatcher code will 613 * always return us to the weakbound cpu) 614 * . preemption was disabled even before we disabled it above 615 * . we are strongbound to this cpu (if we're strongbound to 616 * another and not yet running there the trip through the 617 * dispatcher will move us to the strongbound cpu and we 618 * will grant the weak binding there) 619 */ 620 if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 || 621 t->t_bound_cpu == cp) { 622 /* 623 * Don't be tempted to store to t_weakbound_cpu only on 624 * the first nested bind request - if we're interrupted 625 * after the increment of t_nomigrate and before the 626 * store to t_weakbound_cpu and the interrupt calls 627 * thread_nomigrate then the assertion in thread_allowmigrate 628 * would fail. 629 */ 630 t->t_nomigrate++; 631 t->t_weakbound_cpu = cp; 632 membar_producer(); 633 thread_unlock(curthread); 634 /* 635 * Now that we have dropped the thread_lock another thread 636 * can set our t_weakbound_cpu, and will try to migrate us 637 * to the strongbound cpu (which will not be prevented by 638 * preemption being disabled since we're about to enable 639 * preemption). We have granted the weakbinding to the current 640 * cpu, so again we are in the position that is is is possible 641 * that our weak and strong bindings differ. Again this 642 * is catered for by dispatcher code which will favour our 643 * weak binding. 644 */ 645 kpreempt_enable(); 646 } else { 647 /* 648 * Move to another cpu before granting the request by 649 * forcing this thread through preemption code. When we 650 * get to set{front,back}dq called from CL_PREEMPT() 651 * cpu_choose() will be used to select a cpu to queue 652 * us on - that will see cpu_inmotion and take 653 * steps to avoid returning us to this cpu. 654 */ 655 cp->cpu_kprunrun = 1; 656 thread_unlock(curthread); 657 kpreempt_enable(); /* will call preempt() */ 658 goto again; 659 } 660 } 661 662 void 663 thread_allowmigrate(void) 664 { 665 kthread_id_t t = curthread; 666 667 ASSERT(t->t_weakbound_cpu == CPU || 668 (t->t_nomigrate < 0 && t->t_preempt > 0) || 669 CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD || 670 getpil() >= DISP_LEVEL); 671 672 if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) || 673 getpil() >= DISP_LEVEL) 674 return; 675 676 if (t->t_nomigrate < 0) { 677 /* 678 * This thread was granted "weak binding" in the 679 * stronger form of kernel preemption disabling. 680 * Undo a level of nesting for both t_nomigrate 681 * and t_preempt. 682 */ 683 ++t->t_nomigrate; 684 kpreempt_enable(); 685 } else if (--t->t_nomigrate == 0) { 686 /* 687 * Time to drop the weak binding. We need to cater 688 * for the case where we're weakbound to a different 689 * cpu than that to which we're strongbound (a very 690 * temporary arrangement that must only persist until 691 * weak binding drops). We don't acquire thread_lock 692 * here so even as this code executes t_bound_cpu 693 * may be changing. So we disable preemption and 694 * a) in the case that t_bound_cpu changes while we 695 * have preemption disabled kprunrun will be set 696 * asynchronously, and b) if before disabling 697 * preemption we were already on a different cpu to 698 * our t_bound_cpu then we set kprunrun ourselves 699 * to force a trip through the dispatcher when 700 * preemption is enabled. 701 */ 702 kpreempt_disable(); 703 if (t->t_bound_cpu && 704 t->t_weakbound_cpu != t->t_bound_cpu) 705 CPU->cpu_kprunrun = 1; 706 t->t_weakbound_cpu = NULL; 707 membar_producer(); 708 kpreempt_enable(); 709 } 710 } 711 712 /* 713 * weakbinding_stop can be used to temporarily cause weakbindings made 714 * with thread_nomigrate to be satisfied through the stronger action of 715 * kpreempt_disable. weakbinding_start recommences normal weakbinding. 716 */ 717 718 void 719 weakbinding_stop(void) 720 { 721 ASSERT(MUTEX_HELD(&cpu_lock)); 722 weakbindingbarrier = 1; 723 membar_producer(); /* make visible before subsequent thread_lock */ 724 } 725 726 void 727 weakbinding_start(void) 728 { 729 ASSERT(MUTEX_HELD(&cpu_lock)); 730 weakbindingbarrier = 0; 731 } 732 733 /* 734 * This routine is called to place the CPUs in a safe place so that 735 * one of them can be taken off line or placed on line. What we are 736 * trying to do here is prevent a thread from traversing the list 737 * of active CPUs while we are changing it or from getting placed on 738 * the run queue of a CPU that has just gone off line. We do this by 739 * creating a thread with the highest possible prio for each CPU and 740 * having it call this routine. The advantage of this method is that 741 * we can eliminate all checks for CPU_ACTIVE in the disp routines. 742 * This makes disp faster at the expense of making p_online() slower 743 * which is a good trade off. 744 */ 745 static void 746 cpu_pause(volatile char *safe) 747 { 748 int s; 749 struct _cpu_pause_info *cpi = &cpu_pause_info; 750 751 ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE)); 752 753 while (*safe != PAUSE_DIE) { 754 *safe = PAUSE_READY; 755 membar_enter(); /* make sure stores are flushed */ 756 sema_v(&cpi->cp_sem); /* signal requesting thread */ 757 758 /* 759 * Wait here until all pause threads are running. That 760 * indicates that it's safe to do the spl. Until 761 * cpu_pause_info.cp_go is set, we don't want to spl 762 * because that might block clock interrupts needed 763 * to preempt threads on other CPUs. 764 */ 765 while (cpi->cp_go == 0) 766 ; 767 /* 768 * Even though we are at the highest disp prio, we need 769 * to block out all interrupts below LOCK_LEVEL so that 770 * an intr doesn't come in, wake up a thread, and call 771 * setbackdq/setfrontdq. 772 */ 773 s = splhigh(); 774 /* 775 * This cpu is now safe. 776 */ 777 *safe = PAUSE_WAIT; 778 membar_enter(); /* make sure stores are flushed */ 779 780 /* 781 * Now we wait. When we are allowed to continue, safe will 782 * be set to PAUSE_IDLE. 783 */ 784 while (*safe != PAUSE_IDLE) 785 ; 786 787 splx(s); 788 /* 789 * Waiting is at an end. Switch out of cpu_pause 790 * loop and resume useful work. 791 */ 792 swtch(); 793 } 794 795 mutex_enter(&pause_free_mutex); 796 *safe = PAUSE_DEAD; 797 cv_broadcast(&pause_free_cv); 798 mutex_exit(&pause_free_mutex); 799 } 800 801 /* 802 * Allow the cpus to start running again. 803 */ 804 void 805 start_cpus() 806 { 807 int i; 808 809 ASSERT(MUTEX_HELD(&cpu_lock)); 810 ASSERT(cpu_pause_info.cp_paused); 811 cpu_pause_info.cp_paused = NULL; 812 for (i = 0; i < NCPU; i++) 813 safe_list[i] = PAUSE_IDLE; 814 membar_enter(); /* make sure stores are flushed */ 815 affinity_clear(); 816 splx(cpu_pause_info.cp_spl); 817 kpreempt_enable(); 818 } 819 820 /* 821 * Allocate a pause thread for a CPU. 822 */ 823 static void 824 cpu_pause_alloc(cpu_t *cp) 825 { 826 kthread_id_t t; 827 int cpun = cp->cpu_id; 828 829 /* 830 * Note, v.v_nglobpris will not change value as long as I hold 831 * cpu_lock. 832 */ 833 t = thread_create(NULL, 0, cpu_pause, (caddr_t)&safe_list[cpun], 834 0, &p0, TS_STOPPED, v.v_nglobpris - 1); 835 thread_lock(t); 836 t->t_bound_cpu = cp; 837 t->t_disp_queue = cp->cpu_disp; 838 t->t_affinitycnt = 1; 839 t->t_preempt = 1; 840 thread_unlock(t); 841 cp->cpu_pause_thread = t; 842 /* 843 * Registering a thread in the callback table is usually done 844 * in the initialization code of the thread. In this 845 * case, we do it right after thread creation because the 846 * thread itself may never run, and we need to register the 847 * fact that it is safe for cpr suspend. 848 */ 849 CALLB_CPR_INIT_SAFE(t, "cpu_pause"); 850 } 851 852 /* 853 * Free a pause thread for a CPU. 854 */ 855 static void 856 cpu_pause_free(cpu_t *cp) 857 { 858 kthread_id_t t; 859 int cpun = cp->cpu_id; 860 861 ASSERT(MUTEX_HELD(&cpu_lock)); 862 /* 863 * We have to get the thread and tell him to die. 864 */ 865 if ((t = cp->cpu_pause_thread) == NULL) { 866 ASSERT(safe_list[cpun] == PAUSE_IDLE); 867 return; 868 } 869 thread_lock(t); 870 t->t_cpu = CPU; /* disp gets upset if last cpu is quiesced. */ 871 t->t_bound_cpu = NULL; /* Must un-bind; cpu may not be running. */ 872 t->t_pri = v.v_nglobpris - 1; 873 ASSERT(safe_list[cpun] == PAUSE_IDLE); 874 safe_list[cpun] = PAUSE_DIE; 875 THREAD_TRANSITION(t); 876 setbackdq(t); 877 thread_unlock_nopreempt(t); 878 879 /* 880 * If we don't wait for the thread to actually die, it may try to 881 * run on the wrong cpu as part of an actual call to pause_cpus(). 882 */ 883 mutex_enter(&pause_free_mutex); 884 while (safe_list[cpun] != PAUSE_DEAD) { 885 cv_wait(&pause_free_cv, &pause_free_mutex); 886 } 887 mutex_exit(&pause_free_mutex); 888 safe_list[cpun] = PAUSE_IDLE; 889 890 cp->cpu_pause_thread = NULL; 891 } 892 893 /* 894 * Initialize basic structures for pausing CPUs. 895 */ 896 void 897 cpu_pause_init() 898 { 899 sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL); 900 /* 901 * Create initial CPU pause thread. 902 */ 903 cpu_pause_alloc(CPU); 904 } 905 906 /* 907 * Start the threads used to pause another CPU. 908 */ 909 static int 910 cpu_pause_start(processorid_t cpu_id) 911 { 912 int i; 913 int cpu_count = 0; 914 915 for (i = 0; i < NCPU; i++) { 916 cpu_t *cp; 917 kthread_id_t t; 918 919 cp = cpu[i]; 920 if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) { 921 safe_list[i] = PAUSE_WAIT; 922 continue; 923 } 924 925 /* 926 * Skip CPU if it is quiesced or not yet started. 927 */ 928 if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) { 929 safe_list[i] = PAUSE_WAIT; 930 continue; 931 } 932 933 /* 934 * Start this CPU's pause thread. 935 */ 936 t = cp->cpu_pause_thread; 937 thread_lock(t); 938 /* 939 * Reset the priority, since nglobpris may have 940 * changed since the thread was created, if someone 941 * has loaded the RT (or some other) scheduling 942 * class. 943 */ 944 t->t_pri = v.v_nglobpris - 1; 945 THREAD_TRANSITION(t); 946 setbackdq(t); 947 thread_unlock_nopreempt(t); 948 ++cpu_count; 949 } 950 return (cpu_count); 951 } 952 953 954 /* 955 * Pause all of the CPUs except the one we are on by creating a high 956 * priority thread bound to those CPUs. 957 * 958 * Note that one must be extremely careful regarding code 959 * executed while CPUs are paused. Since a CPU may be paused 960 * while a thread scheduling on that CPU is holding an adaptive 961 * lock, code executed with CPUs paused must not acquire adaptive 962 * (or low-level spin) locks. Also, such code must not block, 963 * since the thread that is supposed to initiate the wakeup may 964 * never run. 965 * 966 * With a few exceptions, the restrictions on code executed with CPUs 967 * paused match those for code executed at high-level interrupt 968 * context. 969 */ 970 void 971 pause_cpus(cpu_t *off_cp) 972 { 973 processorid_t cpu_id; 974 int i; 975 struct _cpu_pause_info *cpi = &cpu_pause_info; 976 977 ASSERT(MUTEX_HELD(&cpu_lock)); 978 ASSERT(cpi->cp_paused == NULL); 979 cpi->cp_count = 0; 980 cpi->cp_go = 0; 981 for (i = 0; i < NCPU; i++) 982 safe_list[i] = PAUSE_IDLE; 983 kpreempt_disable(); 984 985 /* 986 * If running on the cpu that is going offline, get off it. 987 * This is so that it won't be necessary to rechoose a CPU 988 * when done. 989 */ 990 if (CPU == off_cp) 991 cpu_id = off_cp->cpu_next_part->cpu_id; 992 else 993 cpu_id = CPU->cpu_id; 994 affinity_set(cpu_id); 995 996 /* 997 * Start the pause threads and record how many were started 998 */ 999 cpi->cp_count = cpu_pause_start(cpu_id); 1000 1001 /* 1002 * Now wait for all CPUs to be running the pause thread. 1003 */ 1004 while (cpi->cp_count > 0) { 1005 /* 1006 * Spin reading the count without grabbing the disp 1007 * lock to make sure we don't prevent the pause 1008 * threads from getting the lock. 1009 */ 1010 while (sema_held(&cpi->cp_sem)) 1011 ; 1012 if (sema_tryp(&cpi->cp_sem)) 1013 --cpi->cp_count; 1014 } 1015 cpi->cp_go = 1; /* all have reached cpu_pause */ 1016 1017 /* 1018 * Now wait for all CPUs to spl. (Transition from PAUSE_READY 1019 * to PAUSE_WAIT.) 1020 */ 1021 for (i = 0; i < NCPU; i++) { 1022 while (safe_list[i] != PAUSE_WAIT) 1023 ; 1024 } 1025 cpi->cp_spl = splhigh(); /* block dispatcher on this CPU */ 1026 cpi->cp_paused = curthread; 1027 } 1028 1029 /* 1030 * Check whether the current thread has CPUs paused 1031 */ 1032 int 1033 cpus_paused(void) 1034 { 1035 if (cpu_pause_info.cp_paused != NULL) { 1036 ASSERT(cpu_pause_info.cp_paused == curthread); 1037 return (1); 1038 } 1039 return (0); 1040 } 1041 1042 static cpu_t * 1043 cpu_get_all(processorid_t cpun) 1044 { 1045 ASSERT(MUTEX_HELD(&cpu_lock)); 1046 1047 if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun)) 1048 return (NULL); 1049 return (cpu[cpun]); 1050 } 1051 1052 /* 1053 * Check whether cpun is a valid processor id and whether it should be 1054 * visible from the current zone. If it is, return a pointer to the 1055 * associated CPU structure. 1056 */ 1057 cpu_t * 1058 cpu_get(processorid_t cpun) 1059 { 1060 cpu_t *c; 1061 1062 ASSERT(MUTEX_HELD(&cpu_lock)); 1063 c = cpu_get_all(cpun); 1064 if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() && 1065 zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c)) 1066 return (NULL); 1067 return (c); 1068 } 1069 1070 /* 1071 * The following functions should be used to check CPU states in the kernel. 1072 * They should be invoked with cpu_lock held. Kernel subsystems interested 1073 * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc 1074 * states. Those are for user-land (and system call) use only. 1075 */ 1076 1077 /* 1078 * Determine whether the CPU is online and handling interrupts. 1079 */ 1080 int 1081 cpu_is_online(cpu_t *cpu) 1082 { 1083 ASSERT(MUTEX_HELD(&cpu_lock)); 1084 return (cpu_flagged_online(cpu->cpu_flags)); 1085 } 1086 1087 /* 1088 * Determine whether the CPU is offline (this includes spare and faulted). 1089 */ 1090 int 1091 cpu_is_offline(cpu_t *cpu) 1092 { 1093 ASSERT(MUTEX_HELD(&cpu_lock)); 1094 return (cpu_flagged_offline(cpu->cpu_flags)); 1095 } 1096 1097 /* 1098 * Determine whether the CPU is powered off. 1099 */ 1100 int 1101 cpu_is_poweredoff(cpu_t *cpu) 1102 { 1103 ASSERT(MUTEX_HELD(&cpu_lock)); 1104 return (cpu_flagged_poweredoff(cpu->cpu_flags)); 1105 } 1106 1107 /* 1108 * Determine whether the CPU is handling interrupts. 1109 */ 1110 int 1111 cpu_is_nointr(cpu_t *cpu) 1112 { 1113 ASSERT(MUTEX_HELD(&cpu_lock)); 1114 return (cpu_flagged_nointr(cpu->cpu_flags)); 1115 } 1116 1117 /* 1118 * Determine whether the CPU is active (scheduling threads). 1119 */ 1120 int 1121 cpu_is_active(cpu_t *cpu) 1122 { 1123 ASSERT(MUTEX_HELD(&cpu_lock)); 1124 return (cpu_flagged_active(cpu->cpu_flags)); 1125 } 1126 1127 /* 1128 * Same as above, but these require cpu_flags instead of cpu_t pointers. 1129 */ 1130 int 1131 cpu_flagged_online(cpu_flag_t cpu_flags) 1132 { 1133 return (cpu_flagged_active(cpu_flags) && 1134 (cpu_flags & CPU_ENABLE)); 1135 } 1136 1137 int 1138 cpu_flagged_offline(cpu_flag_t cpu_flags) 1139 { 1140 return (((cpu_flags & CPU_POWEROFF) == 0) && 1141 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)); 1142 } 1143 1144 int 1145 cpu_flagged_poweredoff(cpu_flag_t cpu_flags) 1146 { 1147 return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF); 1148 } 1149 1150 int 1151 cpu_flagged_nointr(cpu_flag_t cpu_flags) 1152 { 1153 return (cpu_flagged_active(cpu_flags) && 1154 (cpu_flags & CPU_ENABLE) == 0); 1155 } 1156 1157 int 1158 cpu_flagged_active(cpu_flag_t cpu_flags) 1159 { 1160 return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) && 1161 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY)); 1162 } 1163 1164 /* 1165 * Bring the indicated CPU online. 1166 */ 1167 int 1168 cpu_online(cpu_t *cp) 1169 { 1170 int error = 0; 1171 1172 /* 1173 * Handle on-line request. 1174 * This code must put the new CPU on the active list before 1175 * starting it because it will not be paused, and will start 1176 * using the active list immediately. The real start occurs 1177 * when the CPU_QUIESCED flag is turned off. 1178 */ 1179 1180 ASSERT(MUTEX_HELD(&cpu_lock)); 1181 1182 /* 1183 * Put all the cpus into a known safe place. 1184 * No mutexes can be entered while CPUs are paused. 1185 */ 1186 error = mp_cpu_start(cp); /* arch-dep hook */ 1187 if (error == 0) { 1188 pause_cpus(NULL); 1189 cpu_add_active_internal(cp); 1190 if (cp->cpu_flags & CPU_FAULTED) { 1191 cp->cpu_flags &= ~CPU_FAULTED; 1192 mp_cpu_faulted_exit(cp); 1193 } 1194 cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN | 1195 CPU_SPARE); 1196 start_cpus(); 1197 cpu_stats_kstat_create(cp); 1198 cpu_create_intrstat(cp); 1199 lgrp_kstat_create(cp); 1200 cpu_state_change_notify(cp->cpu_id, CPU_ON); 1201 cpu_intr_enable(cp); /* arch-dep hook */ 1202 cyclic_online(cp); 1203 poke_cpu(cp->cpu_id); 1204 } 1205 1206 return (error); 1207 } 1208 1209 /* 1210 * Take the indicated CPU offline. 1211 */ 1212 int 1213 cpu_offline(cpu_t *cp, int flags) 1214 { 1215 cpupart_t *pp; 1216 int error = 0; 1217 cpu_t *ncp; 1218 int intr_enable; 1219 int cyclic_off = 0; 1220 int loop_count; 1221 int no_quiesce = 0; 1222 int (*bound_func)(struct cpu *, int); 1223 kthread_t *t; 1224 lpl_t *cpu_lpl; 1225 proc_t *p; 1226 int lgrp_diff_lpl; 1227 1228 ASSERT(MUTEX_HELD(&cpu_lock)); 1229 1230 /* 1231 * If we're going from faulted or spare to offline, just 1232 * clear these flags and update CPU state. 1233 */ 1234 if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) { 1235 if (cp->cpu_flags & CPU_FAULTED) { 1236 cp->cpu_flags &= ~CPU_FAULTED; 1237 mp_cpu_faulted_exit(cp); 1238 } 1239 cp->cpu_flags &= ~CPU_SPARE; 1240 cpu_set_state(cp); 1241 return (0); 1242 } 1243 1244 /* 1245 * Handle off-line request. 1246 */ 1247 pp = cp->cpu_part; 1248 /* 1249 * Don't offline last online CPU in partition 1250 */ 1251 if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2) 1252 return (EBUSY); 1253 /* 1254 * Unbind all thread bound to our CPU if we were asked to. 1255 */ 1256 if (flags & CPU_FORCED && (error = cpu_unbind(cp->cpu_id)) != 0) 1257 return (error); 1258 /* 1259 * We shouldn't be bound to this CPU ourselves. 1260 */ 1261 if (curthread->t_bound_cpu == cp) 1262 return (EBUSY); 1263 1264 /* 1265 * Tell interested parties that this CPU is going offline. 1266 */ 1267 cpu_state_change_notify(cp->cpu_id, CPU_OFF); 1268 1269 /* 1270 * Take the CPU out of interrupt participation so we won't find 1271 * bound kernel threads. If the architecture cannot completely 1272 * shut off interrupts on the CPU, don't quiesce it, but don't 1273 * run anything but interrupt thread... this is indicated by 1274 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being 1275 * off. 1276 */ 1277 intr_enable = cp->cpu_flags & CPU_ENABLE; 1278 if (intr_enable) 1279 no_quiesce = cpu_intr_disable(cp); 1280 1281 /* 1282 * Record that we are aiming to offline this cpu. This acts as 1283 * a barrier to further weak binding requests in thread_nomigrate 1284 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to 1285 * lean away from this cpu. Further strong bindings are already 1286 * avoided since we hold cpu_lock. Since threads that are set 1287 * runnable around now and others coming off the target cpu are 1288 * directed away from the target, existing strong and weak bindings 1289 * (especially the latter) to the target cpu stand maximum chance of 1290 * being able to unbind during the short delay loop below (if other 1291 * unbound threads compete they may not see cpu in time to unbind 1292 * even if they would do so immediately. 1293 */ 1294 cpu_inmotion = cp; 1295 membar_enter(); 1296 1297 /* 1298 * Check for kernel threads (strong or weak) bound to that CPU. 1299 * Strongly bound threads may not unbind, and we'll have to return 1300 * EBUSY. Weakly bound threads should always disappear - we've 1301 * stopped more weak binding with cpu_inmotion and existing 1302 * bindings will drain imminently (they may not block). Nonetheless 1303 * we will wait for a fixed period for all bound threads to disappear. 1304 * Inactive interrupt threads are OK (they'll be in TS_FREE 1305 * state). If test finds some bound threads, wait a few ticks 1306 * to give short-lived threads (such as interrupts) chance to 1307 * complete. Note that if no_quiesce is set, i.e. this cpu 1308 * is required to service interrupts, then we take the route 1309 * that permits interrupt threads to be active (or bypassed). 1310 */ 1311 bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads; 1312 1313 again: for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) { 1314 if (loop_count >= 5) { 1315 error = EBUSY; /* some threads still bound */ 1316 break; 1317 } 1318 1319 /* 1320 * If some threads were assigned, give them 1321 * a chance to complete or move. 1322 * 1323 * This assumes that the clock_thread is not bound 1324 * to any CPU, because the clock_thread is needed to 1325 * do the delay(hz/100). 1326 * 1327 * Note: we still hold the cpu_lock while waiting for 1328 * the next clock tick. This is OK since it isn't 1329 * needed for anything else except processor_bind(2), 1330 * and system initialization. If we drop the lock, 1331 * we would risk another p_online disabling the last 1332 * processor. 1333 */ 1334 delay(hz/100); 1335 } 1336 1337 if (error == 0 && cyclic_off == 0) { 1338 if (!cyclic_offline(cp)) { 1339 /* 1340 * We must have bound cyclics... 1341 */ 1342 error = EBUSY; 1343 goto out; 1344 } 1345 cyclic_off = 1; 1346 } 1347 1348 /* 1349 * Call mp_cpu_stop() to perform any special operations 1350 * needed for this machine architecture to offline a CPU. 1351 */ 1352 if (error == 0) 1353 error = mp_cpu_stop(cp); /* arch-dep hook */ 1354 1355 /* 1356 * If that all worked, take the CPU offline and decrement 1357 * ncpus_online. 1358 */ 1359 if (error == 0) { 1360 /* 1361 * Put all the cpus into a known safe place. 1362 * No mutexes can be entered while CPUs are paused. 1363 */ 1364 pause_cpus(cp); 1365 /* 1366 * Repeat the operation, if necessary, to make sure that 1367 * all outstanding low-level interrupts run to completion 1368 * before we set the CPU_QUIESCED flag. It's also possible 1369 * that a thread has weak bound to the cpu despite our raising 1370 * cpu_inmotion above since it may have loaded that 1371 * value before the barrier became visible (this would have 1372 * to be the thread that was on the target cpu at the time 1373 * we raised the barrier). 1374 */ 1375 if ((!no_quiesce && cp->cpu_intr_actv != 0) || 1376 (*bound_func)(cp, 1)) { 1377 start_cpus(); 1378 (void) mp_cpu_start(cp); 1379 goto again; 1380 } 1381 ncp = cp->cpu_next_part; 1382 cpu_lpl = cp->cpu_lpl; 1383 ASSERT(cpu_lpl != NULL); 1384 1385 /* 1386 * Remove the CPU from the list of active CPUs. 1387 */ 1388 cpu_remove_active(cp); 1389 1390 /* 1391 * Walk the active process list and look for threads 1392 * whose home lgroup needs to be updated, or 1393 * the last CPU they run on is the one being offlined now. 1394 */ 1395 1396 ASSERT(curthread->t_cpu != cp); 1397 for (p = practive; p != NULL; p = p->p_next) { 1398 1399 t = p->p_tlist; 1400 1401 if (t == NULL) 1402 continue; 1403 1404 lgrp_diff_lpl = 0; 1405 1406 do { 1407 ASSERT(t->t_lpl != NULL); 1408 /* 1409 * Taking last CPU in lpl offline 1410 * Rehome thread if it is in this lpl 1411 * Otherwise, update the count of how many 1412 * threads are in this CPU's lgroup but have 1413 * a different lpl. 1414 */ 1415 1416 if (cpu_lpl->lpl_ncpu == 0) { 1417 if (t->t_lpl == cpu_lpl) 1418 lgrp_move_thread(t, 1419 lgrp_choose(t, 1420 t->t_cpupart), 0); 1421 else if (t->t_lpl->lpl_lgrpid == 1422 cpu_lpl->lpl_lgrpid) 1423 lgrp_diff_lpl++; 1424 } 1425 ASSERT(t->t_lpl->lpl_ncpu > 0); 1426 1427 /* 1428 * Update CPU last ran on if it was this CPU 1429 */ 1430 if (t->t_cpu == cp && t->t_bound_cpu != cp) 1431 t->t_cpu = disp_lowpri_cpu(ncp, t->t_lpl, 1432 t->t_pri, NULL); 1433 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp || 1434 t->t_weakbound_cpu == cp); 1435 1436 t = t->t_forw; 1437 } while (t != p->p_tlist); 1438 1439 /* 1440 * Didn't find any threads in the same lgroup as this 1441 * CPU with a different lpl, so remove the lgroup from 1442 * the process lgroup bitmask. 1443 */ 1444 1445 if (lgrp_diff_lpl == 0) 1446 klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid); 1447 } 1448 1449 /* 1450 * Walk thread list looking for threads that need to be 1451 * rehomed, since there are some threads that are not in 1452 * their process's p_tlist. 1453 */ 1454 1455 t = curthread; 1456 do { 1457 ASSERT(t != NULL && t->t_lpl != NULL); 1458 1459 /* 1460 * Rehome threads with same lpl as this CPU when this 1461 * is the last CPU in the lpl. 1462 */ 1463 1464 if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl)) 1465 lgrp_move_thread(t, 1466 lgrp_choose(t, t->t_cpupart), 1); 1467 1468 ASSERT(t->t_lpl->lpl_ncpu > 0); 1469 1470 /* 1471 * Update CPU last ran on if it was this CPU 1472 */ 1473 1474 if (t->t_cpu == cp && t->t_bound_cpu != cp) { 1475 t->t_cpu = disp_lowpri_cpu(ncp, 1476 t->t_lpl, t->t_pri, NULL); 1477 } 1478 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp || 1479 t->t_weakbound_cpu == cp); 1480 t = t->t_next; 1481 1482 } while (t != curthread); 1483 ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0); 1484 cp->cpu_flags |= CPU_OFFLINE; 1485 disp_cpu_inactive(cp); 1486 if (!no_quiesce) 1487 cp->cpu_flags |= CPU_QUIESCED; 1488 ncpus_online--; 1489 cpu_set_state(cp); 1490 cpu_inmotion = NULL; 1491 start_cpus(); 1492 cpu_stats_kstat_destroy(cp); 1493 cpu_delete_intrstat(cp); 1494 lgrp_kstat_destroy(cp); 1495 } 1496 1497 out: 1498 cpu_inmotion = NULL; 1499 1500 /* 1501 * If we failed, re-enable interrupts. 1502 * Do this even if cpu_intr_disable returned an error, because 1503 * it may have partially disabled interrupts. 1504 */ 1505 if (error && intr_enable) 1506 cpu_intr_enable(cp); 1507 1508 /* 1509 * If we failed, but managed to offline the cyclic subsystem on this 1510 * CPU, bring it back online. 1511 */ 1512 if (error && cyclic_off) 1513 cyclic_online(cp); 1514 1515 /* 1516 * If we failed, we need to notify everyone that this CPU is back on. 1517 */ 1518 if (error != 0) 1519 cpu_state_change_notify(cp->cpu_id, CPU_ON); 1520 1521 return (error); 1522 } 1523 1524 /* 1525 * Mark the indicated CPU as faulted, taking it offline. 1526 */ 1527 int 1528 cpu_faulted(cpu_t *cp, int flags) 1529 { 1530 int error = 0; 1531 1532 ASSERT(MUTEX_HELD(&cpu_lock)); 1533 ASSERT(!cpu_is_poweredoff(cp)); 1534 1535 if (cpu_is_offline(cp)) { 1536 cp->cpu_flags &= ~CPU_SPARE; 1537 cp->cpu_flags |= CPU_FAULTED; 1538 mp_cpu_faulted_enter(cp); 1539 cpu_set_state(cp); 1540 return (0); 1541 } 1542 1543 if ((error = cpu_offline(cp, flags)) == 0) { 1544 cp->cpu_flags |= CPU_FAULTED; 1545 mp_cpu_faulted_enter(cp); 1546 cpu_set_state(cp); 1547 } 1548 1549 return (error); 1550 } 1551 1552 /* 1553 * Mark the indicated CPU as a spare, taking it offline. 1554 */ 1555 int 1556 cpu_spare(cpu_t *cp, int flags) 1557 { 1558 int error = 0; 1559 1560 ASSERT(MUTEX_HELD(&cpu_lock)); 1561 ASSERT(!cpu_is_poweredoff(cp)); 1562 1563 if (cpu_is_offline(cp)) { 1564 if (cp->cpu_flags & CPU_FAULTED) { 1565 cp->cpu_flags &= ~CPU_FAULTED; 1566 mp_cpu_faulted_exit(cp); 1567 } 1568 cp->cpu_flags |= CPU_SPARE; 1569 cpu_set_state(cp); 1570 return (0); 1571 } 1572 1573 if ((error = cpu_offline(cp, flags)) == 0) { 1574 cp->cpu_flags |= CPU_SPARE; 1575 cpu_set_state(cp); 1576 } 1577 1578 return (error); 1579 } 1580 1581 /* 1582 * Take the indicated CPU from poweroff to offline. 1583 */ 1584 int 1585 cpu_poweron(cpu_t *cp) 1586 { 1587 int error = ENOTSUP; 1588 1589 ASSERT(MUTEX_HELD(&cpu_lock)); 1590 ASSERT(cpu_is_poweredoff(cp)); 1591 1592 error = mp_cpu_poweron(cp); /* arch-dep hook */ 1593 if (error == 0) 1594 cpu_set_state(cp); 1595 1596 return (error); 1597 } 1598 1599 /* 1600 * Take the indicated CPU from any inactive state to powered off. 1601 */ 1602 int 1603 cpu_poweroff(cpu_t *cp) 1604 { 1605 int error = ENOTSUP; 1606 1607 ASSERT(MUTEX_HELD(&cpu_lock)); 1608 ASSERT(cpu_is_offline(cp)); 1609 1610 if (!(cp->cpu_flags & CPU_QUIESCED)) 1611 return (EBUSY); /* not completely idle */ 1612 1613 error = mp_cpu_poweroff(cp); /* arch-dep hook */ 1614 if (error == 0) 1615 cpu_set_state(cp); 1616 1617 return (error); 1618 } 1619 1620 /* 1621 * Initialize the CPU lists for the first CPU. 1622 */ 1623 void 1624 cpu_list_init(cpu_t *cp) 1625 { 1626 cp->cpu_next = cp; 1627 cp->cpu_prev = cp; 1628 cpu_list = cp; 1629 1630 cp->cpu_next_onln = cp; 1631 cp->cpu_prev_onln = cp; 1632 cpu_active = cp; 1633 1634 cp->cpu_seqid = 0; 1635 CPUSET_ADD(cpu_seqid_inuse, 0); 1636 cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid); 1637 cp_default.cp_cpulist = cp; 1638 cp_default.cp_ncpus = 1; 1639 cp->cpu_next_part = cp; 1640 cp->cpu_prev_part = cp; 1641 cp->cpu_part = &cp_default; 1642 1643 CPUSET_ADD(cpu_available, cp->cpu_id); 1644 } 1645 1646 /* 1647 * Insert a CPU into the list of available CPUs. 1648 */ 1649 void 1650 cpu_add_unit(cpu_t *cp) 1651 { 1652 int seqid; 1653 1654 ASSERT(MUTEX_HELD(&cpu_lock)); 1655 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */ 1656 1657 lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0); 1658 1659 /* 1660 * Note: most users of the cpu_list will grab the 1661 * cpu_lock to insure that it isn't modified. However, 1662 * certain users can't or won't do that. To allow this 1663 * we pause the other cpus. Users who walk the list 1664 * without cpu_lock, must disable kernel preemption 1665 * to insure that the list isn't modified underneath 1666 * them. Also, any cached pointers to cpu structures 1667 * must be revalidated by checking to see if the 1668 * cpu_next pointer points to itself. This check must 1669 * be done with the cpu_lock held or kernel preemption 1670 * disabled. This check relies upon the fact that 1671 * old cpu structures are not free'ed or cleared after 1672 * then are removed from the cpu_list. 1673 * 1674 * Note that the clock code walks the cpu list dereferencing 1675 * the cpu_part pointer, so we need to initialize it before 1676 * adding the cpu to the list. 1677 */ 1678 cp->cpu_part = &cp_default; 1679 (void) pause_cpus(NULL); 1680 cp->cpu_next = cpu_list; 1681 cp->cpu_prev = cpu_list->cpu_prev; 1682 cpu_list->cpu_prev->cpu_next = cp; 1683 cpu_list->cpu_prev = cp; 1684 start_cpus(); 1685 1686 for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++) 1687 continue; 1688 CPUSET_ADD(cpu_seqid_inuse, seqid); 1689 cp->cpu_seqid = seqid; 1690 ASSERT(ncpus < max_ncpus); 1691 ncpus++; 1692 cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid); 1693 cpu[cp->cpu_id] = cp; 1694 CPUSET_ADD(cpu_available, cp->cpu_id); 1695 1696 /* 1697 * allocate a pause thread for this CPU. 1698 */ 1699 cpu_pause_alloc(cp); 1700 1701 /* 1702 * So that new CPUs won't have NULL prev_onln and next_onln pointers, 1703 * link them into a list of just that CPU. 1704 * This is so that disp_lowpri_cpu will work for thread_create in 1705 * pause_cpus() when called from the startup thread in a new CPU. 1706 */ 1707 cp->cpu_next_onln = cp; 1708 cp->cpu_prev_onln = cp; 1709 cpu_info_kstat_create(cp); 1710 cp->cpu_next_part = cp; 1711 cp->cpu_prev_part = cp; 1712 1713 init_cpu_mstate(cp, CMS_SYSTEM); 1714 1715 pool_pset_mod = gethrtime(); 1716 } 1717 1718 /* 1719 * Do the opposite of cpu_add_unit(). 1720 */ 1721 void 1722 cpu_del_unit(int cpuid) 1723 { 1724 struct cpu *cp, *cpnext; 1725 1726 ASSERT(MUTEX_HELD(&cpu_lock)); 1727 cp = cpu[cpuid]; 1728 ASSERT(cp != NULL); 1729 1730 ASSERT(cp->cpu_next_onln == cp); 1731 ASSERT(cp->cpu_prev_onln == cp); 1732 ASSERT(cp->cpu_next_part == cp); 1733 ASSERT(cp->cpu_prev_part == cp); 1734 1735 chip_cpu_fini(cp); 1736 1737 /* 1738 * Destroy kstat stuff. 1739 */ 1740 cpu_info_kstat_destroy(cp); 1741 term_cpu_mstate(cp); 1742 /* 1743 * Free up pause thread. 1744 */ 1745 cpu_pause_free(cp); 1746 CPUSET_DEL(cpu_available, cp->cpu_id); 1747 cpu[cp->cpu_id] = NULL; 1748 /* 1749 * The clock thread and mutex_vector_enter cannot hold the 1750 * cpu_lock while traversing the cpu list, therefore we pause 1751 * all other threads by pausing the other cpus. These, and any 1752 * other routines holding cpu pointers while possibly sleeping 1753 * must be sure to call kpreempt_disable before processing the 1754 * list and be sure to check that the cpu has not been deleted 1755 * after any sleeps (check cp->cpu_next != NULL). We guarantee 1756 * to keep the deleted cpu structure around. 1757 * 1758 * Note that this MUST be done AFTER cpu_available 1759 * has been updated so that we don't waste time 1760 * trying to pause the cpu we're trying to delete. 1761 */ 1762 (void) pause_cpus(NULL); 1763 1764 cpnext = cp->cpu_next; 1765 cp->cpu_prev->cpu_next = cp->cpu_next; 1766 cp->cpu_next->cpu_prev = cp->cpu_prev; 1767 if (cp == cpu_list) 1768 cpu_list = cpnext; 1769 1770 /* 1771 * Signals that the cpu has been deleted (see above). 1772 */ 1773 cp->cpu_next = NULL; 1774 cp->cpu_prev = NULL; 1775 1776 start_cpus(); 1777 1778 CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid); 1779 ncpus--; 1780 lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0); 1781 1782 pool_pset_mod = gethrtime(); 1783 } 1784 1785 /* 1786 * Add a CPU to the list of active CPUs. 1787 * This routine must not get any locks, because other CPUs are paused. 1788 */ 1789 static void 1790 cpu_add_active_internal(cpu_t *cp) 1791 { 1792 cpupart_t *pp = cp->cpu_part; 1793 1794 ASSERT(MUTEX_HELD(&cpu_lock)); 1795 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */ 1796 1797 ncpus_online++; 1798 cpu_set_state(cp); 1799 cp->cpu_next_onln = cpu_active; 1800 cp->cpu_prev_onln = cpu_active->cpu_prev_onln; 1801 cpu_active->cpu_prev_onln->cpu_next_onln = cp; 1802 cpu_active->cpu_prev_onln = cp; 1803 1804 if (pp->cp_cpulist) { 1805 cp->cpu_next_part = pp->cp_cpulist; 1806 cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part; 1807 pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp; 1808 pp->cp_cpulist->cpu_prev_part = cp; 1809 } else { 1810 ASSERT(pp->cp_ncpus == 0); 1811 pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp; 1812 } 1813 pp->cp_ncpus++; 1814 if (pp->cp_ncpus == 1) { 1815 cp_numparts_nonempty++; 1816 ASSERT(cp_numparts_nonempty != 0); 1817 } 1818 1819 chip_cpu_assign(cp); 1820 1821 lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0); 1822 1823 bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg)); 1824 } 1825 1826 /* 1827 * Add a CPU to the list of active CPUs. 1828 * This is called from machine-dependent layers when a new CPU is started. 1829 */ 1830 void 1831 cpu_add_active(cpu_t *cp) 1832 { 1833 pause_cpus(NULL); 1834 cpu_add_active_internal(cp); 1835 start_cpus(); 1836 cpu_stats_kstat_create(cp); 1837 cpu_create_intrstat(cp); 1838 lgrp_kstat_create(cp); 1839 cpu_state_change_notify(cp->cpu_id, CPU_INIT); 1840 } 1841 1842 1843 /* 1844 * Remove a CPU from the list of active CPUs. 1845 * This routine must not get any locks, because other CPUs are paused. 1846 */ 1847 /* ARGSUSED */ 1848 static void 1849 cpu_remove_active(cpu_t *cp) 1850 { 1851 cpupart_t *pp = cp->cpu_part; 1852 1853 ASSERT(MUTEX_HELD(&cpu_lock)); 1854 ASSERT(cp->cpu_next_onln != cp); /* not the last one */ 1855 ASSERT(cp->cpu_prev_onln != cp); /* not the last one */ 1856 1857 chip_cpu_unassign(cp); 1858 1859 lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0); 1860 1861 cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln; 1862 cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln; 1863 if (cpu_active == cp) { 1864 cpu_active = cp->cpu_next_onln; 1865 } 1866 cp->cpu_next_onln = cp; 1867 cp->cpu_prev_onln = cp; 1868 1869 cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part; 1870 cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part; 1871 if (pp->cp_cpulist == cp) { 1872 pp->cp_cpulist = cp->cpu_next_part; 1873 ASSERT(pp->cp_cpulist != cp); 1874 } 1875 cp->cpu_next_part = cp; 1876 cp->cpu_prev_part = cp; 1877 pp->cp_ncpus--; 1878 if (pp->cp_ncpus == 0) { 1879 cp_numparts_nonempty--; 1880 ASSERT(cp_numparts_nonempty != 0); 1881 } 1882 } 1883 1884 /* 1885 * Routine used to setup a newly inserted CPU in preparation for starting 1886 * it running code. 1887 */ 1888 int 1889 cpu_configure(int cpuid) 1890 { 1891 int retval = 0; 1892 1893 ASSERT(MUTEX_HELD(&cpu_lock)); 1894 1895 /* 1896 * Some structures are statically allocated based upon 1897 * the maximum number of cpus the system supports. Do not 1898 * try to add anything beyond this limit. 1899 */ 1900 if (cpuid < 0 || cpuid >= NCPU) { 1901 return (EINVAL); 1902 } 1903 1904 if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) { 1905 return (EALREADY); 1906 } 1907 1908 if ((retval = mp_cpu_configure(cpuid)) != 0) { 1909 return (retval); 1910 } 1911 1912 cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF; 1913 cpu_set_state(cpu[cpuid]); 1914 retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG); 1915 if (retval != 0) 1916 (void) mp_cpu_unconfigure(cpuid); 1917 1918 return (retval); 1919 } 1920 1921 /* 1922 * Routine used to cleanup a CPU that has been powered off. This will 1923 * destroy all per-cpu information related to this cpu. 1924 */ 1925 int 1926 cpu_unconfigure(int cpuid) 1927 { 1928 int error; 1929 1930 ASSERT(MUTEX_HELD(&cpu_lock)); 1931 1932 if (cpu[cpuid] == NULL) { 1933 return (ENODEV); 1934 } 1935 1936 if (cpu[cpuid]->cpu_flags == 0) { 1937 return (EALREADY); 1938 } 1939 1940 if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) { 1941 return (EBUSY); 1942 } 1943 1944 if (cpu[cpuid]->cpu_props != NULL) { 1945 (void) nvlist_free(cpu[cpuid]->cpu_props); 1946 cpu[cpuid]->cpu_props = NULL; 1947 } 1948 1949 error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG); 1950 1951 if (error != 0) 1952 return (error); 1953 1954 return (mp_cpu_unconfigure(cpuid)); 1955 } 1956 1957 /* 1958 * Routines for registering and de-registering cpu_setup callback functions. 1959 * 1960 * Caller's context 1961 * These routines must not be called from a driver's attach(9E) or 1962 * detach(9E) entry point. 1963 * 1964 * NOTE: CPU callbacks should not block. They are called with cpu_lock held. 1965 */ 1966 1967 /* 1968 * Ideally, these would be dynamically allocated and put into a linked 1969 * list; however that is not feasible because the registration routine 1970 * has to be available before the kmem allocator is working (in fact, 1971 * it is called by the kmem allocator init code). In any case, there 1972 * are quite a few extra entries for future users. 1973 */ 1974 #define NCPU_SETUPS 20 1975 1976 struct cpu_setup { 1977 cpu_setup_func_t *func; 1978 void *arg; 1979 } cpu_setups[NCPU_SETUPS]; 1980 1981 void 1982 register_cpu_setup_func(cpu_setup_func_t *func, void *arg) 1983 { 1984 int i; 1985 1986 ASSERT(MUTEX_HELD(&cpu_lock)); 1987 1988 for (i = 0; i < NCPU_SETUPS; i++) 1989 if (cpu_setups[i].func == NULL) 1990 break; 1991 if (i >= NCPU_SETUPS) 1992 cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries"); 1993 1994 cpu_setups[i].func = func; 1995 cpu_setups[i].arg = arg; 1996 } 1997 1998 void 1999 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg) 2000 { 2001 int i; 2002 2003 ASSERT(MUTEX_HELD(&cpu_lock)); 2004 2005 for (i = 0; i < NCPU_SETUPS; i++) 2006 if ((cpu_setups[i].func == func) && 2007 (cpu_setups[i].arg == arg)) 2008 break; 2009 if (i >= NCPU_SETUPS) 2010 cmn_err(CE_PANIC, "Could not find cpu_setup callback to " 2011 "deregister"); 2012 2013 cpu_setups[i].func = NULL; 2014 cpu_setups[i].arg = 0; 2015 } 2016 2017 /* 2018 * Call any state change hooks for this CPU, ignore any errors. 2019 */ 2020 void 2021 cpu_state_change_notify(int id, cpu_setup_t what) 2022 { 2023 int i; 2024 2025 ASSERT(MUTEX_HELD(&cpu_lock)); 2026 2027 for (i = 0; i < NCPU_SETUPS; i++) { 2028 if (cpu_setups[i].func != NULL) { 2029 cpu_setups[i].func(what, id, cpu_setups[i].arg); 2030 } 2031 } 2032 } 2033 2034 /* 2035 * Call any state change hooks for this CPU, undo it if error found. 2036 */ 2037 static int 2038 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo) 2039 { 2040 int i; 2041 int retval = 0; 2042 2043 ASSERT(MUTEX_HELD(&cpu_lock)); 2044 2045 for (i = 0; i < NCPU_SETUPS; i++) { 2046 if (cpu_setups[i].func != NULL) { 2047 retval = cpu_setups[i].func(what, id, 2048 cpu_setups[i].arg); 2049 if (retval) { 2050 for (i--; i >= 0; i--) { 2051 if (cpu_setups[i].func != NULL) 2052 cpu_setups[i].func(undo, 2053 id, cpu_setups[i].arg); 2054 } 2055 break; 2056 } 2057 } 2058 } 2059 return (retval); 2060 } 2061 2062 /* 2063 * Export information about this CPU via the kstat mechanism. 2064 */ 2065 static struct { 2066 kstat_named_t ci_state; 2067 kstat_named_t ci_state_begin; 2068 kstat_named_t ci_cpu_type; 2069 kstat_named_t ci_fpu_type; 2070 kstat_named_t ci_clock_MHz; 2071 kstat_named_t ci_chip_id; 2072 kstat_named_t ci_implementation; 2073 kstat_named_t ci_brandstr; 2074 kstat_named_t ci_core_id; 2075 #if defined(__sparcv9) 2076 kstat_named_t ci_device_ID; 2077 kstat_named_t ci_cpu_fru; 2078 #endif 2079 #if defined(__i386) || defined(__amd64) 2080 kstat_named_t ci_vendorstr; 2081 kstat_named_t ci_family; 2082 kstat_named_t ci_model; 2083 kstat_named_t ci_step; 2084 kstat_named_t ci_clogid; 2085 #endif 2086 } cpu_info_template = { 2087 { "state", KSTAT_DATA_CHAR }, 2088 { "state_begin", KSTAT_DATA_LONG }, 2089 { "cpu_type", KSTAT_DATA_CHAR }, 2090 { "fpu_type", KSTAT_DATA_CHAR }, 2091 { "clock_MHz", KSTAT_DATA_LONG }, 2092 { "chip_id", KSTAT_DATA_LONG }, 2093 { "implementation", KSTAT_DATA_STRING }, 2094 { "brand", KSTAT_DATA_STRING }, 2095 { "core_id", KSTAT_DATA_LONG }, 2096 #if defined(__sparcv9) 2097 { "device_ID", KSTAT_DATA_UINT64 }, 2098 { "cpu_fru", KSTAT_DATA_STRING }, 2099 #endif 2100 #if defined(__i386) || defined(__amd64) 2101 { "vendor_id", KSTAT_DATA_STRING }, 2102 { "family", KSTAT_DATA_INT32 }, 2103 { "model", KSTAT_DATA_INT32 }, 2104 { "stepping", KSTAT_DATA_INT32 }, 2105 { "clog_id", KSTAT_DATA_INT32 }, 2106 #endif 2107 }; 2108 2109 static kmutex_t cpu_info_template_lock; 2110 2111 static int 2112 cpu_info_kstat_update(kstat_t *ksp, int rw) 2113 { 2114 cpu_t *cp = ksp->ks_private; 2115 const char *pi_state; 2116 2117 if (rw == KSTAT_WRITE) 2118 return (EACCES); 2119 2120 switch (cp->cpu_type_info.pi_state) { 2121 case P_ONLINE: 2122 pi_state = PS_ONLINE; 2123 break; 2124 case P_POWEROFF: 2125 pi_state = PS_POWEROFF; 2126 break; 2127 case P_NOINTR: 2128 pi_state = PS_NOINTR; 2129 break; 2130 case P_FAULTED: 2131 pi_state = PS_FAULTED; 2132 break; 2133 case P_SPARE: 2134 pi_state = PS_SPARE; 2135 break; 2136 case P_OFFLINE: 2137 pi_state = PS_OFFLINE; 2138 break; 2139 default: 2140 pi_state = "unknown"; 2141 } 2142 (void) strcpy(cpu_info_template.ci_state.value.c, pi_state); 2143 cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin; 2144 (void) strncpy(cpu_info_template.ci_cpu_type.value.c, 2145 cp->cpu_type_info.pi_processor_type, 15); 2146 (void) strncpy(cpu_info_template.ci_fpu_type.value.c, 2147 cp->cpu_type_info.pi_fputypes, 15); 2148 cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock; 2149 cpu_info_template.ci_chip_id.value.l = chip_plat_get_chipid(cp); 2150 kstat_named_setstr(&cpu_info_template.ci_implementation, 2151 cp->cpu_idstr); 2152 kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr); 2153 2154 #if defined(__sparcv9) 2155 cpu_info_template.ci_device_ID.value.ui64 = 2156 cpunodes[cp->cpu_id].device_id; 2157 kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp)); 2158 #endif 2159 #if defined(__i386) || defined(__amd64) 2160 cpu_info_template.ci_core_id.value.l = chip_plat_get_coreid(cp); 2161 kstat_named_setstr(&cpu_info_template.ci_vendorstr, 2162 cpuid_getvendorstr(cp)); 2163 cpu_info_template.ci_family.value.l = cpuid_getfamily(cp); 2164 cpu_info_template.ci_model.value.l = cpuid_getmodel(cp); 2165 cpu_info_template.ci_step.value.l = cpuid_getstep(cp); 2166 cpu_info_template.ci_clogid.value.l = chip_plat_get_clogid(cp); 2167 #endif 2168 2169 return (0); 2170 } 2171 2172 static void 2173 cpu_info_kstat_create(cpu_t *cp) 2174 { 2175 zoneid_t zoneid; 2176 2177 ASSERT(MUTEX_HELD(&cpu_lock)); 2178 2179 if (pool_pset_enabled()) 2180 zoneid = GLOBAL_ZONEID; 2181 else 2182 zoneid = ALL_ZONES; 2183 if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id, 2184 NULL, "misc", KSTAT_TYPE_NAMED, 2185 sizeof (cpu_info_template) / sizeof (kstat_named_t), 2186 KSTAT_FLAG_VIRTUAL, zoneid)) != NULL) { 2187 cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN; 2188 #if defined(__sparcv9) 2189 cp->cpu_info_kstat->ks_data_size += 2190 strlen(cpu_fru_fmri(cp)) + 1; 2191 #endif 2192 #if defined(__i386) || defined(__amd64) 2193 cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN; 2194 #endif 2195 cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock; 2196 cp->cpu_info_kstat->ks_data = &cpu_info_template; 2197 cp->cpu_info_kstat->ks_private = cp; 2198 cp->cpu_info_kstat->ks_update = cpu_info_kstat_update; 2199 kstat_install(cp->cpu_info_kstat); 2200 } 2201 } 2202 2203 static void 2204 cpu_info_kstat_destroy(cpu_t *cp) 2205 { 2206 ASSERT(MUTEX_HELD(&cpu_lock)); 2207 2208 kstat_delete(cp->cpu_info_kstat); 2209 cp->cpu_info_kstat = NULL; 2210 } 2211 2212 /* 2213 * Create and install kstats for the boot CPU. 2214 */ 2215 void 2216 cpu_kstat_init(cpu_t *cp) 2217 { 2218 mutex_enter(&cpu_lock); 2219 cpu_info_kstat_create(cp); 2220 cpu_stats_kstat_create(cp); 2221 cpu_create_intrstat(cp); 2222 chip_kstat_create(cp->cpu_chip); 2223 cpu_set_state(cp); 2224 mutex_exit(&cpu_lock); 2225 } 2226 2227 /* 2228 * Make visible to the zone that subset of the cpu information that would be 2229 * initialized when a cpu is configured (but still offline). 2230 */ 2231 void 2232 cpu_visibility_configure(cpu_t *cp, zone_t *zone) 2233 { 2234 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2235 2236 ASSERT(MUTEX_HELD(&cpu_lock)); 2237 ASSERT(pool_pset_enabled()); 2238 ASSERT(cp != NULL); 2239 2240 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2241 zone->zone_ncpus++; 2242 ASSERT(zone->zone_ncpus <= ncpus); 2243 } 2244 if (cp->cpu_info_kstat != NULL) 2245 kstat_zone_add(cp->cpu_info_kstat, zoneid); 2246 } 2247 2248 /* 2249 * Make visible to the zone that subset of the cpu information that would be 2250 * initialized when a previously configured cpu is onlined. 2251 */ 2252 void 2253 cpu_visibility_online(cpu_t *cp, zone_t *zone) 2254 { 2255 kstat_t *ksp; 2256 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */ 2257 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2258 processorid_t cpun; 2259 2260 ASSERT(MUTEX_HELD(&cpu_lock)); 2261 ASSERT(pool_pset_enabled()); 2262 ASSERT(cp != NULL); 2263 ASSERT(cpu_is_active(cp)); 2264 2265 cpun = cp->cpu_id; 2266 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2267 zone->zone_ncpus_online++; 2268 ASSERT(zone->zone_ncpus_online <= ncpus_online); 2269 } 2270 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun); 2271 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES)) 2272 != NULL) { 2273 kstat_zone_add(ksp, zoneid); 2274 kstat_rele(ksp); 2275 } 2276 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) { 2277 kstat_zone_add(ksp, zoneid); 2278 kstat_rele(ksp); 2279 } 2280 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) { 2281 kstat_zone_add(ksp, zoneid); 2282 kstat_rele(ksp); 2283 } 2284 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) != 2285 NULL) { 2286 kstat_zone_add(ksp, zoneid); 2287 kstat_rele(ksp); 2288 } 2289 } 2290 2291 /* 2292 * Update relevant kstats such that cpu is now visible to processes 2293 * executing in specified zone. 2294 */ 2295 void 2296 cpu_visibility_add(cpu_t *cp, zone_t *zone) 2297 { 2298 cpu_visibility_configure(cp, zone); 2299 if (cpu_is_active(cp)) 2300 cpu_visibility_online(cp, zone); 2301 } 2302 2303 /* 2304 * Make invisible to the zone that subset of the cpu information that would be 2305 * torn down when a previously offlined cpu is unconfigured. 2306 */ 2307 void 2308 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone) 2309 { 2310 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2311 2312 ASSERT(MUTEX_HELD(&cpu_lock)); 2313 ASSERT(pool_pset_enabled()); 2314 ASSERT(cp != NULL); 2315 2316 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2317 ASSERT(zone->zone_ncpus != 0); 2318 zone->zone_ncpus--; 2319 } 2320 if (cp->cpu_info_kstat) 2321 kstat_zone_remove(cp->cpu_info_kstat, zoneid); 2322 } 2323 2324 /* 2325 * Make invisible to the zone that subset of the cpu information that would be 2326 * torn down when a cpu is offlined (but still configured). 2327 */ 2328 void 2329 cpu_visibility_offline(cpu_t *cp, zone_t *zone) 2330 { 2331 kstat_t *ksp; 2332 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */ 2333 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2334 processorid_t cpun; 2335 2336 ASSERT(MUTEX_HELD(&cpu_lock)); 2337 ASSERT(pool_pset_enabled()); 2338 ASSERT(cp != NULL); 2339 ASSERT(cpu_is_active(cp)); 2340 2341 cpun = cp->cpu_id; 2342 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2343 ASSERT(zone->zone_ncpus_online != 0); 2344 zone->zone_ncpus_online--; 2345 } 2346 2347 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) != 2348 NULL) { 2349 kstat_zone_remove(ksp, zoneid); 2350 kstat_rele(ksp); 2351 } 2352 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) { 2353 kstat_zone_remove(ksp, zoneid); 2354 kstat_rele(ksp); 2355 } 2356 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) { 2357 kstat_zone_remove(ksp, zoneid); 2358 kstat_rele(ksp); 2359 } 2360 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun); 2361 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES)) 2362 != NULL) { 2363 kstat_zone_remove(ksp, zoneid); 2364 kstat_rele(ksp); 2365 } 2366 } 2367 2368 /* 2369 * Update relevant kstats such that cpu is no longer visible to processes 2370 * executing in specified zone. 2371 */ 2372 void 2373 cpu_visibility_remove(cpu_t *cp, zone_t *zone) 2374 { 2375 if (cpu_is_active(cp)) 2376 cpu_visibility_offline(cp, zone); 2377 cpu_visibility_unconfigure(cp, zone); 2378 } 2379 2380 /* 2381 * Bind a thread to a CPU as requested. 2382 */ 2383 int 2384 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind, 2385 int *error) 2386 { 2387 processorid_t binding; 2388 cpu_t *cp; 2389 2390 ASSERT(MUTEX_HELD(&cpu_lock)); 2391 ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock)); 2392 2393 thread_lock(tp); 2394 2395 /* 2396 * Record old binding, but change the obind, which was initialized 2397 * to PBIND_NONE, only if this thread has a binding. This avoids 2398 * reporting PBIND_NONE for a process when some LWPs are bound. 2399 */ 2400 binding = tp->t_bind_cpu; 2401 if (binding != PBIND_NONE) 2402 *obind = binding; /* record old binding */ 2403 2404 if (bind == PBIND_QUERY) { 2405 thread_unlock(tp); 2406 return (0); 2407 } 2408 2409 /* 2410 * If this thread/LWP cannot be bound because of permission 2411 * problems, just note that and return success so that the 2412 * other threads/LWPs will be bound. This is the way 2413 * processor_bind() is defined to work. 2414 * 2415 * Binding will get EPERM if the thread is of system class 2416 * or hasprocperm() fails. 2417 */ 2418 if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) { 2419 *error = EPERM; 2420 thread_unlock(tp); 2421 return (0); 2422 } 2423 2424 binding = bind; 2425 if (binding != PBIND_NONE) { 2426 cp = cpu[binding]; 2427 /* 2428 * Make sure binding is in right partition. 2429 */ 2430 if (tp->t_cpupart != cp->cpu_part) { 2431 *error = EINVAL; 2432 thread_unlock(tp); 2433 return (0); 2434 } 2435 } 2436 tp->t_bind_cpu = binding; /* set new binding */ 2437 2438 /* 2439 * If there is no system-set reason for affinity, set 2440 * the t_bound_cpu field to reflect the binding. 2441 */ 2442 if (tp->t_affinitycnt == 0) { 2443 if (binding == PBIND_NONE) { 2444 /* 2445 * We may need to adjust disp_max_unbound_pri 2446 * since we're becoming unbound. 2447 */ 2448 disp_adjust_unbound_pri(tp); 2449 2450 tp->t_bound_cpu = NULL; /* set new binding */ 2451 2452 /* 2453 * Move thread to lgroup with strongest affinity 2454 * after unbinding 2455 */ 2456 if (tp->t_lgrp_affinity) 2457 lgrp_move_thread(tp, 2458 lgrp_choose(tp, tp->t_cpupart), 1); 2459 2460 if (tp->t_state == TS_ONPROC && 2461 tp->t_cpu->cpu_part != tp->t_cpupart) 2462 cpu_surrender(tp); 2463 } else { 2464 lpl_t *lpl; 2465 2466 tp->t_bound_cpu = cp; 2467 ASSERT(cp->cpu_lpl != NULL); 2468 2469 /* 2470 * Set home to lgroup with most affinity containing CPU 2471 * that thread is being bound or minimum bounding 2472 * lgroup if no affinities set 2473 */ 2474 if (tp->t_lgrp_affinity) 2475 lpl = lgrp_affinity_best(tp, tp->t_cpupart, 0); 2476 else 2477 lpl = cp->cpu_lpl; 2478 2479 if (tp->t_lpl != lpl) { 2480 /* can't grab cpu_lock */ 2481 lgrp_move_thread(tp, lpl, 1); 2482 } 2483 2484 /* 2485 * Make the thread switch to the bound CPU. 2486 * If the thread is runnable, we need to 2487 * requeue it even if t_cpu is already set 2488 * to the right CPU, since it may be on a 2489 * kpreempt queue and need to move to a local 2490 * queue. We could check t_disp_queue to 2491 * avoid unnecessary overhead if it's already 2492 * on the right queue, but since this isn't 2493 * a performance-critical operation it doesn't 2494 * seem worth the extra code and complexity. 2495 * 2496 * If the thread is weakbound to the cpu then it will 2497 * resist the new binding request until the weak 2498 * binding drops. The cpu_surrender or requeueing 2499 * below could be skipped in such cases (since it 2500 * will have no effect), but that would require 2501 * thread_allowmigrate to acquire thread_lock so 2502 * we'll take the very occasional hit here instead. 2503 */ 2504 if (tp->t_state == TS_ONPROC) { 2505 cpu_surrender(tp); 2506 } else if (tp->t_state == TS_RUN) { 2507 cpu_t *ocp = tp->t_cpu; 2508 2509 (void) dispdeq(tp); 2510 setbackdq(tp); 2511 /* 2512 * Either on the bound CPU's disp queue now, 2513 * or swapped out or on the swap queue. 2514 */ 2515 ASSERT(tp->t_disp_queue == cp->cpu_disp || 2516 tp->t_weakbound_cpu == ocp || 2517 (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) 2518 != TS_LOAD); 2519 } 2520 } 2521 } 2522 2523 /* 2524 * Our binding has changed; set TP_CHANGEBIND. 2525 */ 2526 tp->t_proc_flag |= TP_CHANGEBIND; 2527 aston(tp); 2528 2529 thread_unlock(tp); 2530 2531 return (0); 2532 } 2533 2534 #if CPUSET_WORDS > 1 2535 2536 /* 2537 * Functions for implementing cpuset operations when a cpuset is more 2538 * than one word. On platforms where a cpuset is a single word these 2539 * are implemented as macros in cpuvar.h. 2540 */ 2541 2542 void 2543 cpuset_all(cpuset_t *s) 2544 { 2545 int i; 2546 2547 for (i = 0; i < CPUSET_WORDS; i++) 2548 s->cpub[i] = ~0UL; 2549 } 2550 2551 void 2552 cpuset_all_but(cpuset_t *s, uint_t cpu) 2553 { 2554 cpuset_all(s); 2555 CPUSET_DEL(*s, cpu); 2556 } 2557 2558 void 2559 cpuset_only(cpuset_t *s, uint_t cpu) 2560 { 2561 CPUSET_ZERO(*s); 2562 CPUSET_ADD(*s, cpu); 2563 } 2564 2565 int 2566 cpuset_isnull(cpuset_t *s) 2567 { 2568 int i; 2569 2570 for (i = 0; i < CPUSET_WORDS; i++) 2571 if (s->cpub[i] != 0) 2572 return (0); 2573 return (1); 2574 } 2575 2576 int 2577 cpuset_cmp(cpuset_t *s1, cpuset_t *s2) 2578 { 2579 int i; 2580 2581 for (i = 0; i < CPUSET_WORDS; i++) 2582 if (s1->cpub[i] != s2->cpub[i]) 2583 return (0); 2584 return (1); 2585 } 2586 2587 uint_t 2588 cpuset_find(cpuset_t *s) 2589 { 2590 2591 uint_t i; 2592 uint_t cpu = (uint_t)-1; 2593 2594 /* 2595 * Find a cpu in the cpuset 2596 */ 2597 for (i = 0; (i < CPUSET_WORDS && cpu == (uint_t)-1); i++) 2598 cpu = (uint_t)(lowbit(s->cpub[i]) - 1); 2599 return (cpu); 2600 } 2601 2602 #endif /* CPUSET_WORDS */ 2603 2604 /* 2605 * Unbind all user threads bound to a given CPU. 2606 */ 2607 int 2608 cpu_unbind(processorid_t cpu) 2609 { 2610 processorid_t obind; 2611 kthread_t *tp; 2612 int ret = 0; 2613 proc_t *pp; 2614 int err, berr = 0; 2615 2616 ASSERT(MUTEX_HELD(&cpu_lock)); 2617 2618 mutex_enter(&pidlock); 2619 for (pp = practive; pp != NULL; pp = pp->p_next) { 2620 mutex_enter(&pp->p_lock); 2621 tp = pp->p_tlist; 2622 /* 2623 * Skip zombies, kernel processes, and processes in 2624 * other zones, if called from a non-global zone. 2625 */ 2626 if (tp == NULL || (pp->p_flag & SSYS) || 2627 !HASZONEACCESS(curproc, pp->p_zone->zone_id)) { 2628 mutex_exit(&pp->p_lock); 2629 continue; 2630 } 2631 do { 2632 if (tp->t_bind_cpu != cpu) 2633 continue; 2634 err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr); 2635 if (ret == 0) 2636 ret = err; 2637 } while ((tp = tp->t_forw) != pp->p_tlist); 2638 mutex_exit(&pp->p_lock); 2639 } 2640 mutex_exit(&pidlock); 2641 if (ret == 0) 2642 ret = berr; 2643 return (ret); 2644 } 2645 2646 2647 /* 2648 * Destroy all remaining bound threads on a cpu. 2649 */ 2650 void 2651 cpu_destroy_bound_threads(cpu_t *cp) 2652 { 2653 extern id_t syscid; 2654 register kthread_id_t t, tlist, tnext; 2655 2656 /* 2657 * Destroy all remaining bound threads on the cpu. This 2658 * should include both the interrupt threads and the idle thread. 2659 * This requires some care, since we need to traverse the 2660 * thread list with the pidlock mutex locked, but thread_free 2661 * also locks the pidlock mutex. So, we collect the threads 2662 * we're going to reap in a list headed by "tlist", then we 2663 * unlock the pidlock mutex and traverse the tlist list, 2664 * doing thread_free's on the thread's. Simple, n'est pas? 2665 * Also, this depends on thread_free not mucking with the 2666 * t_next and t_prev links of the thread. 2667 */ 2668 2669 if ((t = curthread) != NULL) { 2670 2671 tlist = NULL; 2672 mutex_enter(&pidlock); 2673 do { 2674 tnext = t->t_next; 2675 if (t->t_bound_cpu == cp) { 2676 2677 /* 2678 * We've found a bound thread, carefully unlink 2679 * it out of the thread list, and add it to 2680 * our "tlist". We "know" we don't have to 2681 * worry about unlinking curthread (the thread 2682 * that is executing this code). 2683 */ 2684 t->t_next->t_prev = t->t_prev; 2685 t->t_prev->t_next = t->t_next; 2686 t->t_next = tlist; 2687 tlist = t; 2688 ASSERT(t->t_cid == syscid); 2689 /* wake up anyone blocked in thread_join */ 2690 cv_broadcast(&t->t_joincv); 2691 /* 2692 * t_lwp set by interrupt threads and not 2693 * cleared. 2694 */ 2695 t->t_lwp = NULL; 2696 /* 2697 * Pause and idle threads always have 2698 * t_state set to TS_ONPROC. 2699 */ 2700 t->t_state = TS_FREE; 2701 t->t_prev = NULL; /* Just in case */ 2702 } 2703 2704 } while ((t = tnext) != curthread); 2705 2706 mutex_exit(&pidlock); 2707 2708 2709 for (t = tlist; t != NULL; t = tnext) { 2710 tnext = t->t_next; 2711 thread_free(t); 2712 } 2713 } 2714 } 2715 2716 /* 2717 * processor_info(2) and p_online(2) status support functions 2718 * The constants returned by the cpu_get_state() and cpu_get_state_str() are 2719 * for use in communicating processor state information to userland. Kernel 2720 * subsystems should only be using the cpu_flags value directly. Subsystems 2721 * modifying cpu_flags should record the state change via a call to the 2722 * cpu_set_state(). 2723 */ 2724 2725 /* 2726 * Update the pi_state of this CPU. This function provides the CPU status for 2727 * the information returned by processor_info(2). 2728 */ 2729 void 2730 cpu_set_state(cpu_t *cpu) 2731 { 2732 ASSERT(MUTEX_HELD(&cpu_lock)); 2733 cpu->cpu_type_info.pi_state = cpu_get_state(cpu); 2734 cpu->cpu_state_begin = gethrestime_sec(); 2735 pool_cpu_mod = gethrtime(); 2736 } 2737 2738 /* 2739 * Return offline/online/other status for the indicated CPU. Use only for 2740 * communication with user applications; cpu_flags provides the in-kernel 2741 * interface. 2742 */ 2743 int 2744 cpu_get_state(cpu_t *cpu) 2745 { 2746 ASSERT(MUTEX_HELD(&cpu_lock)); 2747 if (cpu->cpu_flags & CPU_POWEROFF) 2748 return (P_POWEROFF); 2749 else if (cpu->cpu_flags & CPU_FAULTED) 2750 return (P_FAULTED); 2751 else if (cpu->cpu_flags & CPU_SPARE) 2752 return (P_SPARE); 2753 else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY) 2754 return (P_OFFLINE); 2755 else if (cpu->cpu_flags & CPU_ENABLE) 2756 return (P_ONLINE); 2757 else 2758 return (P_NOINTR); 2759 } 2760 2761 /* 2762 * Return processor_info(2) state as a string. 2763 */ 2764 const char * 2765 cpu_get_state_str(cpu_t *cpu) 2766 { 2767 const char *string; 2768 2769 switch (cpu_get_state(cpu)) { 2770 case P_ONLINE: 2771 string = PS_ONLINE; 2772 break; 2773 case P_POWEROFF: 2774 string = PS_POWEROFF; 2775 break; 2776 case P_NOINTR: 2777 string = PS_NOINTR; 2778 break; 2779 case P_SPARE: 2780 string = PS_SPARE; 2781 break; 2782 case P_FAULTED: 2783 string = PS_FAULTED; 2784 break; 2785 case P_OFFLINE: 2786 string = PS_OFFLINE; 2787 break; 2788 default: 2789 string = "unknown"; 2790 break; 2791 } 2792 return (string); 2793 } 2794 2795 /* 2796 * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named 2797 * kstats, respectively. This is done when a CPU is initialized or placed 2798 * online via p_online(2). 2799 */ 2800 static void 2801 cpu_stats_kstat_create(cpu_t *cp) 2802 { 2803 int instance = cp->cpu_id; 2804 char *module = "cpu"; 2805 char *class = "misc"; 2806 kstat_t *ksp; 2807 zoneid_t zoneid; 2808 2809 ASSERT(MUTEX_HELD(&cpu_lock)); 2810 2811 if (pool_pset_enabled()) 2812 zoneid = GLOBAL_ZONEID; 2813 else 2814 zoneid = ALL_ZONES; 2815 /* 2816 * Create named kstats 2817 */ 2818 #define CPU_STATS_KS_CREATE(name, tsize, update_func) \ 2819 ksp = kstat_create_zone(module, instance, (name), class, \ 2820 KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0, \ 2821 zoneid); \ 2822 if (ksp != NULL) { \ 2823 ksp->ks_private = cp; \ 2824 ksp->ks_update = (update_func); \ 2825 kstat_install(ksp); \ 2826 } else \ 2827 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \ 2828 module, instance, (name)); 2829 2830 CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template), 2831 cpu_sys_stats_ks_update); 2832 CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template), 2833 cpu_vm_stats_ks_update); 2834 2835 /* 2836 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat. 2837 */ 2838 ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL, 2839 "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid); 2840 if (ksp != NULL) { 2841 ksp->ks_update = cpu_stat_ks_update; 2842 ksp->ks_private = cp; 2843 kstat_install(ksp); 2844 } 2845 } 2846 2847 static void 2848 cpu_stats_kstat_destroy(cpu_t *cp) 2849 { 2850 char ks_name[KSTAT_STRLEN]; 2851 2852 (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id); 2853 kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name); 2854 2855 kstat_delete_byname("cpu", cp->cpu_id, "sys"); 2856 kstat_delete_byname("cpu", cp->cpu_id, "vm"); 2857 } 2858 2859 static int 2860 cpu_sys_stats_ks_update(kstat_t *ksp, int rw) 2861 { 2862 cpu_t *cp = (cpu_t *)ksp->ks_private; 2863 struct cpu_sys_stats_ks_data *csskd; 2864 cpu_sys_stats_t *css; 2865 hrtime_t msnsecs[NCMSTATES]; 2866 int i; 2867 2868 if (rw == KSTAT_WRITE) 2869 return (EACCES); 2870 2871 csskd = ksp->ks_data; 2872 css = &cp->cpu_stats.sys; 2873 2874 /* 2875 * Read CPU mstate, but compare with the last values we 2876 * received to make sure that the returned kstats never 2877 * decrease. 2878 */ 2879 2880 get_cpu_mstate(cp, msnsecs); 2881 if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE]) 2882 msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64; 2883 if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER]) 2884 msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64; 2885 if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM]) 2886 msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64; 2887 2888 bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data, 2889 sizeof (cpu_sys_stats_ks_data_template)); 2890 2891 csskd->cpu_ticks_wait.value.ui64 = 0; 2892 csskd->wait_ticks_io.value.ui64 = 0; 2893 2894 csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE]; 2895 csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER]; 2896 csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM]; 2897 csskd->cpu_ticks_idle.value.ui64 = 2898 NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64); 2899 csskd->cpu_ticks_user.value.ui64 = 2900 NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64); 2901 csskd->cpu_ticks_kernel.value.ui64 = 2902 NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64); 2903 csskd->bread.value.ui64 = css->bread; 2904 csskd->bwrite.value.ui64 = css->bwrite; 2905 csskd->lread.value.ui64 = css->lread; 2906 csskd->lwrite.value.ui64 = css->lwrite; 2907 csskd->phread.value.ui64 = css->phread; 2908 csskd->phwrite.value.ui64 = css->phwrite; 2909 csskd->pswitch.value.ui64 = css->pswitch; 2910 csskd->trap.value.ui64 = css->trap; 2911 csskd->intr.value.ui64 = 0; 2912 for (i = 0; i < PIL_MAX; i++) 2913 csskd->intr.value.ui64 += css->intr[i]; 2914 csskd->syscall.value.ui64 = css->syscall; 2915 csskd->sysread.value.ui64 = css->sysread; 2916 csskd->syswrite.value.ui64 = css->syswrite; 2917 csskd->sysfork.value.ui64 = css->sysfork; 2918 csskd->sysvfork.value.ui64 = css->sysvfork; 2919 csskd->sysexec.value.ui64 = css->sysexec; 2920 csskd->readch.value.ui64 = css->readch; 2921 csskd->writech.value.ui64 = css->writech; 2922 csskd->rcvint.value.ui64 = css->rcvint; 2923 csskd->xmtint.value.ui64 = css->xmtint; 2924 csskd->mdmint.value.ui64 = css->mdmint; 2925 csskd->rawch.value.ui64 = css->rawch; 2926 csskd->canch.value.ui64 = css->canch; 2927 csskd->outch.value.ui64 = css->outch; 2928 csskd->msg.value.ui64 = css->msg; 2929 csskd->sema.value.ui64 = css->sema; 2930 csskd->namei.value.ui64 = css->namei; 2931 csskd->ufsiget.value.ui64 = css->ufsiget; 2932 csskd->ufsdirblk.value.ui64 = css->ufsdirblk; 2933 csskd->ufsipage.value.ui64 = css->ufsipage; 2934 csskd->ufsinopage.value.ui64 = css->ufsinopage; 2935 csskd->procovf.value.ui64 = css->procovf; 2936 csskd->intrthread.value.ui64 = 0; 2937 for (i = 0; i < LOCK_LEVEL; i++) 2938 csskd->intrthread.value.ui64 += css->intr[i]; 2939 csskd->intrblk.value.ui64 = css->intrblk; 2940 csskd->intrunpin.value.ui64 = css->intrunpin; 2941 csskd->idlethread.value.ui64 = css->idlethread; 2942 csskd->inv_swtch.value.ui64 = css->inv_swtch; 2943 csskd->nthreads.value.ui64 = css->nthreads; 2944 csskd->cpumigrate.value.ui64 = css->cpumigrate; 2945 csskd->xcalls.value.ui64 = css->xcalls; 2946 csskd->mutex_adenters.value.ui64 = css->mutex_adenters; 2947 csskd->rw_rdfails.value.ui64 = css->rw_rdfails; 2948 csskd->rw_wrfails.value.ui64 = css->rw_wrfails; 2949 csskd->modload.value.ui64 = css->modload; 2950 csskd->modunload.value.ui64 = css->modunload; 2951 csskd->bawrite.value.ui64 = css->bawrite; 2952 csskd->iowait.value.ui64 = 0; 2953 2954 return (0); 2955 } 2956 2957 static int 2958 cpu_vm_stats_ks_update(kstat_t *ksp, int rw) 2959 { 2960 cpu_t *cp = (cpu_t *)ksp->ks_private; 2961 struct cpu_vm_stats_ks_data *cvskd; 2962 cpu_vm_stats_t *cvs; 2963 2964 if (rw == KSTAT_WRITE) 2965 return (EACCES); 2966 2967 cvs = &cp->cpu_stats.vm; 2968 cvskd = ksp->ks_data; 2969 2970 bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data, 2971 sizeof (cpu_vm_stats_ks_data_template)); 2972 cvskd->pgrec.value.ui64 = cvs->pgrec; 2973 cvskd->pgfrec.value.ui64 = cvs->pgfrec; 2974 cvskd->pgin.value.ui64 = cvs->pgin; 2975 cvskd->pgpgin.value.ui64 = cvs->pgpgin; 2976 cvskd->pgout.value.ui64 = cvs->pgout; 2977 cvskd->pgpgout.value.ui64 = cvs->pgpgout; 2978 cvskd->swapin.value.ui64 = cvs->swapin; 2979 cvskd->pgswapin.value.ui64 = cvs->pgswapin; 2980 cvskd->swapout.value.ui64 = cvs->swapout; 2981 cvskd->pgswapout.value.ui64 = cvs->pgswapout; 2982 cvskd->zfod.value.ui64 = cvs->zfod; 2983 cvskd->dfree.value.ui64 = cvs->dfree; 2984 cvskd->scan.value.ui64 = cvs->scan; 2985 cvskd->rev.value.ui64 = cvs->rev; 2986 cvskd->hat_fault.value.ui64 = cvs->hat_fault; 2987 cvskd->as_fault.value.ui64 = cvs->as_fault; 2988 cvskd->maj_fault.value.ui64 = cvs->maj_fault; 2989 cvskd->cow_fault.value.ui64 = cvs->cow_fault; 2990 cvskd->prot_fault.value.ui64 = cvs->prot_fault; 2991 cvskd->softlock.value.ui64 = cvs->softlock; 2992 cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt; 2993 cvskd->pgrrun.value.ui64 = cvs->pgrrun; 2994 cvskd->execpgin.value.ui64 = cvs->execpgin; 2995 cvskd->execpgout.value.ui64 = cvs->execpgout; 2996 cvskd->execfree.value.ui64 = cvs->execfree; 2997 cvskd->anonpgin.value.ui64 = cvs->anonpgin; 2998 cvskd->anonpgout.value.ui64 = cvs->anonpgout; 2999 cvskd->anonfree.value.ui64 = cvs->anonfree; 3000 cvskd->fspgin.value.ui64 = cvs->fspgin; 3001 cvskd->fspgout.value.ui64 = cvs->fspgout; 3002 cvskd->fsfree.value.ui64 = cvs->fsfree; 3003 3004 return (0); 3005 } 3006 3007 static int 3008 cpu_stat_ks_update(kstat_t *ksp, int rw) 3009 { 3010 cpu_stat_t *cso; 3011 cpu_t *cp; 3012 int i; 3013 hrtime_t msnsecs[NCMSTATES]; 3014 3015 cso = (cpu_stat_t *)ksp->ks_data; 3016 cp = (cpu_t *)ksp->ks_private; 3017 3018 if (rw == KSTAT_WRITE) 3019 return (EACCES); 3020 3021 /* 3022 * Read CPU mstate, but compare with the last values we 3023 * received to make sure that the returned kstats never 3024 * decrease. 3025 */ 3026 3027 get_cpu_mstate(cp, msnsecs); 3028 msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]); 3029 msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]); 3030 msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]); 3031 if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE]) 3032 cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE]; 3033 if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER]) 3034 cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER]; 3035 if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM]) 3036 cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM]; 3037 cso->cpu_sysinfo.cpu[CPU_WAIT] = 0; 3038 cso->cpu_sysinfo.wait[W_IO] = 0; 3039 cso->cpu_sysinfo.wait[W_SWAP] = 0; 3040 cso->cpu_sysinfo.wait[W_PIO] = 0; 3041 cso->cpu_sysinfo.bread = CPU_STATS(cp, sys.bread); 3042 cso->cpu_sysinfo.bwrite = CPU_STATS(cp, sys.bwrite); 3043 cso->cpu_sysinfo.lread = CPU_STATS(cp, sys.lread); 3044 cso->cpu_sysinfo.lwrite = CPU_STATS(cp, sys.lwrite); 3045 cso->cpu_sysinfo.phread = CPU_STATS(cp, sys.phread); 3046 cso->cpu_sysinfo.phwrite = CPU_STATS(cp, sys.phwrite); 3047 cso->cpu_sysinfo.pswitch = CPU_STATS(cp, sys.pswitch); 3048 cso->cpu_sysinfo.trap = CPU_STATS(cp, sys.trap); 3049 cso->cpu_sysinfo.intr = 0; 3050 for (i = 0; i < PIL_MAX; i++) 3051 cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]); 3052 cso->cpu_sysinfo.syscall = CPU_STATS(cp, sys.syscall); 3053 cso->cpu_sysinfo.sysread = CPU_STATS(cp, sys.sysread); 3054 cso->cpu_sysinfo.syswrite = CPU_STATS(cp, sys.syswrite); 3055 cso->cpu_sysinfo.sysfork = CPU_STATS(cp, sys.sysfork); 3056 cso->cpu_sysinfo.sysvfork = CPU_STATS(cp, sys.sysvfork); 3057 cso->cpu_sysinfo.sysexec = CPU_STATS(cp, sys.sysexec); 3058 cso->cpu_sysinfo.readch = CPU_STATS(cp, sys.readch); 3059 cso->cpu_sysinfo.writech = CPU_STATS(cp, sys.writech); 3060 cso->cpu_sysinfo.rcvint = CPU_STATS(cp, sys.rcvint); 3061 cso->cpu_sysinfo.xmtint = CPU_STATS(cp, sys.xmtint); 3062 cso->cpu_sysinfo.mdmint = CPU_STATS(cp, sys.mdmint); 3063 cso->cpu_sysinfo.rawch = CPU_STATS(cp, sys.rawch); 3064 cso->cpu_sysinfo.canch = CPU_STATS(cp, sys.canch); 3065 cso->cpu_sysinfo.outch = CPU_STATS(cp, sys.outch); 3066 cso->cpu_sysinfo.msg = CPU_STATS(cp, sys.msg); 3067 cso->cpu_sysinfo.sema = CPU_STATS(cp, sys.sema); 3068 cso->cpu_sysinfo.namei = CPU_STATS(cp, sys.namei); 3069 cso->cpu_sysinfo.ufsiget = CPU_STATS(cp, sys.ufsiget); 3070 cso->cpu_sysinfo.ufsdirblk = CPU_STATS(cp, sys.ufsdirblk); 3071 cso->cpu_sysinfo.ufsipage = CPU_STATS(cp, sys.ufsipage); 3072 cso->cpu_sysinfo.ufsinopage = CPU_STATS(cp, sys.ufsinopage); 3073 cso->cpu_sysinfo.inodeovf = 0; 3074 cso->cpu_sysinfo.fileovf = 0; 3075 cso->cpu_sysinfo.procovf = CPU_STATS(cp, sys.procovf); 3076 cso->cpu_sysinfo.intrthread = 0; 3077 for (i = 0; i < LOCK_LEVEL; i++) 3078 cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]); 3079 cso->cpu_sysinfo.intrblk = CPU_STATS(cp, sys.intrblk); 3080 cso->cpu_sysinfo.idlethread = CPU_STATS(cp, sys.idlethread); 3081 cso->cpu_sysinfo.inv_swtch = CPU_STATS(cp, sys.inv_swtch); 3082 cso->cpu_sysinfo.nthreads = CPU_STATS(cp, sys.nthreads); 3083 cso->cpu_sysinfo.cpumigrate = CPU_STATS(cp, sys.cpumigrate); 3084 cso->cpu_sysinfo.xcalls = CPU_STATS(cp, sys.xcalls); 3085 cso->cpu_sysinfo.mutex_adenters = CPU_STATS(cp, sys.mutex_adenters); 3086 cso->cpu_sysinfo.rw_rdfails = CPU_STATS(cp, sys.rw_rdfails); 3087 cso->cpu_sysinfo.rw_wrfails = CPU_STATS(cp, sys.rw_wrfails); 3088 cso->cpu_sysinfo.modload = CPU_STATS(cp, sys.modload); 3089 cso->cpu_sysinfo.modunload = CPU_STATS(cp, sys.modunload); 3090 cso->cpu_sysinfo.bawrite = CPU_STATS(cp, sys.bawrite); 3091 cso->cpu_sysinfo.rw_enters = 0; 3092 cso->cpu_sysinfo.win_uo_cnt = 0; 3093 cso->cpu_sysinfo.win_uu_cnt = 0; 3094 cso->cpu_sysinfo.win_so_cnt = 0; 3095 cso->cpu_sysinfo.win_su_cnt = 0; 3096 cso->cpu_sysinfo.win_suo_cnt = 0; 3097 3098 cso->cpu_syswait.iowait = 0; 3099 cso->cpu_syswait.swap = 0; 3100 cso->cpu_syswait.physio = 0; 3101 3102 cso->cpu_vminfo.pgrec = CPU_STATS(cp, vm.pgrec); 3103 cso->cpu_vminfo.pgfrec = CPU_STATS(cp, vm.pgfrec); 3104 cso->cpu_vminfo.pgin = CPU_STATS(cp, vm.pgin); 3105 cso->cpu_vminfo.pgpgin = CPU_STATS(cp, vm.pgpgin); 3106 cso->cpu_vminfo.pgout = CPU_STATS(cp, vm.pgout); 3107 cso->cpu_vminfo.pgpgout = CPU_STATS(cp, vm.pgpgout); 3108 cso->cpu_vminfo.swapin = CPU_STATS(cp, vm.swapin); 3109 cso->cpu_vminfo.pgswapin = CPU_STATS(cp, vm.pgswapin); 3110 cso->cpu_vminfo.swapout = CPU_STATS(cp, vm.swapout); 3111 cso->cpu_vminfo.pgswapout = CPU_STATS(cp, vm.pgswapout); 3112 cso->cpu_vminfo.zfod = CPU_STATS(cp, vm.zfod); 3113 cso->cpu_vminfo.dfree = CPU_STATS(cp, vm.dfree); 3114 cso->cpu_vminfo.scan = CPU_STATS(cp, vm.scan); 3115 cso->cpu_vminfo.rev = CPU_STATS(cp, vm.rev); 3116 cso->cpu_vminfo.hat_fault = CPU_STATS(cp, vm.hat_fault); 3117 cso->cpu_vminfo.as_fault = CPU_STATS(cp, vm.as_fault); 3118 cso->cpu_vminfo.maj_fault = CPU_STATS(cp, vm.maj_fault); 3119 cso->cpu_vminfo.cow_fault = CPU_STATS(cp, vm.cow_fault); 3120 cso->cpu_vminfo.prot_fault = CPU_STATS(cp, vm.prot_fault); 3121 cso->cpu_vminfo.softlock = CPU_STATS(cp, vm.softlock); 3122 cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt); 3123 cso->cpu_vminfo.pgrrun = CPU_STATS(cp, vm.pgrrun); 3124 cso->cpu_vminfo.execpgin = CPU_STATS(cp, vm.execpgin); 3125 cso->cpu_vminfo.execpgout = CPU_STATS(cp, vm.execpgout); 3126 cso->cpu_vminfo.execfree = CPU_STATS(cp, vm.execfree); 3127 cso->cpu_vminfo.anonpgin = CPU_STATS(cp, vm.anonpgin); 3128 cso->cpu_vminfo.anonpgout = CPU_STATS(cp, vm.anonpgout); 3129 cso->cpu_vminfo.anonfree = CPU_STATS(cp, vm.anonfree); 3130 cso->cpu_vminfo.fspgin = CPU_STATS(cp, vm.fspgin); 3131 cso->cpu_vminfo.fspgout = CPU_STATS(cp, vm.fspgout); 3132 cso->cpu_vminfo.fsfree = CPU_STATS(cp, vm.fsfree); 3133 3134 return (0); 3135 } 3136