1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * Architecture-independent CPU control functions. 31 */ 32 33 #include <sys/types.h> 34 #include <sys/param.h> 35 #include <sys/var.h> 36 #include <sys/thread.h> 37 #include <sys/cpuvar.h> 38 #include <sys/kstat.h> 39 #include <sys/uadmin.h> 40 #include <sys/systm.h> 41 #include <sys/errno.h> 42 #include <sys/cmn_err.h> 43 #include <sys/procset.h> 44 #include <sys/processor.h> 45 #include <sys/debug.h> 46 #include <sys/cpupart.h> 47 #include <sys/lgrp.h> 48 #include <sys/pset.h> 49 #include <sys/chip.h> 50 #include <sys/kmem.h> 51 #include <sys/kmem_impl.h> /* to set per-cpu kmem_cache offset */ 52 #include <sys/atomic.h> 53 #include <sys/callb.h> 54 #include <sys/vtrace.h> 55 #include <sys/cyclic.h> 56 #include <sys/bitmap.h> 57 #include <sys/nvpair.h> 58 #include <sys/pool_pset.h> 59 #include <sys/msacct.h> 60 #include <sys/time.h> 61 #include <sys/archsystm.h> 62 63 extern int mp_cpu_start(cpu_t *); 64 extern int mp_cpu_stop(cpu_t *); 65 extern int mp_cpu_poweron(cpu_t *); 66 extern int mp_cpu_poweroff(cpu_t *); 67 extern int mp_cpu_configure(int); 68 extern int mp_cpu_unconfigure(int); 69 extern void mp_cpu_faulted_enter(cpu_t *); 70 extern void mp_cpu_faulted_exit(cpu_t *); 71 72 extern int cmp_cpu_to_chip(processorid_t cpuid); 73 #ifdef __sparcv9 74 extern char *cpu_fru_fmri(cpu_t *cp); 75 #endif 76 77 static void cpu_add_active_internal(cpu_t *cp); 78 static void cpu_remove_active(cpu_t *cp); 79 static void cpu_info_kstat_create(cpu_t *cp); 80 static void cpu_info_kstat_destroy(cpu_t *cp); 81 static void cpu_stats_kstat_create(cpu_t *cp); 82 static void cpu_stats_kstat_destroy(cpu_t *cp); 83 84 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw); 85 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw); 86 static int cpu_stat_ks_update(kstat_t *ksp, int rw); 87 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t); 88 89 /* 90 * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active, 91 * and dispatch queue reallocations. The lock ordering with respect to 92 * related locks is: 93 * 94 * cpu_lock --> thread_free_lock ---> p_lock ---> thread_lock() 95 * 96 * Warning: Certain sections of code do not use the cpu_lock when 97 * traversing the cpu_list (e.g. mutex_vector_enter(), clock()). Since 98 * all cpus are paused during modifications to this list, a solution 99 * to protect the list is too either disable kernel preemption while 100 * walking the list, *or* recheck the cpu_next pointer at each 101 * iteration in the loop. Note that in no cases can any cached 102 * copies of the cpu pointers be kept as they may become invalid. 103 */ 104 kmutex_t cpu_lock; 105 cpu_t *cpu_list; /* list of all CPUs */ 106 cpu_t *cpu_active; /* list of active CPUs */ 107 static cpuset_t cpu_available; /* set of available CPUs */ 108 cpuset_t cpu_seqid_inuse; /* which cpu_seqids are in use */ 109 110 /* 111 * max_ncpus keeps the max cpus the system can have. Initially 112 * it's NCPU, but since most archs scan the devtree for cpus 113 * fairly early on during boot, the real max can be known before 114 * ncpus is set (useful for early NCPU based allocations). 115 */ 116 int max_ncpus = NCPU; 117 /* 118 * platforms that set max_ncpus to maxiumum number of cpus that can be 119 * dynamically added will set boot_max_ncpus to the number of cpus found 120 * at device tree scan time during boot. 121 */ 122 int boot_max_ncpus = -1; 123 /* 124 * Maximum possible CPU id. This can never be >= NCPU since NCPU is 125 * used to size arrays that are indexed by CPU id. 126 */ 127 processorid_t max_cpuid = NCPU - 1; 128 129 int ncpus = 1; 130 int ncpus_online = 1; 131 132 /* 133 * CPU that we're trying to offline. Protected by cpu_lock. 134 */ 135 cpu_t *cpu_inmotion; 136 137 /* 138 * Can be raised to suppress further weakbinding, which are instead 139 * satisfied by disabling preemption. Must be raised/lowered under cpu_lock, 140 * while individual thread weakbinding synchronisation is done under thread 141 * lock. 142 */ 143 int weakbindingbarrier; 144 145 /* 146 * values for safe_list. Pause state that CPUs are in. 147 */ 148 #define PAUSE_IDLE 0 /* normal state */ 149 #define PAUSE_READY 1 /* paused thread ready to spl */ 150 #define PAUSE_WAIT 2 /* paused thread is spl-ed high */ 151 #define PAUSE_DIE 3 /* tell pause thread to leave */ 152 #define PAUSE_DEAD 4 /* pause thread has left */ 153 154 /* 155 * Variables used in pause_cpus(). 156 */ 157 static volatile char safe_list[NCPU]; 158 159 static struct _cpu_pause_info { 160 int cp_spl; /* spl saved in pause_cpus() */ 161 volatile int cp_go; /* Go signal sent after all ready */ 162 int cp_count; /* # of CPUs to pause */ 163 ksema_t cp_sem; /* synch pause_cpus & cpu_pause */ 164 kthread_id_t cp_paused; 165 } cpu_pause_info; 166 167 static kmutex_t pause_free_mutex; 168 static kcondvar_t pause_free_cv; 169 170 static struct cpu_sys_stats_ks_data { 171 kstat_named_t cpu_ticks_idle; 172 kstat_named_t cpu_ticks_user; 173 kstat_named_t cpu_ticks_kernel; 174 kstat_named_t cpu_ticks_wait; 175 kstat_named_t cpu_nsec_idle; 176 kstat_named_t cpu_nsec_user; 177 kstat_named_t cpu_nsec_kernel; 178 kstat_named_t wait_ticks_io; 179 kstat_named_t bread; 180 kstat_named_t bwrite; 181 kstat_named_t lread; 182 kstat_named_t lwrite; 183 kstat_named_t phread; 184 kstat_named_t phwrite; 185 kstat_named_t pswitch; 186 kstat_named_t trap; 187 kstat_named_t intr; 188 kstat_named_t syscall; 189 kstat_named_t sysread; 190 kstat_named_t syswrite; 191 kstat_named_t sysfork; 192 kstat_named_t sysvfork; 193 kstat_named_t sysexec; 194 kstat_named_t readch; 195 kstat_named_t writech; 196 kstat_named_t rcvint; 197 kstat_named_t xmtint; 198 kstat_named_t mdmint; 199 kstat_named_t rawch; 200 kstat_named_t canch; 201 kstat_named_t outch; 202 kstat_named_t msg; 203 kstat_named_t sema; 204 kstat_named_t namei; 205 kstat_named_t ufsiget; 206 kstat_named_t ufsdirblk; 207 kstat_named_t ufsipage; 208 kstat_named_t ufsinopage; 209 kstat_named_t procovf; 210 kstat_named_t intrthread; 211 kstat_named_t intrblk; 212 kstat_named_t intrunpin; 213 kstat_named_t idlethread; 214 kstat_named_t inv_swtch; 215 kstat_named_t nthreads; 216 kstat_named_t cpumigrate; 217 kstat_named_t xcalls; 218 kstat_named_t mutex_adenters; 219 kstat_named_t rw_rdfails; 220 kstat_named_t rw_wrfails; 221 kstat_named_t modload; 222 kstat_named_t modunload; 223 kstat_named_t bawrite; 224 kstat_named_t iowait; 225 } cpu_sys_stats_ks_data_template = { 226 { "cpu_ticks_idle", KSTAT_DATA_UINT64 }, 227 { "cpu_ticks_user", KSTAT_DATA_UINT64 }, 228 { "cpu_ticks_kernel", KSTAT_DATA_UINT64 }, 229 { "cpu_ticks_wait", KSTAT_DATA_UINT64 }, 230 { "cpu_nsec_idle", KSTAT_DATA_UINT64 }, 231 { "cpu_nsec_user", KSTAT_DATA_UINT64 }, 232 { "cpu_nsec_kernel", KSTAT_DATA_UINT64 }, 233 { "wait_ticks_io", KSTAT_DATA_UINT64 }, 234 { "bread", KSTAT_DATA_UINT64 }, 235 { "bwrite", KSTAT_DATA_UINT64 }, 236 { "lread", KSTAT_DATA_UINT64 }, 237 { "lwrite", KSTAT_DATA_UINT64 }, 238 { "phread", KSTAT_DATA_UINT64 }, 239 { "phwrite", KSTAT_DATA_UINT64 }, 240 { "pswitch", KSTAT_DATA_UINT64 }, 241 { "trap", KSTAT_DATA_UINT64 }, 242 { "intr", KSTAT_DATA_UINT64 }, 243 { "syscall", KSTAT_DATA_UINT64 }, 244 { "sysread", KSTAT_DATA_UINT64 }, 245 { "syswrite", KSTAT_DATA_UINT64 }, 246 { "sysfork", KSTAT_DATA_UINT64 }, 247 { "sysvfork", KSTAT_DATA_UINT64 }, 248 { "sysexec", KSTAT_DATA_UINT64 }, 249 { "readch", KSTAT_DATA_UINT64 }, 250 { "writech", KSTAT_DATA_UINT64 }, 251 { "rcvint", KSTAT_DATA_UINT64 }, 252 { "xmtint", KSTAT_DATA_UINT64 }, 253 { "mdmint", KSTAT_DATA_UINT64 }, 254 { "rawch", KSTAT_DATA_UINT64 }, 255 { "canch", KSTAT_DATA_UINT64 }, 256 { "outch", KSTAT_DATA_UINT64 }, 257 { "msg", KSTAT_DATA_UINT64 }, 258 { "sema", KSTAT_DATA_UINT64 }, 259 { "namei", KSTAT_DATA_UINT64 }, 260 { "ufsiget", KSTAT_DATA_UINT64 }, 261 { "ufsdirblk", KSTAT_DATA_UINT64 }, 262 { "ufsipage", KSTAT_DATA_UINT64 }, 263 { "ufsinopage", KSTAT_DATA_UINT64 }, 264 { "procovf", KSTAT_DATA_UINT64 }, 265 { "intrthread", KSTAT_DATA_UINT64 }, 266 { "intrblk", KSTAT_DATA_UINT64 }, 267 { "intrunpin", KSTAT_DATA_UINT64 }, 268 { "idlethread", KSTAT_DATA_UINT64 }, 269 { "inv_swtch", KSTAT_DATA_UINT64 }, 270 { "nthreads", KSTAT_DATA_UINT64 }, 271 { "cpumigrate", KSTAT_DATA_UINT64 }, 272 { "xcalls", KSTAT_DATA_UINT64 }, 273 { "mutex_adenters", KSTAT_DATA_UINT64 }, 274 { "rw_rdfails", KSTAT_DATA_UINT64 }, 275 { "rw_wrfails", KSTAT_DATA_UINT64 }, 276 { "modload", KSTAT_DATA_UINT64 }, 277 { "modunload", KSTAT_DATA_UINT64 }, 278 { "bawrite", KSTAT_DATA_UINT64 }, 279 { "iowait", KSTAT_DATA_UINT64 }, 280 }; 281 282 static struct cpu_vm_stats_ks_data { 283 kstat_named_t pgrec; 284 kstat_named_t pgfrec; 285 kstat_named_t pgin; 286 kstat_named_t pgpgin; 287 kstat_named_t pgout; 288 kstat_named_t pgpgout; 289 kstat_named_t swapin; 290 kstat_named_t pgswapin; 291 kstat_named_t swapout; 292 kstat_named_t pgswapout; 293 kstat_named_t zfod; 294 kstat_named_t dfree; 295 kstat_named_t scan; 296 kstat_named_t rev; 297 kstat_named_t hat_fault; 298 kstat_named_t as_fault; 299 kstat_named_t maj_fault; 300 kstat_named_t cow_fault; 301 kstat_named_t prot_fault; 302 kstat_named_t softlock; 303 kstat_named_t kernel_asflt; 304 kstat_named_t pgrrun; 305 kstat_named_t execpgin; 306 kstat_named_t execpgout; 307 kstat_named_t execfree; 308 kstat_named_t anonpgin; 309 kstat_named_t anonpgout; 310 kstat_named_t anonfree; 311 kstat_named_t fspgin; 312 kstat_named_t fspgout; 313 kstat_named_t fsfree; 314 } cpu_vm_stats_ks_data_template = { 315 { "pgrec", KSTAT_DATA_UINT64 }, 316 { "pgfrec", KSTAT_DATA_UINT64 }, 317 { "pgin", KSTAT_DATA_UINT64 }, 318 { "pgpgin", KSTAT_DATA_UINT64 }, 319 { "pgout", KSTAT_DATA_UINT64 }, 320 { "pgpgout", KSTAT_DATA_UINT64 }, 321 { "swapin", KSTAT_DATA_UINT64 }, 322 { "pgswapin", KSTAT_DATA_UINT64 }, 323 { "swapout", KSTAT_DATA_UINT64 }, 324 { "pgswapout", KSTAT_DATA_UINT64 }, 325 { "zfod", KSTAT_DATA_UINT64 }, 326 { "dfree", KSTAT_DATA_UINT64 }, 327 { "scan", KSTAT_DATA_UINT64 }, 328 { "rev", KSTAT_DATA_UINT64 }, 329 { "hat_fault", KSTAT_DATA_UINT64 }, 330 { "as_fault", KSTAT_DATA_UINT64 }, 331 { "maj_fault", KSTAT_DATA_UINT64 }, 332 { "cow_fault", KSTAT_DATA_UINT64 }, 333 { "prot_fault", KSTAT_DATA_UINT64 }, 334 { "softlock", KSTAT_DATA_UINT64 }, 335 { "kernel_asflt", KSTAT_DATA_UINT64 }, 336 { "pgrrun", KSTAT_DATA_UINT64 }, 337 { "execpgin", KSTAT_DATA_UINT64 }, 338 { "execpgout", KSTAT_DATA_UINT64 }, 339 { "execfree", KSTAT_DATA_UINT64 }, 340 { "anonpgin", KSTAT_DATA_UINT64 }, 341 { "anonpgout", KSTAT_DATA_UINT64 }, 342 { "anonfree", KSTAT_DATA_UINT64 }, 343 { "fspgin", KSTAT_DATA_UINT64 }, 344 { "fspgout", KSTAT_DATA_UINT64 }, 345 { "fsfree", KSTAT_DATA_UINT64 }, 346 }; 347 348 /* 349 * Force the specified thread to migrate to the appropriate processor. 350 * Called with thread lock held, returns with it dropped. 351 */ 352 static void 353 force_thread_migrate(kthread_id_t tp) 354 { 355 ASSERT(THREAD_LOCK_HELD(tp)); 356 if (tp == curthread) { 357 THREAD_TRANSITION(tp); 358 CL_SETRUN(tp); 359 thread_unlock_nopreempt(tp); 360 swtch(); 361 } else { 362 if (tp->t_state == TS_ONPROC) { 363 cpu_surrender(tp); 364 } else if (tp->t_state == TS_RUN) { 365 (void) dispdeq(tp); 366 setbackdq(tp); 367 } 368 thread_unlock(tp); 369 } 370 } 371 372 /* 373 * Set affinity for a specified CPU. 374 * A reference count is incremented and the affinity is held until the 375 * reference count is decremented to zero by thread_affinity_clear(). 376 * This is so regions of code requiring affinity can be nested. 377 * Caller needs to ensure that cpu_id remains valid, which can be 378 * done by holding cpu_lock across this call, unless the caller 379 * specifies CPU_CURRENT in which case the cpu_lock will be acquired 380 * by thread_affinity_set and CPU->cpu_id will be the target CPU. 381 */ 382 void 383 thread_affinity_set(kthread_id_t t, int cpu_id) 384 { 385 cpu_t *cp; 386 int c; 387 388 ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL)); 389 390 if ((c = cpu_id) == CPU_CURRENT) { 391 mutex_enter(&cpu_lock); 392 cpu_id = CPU->cpu_id; 393 } 394 /* 395 * We should be asserting that cpu_lock is held here, but 396 * the NCA code doesn't acquire it. The following assert 397 * should be uncommented when the NCA code is fixed. 398 * 399 * ASSERT(MUTEX_HELD(&cpu_lock)); 400 */ 401 ASSERT((cpu_id >= 0) && (cpu_id < NCPU)); 402 cp = cpu[cpu_id]; 403 ASSERT(cp != NULL); /* user must provide a good cpu_id */ 404 /* 405 * If there is already a hard affinity requested, and this affinity 406 * conflicts with that, panic. 407 */ 408 thread_lock(t); 409 if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) { 410 panic("affinity_set: setting %p but already bound to %p", 411 (void *)cp, (void *)t->t_bound_cpu); 412 } 413 t->t_affinitycnt++; 414 t->t_bound_cpu = cp; 415 416 /* 417 * Make sure we're running on the right CPU. 418 */ 419 if (cp != t->t_cpu || t != curthread) { 420 force_thread_migrate(t); /* drops thread lock */ 421 } else { 422 thread_unlock(t); 423 } 424 425 if (c == CPU_CURRENT) 426 mutex_exit(&cpu_lock); 427 } 428 429 /* 430 * Wrapper for backward compatibility. 431 */ 432 void 433 affinity_set(int cpu_id) 434 { 435 thread_affinity_set(curthread, cpu_id); 436 } 437 438 /* 439 * Decrement the affinity reservation count and if it becomes zero, 440 * clear the CPU affinity for the current thread, or set it to the user's 441 * software binding request. 442 */ 443 void 444 thread_affinity_clear(kthread_id_t t) 445 { 446 register processorid_t binding; 447 448 thread_lock(t); 449 if (--t->t_affinitycnt == 0) { 450 if ((binding = t->t_bind_cpu) == PBIND_NONE) { 451 /* 452 * Adjust disp_max_unbound_pri if necessary. 453 */ 454 disp_adjust_unbound_pri(t); 455 t->t_bound_cpu = NULL; 456 if (t->t_cpu->cpu_part != t->t_cpupart) { 457 force_thread_migrate(t); 458 return; 459 } 460 } else { 461 t->t_bound_cpu = cpu[binding]; 462 /* 463 * Make sure the thread is running on the bound CPU. 464 */ 465 if (t->t_cpu != t->t_bound_cpu) { 466 force_thread_migrate(t); 467 return; /* already dropped lock */ 468 } 469 } 470 } 471 thread_unlock(t); 472 } 473 474 /* 475 * Wrapper for backward compatibility. 476 */ 477 void 478 affinity_clear(void) 479 { 480 thread_affinity_clear(curthread); 481 } 482 483 /* 484 * Weak cpu affinity. Bind to the "current" cpu for short periods 485 * of time during which the thread must not block (but may be preempted). 486 * Use this instead of kpreempt_disable() when it is only "no migration" 487 * rather than "no preemption" semantics that are required - disabling 488 * preemption holds higher priority threads off of cpu and if the 489 * operation that is protected is more than momentary this is not good 490 * for realtime etc. 491 * 492 * Weakly bound threads will not prevent a cpu from being offlined - 493 * we'll only run them on the cpu to which they are weakly bound but 494 * (because they do not block) we'll always be able to move them on to 495 * another cpu at offline time if we give them just a short moment to 496 * run during which they will unbind. To give a cpu a chance of offlining, 497 * however, we require a barrier to weak bindings that may be raised for a 498 * given cpu (offline/move code may set this and then wait a short time for 499 * existing weak bindings to drop); the cpu_inmotion pointer is that barrier. 500 * 501 * There are few restrictions on the calling context of thread_nomigrate. 502 * The caller must not hold the thread lock. Calls may be nested. 503 * 504 * After weakbinding a thread must not perform actions that may block. 505 * In particular it must not call thread_affinity_set; calling that when 506 * already weakbound is nonsensical anyway. 507 * 508 * If curthread is prevented from migrating for other reasons 509 * (kernel preemption disabled; high pil; strongly bound; interrupt thread) 510 * then the weak binding will succeed even if this cpu is the target of an 511 * offline/move request. 512 */ 513 void 514 thread_nomigrate(void) 515 { 516 cpu_t *cp; 517 kthread_id_t t = curthread; 518 519 again: 520 kpreempt_disable(); 521 cp = CPU; 522 523 /* 524 * A highlevel interrupt must not modify t_nomigrate or 525 * t_weakbound_cpu of the thread it has interrupted. A lowlevel 526 * interrupt thread cannot migrate and we can avoid the 527 * thread_lock call below by short-circuiting here. In either 528 * case we can just return since no migration is possible and 529 * the condition will persist (ie, when we test for these again 530 * in thread_allowmigrate they can't have changed). Migration 531 * is also impossible if we're at or above DISP_LEVEL pil. 532 */ 533 if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD || 534 getpil() >= DISP_LEVEL) { 535 kpreempt_enable(); 536 return; 537 } 538 539 /* 540 * We must be consistent with existing weak bindings. Since we 541 * may be interrupted between the increment of t_nomigrate and 542 * the store to t_weakbound_cpu below we cannot assume that 543 * t_weakbound_cpu will be set if t_nomigrate is. Note that we 544 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not 545 * always the case. 546 */ 547 if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) { 548 if (!panicstr) 549 panic("thread_nomigrate: binding to %p but already " 550 "bound to %p", (void *)cp, 551 (void *)t->t_weakbound_cpu); 552 } 553 554 /* 555 * At this point we have preemption disabled and we don't yet hold 556 * the thread lock. So it's possible that somebody else could 557 * set t_bind_cpu here and not be able to force us across to the 558 * new cpu (since we have preemption disabled). 559 */ 560 thread_lock(curthread); 561 562 /* 563 * If further weak bindings are being (temporarily) suppressed then 564 * we'll settle for disabling kernel preemption (which assures 565 * no migration provided the thread does not block which it is 566 * not allowed to if using thread_nomigrate). We must remember 567 * this disposition so we can take appropriate action in 568 * thread_allowmigrate. If this is a nested call and the 569 * thread is already weakbound then fall through as normal. 570 * We remember the decision to settle for kpreempt_disable through 571 * negative nesting counting in t_nomigrate. Once a thread has had one 572 * weakbinding request satisfied in this way any further (nested) 573 * requests will continue to be satisfied in the same way, 574 * even if weak bindings have recommenced. 575 */ 576 if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) { 577 --t->t_nomigrate; 578 thread_unlock(curthread); 579 return; /* with kpreempt_disable still active */ 580 } 581 582 /* 583 * We hold thread_lock so t_bind_cpu cannot change. We could, 584 * however, be running on a different cpu to which we are t_bound_cpu 585 * to (as explained above). If we grant the weak binding request 586 * in that case then the dispatcher must favour our weak binding 587 * over our strong (in which case, just as when preemption is 588 * disabled, we can continue to run on a cpu other than the one to 589 * which we are strongbound; the difference in this case is that 590 * this thread can be preempted and so can appear on the dispatch 591 * queues of a cpu other than the one it is strongbound to). 592 * 593 * If the cpu we are running on does not appear to be a current 594 * offline target (we check cpu_inmotion to determine this - since 595 * we don't hold cpu_lock we may not see a recent store to that, 596 * so it's possible that we at times can grant a weak binding to a 597 * cpu that is an offline target, but that one request will not 598 * prevent the offline from succeeding) then we will always grant 599 * the weak binding request. This includes the case above where 600 * we grant a weakbinding not commensurate with our strong binding. 601 * 602 * If our cpu does appear to be an offline target then we're inclined 603 * not to grant the weakbinding request just yet - we'd prefer to 604 * migrate to another cpu and grant the request there. The 605 * exceptions are those cases where going through preemption code 606 * will not result in us changing cpu: 607 * 608 * . interrupts have already bypassed this case (see above) 609 * . we are already weakbound to this cpu (dispatcher code will 610 * always return us to the weakbound cpu) 611 * . preemption was disabled even before we disabled it above 612 * . we are strongbound to this cpu (if we're strongbound to 613 * another and not yet running there the trip through the 614 * dispatcher will move us to the strongbound cpu and we 615 * will grant the weak binding there) 616 */ 617 if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 || 618 t->t_bound_cpu == cp) { 619 /* 620 * Don't be tempted to store to t_weakbound_cpu only on 621 * the first nested bind request - if we're interrupted 622 * after the increment of t_nomigrate and before the 623 * store to t_weakbound_cpu and the interrupt calls 624 * thread_nomigrate then the assertion in thread_allowmigrate 625 * would fail. 626 */ 627 t->t_nomigrate++; 628 t->t_weakbound_cpu = cp; 629 membar_producer(); 630 thread_unlock(curthread); 631 /* 632 * Now that we have dropped the thread_lock another thread 633 * can set our t_weakbound_cpu, and will try to migrate us 634 * to the strongbound cpu (which will not be prevented by 635 * preemption being disabled since we're about to enable 636 * preemption). We have granted the weakbinding to the current 637 * cpu, so again we are in the position that is is is possible 638 * that our weak and strong bindings differ. Again this 639 * is catered for by dispatcher code which will favour our 640 * weak binding. 641 */ 642 kpreempt_enable(); 643 } else { 644 /* 645 * Move to another cpu before granting the request by 646 * forcing this thread through preemption code. When we 647 * get to set{front,back}dq called from CL_PREEMPT() 648 * cpu_choose() will be used to select a cpu to queue 649 * us on - that will see cpu_inmotion and take 650 * steps to avoid returning us to this cpu. 651 */ 652 cp->cpu_kprunrun = 1; 653 thread_unlock(curthread); 654 kpreempt_enable(); /* will call preempt() */ 655 goto again; 656 } 657 } 658 659 void 660 thread_allowmigrate(void) 661 { 662 kthread_id_t t = curthread; 663 664 ASSERT(t->t_weakbound_cpu == CPU || 665 (t->t_nomigrate < 0 && t->t_preempt > 0) || 666 CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD || 667 getpil() >= DISP_LEVEL); 668 669 if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) || 670 getpil() >= DISP_LEVEL) 671 return; 672 673 if (t->t_nomigrate < 0) { 674 /* 675 * This thread was granted "weak binding" in the 676 * stronger form of kernel preemption disabling. 677 * Undo a level of nesting for both t_nomigrate 678 * and t_preempt. 679 */ 680 ++t->t_nomigrate; 681 kpreempt_enable(); 682 } else if (--t->t_nomigrate == 0) { 683 /* 684 * Time to drop the weak binding. We need to cater 685 * for the case where we're weakbound to a different 686 * cpu than that to which we're strongbound (a very 687 * temporary arrangement that must only persist until 688 * weak binding drops). We don't acquire thread_lock 689 * here so even as this code executes t_bound_cpu 690 * may be changing. So we disable preemption and 691 * a) in the case that t_bound_cpu changes while we 692 * have preemption disabled kprunrun will be set 693 * asynchronously, and b) if before disabling 694 * preemption we were already on a different cpu to 695 * our t_bound_cpu then we set kprunrun ourselves 696 * to force a trip through the dispatcher when 697 * preemption is enabled. 698 */ 699 kpreempt_disable(); 700 if (t->t_bound_cpu && 701 t->t_weakbound_cpu != t->t_bound_cpu) 702 CPU->cpu_kprunrun = 1; 703 t->t_weakbound_cpu = NULL; 704 membar_producer(); 705 kpreempt_enable(); 706 } 707 } 708 709 /* 710 * weakbinding_stop can be used to temporarily cause weakbindings made 711 * with thread_nomigrate to be satisfied through the stronger action of 712 * kpreempt_disable. weakbinding_start recommences normal weakbinding. 713 */ 714 715 void 716 weakbinding_stop(void) 717 { 718 ASSERT(MUTEX_HELD(&cpu_lock)); 719 weakbindingbarrier = 1; 720 membar_producer(); /* make visible before subsequent thread_lock */ 721 } 722 723 void 724 weakbinding_start(void) 725 { 726 ASSERT(MUTEX_HELD(&cpu_lock)); 727 weakbindingbarrier = 0; 728 } 729 730 /* 731 * This routine is called to place the CPUs in a safe place so that 732 * one of them can be taken off line or placed on line. What we are 733 * trying to do here is prevent a thread from traversing the list 734 * of active CPUs while we are changing it or from getting placed on 735 * the run queue of a CPU that has just gone off line. We do this by 736 * creating a thread with the highest possible prio for each CPU and 737 * having it call this routine. The advantage of this method is that 738 * we can eliminate all checks for CPU_ACTIVE in the disp routines. 739 * This makes disp faster at the expense of making p_online() slower 740 * which is a good trade off. 741 */ 742 static void 743 cpu_pause(volatile char *safe) 744 { 745 int s; 746 struct _cpu_pause_info *cpi = &cpu_pause_info; 747 748 ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE)); 749 750 while (*safe != PAUSE_DIE) { 751 *safe = PAUSE_READY; 752 membar_enter(); /* make sure stores are flushed */ 753 sema_v(&cpi->cp_sem); /* signal requesting thread */ 754 755 /* 756 * Wait here until all pause threads are running. That 757 * indicates that it's safe to do the spl. Until 758 * cpu_pause_info.cp_go is set, we don't want to spl 759 * because that might block clock interrupts needed 760 * to preempt threads on other CPUs. 761 */ 762 while (cpi->cp_go == 0) 763 ; 764 /* 765 * Even though we are at the highest disp prio, we need 766 * to block out all interrupts below LOCK_LEVEL so that 767 * an intr doesn't come in, wake up a thread, and call 768 * setbackdq/setfrontdq. 769 */ 770 s = splhigh(); 771 /* 772 * This cpu is now safe. 773 */ 774 *safe = PAUSE_WAIT; 775 membar_enter(); /* make sure stores are flushed */ 776 777 /* 778 * Now we wait. When we are allowed to continue, safe will 779 * be set to PAUSE_IDLE. 780 */ 781 while (*safe != PAUSE_IDLE) 782 ; 783 784 splx(s); 785 /* 786 * Waiting is at an end. Switch out of cpu_pause 787 * loop and resume useful work. 788 */ 789 swtch(); 790 } 791 792 mutex_enter(&pause_free_mutex); 793 *safe = PAUSE_DEAD; 794 cv_broadcast(&pause_free_cv); 795 mutex_exit(&pause_free_mutex); 796 } 797 798 /* 799 * Allow the cpus to start running again. 800 */ 801 void 802 start_cpus() 803 { 804 int i; 805 806 ASSERT(MUTEX_HELD(&cpu_lock)); 807 ASSERT(cpu_pause_info.cp_paused); 808 cpu_pause_info.cp_paused = NULL; 809 for (i = 0; i < NCPU; i++) 810 safe_list[i] = PAUSE_IDLE; 811 membar_enter(); /* make sure stores are flushed */ 812 affinity_clear(); 813 splx(cpu_pause_info.cp_spl); 814 kpreempt_enable(); 815 } 816 817 /* 818 * Allocate a pause thread for a CPU. 819 */ 820 static void 821 cpu_pause_alloc(cpu_t *cp) 822 { 823 kthread_id_t t; 824 int cpun = cp->cpu_id; 825 826 /* 827 * Note, v.v_nglobpris will not change value as long as I hold 828 * cpu_lock. 829 */ 830 t = thread_create(NULL, 0, cpu_pause, (caddr_t)&safe_list[cpun], 831 0, &p0, TS_STOPPED, v.v_nglobpris - 1); 832 thread_lock(t); 833 t->t_bound_cpu = cp; 834 t->t_disp_queue = cp->cpu_disp; 835 t->t_affinitycnt = 1; 836 t->t_preempt = 1; 837 thread_unlock(t); 838 cp->cpu_pause_thread = t; 839 /* 840 * Registering a thread in the callback table is usually done 841 * in the initialization code of the thread. In this 842 * case, we do it right after thread creation because the 843 * thread itself may never run, and we need to register the 844 * fact that it is safe for cpr suspend. 845 */ 846 CALLB_CPR_INIT_SAFE(t, "cpu_pause"); 847 } 848 849 /* 850 * Free a pause thread for a CPU. 851 */ 852 static void 853 cpu_pause_free(cpu_t *cp) 854 { 855 kthread_id_t t; 856 int cpun = cp->cpu_id; 857 858 ASSERT(MUTEX_HELD(&cpu_lock)); 859 /* 860 * We have to get the thread and tell him to die. 861 */ 862 if ((t = cp->cpu_pause_thread) == NULL) { 863 ASSERT(safe_list[cpun] == PAUSE_IDLE); 864 return; 865 } 866 thread_lock(t); 867 t->t_cpu = CPU; /* disp gets upset if last cpu is quiesced. */ 868 t->t_bound_cpu = NULL; /* Must un-bind; cpu may not be running. */ 869 t->t_pri = v.v_nglobpris - 1; 870 ASSERT(safe_list[cpun] == PAUSE_IDLE); 871 safe_list[cpun] = PAUSE_DIE; 872 THREAD_TRANSITION(t); 873 setbackdq(t); 874 thread_unlock_nopreempt(t); 875 876 /* 877 * If we don't wait for the thread to actually die, it may try to 878 * run on the wrong cpu as part of an actual call to pause_cpus(). 879 */ 880 mutex_enter(&pause_free_mutex); 881 while (safe_list[cpun] != PAUSE_DEAD) { 882 cv_wait(&pause_free_cv, &pause_free_mutex); 883 } 884 mutex_exit(&pause_free_mutex); 885 safe_list[cpun] = PAUSE_IDLE; 886 887 cp->cpu_pause_thread = NULL; 888 } 889 890 /* 891 * Initialize basic structures for pausing CPUs. 892 */ 893 void 894 cpu_pause_init() 895 { 896 sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL); 897 /* 898 * Create initial CPU pause thread. 899 */ 900 cpu_pause_alloc(CPU); 901 } 902 903 /* 904 * Start the threads used to pause another CPU. 905 */ 906 static int 907 cpu_pause_start(processorid_t cpu_id) 908 { 909 int i; 910 int cpu_count = 0; 911 912 for (i = 0; i < NCPU; i++) { 913 cpu_t *cp; 914 kthread_id_t t; 915 916 cp = cpu[i]; 917 if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) { 918 safe_list[i] = PAUSE_WAIT; 919 continue; 920 } 921 922 /* 923 * Skip CPU if it is quiesced or not yet started. 924 */ 925 if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) { 926 safe_list[i] = PAUSE_WAIT; 927 continue; 928 } 929 930 /* 931 * Start this CPU's pause thread. 932 */ 933 t = cp->cpu_pause_thread; 934 thread_lock(t); 935 /* 936 * Reset the priority, since nglobpris may have 937 * changed since the thread was created, if someone 938 * has loaded the RT (or some other) scheduling 939 * class. 940 */ 941 t->t_pri = v.v_nglobpris - 1; 942 THREAD_TRANSITION(t); 943 setbackdq(t); 944 thread_unlock_nopreempt(t); 945 ++cpu_count; 946 } 947 return (cpu_count); 948 } 949 950 951 /* 952 * Pause all of the CPUs except the one we are on by creating a high 953 * priority thread bound to those CPUs. 954 * 955 * Note that one must be extremely careful regarding code 956 * executed while CPUs are paused. Since a CPU may be paused 957 * while a thread scheduling on that CPU is holding an adaptive 958 * lock, code executed with CPUs paused must not acquire adaptive 959 * (or low-level spin) locks. Also, such code must not block, 960 * since the thread that is supposed to initiate the wakeup may 961 * never run. 962 * 963 * With a few exceptions, the restrictions on code executed with CPUs 964 * paused match those for code executed at high-level interrupt 965 * context. 966 */ 967 void 968 pause_cpus(cpu_t *off_cp) 969 { 970 processorid_t cpu_id; 971 int i; 972 struct _cpu_pause_info *cpi = &cpu_pause_info; 973 974 ASSERT(MUTEX_HELD(&cpu_lock)); 975 ASSERT(cpi->cp_paused == NULL); 976 cpi->cp_count = 0; 977 cpi->cp_go = 0; 978 for (i = 0; i < NCPU; i++) 979 safe_list[i] = PAUSE_IDLE; 980 kpreempt_disable(); 981 982 /* 983 * If running on the cpu that is going offline, get off it. 984 * This is so that it won't be necessary to rechoose a CPU 985 * when done. 986 */ 987 if (CPU == off_cp) 988 cpu_id = off_cp->cpu_next_part->cpu_id; 989 else 990 cpu_id = CPU->cpu_id; 991 affinity_set(cpu_id); 992 993 /* 994 * Start the pause threads and record how many were started 995 */ 996 cpi->cp_count = cpu_pause_start(cpu_id); 997 998 /* 999 * Now wait for all CPUs to be running the pause thread. 1000 */ 1001 while (cpi->cp_count > 0) { 1002 /* 1003 * Spin reading the count without grabbing the disp 1004 * lock to make sure we don't prevent the pause 1005 * threads from getting the lock. 1006 */ 1007 while (sema_held(&cpi->cp_sem)) 1008 ; 1009 if (sema_tryp(&cpi->cp_sem)) 1010 --cpi->cp_count; 1011 } 1012 cpi->cp_go = 1; /* all have reached cpu_pause */ 1013 1014 /* 1015 * Now wait for all CPUs to spl. (Transition from PAUSE_READY 1016 * to PAUSE_WAIT.) 1017 */ 1018 for (i = 0; i < NCPU; i++) { 1019 while (safe_list[i] != PAUSE_WAIT) 1020 ; 1021 } 1022 cpi->cp_spl = splhigh(); /* block dispatcher on this CPU */ 1023 cpi->cp_paused = curthread; 1024 } 1025 1026 /* 1027 * Check whether the current thread has CPUs paused 1028 */ 1029 int 1030 cpus_paused(void) 1031 { 1032 if (cpu_pause_info.cp_paused != NULL) { 1033 ASSERT(cpu_pause_info.cp_paused == curthread); 1034 return (1); 1035 } 1036 return (0); 1037 } 1038 1039 static cpu_t * 1040 cpu_get_all(processorid_t cpun) 1041 { 1042 ASSERT(MUTEX_HELD(&cpu_lock)); 1043 1044 if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun)) 1045 return (NULL); 1046 return (cpu[cpun]); 1047 } 1048 1049 /* 1050 * Check whether cpun is a valid processor id and whether it should be 1051 * visible from the current zone. If it is, return a pointer to the 1052 * associated CPU structure. 1053 */ 1054 cpu_t * 1055 cpu_get(processorid_t cpun) 1056 { 1057 cpu_t *c; 1058 1059 ASSERT(MUTEX_HELD(&cpu_lock)); 1060 c = cpu_get_all(cpun); 1061 if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() && 1062 zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c)) 1063 return (NULL); 1064 return (c); 1065 } 1066 1067 /* 1068 * The following functions should be used to check CPU states in the kernel. 1069 * They should be invoked with cpu_lock held. Kernel subsystems interested 1070 * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc 1071 * states. Those are for user-land (and system call) use only. 1072 */ 1073 1074 /* 1075 * Determine whether the CPU is online and handling interrupts. 1076 */ 1077 int 1078 cpu_is_online(cpu_t *cpu) 1079 { 1080 ASSERT(MUTEX_HELD(&cpu_lock)); 1081 return (cpu_flagged_online(cpu->cpu_flags)); 1082 } 1083 1084 /* 1085 * Determine whether the CPU is offline (this includes spare and faulted). 1086 */ 1087 int 1088 cpu_is_offline(cpu_t *cpu) 1089 { 1090 ASSERT(MUTEX_HELD(&cpu_lock)); 1091 return (cpu_flagged_offline(cpu->cpu_flags)); 1092 } 1093 1094 /* 1095 * Determine whether the CPU is powered off. 1096 */ 1097 int 1098 cpu_is_poweredoff(cpu_t *cpu) 1099 { 1100 ASSERT(MUTEX_HELD(&cpu_lock)); 1101 return (cpu_flagged_poweredoff(cpu->cpu_flags)); 1102 } 1103 1104 /* 1105 * Determine whether the CPU is handling interrupts. 1106 */ 1107 int 1108 cpu_is_nointr(cpu_t *cpu) 1109 { 1110 ASSERT(MUTEX_HELD(&cpu_lock)); 1111 return (cpu_flagged_nointr(cpu->cpu_flags)); 1112 } 1113 1114 /* 1115 * Determine whether the CPU is active (scheduling threads). 1116 */ 1117 int 1118 cpu_is_active(cpu_t *cpu) 1119 { 1120 ASSERT(MUTEX_HELD(&cpu_lock)); 1121 return (cpu_flagged_active(cpu->cpu_flags)); 1122 } 1123 1124 /* 1125 * Same as above, but these require cpu_flags instead of cpu_t pointers. 1126 */ 1127 int 1128 cpu_flagged_online(cpu_flag_t cpu_flags) 1129 { 1130 return (cpu_flagged_active(cpu_flags) && 1131 (cpu_flags & CPU_ENABLE)); 1132 } 1133 1134 int 1135 cpu_flagged_offline(cpu_flag_t cpu_flags) 1136 { 1137 return (((cpu_flags & CPU_POWEROFF) == 0) && 1138 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)); 1139 } 1140 1141 int 1142 cpu_flagged_poweredoff(cpu_flag_t cpu_flags) 1143 { 1144 return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF); 1145 } 1146 1147 int 1148 cpu_flagged_nointr(cpu_flag_t cpu_flags) 1149 { 1150 return (cpu_flagged_active(cpu_flags) && 1151 (cpu_flags & CPU_ENABLE) == 0); 1152 } 1153 1154 int 1155 cpu_flagged_active(cpu_flag_t cpu_flags) 1156 { 1157 return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) && 1158 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY)); 1159 } 1160 1161 /* 1162 * Bring the indicated CPU online. 1163 */ 1164 int 1165 cpu_online(cpu_t *cp) 1166 { 1167 int error = 0; 1168 1169 /* 1170 * Handle on-line request. 1171 * This code must put the new CPU on the active list before 1172 * starting it because it will not be paused, and will start 1173 * using the active list immediately. The real start occurs 1174 * when the CPU_QUIESCED flag is turned off. 1175 */ 1176 1177 ASSERT(MUTEX_HELD(&cpu_lock)); 1178 1179 /* 1180 * Put all the cpus into a known safe place. 1181 * No mutexes can be entered while CPUs are paused. 1182 */ 1183 error = mp_cpu_start(cp); /* arch-dep hook */ 1184 if (error == 0) { 1185 pause_cpus(NULL); 1186 cpu_add_active_internal(cp); 1187 if (cp->cpu_flags & CPU_FAULTED) { 1188 cp->cpu_flags &= ~CPU_FAULTED; 1189 mp_cpu_faulted_exit(cp); 1190 } 1191 cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN | 1192 CPU_SPARE); 1193 start_cpus(); 1194 cpu_stats_kstat_create(cp); 1195 cpu_create_intrstat(cp); 1196 lgrp_kstat_create(cp); 1197 cpu_state_change_notify(cp->cpu_id, CPU_ON); 1198 cpu_intr_enable(cp); /* arch-dep hook */ 1199 cyclic_online(cp); 1200 poke_cpu(cp->cpu_id); 1201 } 1202 1203 return (error); 1204 } 1205 1206 /* 1207 * Take the indicated CPU offline. 1208 */ 1209 int 1210 cpu_offline(cpu_t *cp, int flags) 1211 { 1212 cpupart_t *pp; 1213 int error = 0; 1214 cpu_t *ncp; 1215 int intr_enable; 1216 int cyclic_off = 0; 1217 int loop_count; 1218 int no_quiesce = 0; 1219 int (*bound_func)(struct cpu *, int); 1220 kthread_t *t; 1221 lpl_t *cpu_lpl; 1222 proc_t *p; 1223 int lgrp_diff_lpl; 1224 1225 ASSERT(MUTEX_HELD(&cpu_lock)); 1226 1227 /* 1228 * If we're going from faulted or spare to offline, just 1229 * clear these flags and update CPU state. 1230 */ 1231 if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) { 1232 if (cp->cpu_flags & CPU_FAULTED) { 1233 cp->cpu_flags &= ~CPU_FAULTED; 1234 mp_cpu_faulted_exit(cp); 1235 } 1236 cp->cpu_flags &= ~CPU_SPARE; 1237 cpu_set_state(cp); 1238 return (0); 1239 } 1240 1241 /* 1242 * Handle off-line request. 1243 */ 1244 pp = cp->cpu_part; 1245 /* 1246 * Don't offline last online CPU in partition 1247 */ 1248 if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2) 1249 return (EBUSY); 1250 /* 1251 * Unbind all thread bound to our CPU if we were asked to. 1252 */ 1253 if (flags & CPU_FORCED && (error = cpu_unbind(cp->cpu_id)) != 0) 1254 return (error); 1255 /* 1256 * We shouldn't be bound to this CPU ourselves. 1257 */ 1258 if (curthread->t_bound_cpu == cp) 1259 return (EBUSY); 1260 1261 /* 1262 * Tell interested parties that this CPU is going offline. 1263 */ 1264 cpu_state_change_notify(cp->cpu_id, CPU_OFF); 1265 1266 /* 1267 * Take the CPU out of interrupt participation so we won't find 1268 * bound kernel threads. If the architecture cannot completely 1269 * shut off interrupts on the CPU, don't quiesce it, but don't 1270 * run anything but interrupt thread... this is indicated by 1271 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being 1272 * off. 1273 */ 1274 intr_enable = cp->cpu_flags & CPU_ENABLE; 1275 if (intr_enable) 1276 no_quiesce = cpu_intr_disable(cp); 1277 1278 /* 1279 * Record that we are aiming to offline this cpu. This acts as 1280 * a barrier to further weak binding requests in thread_nomigrate 1281 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to 1282 * lean away from this cpu. Further strong bindings are already 1283 * avoided since we hold cpu_lock. Since threads that are set 1284 * runnable around now and others coming off the target cpu are 1285 * directed away from the target, existing strong and weak bindings 1286 * (especially the latter) to the target cpu stand maximum chance of 1287 * being able to unbind during the short delay loop below (if other 1288 * unbound threads compete they may not see cpu in time to unbind 1289 * even if they would do so immediately. 1290 */ 1291 cpu_inmotion = cp; 1292 membar_enter(); 1293 1294 /* 1295 * Check for kernel threads (strong or weak) bound to that CPU. 1296 * Strongly bound threads may not unbind, and we'll have to return 1297 * EBUSY. Weakly bound threads should always disappear - we've 1298 * stopped more weak binding with cpu_inmotion and existing 1299 * bindings will drain imminently (they may not block). Nonetheless 1300 * we will wait for a fixed period for all bound threads to disappear. 1301 * Inactive interrupt threads are OK (they'll be in TS_FREE 1302 * state). If test finds some bound threads, wait a few ticks 1303 * to give short-lived threads (such as interrupts) chance to 1304 * complete. Note that if no_quiesce is set, i.e. this cpu 1305 * is required to service interrupts, then we take the route 1306 * that permits interrupt threads to be active (or bypassed). 1307 */ 1308 bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads; 1309 1310 again: for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) { 1311 if (loop_count >= 5) { 1312 error = EBUSY; /* some threads still bound */ 1313 break; 1314 } 1315 1316 /* 1317 * If some threads were assigned, give them 1318 * a chance to complete or move. 1319 * 1320 * This assumes that the clock_thread is not bound 1321 * to any CPU, because the clock_thread is needed to 1322 * do the delay(hz/100). 1323 * 1324 * Note: we still hold the cpu_lock while waiting for 1325 * the next clock tick. This is OK since it isn't 1326 * needed for anything else except processor_bind(2), 1327 * and system initialization. If we drop the lock, 1328 * we would risk another p_online disabling the last 1329 * processor. 1330 */ 1331 delay(hz/100); 1332 } 1333 1334 if (error == 0 && cyclic_off == 0) { 1335 if (!cyclic_offline(cp)) { 1336 /* 1337 * We must have bound cyclics... 1338 */ 1339 error = EBUSY; 1340 goto out; 1341 } 1342 cyclic_off = 1; 1343 } 1344 1345 /* 1346 * Call mp_cpu_stop() to perform any special operations 1347 * needed for this machine architecture to offline a CPU. 1348 */ 1349 if (error == 0) 1350 error = mp_cpu_stop(cp); /* arch-dep hook */ 1351 1352 /* 1353 * If that all worked, take the CPU offline and decrement 1354 * ncpus_online. 1355 */ 1356 if (error == 0) { 1357 /* 1358 * Put all the cpus into a known safe place. 1359 * No mutexes can be entered while CPUs are paused. 1360 */ 1361 pause_cpus(cp); 1362 /* 1363 * Repeat the operation, if necessary, to make sure that 1364 * all outstanding low-level interrupts run to completion 1365 * before we set the CPU_QUIESCED flag. It's also possible 1366 * that a thread has weak bound to the cpu despite our raising 1367 * cpu_inmotion above since it may have loaded that 1368 * value before the barrier became visible (this would have 1369 * to be the thread that was on the target cpu at the time 1370 * we raised the barrier). 1371 */ 1372 if ((!no_quiesce && cp->cpu_intr_actv != 0) || 1373 (*bound_func)(cp, 1)) { 1374 start_cpus(); 1375 (void) mp_cpu_start(cp); 1376 goto again; 1377 } 1378 ncp = cp->cpu_next_part; 1379 cpu_lpl = cp->cpu_lpl; 1380 ASSERT(cpu_lpl != NULL); 1381 1382 /* 1383 * Remove the CPU from the list of active CPUs. 1384 */ 1385 cpu_remove_active(cp); 1386 1387 /* 1388 * Walk the active process list and look for threads 1389 * whose home lgroup needs to be updated, or 1390 * the last CPU they run on is the one being offlined now. 1391 */ 1392 1393 ASSERT(curthread->t_cpu != cp); 1394 for (p = practive; p != NULL; p = p->p_next) { 1395 1396 t = p->p_tlist; 1397 1398 if (t == NULL) 1399 continue; 1400 1401 lgrp_diff_lpl = 0; 1402 1403 do { 1404 ASSERT(t->t_lpl != NULL); 1405 /* 1406 * Taking last CPU in lpl offline 1407 * Rehome thread if it is in this lpl 1408 * Otherwise, update the count of how many 1409 * threads are in this CPU's lgroup but have 1410 * a different lpl. 1411 */ 1412 1413 if (cpu_lpl->lpl_ncpu == 0) { 1414 if (t->t_lpl == cpu_lpl) 1415 lgrp_move_thread(t, 1416 lgrp_choose(t, 1417 t->t_cpupart), 0); 1418 else if (t->t_lpl->lpl_lgrpid == 1419 cpu_lpl->lpl_lgrpid) 1420 lgrp_diff_lpl++; 1421 } 1422 ASSERT(t->t_lpl->lpl_ncpu > 0); 1423 1424 /* 1425 * Update CPU last ran on if it was this CPU 1426 */ 1427 if (t->t_cpu == cp && t->t_bound_cpu != cp) 1428 t->t_cpu = disp_lowpri_cpu(ncp, t->t_lpl, 1429 t->t_pri, NULL); 1430 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp || 1431 t->t_weakbound_cpu == cp); 1432 1433 t = t->t_forw; 1434 } while (t != p->p_tlist); 1435 1436 /* 1437 * Didn't find any threads in the same lgroup as this 1438 * CPU with a different lpl, so remove the lgroup from 1439 * the process lgroup bitmask. 1440 */ 1441 1442 if (lgrp_diff_lpl == 0) 1443 klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid); 1444 } 1445 1446 /* 1447 * Walk thread list looking for threads that need to be 1448 * rehomed, since there are some threads that are not in 1449 * their process's p_tlist. 1450 */ 1451 1452 t = curthread; 1453 do { 1454 ASSERT(t != NULL && t->t_lpl != NULL); 1455 1456 /* 1457 * Rehome threads with same lpl as this CPU when this 1458 * is the last CPU in the lpl. 1459 */ 1460 1461 if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl)) 1462 lgrp_move_thread(t, 1463 lgrp_choose(t, t->t_cpupart), 1); 1464 1465 ASSERT(t->t_lpl->lpl_ncpu > 0); 1466 1467 /* 1468 * Update CPU last ran on if it was this CPU 1469 */ 1470 1471 if (t->t_cpu == cp && t->t_bound_cpu != cp) { 1472 t->t_cpu = disp_lowpri_cpu(ncp, 1473 t->t_lpl, t->t_pri, NULL); 1474 } 1475 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp || 1476 t->t_weakbound_cpu == cp); 1477 t = t->t_next; 1478 1479 } while (t != curthread); 1480 ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0); 1481 cp->cpu_flags |= CPU_OFFLINE; 1482 disp_cpu_inactive(cp); 1483 if (!no_quiesce) 1484 cp->cpu_flags |= CPU_QUIESCED; 1485 ncpus_online--; 1486 cpu_set_state(cp); 1487 cpu_inmotion = NULL; 1488 start_cpus(); 1489 cpu_stats_kstat_destroy(cp); 1490 cpu_delete_intrstat(cp); 1491 lgrp_kstat_destroy(cp); 1492 } 1493 1494 out: 1495 cpu_inmotion = NULL; 1496 1497 /* 1498 * If we failed, re-enable interrupts. 1499 * Do this even if cpu_intr_disable returned an error, because 1500 * it may have partially disabled interrupts. 1501 */ 1502 if (error && intr_enable) 1503 cpu_intr_enable(cp); 1504 1505 /* 1506 * If we failed, but managed to offline the cyclic subsystem on this 1507 * CPU, bring it back online. 1508 */ 1509 if (error && cyclic_off) 1510 cyclic_online(cp); 1511 1512 /* 1513 * If we failed, we need to notify everyone that this CPU is back on. 1514 */ 1515 if (error != 0) 1516 cpu_state_change_notify(cp->cpu_id, CPU_ON); 1517 1518 return (error); 1519 } 1520 1521 /* 1522 * Mark the indicated CPU as faulted, taking it offline. 1523 */ 1524 int 1525 cpu_faulted(cpu_t *cp, int flags) 1526 { 1527 int error = 0; 1528 1529 ASSERT(MUTEX_HELD(&cpu_lock)); 1530 ASSERT(!cpu_is_poweredoff(cp)); 1531 1532 if (cpu_is_offline(cp)) { 1533 cp->cpu_flags &= ~CPU_SPARE; 1534 cp->cpu_flags |= CPU_FAULTED; 1535 mp_cpu_faulted_enter(cp); 1536 cpu_set_state(cp); 1537 return (0); 1538 } 1539 1540 if ((error = cpu_offline(cp, flags)) == 0) { 1541 cp->cpu_flags |= CPU_FAULTED; 1542 mp_cpu_faulted_enter(cp); 1543 cpu_set_state(cp); 1544 } 1545 1546 return (error); 1547 } 1548 1549 /* 1550 * Mark the indicated CPU as a spare, taking it offline. 1551 */ 1552 int 1553 cpu_spare(cpu_t *cp, int flags) 1554 { 1555 int error = 0; 1556 1557 ASSERT(MUTEX_HELD(&cpu_lock)); 1558 ASSERT(!cpu_is_poweredoff(cp)); 1559 1560 if (cpu_is_offline(cp)) { 1561 if (cp->cpu_flags & CPU_FAULTED) { 1562 cp->cpu_flags &= ~CPU_FAULTED; 1563 mp_cpu_faulted_exit(cp); 1564 } 1565 cp->cpu_flags |= CPU_SPARE; 1566 cpu_set_state(cp); 1567 return (0); 1568 } 1569 1570 if ((error = cpu_offline(cp, flags)) == 0) { 1571 cp->cpu_flags |= CPU_SPARE; 1572 cpu_set_state(cp); 1573 } 1574 1575 return (error); 1576 } 1577 1578 /* 1579 * Take the indicated CPU from poweroff to offline. 1580 */ 1581 int 1582 cpu_poweron(cpu_t *cp) 1583 { 1584 int error = ENOTSUP; 1585 1586 ASSERT(MUTEX_HELD(&cpu_lock)); 1587 ASSERT(cpu_is_poweredoff(cp)); 1588 1589 error = mp_cpu_poweron(cp); /* arch-dep hook */ 1590 if (error == 0) 1591 cpu_set_state(cp); 1592 1593 return (error); 1594 } 1595 1596 /* 1597 * Take the indicated CPU from any inactive state to powered off. 1598 */ 1599 int 1600 cpu_poweroff(cpu_t *cp) 1601 { 1602 int error = ENOTSUP; 1603 1604 ASSERT(MUTEX_HELD(&cpu_lock)); 1605 ASSERT(cpu_is_offline(cp)); 1606 1607 if (!(cp->cpu_flags & CPU_QUIESCED)) 1608 return (EBUSY); /* not completely idle */ 1609 1610 error = mp_cpu_poweroff(cp); /* arch-dep hook */ 1611 if (error == 0) 1612 cpu_set_state(cp); 1613 1614 return (error); 1615 } 1616 1617 /* 1618 * Initialize the CPU lists for the first CPU. 1619 */ 1620 void 1621 cpu_list_init(cpu_t *cp) 1622 { 1623 cp->cpu_next = cp; 1624 cp->cpu_prev = cp; 1625 cpu_list = cp; 1626 1627 cp->cpu_next_onln = cp; 1628 cp->cpu_prev_onln = cp; 1629 cpu_active = cp; 1630 1631 cp->cpu_seqid = 0; 1632 CPUSET_ADD(cpu_seqid_inuse, 0); 1633 cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid); 1634 cp_default.cp_cpulist = cp; 1635 cp_default.cp_ncpus = 1; 1636 cp->cpu_next_lpl = cp; 1637 cp->cpu_prev_lpl = cp; 1638 cp->cpu_next_lgrp = cp; 1639 cp->cpu_prev_lgrp = cp; 1640 cp->cpu_next_part = cp; 1641 cp->cpu_prev_part = cp; 1642 cp->cpu_part = &cp_default; 1643 cp->cpu_lpl = lpl_bootstrap; 1644 1645 CPUSET_ADD(cpu_available, cp->cpu_id); 1646 } 1647 1648 /* 1649 * Insert a CPU into the list of available CPUs. 1650 */ 1651 void 1652 cpu_add_unit(cpu_t *cp) 1653 { 1654 int seqid; 1655 1656 ASSERT(MUTEX_HELD(&cpu_lock)); 1657 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */ 1658 1659 lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0); 1660 1661 /* 1662 * Note: most users of the cpu_list will grab the 1663 * cpu_lock to insure that it isn't modified. However, 1664 * certain users can't or won't do that. To allow this 1665 * we pause the other cpus. Users who walk the list 1666 * without cpu_lock, must disable kernel preemption 1667 * to insure that the list isn't modified underneath 1668 * them. Also, any cached pointers to cpu structures 1669 * must be revalidated by checking to see if the 1670 * cpu_next pointer points to itself. This check must 1671 * be done with the cpu_lock held or kernel preemption 1672 * disabled. This check relies upon the fact that 1673 * old cpu structures are not free'ed or cleared after 1674 * then are removed from the cpu_list. 1675 * 1676 * Note that the clock code walks the cpu list dereferencing 1677 * the cpu_part pointer, so we need to initialize it before 1678 * adding the cpu to the list. 1679 */ 1680 cp->cpu_part = &cp_default; 1681 (void) pause_cpus(NULL); 1682 cp->cpu_next = cpu_list; 1683 cp->cpu_prev = cpu_list->cpu_prev; 1684 cpu_list->cpu_prev->cpu_next = cp; 1685 cpu_list->cpu_prev = cp; 1686 start_cpus(); 1687 1688 for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++) 1689 continue; 1690 CPUSET_ADD(cpu_seqid_inuse, seqid); 1691 cp->cpu_seqid = seqid; 1692 ASSERT(ncpus < max_ncpus); 1693 ncpus++; 1694 cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid); 1695 cpu[cp->cpu_id] = cp; 1696 CPUSET_ADD(cpu_available, cp->cpu_id); 1697 1698 /* 1699 * allocate a pause thread for this CPU. 1700 */ 1701 cpu_pause_alloc(cp); 1702 1703 /* 1704 * So that new CPUs won't have NULL prev_onln and next_onln pointers, 1705 * link them into a list of just that CPU. 1706 * This is so that disp_lowpri_cpu will work for thread_create in 1707 * pause_cpus() when called from the startup thread in a new CPU. 1708 */ 1709 cp->cpu_next_onln = cp; 1710 cp->cpu_prev_onln = cp; 1711 cpu_info_kstat_create(cp); 1712 cp->cpu_next_part = cp; 1713 cp->cpu_prev_part = cp; 1714 1715 cp->cpu_next_lgrp = cp; 1716 cp->cpu_prev_lgrp = cp; 1717 cp->cpu_next_lpl = cp; 1718 cp->cpu_prev_lpl = cp; 1719 cp->cpu_lpl = lpl_bootstrap; 1720 1721 init_cpu_mstate(cp, CMS_SYSTEM); 1722 1723 pool_pset_mod = gethrtime(); 1724 } 1725 1726 /* 1727 * Do the opposite of cpu_add_unit(). 1728 */ 1729 void 1730 cpu_del_unit(int cpuid) 1731 { 1732 struct cpu *cp, *cpnext; 1733 1734 ASSERT(MUTEX_HELD(&cpu_lock)); 1735 cp = cpu[cpuid]; 1736 ASSERT(cp != NULL); 1737 1738 ASSERT(cp->cpu_next_onln == cp); 1739 ASSERT(cp->cpu_prev_onln == cp); 1740 ASSERT(cp->cpu_next_part == cp); 1741 ASSERT(cp->cpu_prev_part == cp); 1742 1743 chip_cpu_fini(cp); 1744 1745 /* 1746 * Destroy kstat stuff. 1747 */ 1748 cpu_info_kstat_destroy(cp); 1749 term_cpu_mstate(cp); 1750 /* 1751 * Free up pause thread. 1752 */ 1753 cpu_pause_free(cp); 1754 CPUSET_DEL(cpu_available, cp->cpu_id); 1755 cpu[cp->cpu_id] = NULL; 1756 /* 1757 * The clock thread and mutex_vector_enter cannot hold the 1758 * cpu_lock while traversing the cpu list, therefore we pause 1759 * all other threads by pausing the other cpus. These, and any 1760 * other routines holding cpu pointers while possibly sleeping 1761 * must be sure to call kpreempt_disable before processing the 1762 * list and be sure to check that the cpu has not been deleted 1763 * after any sleeps (check cp->cpu_next != NULL). We guarantee 1764 * to keep the deleted cpu structure around. 1765 * 1766 * Note that this MUST be done AFTER cpu_available 1767 * has been updated so that we don't waste time 1768 * trying to pause the cpu we're trying to delete. 1769 */ 1770 (void) pause_cpus(NULL); 1771 1772 cpnext = cp->cpu_next; 1773 cp->cpu_prev->cpu_next = cp->cpu_next; 1774 cp->cpu_next->cpu_prev = cp->cpu_prev; 1775 if (cp == cpu_list) 1776 cpu_list = cpnext; 1777 1778 /* 1779 * Signals that the cpu has been deleted (see above). 1780 */ 1781 cp->cpu_next = NULL; 1782 cp->cpu_prev = NULL; 1783 1784 start_cpus(); 1785 1786 CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid); 1787 ncpus--; 1788 lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0); 1789 1790 pool_pset_mod = gethrtime(); 1791 } 1792 1793 /* 1794 * Add a CPU to the list of active CPUs. 1795 * This routine must not get any locks, because other CPUs are paused. 1796 */ 1797 static void 1798 cpu_add_active_internal(cpu_t *cp) 1799 { 1800 cpupart_t *pp = cp->cpu_part; 1801 1802 ASSERT(MUTEX_HELD(&cpu_lock)); 1803 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */ 1804 1805 ncpus_online++; 1806 cpu_set_state(cp); 1807 cp->cpu_next_onln = cpu_active; 1808 cp->cpu_prev_onln = cpu_active->cpu_prev_onln; 1809 cpu_active->cpu_prev_onln->cpu_next_onln = cp; 1810 cpu_active->cpu_prev_onln = cp; 1811 1812 if (pp->cp_cpulist) { 1813 cp->cpu_next_part = pp->cp_cpulist; 1814 cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part; 1815 pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp; 1816 pp->cp_cpulist->cpu_prev_part = cp; 1817 } else { 1818 ASSERT(pp->cp_ncpus == 0); 1819 pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp; 1820 } 1821 pp->cp_ncpus++; 1822 if (pp->cp_ncpus == 1) { 1823 cp_numparts_nonempty++; 1824 ASSERT(cp_numparts_nonempty != 0); 1825 } 1826 1827 chip_cpu_assign(cp); 1828 1829 lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0); 1830 1831 bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg)); 1832 } 1833 1834 /* 1835 * Add a CPU to the list of active CPUs. 1836 * This is called from machine-dependent layers when a new CPU is started. 1837 */ 1838 void 1839 cpu_add_active(cpu_t *cp) 1840 { 1841 pause_cpus(NULL); 1842 cpu_add_active_internal(cp); 1843 start_cpus(); 1844 cpu_stats_kstat_create(cp); 1845 cpu_create_intrstat(cp); 1846 lgrp_kstat_create(cp); 1847 cpu_state_change_notify(cp->cpu_id, CPU_INIT); 1848 } 1849 1850 1851 /* 1852 * Remove a CPU from the list of active CPUs. 1853 * This routine must not get any locks, because other CPUs are paused. 1854 */ 1855 /* ARGSUSED */ 1856 static void 1857 cpu_remove_active(cpu_t *cp) 1858 { 1859 cpupart_t *pp = cp->cpu_part; 1860 1861 ASSERT(MUTEX_HELD(&cpu_lock)); 1862 ASSERT(cp->cpu_next_onln != cp); /* not the last one */ 1863 ASSERT(cp->cpu_prev_onln != cp); /* not the last one */ 1864 1865 chip_cpu_unassign(cp); 1866 1867 lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0); 1868 1869 cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln; 1870 cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln; 1871 if (cpu_active == cp) { 1872 cpu_active = cp->cpu_next_onln; 1873 } 1874 cp->cpu_next_onln = cp; 1875 cp->cpu_prev_onln = cp; 1876 1877 cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part; 1878 cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part; 1879 if (pp->cp_cpulist == cp) { 1880 pp->cp_cpulist = cp->cpu_next_part; 1881 ASSERT(pp->cp_cpulist != cp); 1882 } 1883 cp->cpu_next_part = cp; 1884 cp->cpu_prev_part = cp; 1885 pp->cp_ncpus--; 1886 if (pp->cp_ncpus == 0) { 1887 cp_numparts_nonempty--; 1888 ASSERT(cp_numparts_nonempty != 0); 1889 } 1890 } 1891 1892 /* 1893 * Routine used to setup a newly inserted CPU in preparation for starting 1894 * it running code. 1895 */ 1896 int 1897 cpu_configure(int cpuid) 1898 { 1899 int retval = 0; 1900 1901 ASSERT(MUTEX_HELD(&cpu_lock)); 1902 1903 /* 1904 * Some structures are statically allocated based upon 1905 * the maximum number of cpus the system supports. Do not 1906 * try to add anything beyond this limit. 1907 */ 1908 if (cpuid < 0 || cpuid >= NCPU) { 1909 return (EINVAL); 1910 } 1911 1912 if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) { 1913 return (EALREADY); 1914 } 1915 1916 if ((retval = mp_cpu_configure(cpuid)) != 0) { 1917 return (retval); 1918 } 1919 1920 cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF; 1921 cpu_set_state(cpu[cpuid]); 1922 retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG); 1923 if (retval != 0) 1924 (void) mp_cpu_unconfigure(cpuid); 1925 1926 return (retval); 1927 } 1928 1929 /* 1930 * Routine used to cleanup a CPU that has been powered off. This will 1931 * destroy all per-cpu information related to this cpu. 1932 */ 1933 int 1934 cpu_unconfigure(int cpuid) 1935 { 1936 int error; 1937 1938 ASSERT(MUTEX_HELD(&cpu_lock)); 1939 1940 if (cpu[cpuid] == NULL) { 1941 return (ENODEV); 1942 } 1943 1944 if (cpu[cpuid]->cpu_flags == 0) { 1945 return (EALREADY); 1946 } 1947 1948 if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) { 1949 return (EBUSY); 1950 } 1951 1952 if (cpu[cpuid]->cpu_props != NULL) { 1953 (void) nvlist_free(cpu[cpuid]->cpu_props); 1954 cpu[cpuid]->cpu_props = NULL; 1955 } 1956 1957 error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG); 1958 1959 if (error != 0) 1960 return (error); 1961 1962 return (mp_cpu_unconfigure(cpuid)); 1963 } 1964 1965 /* 1966 * Routines for registering and de-registering cpu_setup callback functions. 1967 * 1968 * Caller's context 1969 * These routines must not be called from a driver's attach(9E) or 1970 * detach(9E) entry point. 1971 * 1972 * NOTE: CPU callbacks should not block. They are called with cpu_lock held. 1973 */ 1974 1975 /* 1976 * Ideally, these would be dynamically allocated and put into a linked 1977 * list; however that is not feasible because the registration routine 1978 * has to be available before the kmem allocator is working (in fact, 1979 * it is called by the kmem allocator init code). In any case, there 1980 * are quite a few extra entries for future users. 1981 */ 1982 #define NCPU_SETUPS 10 1983 1984 struct cpu_setup { 1985 cpu_setup_func_t *func; 1986 void *arg; 1987 } cpu_setups[NCPU_SETUPS]; 1988 1989 void 1990 register_cpu_setup_func(cpu_setup_func_t *func, void *arg) 1991 { 1992 int i; 1993 1994 ASSERT(MUTEX_HELD(&cpu_lock)); 1995 1996 for (i = 0; i < NCPU_SETUPS; i++) 1997 if (cpu_setups[i].func == NULL) 1998 break; 1999 if (i >= NCPU_SETUPS) 2000 cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries"); 2001 2002 cpu_setups[i].func = func; 2003 cpu_setups[i].arg = arg; 2004 } 2005 2006 void 2007 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg) 2008 { 2009 int i; 2010 2011 ASSERT(MUTEX_HELD(&cpu_lock)); 2012 2013 for (i = 0; i < NCPU_SETUPS; i++) 2014 if ((cpu_setups[i].func == func) && 2015 (cpu_setups[i].arg == arg)) 2016 break; 2017 if (i >= NCPU_SETUPS) 2018 cmn_err(CE_PANIC, "Could not find cpu_setup callback to " 2019 "deregister"); 2020 2021 cpu_setups[i].func = NULL; 2022 cpu_setups[i].arg = 0; 2023 } 2024 2025 /* 2026 * Call any state change hooks for this CPU, ignore any errors. 2027 */ 2028 void 2029 cpu_state_change_notify(int id, cpu_setup_t what) 2030 { 2031 int i; 2032 2033 ASSERT(MUTEX_HELD(&cpu_lock)); 2034 2035 for (i = 0; i < NCPU_SETUPS; i++) { 2036 if (cpu_setups[i].func != NULL) { 2037 cpu_setups[i].func(what, id, cpu_setups[i].arg); 2038 } 2039 } 2040 } 2041 2042 /* 2043 * Call any state change hooks for this CPU, undo it if error found. 2044 */ 2045 static int 2046 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo) 2047 { 2048 int i; 2049 int retval = 0; 2050 2051 ASSERT(MUTEX_HELD(&cpu_lock)); 2052 2053 for (i = 0; i < NCPU_SETUPS; i++) { 2054 if (cpu_setups[i].func != NULL) { 2055 retval = cpu_setups[i].func(what, id, 2056 cpu_setups[i].arg); 2057 if (retval) { 2058 for (i--; i >= 0; i--) { 2059 if (cpu_setups[i].func != NULL) 2060 cpu_setups[i].func(undo, 2061 id, cpu_setups[i].arg); 2062 } 2063 break; 2064 } 2065 } 2066 } 2067 return (retval); 2068 } 2069 2070 /* 2071 * Export information about this CPU via the kstat mechanism. 2072 */ 2073 static struct { 2074 kstat_named_t ci_state; 2075 kstat_named_t ci_state_begin; 2076 kstat_named_t ci_cpu_type; 2077 kstat_named_t ci_fpu_type; 2078 kstat_named_t ci_clock_MHz; 2079 kstat_named_t ci_chip_id; 2080 kstat_named_t ci_implementation; 2081 #ifdef __sparcv9 2082 kstat_named_t ci_device_ID; 2083 kstat_named_t ci_cpu_fru; 2084 #endif 2085 kstat_named_t ci_brandstr; 2086 } cpu_info_template = { 2087 { "state", KSTAT_DATA_CHAR }, 2088 { "state_begin", KSTAT_DATA_LONG }, 2089 { "cpu_type", KSTAT_DATA_CHAR }, 2090 { "fpu_type", KSTAT_DATA_CHAR }, 2091 { "clock_MHz", KSTAT_DATA_LONG }, 2092 { "chip_id", KSTAT_DATA_LONG }, 2093 { "implementation", KSTAT_DATA_STRING }, 2094 #ifdef __sparcv9 2095 { "device_ID", KSTAT_DATA_UINT64 }, 2096 { "cpu_fru", KSTAT_DATA_STRING }, 2097 #endif 2098 { "brand", KSTAT_DATA_STRING }, 2099 }; 2100 2101 static kmutex_t cpu_info_template_lock; 2102 2103 static int 2104 cpu_info_kstat_update(kstat_t *ksp, int rw) 2105 { 2106 cpu_t *cp = ksp->ks_private; 2107 const char *pi_state; 2108 2109 if (rw == KSTAT_WRITE) 2110 return (EACCES); 2111 2112 switch (cp->cpu_type_info.pi_state) { 2113 case P_ONLINE: 2114 pi_state = PS_ONLINE; 2115 break; 2116 case P_POWEROFF: 2117 pi_state = PS_POWEROFF; 2118 break; 2119 case P_NOINTR: 2120 pi_state = PS_NOINTR; 2121 break; 2122 case P_FAULTED: 2123 pi_state = PS_FAULTED; 2124 break; 2125 case P_SPARE: 2126 pi_state = PS_SPARE; 2127 break; 2128 case P_OFFLINE: 2129 pi_state = PS_OFFLINE; 2130 break; 2131 default: 2132 pi_state = "unknown"; 2133 } 2134 (void) strcpy(cpu_info_template.ci_state.value.c, pi_state); 2135 cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin; 2136 (void) strncpy(cpu_info_template.ci_cpu_type.value.c, 2137 cp->cpu_type_info.pi_processor_type, 15); 2138 (void) strncpy(cpu_info_template.ci_fpu_type.value.c, 2139 cp->cpu_type_info.pi_fputypes, 15); 2140 cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock; 2141 cpu_info_template.ci_chip_id.value.l = chip_plat_get_chipid(cp); 2142 kstat_named_setstr(&cpu_info_template.ci_implementation, 2143 cp->cpu_idstr); 2144 #ifdef __sparcv9 2145 cpu_info_template.ci_device_ID.value.ui64 = 2146 cpunodes[cp->cpu_id].device_id; 2147 kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp)); 2148 #endif 2149 kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr); 2150 return (0); 2151 } 2152 2153 static void 2154 cpu_info_kstat_create(cpu_t *cp) 2155 { 2156 zoneid_t zoneid; 2157 2158 ASSERT(MUTEX_HELD(&cpu_lock)); 2159 2160 if (pool_pset_enabled()) 2161 zoneid = GLOBAL_ZONEID; 2162 else 2163 zoneid = ALL_ZONES; 2164 if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id, 2165 NULL, "misc", KSTAT_TYPE_NAMED, 2166 sizeof (cpu_info_template) / sizeof (kstat_named_t), 2167 KSTAT_FLAG_VIRTUAL, zoneid)) != NULL) { 2168 cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN; 2169 #ifdef __sparcv9 2170 cp->cpu_info_kstat->ks_data_size += 2171 strlen(cpu_fru_fmri(cp)) + 1; 2172 #endif 2173 cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock; 2174 cp->cpu_info_kstat->ks_data = &cpu_info_template; 2175 cp->cpu_info_kstat->ks_private = cp; 2176 cp->cpu_info_kstat->ks_update = cpu_info_kstat_update; 2177 kstat_install(cp->cpu_info_kstat); 2178 } 2179 } 2180 2181 static void 2182 cpu_info_kstat_destroy(cpu_t *cp) 2183 { 2184 ASSERT(MUTEX_HELD(&cpu_lock)); 2185 2186 kstat_delete(cp->cpu_info_kstat); 2187 cp->cpu_info_kstat = NULL; 2188 } 2189 2190 /* 2191 * Create and install kstats for the boot CPU. 2192 */ 2193 void 2194 cpu_kstat_init(cpu_t *cp) 2195 { 2196 mutex_enter(&cpu_lock); 2197 cpu_info_kstat_create(cp); 2198 cpu_stats_kstat_create(cp); 2199 cpu_create_intrstat(cp); 2200 chip_kstat_create(cp->cpu_chip); 2201 cpu_set_state(cp); 2202 mutex_exit(&cpu_lock); 2203 } 2204 2205 /* 2206 * Make visible to the zone that subset of the cpu information that would be 2207 * initialized when a cpu is configured (but still offline). 2208 */ 2209 void 2210 cpu_visibility_configure(cpu_t *cp, zone_t *zone) 2211 { 2212 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2213 2214 ASSERT(MUTEX_HELD(&cpu_lock)); 2215 ASSERT(pool_pset_enabled()); 2216 ASSERT(cp != NULL); 2217 2218 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2219 zone->zone_ncpus++; 2220 ASSERT(zone->zone_ncpus <= ncpus); 2221 } 2222 if (cp->cpu_info_kstat != NULL) 2223 kstat_zone_add(cp->cpu_info_kstat, zoneid); 2224 } 2225 2226 /* 2227 * Make visible to the zone that subset of the cpu information that would be 2228 * initialized when a previously configured cpu is onlined. 2229 */ 2230 void 2231 cpu_visibility_online(cpu_t *cp, zone_t *zone) 2232 { 2233 kstat_t *ksp; 2234 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */ 2235 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2236 processorid_t cpun; 2237 2238 ASSERT(MUTEX_HELD(&cpu_lock)); 2239 ASSERT(pool_pset_enabled()); 2240 ASSERT(cp != NULL); 2241 ASSERT(cpu_is_active(cp)); 2242 2243 cpun = cp->cpu_id; 2244 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2245 zone->zone_ncpus_online++; 2246 ASSERT(zone->zone_ncpus_online <= ncpus_online); 2247 } 2248 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun); 2249 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES)) 2250 != NULL) { 2251 kstat_zone_add(ksp, zoneid); 2252 kstat_rele(ksp); 2253 } 2254 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) { 2255 kstat_zone_add(ksp, zoneid); 2256 kstat_rele(ksp); 2257 } 2258 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) { 2259 kstat_zone_add(ksp, zoneid); 2260 kstat_rele(ksp); 2261 } 2262 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) != 2263 NULL) { 2264 kstat_zone_add(ksp, zoneid); 2265 kstat_rele(ksp); 2266 } 2267 } 2268 2269 /* 2270 * Update relevant kstats such that cpu is now visible to processes 2271 * executing in specified zone. 2272 */ 2273 void 2274 cpu_visibility_add(cpu_t *cp, zone_t *zone) 2275 { 2276 cpu_visibility_configure(cp, zone); 2277 if (cpu_is_active(cp)) 2278 cpu_visibility_online(cp, zone); 2279 } 2280 2281 /* 2282 * Make invisible to the zone that subset of the cpu information that would be 2283 * torn down when a previously offlined cpu is unconfigured. 2284 */ 2285 void 2286 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone) 2287 { 2288 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2289 2290 ASSERT(MUTEX_HELD(&cpu_lock)); 2291 ASSERT(pool_pset_enabled()); 2292 ASSERT(cp != NULL); 2293 2294 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2295 ASSERT(zone->zone_ncpus != 0); 2296 zone->zone_ncpus--; 2297 } 2298 if (cp->cpu_info_kstat) 2299 kstat_zone_remove(cp->cpu_info_kstat, zoneid); 2300 } 2301 2302 /* 2303 * Make invisible to the zone that subset of the cpu information that would be 2304 * torn down when a cpu is offlined (but still configured). 2305 */ 2306 void 2307 cpu_visibility_offline(cpu_t *cp, zone_t *zone) 2308 { 2309 kstat_t *ksp; 2310 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */ 2311 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2312 processorid_t cpun; 2313 2314 ASSERT(MUTEX_HELD(&cpu_lock)); 2315 ASSERT(pool_pset_enabled()); 2316 ASSERT(cp != NULL); 2317 ASSERT(cpu_is_active(cp)); 2318 2319 cpun = cp->cpu_id; 2320 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2321 ASSERT(zone->zone_ncpus_online != 0); 2322 zone->zone_ncpus_online--; 2323 } 2324 2325 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) != 2326 NULL) { 2327 kstat_zone_remove(ksp, zoneid); 2328 kstat_rele(ksp); 2329 } 2330 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) { 2331 kstat_zone_remove(ksp, zoneid); 2332 kstat_rele(ksp); 2333 } 2334 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) { 2335 kstat_zone_remove(ksp, zoneid); 2336 kstat_rele(ksp); 2337 } 2338 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun); 2339 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES)) 2340 != NULL) { 2341 kstat_zone_remove(ksp, zoneid); 2342 kstat_rele(ksp); 2343 } 2344 } 2345 2346 /* 2347 * Update relevant kstats such that cpu is no longer visible to processes 2348 * executing in specified zone. 2349 */ 2350 void 2351 cpu_visibility_remove(cpu_t *cp, zone_t *zone) 2352 { 2353 if (cpu_is_active(cp)) 2354 cpu_visibility_offline(cp, zone); 2355 cpu_visibility_unconfigure(cp, zone); 2356 } 2357 2358 /* 2359 * Bind a thread to a CPU as requested. 2360 */ 2361 int 2362 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind, 2363 int *error) 2364 { 2365 processorid_t binding; 2366 cpu_t *cp; 2367 2368 ASSERT(MUTEX_HELD(&cpu_lock)); 2369 ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock)); 2370 2371 thread_lock(tp); 2372 2373 /* 2374 * Record old binding, but change the obind, which was initialized 2375 * to PBIND_NONE, only if this thread has a binding. This avoids 2376 * reporting PBIND_NONE for a process when some LWPs are bound. 2377 */ 2378 binding = tp->t_bind_cpu; 2379 if (binding != PBIND_NONE) 2380 *obind = binding; /* record old binding */ 2381 2382 if (bind == PBIND_QUERY) { 2383 thread_unlock(tp); 2384 return (0); 2385 } 2386 2387 /* 2388 * If this thread/LWP cannot be bound because of permission 2389 * problems, just note that and return success so that the 2390 * other threads/LWPs will be bound. This is the way 2391 * processor_bind() is defined to work. 2392 * 2393 * Binding will get EPERM if the thread is of system class 2394 * or hasprocperm() fails. 2395 */ 2396 if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) { 2397 *error = EPERM; 2398 thread_unlock(tp); 2399 return (0); 2400 } 2401 2402 binding = bind; 2403 if (binding != PBIND_NONE) { 2404 cp = cpu[binding]; 2405 /* 2406 * Make sure binding is in right partition. 2407 */ 2408 if (tp->t_cpupart != cp->cpu_part) { 2409 *error = EINVAL; 2410 thread_unlock(tp); 2411 return (0); 2412 } 2413 } 2414 tp->t_bind_cpu = binding; /* set new binding */ 2415 2416 /* 2417 * If there is no system-set reason for affinity, set 2418 * the t_bound_cpu field to reflect the binding. 2419 */ 2420 if (tp->t_affinitycnt == 0) { 2421 if (binding == PBIND_NONE) { 2422 /* 2423 * We may need to adjust disp_max_unbound_pri 2424 * since we're becoming unbound. 2425 */ 2426 disp_adjust_unbound_pri(tp); 2427 2428 tp->t_bound_cpu = NULL; /* set new binding */ 2429 2430 /* 2431 * Move thread to lgroup with strongest affinity 2432 * after unbinding 2433 */ 2434 if (tp->t_lgrp_affinity) 2435 lgrp_move_thread(tp, 2436 lgrp_choose(tp, tp->t_cpupart), 1); 2437 2438 if (tp->t_state == TS_ONPROC && 2439 tp->t_cpu->cpu_part != tp->t_cpupart) 2440 cpu_surrender(tp); 2441 } else { 2442 lpl_t *lpl; 2443 2444 tp->t_bound_cpu = cp; 2445 ASSERT(cp->cpu_lpl != NULL); 2446 2447 /* 2448 * Set home to lgroup with most affinity containing CPU 2449 * that thread is being bound or minimum bounding 2450 * lgroup if no affinities set 2451 */ 2452 if (tp->t_lgrp_affinity) 2453 lpl = lgrp_affinity_best(tp, tp->t_cpupart, 0); 2454 else 2455 lpl = cp->cpu_lpl; 2456 2457 if (tp->t_lpl != lpl) { 2458 /* can't grab cpu_lock */ 2459 lgrp_move_thread(tp, lpl, 1); 2460 } 2461 2462 /* 2463 * Make the thread switch to the bound CPU. 2464 * If the thread is runnable, we need to 2465 * requeue it even if t_cpu is already set 2466 * to the right CPU, since it may be on a 2467 * kpreempt queue and need to move to a local 2468 * queue. We could check t_disp_queue to 2469 * avoid unnecessary overhead if it's already 2470 * on the right queue, but since this isn't 2471 * a performance-critical operation it doesn't 2472 * seem worth the extra code and complexity. 2473 * 2474 * If the thread is weakbound to the cpu then it will 2475 * resist the new binding request until the weak 2476 * binding drops. The cpu_surrender or requeueing 2477 * below could be skipped in such cases (since it 2478 * will have no effect), but that would require 2479 * thread_allowmigrate to acquire thread_lock so 2480 * we'll take the very occasional hit here instead. 2481 */ 2482 if (tp->t_state == TS_ONPROC) { 2483 cpu_surrender(tp); 2484 } else if (tp->t_state == TS_RUN) { 2485 cpu_t *ocp = tp->t_cpu; 2486 2487 (void) dispdeq(tp); 2488 setbackdq(tp); 2489 /* 2490 * Either on the bound CPU's disp queue now, 2491 * or swapped out or on the swap queue. 2492 */ 2493 ASSERT(tp->t_disp_queue == cp->cpu_disp || 2494 tp->t_weakbound_cpu == ocp || 2495 (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) 2496 != TS_LOAD); 2497 } 2498 } 2499 } 2500 2501 /* 2502 * Our binding has changed; set TP_CHANGEBIND. 2503 */ 2504 tp->t_proc_flag |= TP_CHANGEBIND; 2505 aston(tp); 2506 2507 thread_unlock(tp); 2508 2509 return (0); 2510 } 2511 2512 #if CPUSET_WORDS > 1 2513 2514 /* 2515 * Functions for implementing cpuset operations when a cpuset is more 2516 * than one word. On platforms where a cpuset is a single word these 2517 * are implemented as macros in cpuvar.h. 2518 */ 2519 2520 void 2521 cpuset_all(cpuset_t *s) 2522 { 2523 int i; 2524 2525 for (i = 0; i < CPUSET_WORDS; i++) 2526 s->cpub[i] = ~0UL; 2527 } 2528 2529 void 2530 cpuset_all_but(cpuset_t *s, uint_t cpu) 2531 { 2532 cpuset_all(s); 2533 CPUSET_DEL(*s, cpu); 2534 } 2535 2536 void 2537 cpuset_only(cpuset_t *s, uint_t cpu) 2538 { 2539 CPUSET_ZERO(*s); 2540 CPUSET_ADD(*s, cpu); 2541 } 2542 2543 int 2544 cpuset_isnull(cpuset_t *s) 2545 { 2546 int i; 2547 2548 for (i = 0; i < CPUSET_WORDS; i++) 2549 if (s->cpub[i] != 0) 2550 return (0); 2551 return (1); 2552 } 2553 2554 int 2555 cpuset_cmp(cpuset_t *s1, cpuset_t *s2) 2556 { 2557 int i; 2558 2559 for (i = 0; i < CPUSET_WORDS; i++) 2560 if (s1->cpub[i] != s2->cpub[i]) 2561 return (0); 2562 return (1); 2563 } 2564 2565 uint_t 2566 cpuset_find(cpuset_t *s) 2567 { 2568 2569 uint_t i; 2570 uint_t cpu = (uint_t)-1; 2571 2572 /* 2573 * Find a cpu in the cpuset 2574 */ 2575 for (i = 0; (i < CPUSET_WORDS && cpu == (uint_t)-1); i++) 2576 cpu = (uint_t)(lowbit(s->cpub[i]) - 1); 2577 return (cpu); 2578 } 2579 2580 #endif /* CPUSET_WORDS */ 2581 2582 /* 2583 * Unbind all user threads bound to a given CPU. 2584 */ 2585 int 2586 cpu_unbind(processorid_t cpu) 2587 { 2588 processorid_t obind; 2589 kthread_t *tp; 2590 int ret = 0; 2591 proc_t *pp; 2592 int err, berr = 0; 2593 2594 ASSERT(MUTEX_HELD(&cpu_lock)); 2595 2596 mutex_enter(&pidlock); 2597 for (pp = practive; pp != NULL; pp = pp->p_next) { 2598 mutex_enter(&pp->p_lock); 2599 tp = pp->p_tlist; 2600 /* 2601 * Skip zombies, kernel processes, and processes in 2602 * other zones, if called from a non-global zone. 2603 */ 2604 if (tp == NULL || (pp->p_flag & SSYS) || 2605 !HASZONEACCESS(curproc, pp->p_zone->zone_id)) { 2606 mutex_exit(&pp->p_lock); 2607 continue; 2608 } 2609 do { 2610 if (tp->t_bind_cpu != cpu) 2611 continue; 2612 err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr); 2613 if (ret == 0) 2614 ret = err; 2615 } while ((tp = tp->t_forw) != pp->p_tlist); 2616 mutex_exit(&pp->p_lock); 2617 } 2618 mutex_exit(&pidlock); 2619 if (ret == 0) 2620 ret = berr; 2621 return (ret); 2622 } 2623 2624 2625 /* 2626 * Destroy all remaining bound threads on a cpu. 2627 */ 2628 void 2629 cpu_destroy_bound_threads(cpu_t *cp) 2630 { 2631 extern id_t syscid; 2632 register kthread_id_t t, tlist, tnext; 2633 2634 /* 2635 * Destroy all remaining bound threads on the cpu. This 2636 * should include both the interrupt threads and the idle thread. 2637 * This requires some care, since we need to traverse the 2638 * thread list with the pidlock mutex locked, but thread_free 2639 * also locks the pidlock mutex. So, we collect the threads 2640 * we're going to reap in a list headed by "tlist", then we 2641 * unlock the pidlock mutex and traverse the tlist list, 2642 * doing thread_free's on the thread's. Simple, n'est pas? 2643 * Also, this depends on thread_free not mucking with the 2644 * t_next and t_prev links of the thread. 2645 */ 2646 2647 if ((t = curthread) != NULL) { 2648 2649 tlist = NULL; 2650 mutex_enter(&pidlock); 2651 do { 2652 tnext = t->t_next; 2653 if (t->t_bound_cpu == cp) { 2654 2655 /* 2656 * We've found a bound thread, carefully unlink 2657 * it out of the thread list, and add it to 2658 * our "tlist". We "know" we don't have to 2659 * worry about unlinking curthread (the thread 2660 * that is executing this code). 2661 */ 2662 t->t_next->t_prev = t->t_prev; 2663 t->t_prev->t_next = t->t_next; 2664 t->t_next = tlist; 2665 tlist = t; 2666 ASSERT(t->t_cid == syscid); 2667 /* wake up anyone blocked in thread_join */ 2668 cv_broadcast(&t->t_joincv); 2669 /* 2670 * t_lwp set by interrupt threads and not 2671 * cleared. 2672 */ 2673 t->t_lwp = NULL; 2674 /* 2675 * Pause and idle threads always have 2676 * t_state set to TS_ONPROC. 2677 */ 2678 t->t_state = TS_FREE; 2679 t->t_prev = NULL; /* Just in case */ 2680 } 2681 2682 } while ((t = tnext) != curthread); 2683 2684 mutex_exit(&pidlock); 2685 2686 2687 for (t = tlist; t != NULL; t = tnext) { 2688 tnext = t->t_next; 2689 thread_free(t); 2690 } 2691 } 2692 } 2693 2694 /* 2695 * processor_info(2) and p_online(2) status support functions 2696 * The constants returned by the cpu_get_state() and cpu_get_state_str() are 2697 * for use in communicating processor state information to userland. Kernel 2698 * subsystems should only be using the cpu_flags value directly. Subsystems 2699 * modifying cpu_flags should record the state change via a call to the 2700 * cpu_set_state(). 2701 */ 2702 2703 /* 2704 * Update the pi_state of this CPU. This function provides the CPU status for 2705 * the information returned by processor_info(2). 2706 */ 2707 void 2708 cpu_set_state(cpu_t *cpu) 2709 { 2710 ASSERT(MUTEX_HELD(&cpu_lock)); 2711 cpu->cpu_type_info.pi_state = cpu_get_state(cpu); 2712 cpu->cpu_state_begin = gethrestime_sec(); 2713 pool_cpu_mod = gethrtime(); 2714 } 2715 2716 /* 2717 * Return offline/online/other status for the indicated CPU. Use only for 2718 * communication with user applications; cpu_flags provides the in-kernel 2719 * interface. 2720 */ 2721 int 2722 cpu_get_state(cpu_t *cpu) 2723 { 2724 ASSERT(MUTEX_HELD(&cpu_lock)); 2725 if (cpu->cpu_flags & CPU_POWEROFF) 2726 return (P_POWEROFF); 2727 else if (cpu->cpu_flags & CPU_FAULTED) 2728 return (P_FAULTED); 2729 else if (cpu->cpu_flags & CPU_SPARE) 2730 return (P_SPARE); 2731 else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY) 2732 return (P_OFFLINE); 2733 else if (cpu->cpu_flags & CPU_ENABLE) 2734 return (P_ONLINE); 2735 else 2736 return (P_NOINTR); 2737 } 2738 2739 /* 2740 * Return processor_info(2) state as a string. 2741 */ 2742 const char * 2743 cpu_get_state_str(cpu_t *cpu) 2744 { 2745 const char *string; 2746 2747 switch (cpu_get_state(cpu)) { 2748 case P_ONLINE: 2749 string = PS_ONLINE; 2750 break; 2751 case P_POWEROFF: 2752 string = PS_POWEROFF; 2753 break; 2754 case P_NOINTR: 2755 string = PS_NOINTR; 2756 break; 2757 case P_SPARE: 2758 string = PS_SPARE; 2759 break; 2760 case P_FAULTED: 2761 string = PS_FAULTED; 2762 break; 2763 case P_OFFLINE: 2764 string = PS_OFFLINE; 2765 break; 2766 default: 2767 string = "unknown"; 2768 break; 2769 } 2770 return (string); 2771 } 2772 2773 /* 2774 * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named 2775 * kstats, respectively. This is done when a CPU is initialized or placed 2776 * online via p_online(2). 2777 */ 2778 static void 2779 cpu_stats_kstat_create(cpu_t *cp) 2780 { 2781 int instance = cp->cpu_id; 2782 char *module = "cpu"; 2783 char *class = "misc"; 2784 kstat_t *ksp; 2785 zoneid_t zoneid; 2786 2787 ASSERT(MUTEX_HELD(&cpu_lock)); 2788 2789 if (pool_pset_enabled()) 2790 zoneid = GLOBAL_ZONEID; 2791 else 2792 zoneid = ALL_ZONES; 2793 /* 2794 * Create named kstats 2795 */ 2796 #define CPU_STATS_KS_CREATE(name, tsize, update_func) \ 2797 ksp = kstat_create_zone(module, instance, (name), class, \ 2798 KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0, \ 2799 zoneid); \ 2800 if (ksp != NULL) { \ 2801 ksp->ks_private = cp; \ 2802 ksp->ks_update = (update_func); \ 2803 kstat_install(ksp); \ 2804 } else \ 2805 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \ 2806 module, instance, (name)); 2807 2808 CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template), 2809 cpu_sys_stats_ks_update); 2810 CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template), 2811 cpu_vm_stats_ks_update); 2812 2813 /* 2814 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat. 2815 */ 2816 ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL, 2817 "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid); 2818 if (ksp != NULL) { 2819 ksp->ks_update = cpu_stat_ks_update; 2820 ksp->ks_private = cp; 2821 kstat_install(ksp); 2822 } 2823 } 2824 2825 static void 2826 cpu_stats_kstat_destroy(cpu_t *cp) 2827 { 2828 char ks_name[KSTAT_STRLEN]; 2829 2830 (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id); 2831 kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name); 2832 2833 kstat_delete_byname("cpu", cp->cpu_id, "sys"); 2834 kstat_delete_byname("cpu", cp->cpu_id, "vm"); 2835 } 2836 2837 static int 2838 cpu_sys_stats_ks_update(kstat_t *ksp, int rw) 2839 { 2840 cpu_t *cp = (cpu_t *)ksp->ks_private; 2841 struct cpu_sys_stats_ks_data *csskd; 2842 cpu_sys_stats_t *css; 2843 hrtime_t *cmstimep; 2844 hrtime_t newtime; 2845 int i; 2846 2847 if (rw == KSTAT_WRITE) 2848 return (EACCES); 2849 2850 csskd = ksp->ks_data; 2851 css = &cp->cpu_stats.sys; 2852 2853 bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data, 2854 sizeof (cpu_sys_stats_ks_data_template)); 2855 csskd->cpu_ticks_wait.value.ui64 = 0; 2856 csskd->wait_ticks_io.value.ui64 = 0; 2857 csskd->cpu_nsec_idle.value.ui64 = cp->cpu_acct[CMS_IDLE]; 2858 csskd->cpu_nsec_user.value.ui64 = cp->cpu_acct[CMS_USER]; 2859 csskd->cpu_nsec_kernel.value.ui64 = cp->cpu_acct[CMS_SYSTEM]; 2860 2861 /* 2862 * update kstats to include time spent in current state 2863 */ 2864 i = 0; /* set counter to 0 */ 2865 do { 2866 switch (cp->cpu_mstate) { 2867 case CMS_USER: 2868 cmstimep = (hrtime_t *)&csskd->cpu_nsec_user.value.ui64; 2869 break; 2870 case CMS_SYSTEM: 2871 cmstimep = 2872 (hrtime_t *)&csskd->cpu_nsec_kernel.value.ui64; 2873 break; 2874 case CMS_IDLE: 2875 cmstimep = (hrtime_t *)&csskd->cpu_nsec_idle.value.ui64; 2876 break; 2877 case CMS_DISABLED: 2878 panic("cpu_sys_stats_ks_update: disabled state" 2879 " reported!"); 2880 break; 2881 default: 2882 panic("cpu_sys_stats_ks_update: unknown microstate!"); 2883 } 2884 newtime = gethrtime_unscaled() - cp->cpu_mstate_start; 2885 if (newtime < 0) { 2886 newtime = 0; 2887 i++; 2888 } 2889 } while (newtime <= 0 && i < 5); 2890 2891 *cmstimep += newtime; 2892 2893 scalehrtime((hrtime_t *)&csskd->cpu_nsec_idle.value.ui64); 2894 scalehrtime((hrtime_t *)&csskd->cpu_nsec_user.value.ui64); 2895 scalehrtime((hrtime_t *)&csskd->cpu_nsec_kernel.value.ui64); 2896 csskd->cpu_ticks_idle.value.ui64 = 2897 NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64); 2898 csskd->cpu_ticks_user.value.ui64 = 2899 NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64); 2900 csskd->cpu_ticks_kernel.value.ui64 = 2901 NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64); 2902 csskd->bread.value.ui64 = css->bread; 2903 csskd->bwrite.value.ui64 = css->bwrite; 2904 csskd->lread.value.ui64 = css->lread; 2905 csskd->lwrite.value.ui64 = css->lwrite; 2906 csskd->phread.value.ui64 = css->phread; 2907 csskd->phwrite.value.ui64 = css->phwrite; 2908 csskd->pswitch.value.ui64 = css->pswitch; 2909 csskd->trap.value.ui64 = css->trap; 2910 csskd->intr.value.ui64 = 0; 2911 for (i = 0; i < PIL_MAX; i++) 2912 csskd->intr.value.ui64 += css->intr[i]; 2913 csskd->syscall.value.ui64 = css->syscall; 2914 csskd->sysread.value.ui64 = css->sysread; 2915 csskd->syswrite.value.ui64 = css->syswrite; 2916 csskd->sysfork.value.ui64 = css->sysfork; 2917 csskd->sysvfork.value.ui64 = css->sysvfork; 2918 csskd->sysexec.value.ui64 = css->sysexec; 2919 csskd->readch.value.ui64 = css->readch; 2920 csskd->writech.value.ui64 = css->writech; 2921 csskd->rcvint.value.ui64 = css->rcvint; 2922 csskd->xmtint.value.ui64 = css->xmtint; 2923 csskd->mdmint.value.ui64 = css->mdmint; 2924 csskd->rawch.value.ui64 = css->rawch; 2925 csskd->canch.value.ui64 = css->canch; 2926 csskd->outch.value.ui64 = css->outch; 2927 csskd->msg.value.ui64 = css->msg; 2928 csskd->sema.value.ui64 = css->sema; 2929 csskd->namei.value.ui64 = css->namei; 2930 csskd->ufsiget.value.ui64 = css->ufsiget; 2931 csskd->ufsdirblk.value.ui64 = css->ufsdirblk; 2932 csskd->ufsipage.value.ui64 = css->ufsipage; 2933 csskd->ufsinopage.value.ui64 = css->ufsinopage; 2934 csskd->procovf.value.ui64 = css->procovf; 2935 csskd->intrthread.value.ui64 = 0; 2936 for (i = 0; i < LOCK_LEVEL; i++) 2937 csskd->intrthread.value.ui64 += css->intr[i]; 2938 csskd->intrblk.value.ui64 = css->intrblk; 2939 csskd->intrunpin.value.ui64 = css->intrunpin; 2940 csskd->idlethread.value.ui64 = css->idlethread; 2941 csskd->inv_swtch.value.ui64 = css->inv_swtch; 2942 csskd->nthreads.value.ui64 = css->nthreads; 2943 csskd->cpumigrate.value.ui64 = css->cpumigrate; 2944 csskd->xcalls.value.ui64 = css->xcalls; 2945 csskd->mutex_adenters.value.ui64 = css->mutex_adenters; 2946 csskd->rw_rdfails.value.ui64 = css->rw_rdfails; 2947 csskd->rw_wrfails.value.ui64 = css->rw_wrfails; 2948 csskd->modload.value.ui64 = css->modload; 2949 csskd->modunload.value.ui64 = css->modunload; 2950 csskd->bawrite.value.ui64 = css->bawrite; 2951 csskd->iowait.value.ui64 = 0; 2952 2953 return (0); 2954 } 2955 2956 static int 2957 cpu_vm_stats_ks_update(kstat_t *ksp, int rw) 2958 { 2959 cpu_t *cp = (cpu_t *)ksp->ks_private; 2960 struct cpu_vm_stats_ks_data *cvskd; 2961 cpu_vm_stats_t *cvs; 2962 2963 if (rw == KSTAT_WRITE) 2964 return (EACCES); 2965 2966 cvs = &cp->cpu_stats.vm; 2967 cvskd = ksp->ks_data; 2968 2969 bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data, 2970 sizeof (cpu_vm_stats_ks_data_template)); 2971 cvskd->pgrec.value.ui64 = cvs->pgrec; 2972 cvskd->pgfrec.value.ui64 = cvs->pgfrec; 2973 cvskd->pgin.value.ui64 = cvs->pgin; 2974 cvskd->pgpgin.value.ui64 = cvs->pgpgin; 2975 cvskd->pgout.value.ui64 = cvs->pgout; 2976 cvskd->pgpgout.value.ui64 = cvs->pgpgout; 2977 cvskd->swapin.value.ui64 = cvs->swapin; 2978 cvskd->pgswapin.value.ui64 = cvs->pgswapin; 2979 cvskd->swapout.value.ui64 = cvs->swapout; 2980 cvskd->pgswapout.value.ui64 = cvs->pgswapout; 2981 cvskd->zfod.value.ui64 = cvs->zfod; 2982 cvskd->dfree.value.ui64 = cvs->dfree; 2983 cvskd->scan.value.ui64 = cvs->scan; 2984 cvskd->rev.value.ui64 = cvs->rev; 2985 cvskd->hat_fault.value.ui64 = cvs->hat_fault; 2986 cvskd->as_fault.value.ui64 = cvs->as_fault; 2987 cvskd->maj_fault.value.ui64 = cvs->maj_fault; 2988 cvskd->cow_fault.value.ui64 = cvs->cow_fault; 2989 cvskd->prot_fault.value.ui64 = cvs->prot_fault; 2990 cvskd->softlock.value.ui64 = cvs->softlock; 2991 cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt; 2992 cvskd->pgrrun.value.ui64 = cvs->pgrrun; 2993 cvskd->execpgin.value.ui64 = cvs->execpgin; 2994 cvskd->execpgout.value.ui64 = cvs->execpgout; 2995 cvskd->execfree.value.ui64 = cvs->execfree; 2996 cvskd->anonpgin.value.ui64 = cvs->anonpgin; 2997 cvskd->anonpgout.value.ui64 = cvs->anonpgout; 2998 cvskd->anonfree.value.ui64 = cvs->anonfree; 2999 cvskd->fspgin.value.ui64 = cvs->fspgin; 3000 cvskd->fspgout.value.ui64 = cvs->fspgout; 3001 cvskd->fsfree.value.ui64 = cvs->fsfree; 3002 3003 return (0); 3004 } 3005 3006 static int 3007 cpu_stat_ks_update(kstat_t *ksp, int rw) 3008 { 3009 cpu_stat_t *cso; 3010 cpu_t *cp; 3011 int i; 3012 hrtime_t newtime; 3013 hrtime_t cphrt_i; 3014 hrtime_t cphrt_u; 3015 hrtime_t cphrt_s; 3016 hrtime_t *cmstimep; 3017 3018 cso = (cpu_stat_t *)ksp->ks_data; 3019 cp = (cpu_t *)ksp->ks_private; 3020 3021 if (rw == KSTAT_WRITE) 3022 return (EACCES); 3023 3024 cphrt_i = cp->cpu_acct[CMS_IDLE]; 3025 cphrt_u = cp->cpu_acct[CMS_USER]; 3026 cphrt_s = cp->cpu_acct[CMS_SYSTEM]; 3027 3028 /* 3029 * update kstats to include time spent in current state 3030 */ 3031 i = 0; /* set counter to 0 */ 3032 do { 3033 switch (cp->cpu_mstate) { 3034 case CMS_USER: 3035 cmstimep = &cphrt_u; 3036 break; 3037 case CMS_SYSTEM: 3038 cmstimep = &cphrt_s; 3039 break; 3040 case CMS_IDLE: 3041 cmstimep = &cphrt_i; 3042 break; 3043 case CMS_DISABLED: 3044 panic("cpu_stat_ks_update: disabled state reported!"); 3045 break; 3046 default: 3047 panic("cpu_stat_ks_update: unknown microstate!"); 3048 } 3049 newtime = gethrtime_unscaled() - cp->cpu_mstate_start; 3050 if (newtime < 0) { 3051 newtime = 0; 3052 i++; 3053 } 3054 } while (newtime <= 0 && i < 5); 3055 3056 *cmstimep += newtime; 3057 3058 scalehrtime((hrtime_t *)&cphrt_i); 3059 cso->cpu_sysinfo.cpu[CPU_IDLE] = NSEC_TO_TICK(cphrt_i); 3060 3061 scalehrtime((hrtime_t *)&cphrt_u); 3062 cso->cpu_sysinfo.cpu[CPU_USER] = NSEC_TO_TICK(cphrt_u); 3063 3064 scalehrtime((hrtime_t *)&cphrt_s); 3065 cso->cpu_sysinfo.cpu[CPU_KERNEL] = NSEC_TO_TICK(cphrt_s); 3066 3067 cso->cpu_sysinfo.cpu[CPU_WAIT] = 0; 3068 cso->cpu_sysinfo.wait[W_IO] = 0; 3069 cso->cpu_sysinfo.wait[W_SWAP] = 0; 3070 cso->cpu_sysinfo.wait[W_PIO] = 0; 3071 cso->cpu_sysinfo.bread = CPU_STATS(cp, sys.bread); 3072 cso->cpu_sysinfo.bwrite = CPU_STATS(cp, sys.bwrite); 3073 cso->cpu_sysinfo.lread = CPU_STATS(cp, sys.lread); 3074 cso->cpu_sysinfo.lwrite = CPU_STATS(cp, sys.lwrite); 3075 cso->cpu_sysinfo.phread = CPU_STATS(cp, sys.phread); 3076 cso->cpu_sysinfo.phwrite = CPU_STATS(cp, sys.phwrite); 3077 cso->cpu_sysinfo.pswitch = CPU_STATS(cp, sys.pswitch); 3078 cso->cpu_sysinfo.trap = CPU_STATS(cp, sys.trap); 3079 cso->cpu_sysinfo.intr = 0; 3080 for (i = 0; i < PIL_MAX; i++) 3081 cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]); 3082 cso->cpu_sysinfo.syscall = CPU_STATS(cp, sys.syscall); 3083 cso->cpu_sysinfo.sysread = CPU_STATS(cp, sys.sysread); 3084 cso->cpu_sysinfo.syswrite = CPU_STATS(cp, sys.syswrite); 3085 cso->cpu_sysinfo.sysfork = CPU_STATS(cp, sys.sysfork); 3086 cso->cpu_sysinfo.sysvfork = CPU_STATS(cp, sys.sysvfork); 3087 cso->cpu_sysinfo.sysexec = CPU_STATS(cp, sys.sysexec); 3088 cso->cpu_sysinfo.readch = CPU_STATS(cp, sys.readch); 3089 cso->cpu_sysinfo.writech = CPU_STATS(cp, sys.writech); 3090 cso->cpu_sysinfo.rcvint = CPU_STATS(cp, sys.rcvint); 3091 cso->cpu_sysinfo.xmtint = CPU_STATS(cp, sys.xmtint); 3092 cso->cpu_sysinfo.mdmint = CPU_STATS(cp, sys.mdmint); 3093 cso->cpu_sysinfo.rawch = CPU_STATS(cp, sys.rawch); 3094 cso->cpu_sysinfo.canch = CPU_STATS(cp, sys.canch); 3095 cso->cpu_sysinfo.outch = CPU_STATS(cp, sys.outch); 3096 cso->cpu_sysinfo.msg = CPU_STATS(cp, sys.msg); 3097 cso->cpu_sysinfo.sema = CPU_STATS(cp, sys.sema); 3098 cso->cpu_sysinfo.namei = CPU_STATS(cp, sys.namei); 3099 cso->cpu_sysinfo.ufsiget = CPU_STATS(cp, sys.ufsiget); 3100 cso->cpu_sysinfo.ufsdirblk = CPU_STATS(cp, sys.ufsdirblk); 3101 cso->cpu_sysinfo.ufsipage = CPU_STATS(cp, sys.ufsipage); 3102 cso->cpu_sysinfo.ufsinopage = CPU_STATS(cp, sys.ufsinopage); 3103 cso->cpu_sysinfo.inodeovf = 0; 3104 cso->cpu_sysinfo.fileovf = 0; 3105 cso->cpu_sysinfo.procovf = CPU_STATS(cp, sys.procovf); 3106 cso->cpu_sysinfo.intrthread = 0; 3107 for (i = 0; i < LOCK_LEVEL; i++) 3108 cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]); 3109 cso->cpu_sysinfo.intrblk = CPU_STATS(cp, sys.intrblk); 3110 cso->cpu_sysinfo.idlethread = CPU_STATS(cp, sys.idlethread); 3111 cso->cpu_sysinfo.inv_swtch = CPU_STATS(cp, sys.inv_swtch); 3112 cso->cpu_sysinfo.nthreads = CPU_STATS(cp, sys.nthreads); 3113 cso->cpu_sysinfo.cpumigrate = CPU_STATS(cp, sys.cpumigrate); 3114 cso->cpu_sysinfo.xcalls = CPU_STATS(cp, sys.xcalls); 3115 cso->cpu_sysinfo.mutex_adenters = CPU_STATS(cp, sys.mutex_adenters); 3116 cso->cpu_sysinfo.rw_rdfails = CPU_STATS(cp, sys.rw_rdfails); 3117 cso->cpu_sysinfo.rw_wrfails = CPU_STATS(cp, sys.rw_wrfails); 3118 cso->cpu_sysinfo.modload = CPU_STATS(cp, sys.modload); 3119 cso->cpu_sysinfo.modunload = CPU_STATS(cp, sys.modunload); 3120 cso->cpu_sysinfo.bawrite = CPU_STATS(cp, sys.bawrite); 3121 cso->cpu_sysinfo.rw_enters = 0; 3122 cso->cpu_sysinfo.win_uo_cnt = 0; 3123 cso->cpu_sysinfo.win_uu_cnt = 0; 3124 cso->cpu_sysinfo.win_so_cnt = 0; 3125 cso->cpu_sysinfo.win_su_cnt = 0; 3126 cso->cpu_sysinfo.win_suo_cnt = 0; 3127 3128 cso->cpu_syswait.iowait = 0; 3129 cso->cpu_syswait.swap = 0; 3130 cso->cpu_syswait.physio = 0; 3131 3132 cso->cpu_vminfo.pgrec = CPU_STATS(cp, vm.pgrec); 3133 cso->cpu_vminfo.pgfrec = CPU_STATS(cp, vm.pgfrec); 3134 cso->cpu_vminfo.pgin = CPU_STATS(cp, vm.pgin); 3135 cso->cpu_vminfo.pgpgin = CPU_STATS(cp, vm.pgpgin); 3136 cso->cpu_vminfo.pgout = CPU_STATS(cp, vm.pgout); 3137 cso->cpu_vminfo.pgpgout = CPU_STATS(cp, vm.pgpgout); 3138 cso->cpu_vminfo.swapin = CPU_STATS(cp, vm.swapin); 3139 cso->cpu_vminfo.pgswapin = CPU_STATS(cp, vm.pgswapin); 3140 cso->cpu_vminfo.swapout = CPU_STATS(cp, vm.swapout); 3141 cso->cpu_vminfo.pgswapout = CPU_STATS(cp, vm.pgswapout); 3142 cso->cpu_vminfo.zfod = CPU_STATS(cp, vm.zfod); 3143 cso->cpu_vminfo.dfree = CPU_STATS(cp, vm.dfree); 3144 cso->cpu_vminfo.scan = CPU_STATS(cp, vm.scan); 3145 cso->cpu_vminfo.rev = CPU_STATS(cp, vm.rev); 3146 cso->cpu_vminfo.hat_fault = CPU_STATS(cp, vm.hat_fault); 3147 cso->cpu_vminfo.as_fault = CPU_STATS(cp, vm.as_fault); 3148 cso->cpu_vminfo.maj_fault = CPU_STATS(cp, vm.maj_fault); 3149 cso->cpu_vminfo.cow_fault = CPU_STATS(cp, vm.cow_fault); 3150 cso->cpu_vminfo.prot_fault = CPU_STATS(cp, vm.prot_fault); 3151 cso->cpu_vminfo.softlock = CPU_STATS(cp, vm.softlock); 3152 cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt); 3153 cso->cpu_vminfo.pgrrun = CPU_STATS(cp, vm.pgrrun); 3154 cso->cpu_vminfo.execpgin = CPU_STATS(cp, vm.execpgin); 3155 cso->cpu_vminfo.execpgout = CPU_STATS(cp, vm.execpgout); 3156 cso->cpu_vminfo.execfree = CPU_STATS(cp, vm.execfree); 3157 cso->cpu_vminfo.anonpgin = CPU_STATS(cp, vm.anonpgin); 3158 cso->cpu_vminfo.anonpgout = CPU_STATS(cp, vm.anonpgout); 3159 cso->cpu_vminfo.anonfree = CPU_STATS(cp, vm.anonfree); 3160 cso->cpu_vminfo.fspgin = CPU_STATS(cp, vm.fspgin); 3161 cso->cpu_vminfo.fspgout = CPU_STATS(cp, vm.fspgout); 3162 cso->cpu_vminfo.fsfree = CPU_STATS(cp, vm.fsfree); 3163 3164 return (0); 3165 } 3166