1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 28 /* All Rights Reserved */ 29 30 31 #pragma ident "%Z%%M% %I% %E% SMI" 32 33 #include <sys/param.h> 34 #include <sys/types.h> 35 #include <sys/sysmacros.h> 36 #include <sys/systm.h> 37 #include <sys/proc.h> 38 #include <sys/cpuvar.h> 39 #include <sys/var.h> 40 #include <sys/tuneable.h> 41 #include <sys/cmn_err.h> 42 #include <sys/buf.h> 43 #include <sys/disp.h> 44 #include <sys/vmsystm.h> 45 #include <sys/vmparam.h> 46 #include <sys/class.h> 47 #include <sys/vtrace.h> 48 #include <sys/modctl.h> 49 #include <sys/debug.h> 50 #include <sys/tnf_probe.h> 51 #include <sys/procfs.h> 52 53 #include <vm/seg.h> 54 #include <vm/seg_kp.h> 55 #include <vm/as.h> 56 #include <vm/rm.h> 57 #include <vm/seg_kmem.h> 58 #include <sys/callb.h> 59 60 /* 61 * The swapper sleeps on runout when there is no one to swap in. 62 * It sleeps on runin when it could not find space to swap someone 63 * in or after swapping someone in. 64 */ 65 char runout; 66 char runin; 67 char wake_sched; /* flag tells clock to wake swapper on next tick */ 68 char wake_sched_sec; /* flag tells clock to wake swapper after a second */ 69 70 /* 71 * The swapper swaps processes to reduce memory demand and runs 72 * when avefree < desfree. The swapper resorts to SOFTSWAP when 73 * avefree < desfree which results in swapping out all processes 74 * sleeping for more than maxslp seconds. HARDSWAP occurs when the 75 * system is on the verge of thrashing and this results in swapping 76 * out runnable threads or threads sleeping for less than maxslp secs. 77 * 78 * The swapper runs through all the active processes in the system 79 * and invokes the scheduling class specific swapin/swapout routine 80 * for every thread in the process to obtain an effective priority 81 * for the process. A priority of -1 implies that the thread isn't 82 * swappable. This effective priority is used to find the most 83 * eligible process to swapout or swapin. 84 * 85 * NOTE: Threads which have been swapped are not linked on any 86 * queue and their dispatcher lock points at the "swapped_lock". 87 * 88 * Processes containing threads with the TS_DONT_SWAP flag set cannot be 89 * swapped out immediately by the swapper. This is due to the fact that 90 * such threads may be holding locks which may be needed by the swapper 91 * to push its pages out. The TS_SWAPENQ flag is set on such threads 92 * to prevent them running in user mode. When such threads reach a 93 * safe point (i.e., are not holding any locks - CL_TRAPRET), they 94 * queue themseleves onto the swap queue which is processed by the 95 * swapper. This results in reducing memory demand when the system 96 * is desparate for memory as the thread can't run in user mode. 97 * 98 * The swap queue consists of threads, linked via t_link, which are 99 * haven't been swapped, are runnable but not on the run queue. The 100 * swap queue is protected by the "swapped_lock". The dispatcher 101 * lock (t_lockp) of all threads on the swap queue points at the 102 * "swapped_lock". Thus, the entire queue and/or threads on the 103 * queue can be locked by acquiring "swapped_lock". 104 */ 105 static kthread_t *tswap_queue; 106 extern disp_lock_t swapped_lock; /* protects swap queue and threads on it */ 107 108 int maxslp = 0; 109 pgcnt_t avefree; /* 5 sec moving average of free memory */ 110 pgcnt_t avefree30; /* 30 sec moving average of free memory */ 111 112 /* 113 * Minimum size used to decide if sufficient memory is available 114 * before a process is swapped in. This is necessary since in most 115 * cases the actual size of a process (p_swrss) being swapped in 116 * is usually 2 pages (kernel stack pages). This is due to the fact 117 * almost all user pages of a process are stolen by pageout before 118 * the swapper decides to swapout it out. 119 */ 120 int min_procsize = 12; 121 122 static int swapin(proc_t *); 123 static int swapout(proc_t *, uint_t *, int); 124 static void process_swap_queue(); 125 126 #ifdef __sparc 127 extern void lwp_swapin(kthread_t *); 128 #endif /* __sparc */ 129 130 /* 131 * Counters to keep track of the number of swapins or swapouts. 132 */ 133 uint_t tot_swapped_in, tot_swapped_out; 134 uint_t softswap, hardswap, swapqswap; 135 136 /* 137 * Macro to determine if a process is eligble to be swapped. 138 */ 139 #define not_swappable(p) \ 140 (((p)->p_flag & SSYS) || (p)->p_stat == SIDL || \ 141 (p)->p_stat == SZOMB || (p)->p_as == NULL || \ 142 (p)->p_as == &kas) 143 144 /* 145 * Memory scheduler. 146 */ 147 void 148 sched() 149 { 150 kthread_id_t t; 151 pri_t proc_pri; 152 pri_t thread_pri; 153 pri_t swapin_pri; 154 int desperate; 155 pgcnt_t needs; 156 int divisor; 157 proc_t *prp; 158 proc_t *swapout_prp; 159 proc_t *swapin_prp; 160 spgcnt_t avail; 161 int chosen_pri; 162 time_t swapout_time; 163 time_t swapin_proc_time; 164 callb_cpr_t cprinfo; 165 kmutex_t swap_cpr_lock; 166 167 mutex_init(&swap_cpr_lock, NULL, MUTEX_DEFAULT, NULL); 168 CALLB_CPR_INIT(&cprinfo, &swap_cpr_lock, callb_generic_cpr, "sched"); 169 if (maxslp == 0) 170 maxslp = MAXSLP; 171 loop: 172 needs = 0; 173 desperate = 0; 174 175 swapin_pri = v.v_nglobpris; 176 swapin_prp = NULL; 177 chosen_pri = -1; 178 179 process_swap_queue(); 180 181 /* 182 * Set desperate if 183 * 1. At least 2 runnable processes (on average). 184 * 2. Short (5 sec) and longer (30 sec) average is less 185 * than minfree and desfree respectively. 186 * 3. Pagein + pageout rate is excessive. 187 */ 188 if (avenrun[0] >= 2 * FSCALE && 189 (MAX(avefree, avefree30) < desfree) && 190 (pginrate + pgoutrate > maxpgio || avefree < minfree)) { 191 TRACE_4(TR_FAC_SCHED, TR_DESPERATE, 192 "desp:avefree: %d, avefree30: %d, freemem: %d" 193 " pginrate: %d\n", avefree, avefree30, freemem, pginrate); 194 desperate = 1; 195 goto unload; 196 } 197 198 /* 199 * Search list of processes to swapin and swapout deadwood. 200 */ 201 swapin_proc_time = 0; 202 top: 203 mutex_enter(&pidlock); 204 for (prp = practive; prp != NULL; prp = prp->p_next) { 205 if (not_swappable(prp)) 206 continue; 207 208 /* 209 * Look at processes with at least one swapped lwp. 210 */ 211 if (prp->p_swapcnt) { 212 time_t proc_time; 213 214 /* 215 * Higher priority processes are good candidates 216 * to swapin. 217 */ 218 mutex_enter(&prp->p_lock); 219 proc_pri = -1; 220 t = prp->p_tlist; 221 proc_time = 0; 222 do { 223 if (t->t_schedflag & TS_LOAD) 224 continue; 225 226 thread_lock(t); 227 thread_pri = CL_SWAPIN(t, 0); 228 thread_unlock(t); 229 230 if (t->t_stime - proc_time > 0) 231 proc_time = t->t_stime; 232 if (thread_pri > proc_pri) 233 proc_pri = thread_pri; 234 } while ((t = t->t_forw) != prp->p_tlist); 235 mutex_exit(&prp->p_lock); 236 237 if (proc_pri == -1) 238 continue; 239 240 TRACE_3(TR_FAC_SCHED, TR_CHOOSE_SWAPIN, 241 "prp %p epri %d proc_time %d", 242 prp, proc_pri, proc_time); 243 244 /* 245 * Swapin processes with a high effective priority. 246 */ 247 if (swapin_prp == NULL || proc_pri > chosen_pri) { 248 swapin_prp = prp; 249 chosen_pri = proc_pri; 250 swapin_pri = proc_pri; 251 swapin_proc_time = proc_time; 252 } 253 } else { 254 /* 255 * No need to soft swap if we have sufficient 256 * memory. 257 */ 258 if (avefree > desfree || 259 avefree < desfree && freemem > desfree) 260 continue; 261 262 /* 263 * Skip processes which are exiting. This is 264 * determined by checking p_lwpcnt since SZOMB is 265 * set after the addressed space is released. 266 */ 267 mutex_enter(&prp->p_lock); 268 if (prp->p_lwpcnt == 0 || 269 (prp->p_flag & SEXITLWPS) || 270 (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) { 271 mutex_exit(&prp->p_lock); 272 continue; 273 } 274 275 /* 276 * Softswapping to kick out deadwood. 277 */ 278 proc_pri = -1; 279 t = prp->p_tlist; 280 do { 281 if ((t->t_schedflag & (TS_SWAPENQ | 282 TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD) 283 continue; 284 285 thread_lock(t); 286 thread_pri = CL_SWAPOUT(t, SOFTSWAP); 287 thread_unlock(t); 288 if (thread_pri > proc_pri) 289 proc_pri = thread_pri; 290 } while ((t = t->t_forw) != prp->p_tlist); 291 292 if (proc_pri != -1) { 293 uint_t swrss; 294 295 mutex_exit(&pidlock); 296 297 TRACE_1(TR_FAC_SCHED, TR_SOFTSWAP, 298 "softswap:prp %p", prp); 299 300 (void) swapout(prp, &swrss, SOFTSWAP); 301 softswap++; 302 prp->p_swrss += swrss; 303 mutex_exit(&prp->p_lock); 304 goto top; 305 } 306 mutex_exit(&prp->p_lock); 307 } 308 } 309 if (swapin_prp != NULL) 310 mutex_enter(&swapin_prp->p_lock); 311 mutex_exit(&pidlock); 312 313 if (swapin_prp == NULL) { 314 TRACE_3(TR_FAC_SCHED, TR_RUNOUT, 315 "schedrunout:runout nswapped: %d, avefree: %ld freemem: %ld", 316 nswapped, avefree, freemem); 317 318 t = curthread; 319 thread_lock(t); 320 runout++; 321 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART); 322 t->t_whystop = PR_SUSPENDED; 323 t->t_whatstop = SUSPEND_NORMAL; 324 (void) new_mstate(t, LMS_SLEEP); 325 mutex_enter(&swap_cpr_lock); 326 CALLB_CPR_SAFE_BEGIN(&cprinfo); 327 mutex_exit(&swap_cpr_lock); 328 thread_stop(t); /* change state and drop lock */ 329 swtch(); 330 mutex_enter(&swap_cpr_lock); 331 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock); 332 mutex_exit(&swap_cpr_lock); 333 goto loop; 334 } 335 336 /* 337 * Decide how deserving this process is to be brought in. 338 * Needs is an estimate of how much core the process will 339 * need. If the process has been out for a while, then we 340 * will bring it in with 1/2 the core needed, otherwise 341 * we are conservative. 342 */ 343 divisor = 1; 344 swapout_time = (lbolt - swapin_proc_time) / hz; 345 if (swapout_time > maxslp / 2) 346 divisor = 2; 347 348 needs = MIN(swapin_prp->p_swrss, lotsfree); 349 needs = MAX(needs, min_procsize); 350 needs = needs / divisor; 351 352 /* 353 * Use freemem, since we want processes to be swapped 354 * in quickly. 355 */ 356 avail = freemem - deficit; 357 if (avail > (spgcnt_t)needs) { 358 deficit += needs; 359 360 TRACE_2(TR_FAC_SCHED, TR_SWAPIN_VALUES, 361 "swapin_values: prp %p needs %lu", swapin_prp, needs); 362 363 if (swapin(swapin_prp)) { 364 mutex_exit(&swapin_prp->p_lock); 365 goto loop; 366 } 367 deficit -= MIN(needs, deficit); 368 mutex_exit(&swapin_prp->p_lock); 369 } else { 370 mutex_exit(&swapin_prp->p_lock); 371 /* 372 * If deficit is high, too many processes have been 373 * swapped in so wait a sec before attempting to 374 * swapin more. 375 */ 376 if (freemem > needs) { 377 TRACE_2(TR_FAC_SCHED, TR_HIGH_DEFICIT, 378 "deficit: prp %p needs %lu", swapin_prp, needs); 379 goto block; 380 } 381 } 382 383 TRACE_2(TR_FAC_SCHED, TR_UNLOAD, 384 "unload: prp %p needs %lu", swapin_prp, needs); 385 386 unload: 387 /* 388 * Unload all unloadable modules, free all other memory 389 * resources we can find, then look for a thread to hardswap. 390 */ 391 modreap(); 392 segkp_cache_free(); 393 394 swapout_prp = NULL; 395 mutex_enter(&pidlock); 396 for (prp = practive; prp != NULL; prp = prp->p_next) { 397 398 /* 399 * No need to soft swap if we have sufficient 400 * memory. 401 */ 402 if (not_swappable(prp)) 403 continue; 404 405 if (avefree > minfree || 406 avefree < minfree && freemem > desfree) { 407 swapout_prp = NULL; 408 break; 409 } 410 411 /* 412 * Skip processes which are exiting. This is determined 413 * by checking p_lwpcnt since SZOMB is set after the 414 * addressed space is released. 415 */ 416 mutex_enter(&prp->p_lock); 417 if (prp->p_lwpcnt == 0 || 418 (prp->p_flag & SEXITLWPS) || 419 (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) { 420 mutex_exit(&prp->p_lock); 421 continue; 422 } 423 424 proc_pri = -1; 425 t = prp->p_tlist; 426 do { 427 if ((t->t_schedflag & (TS_SWAPENQ | 428 TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD) 429 continue; 430 431 thread_lock(t); 432 thread_pri = CL_SWAPOUT(t, HARDSWAP); 433 thread_unlock(t); 434 if (thread_pri > proc_pri) 435 proc_pri = thread_pri; 436 } while ((t = t->t_forw) != prp->p_tlist); 437 438 mutex_exit(&prp->p_lock); 439 if (proc_pri == -1) 440 continue; 441 442 /* 443 * Swapout processes sleeping with a lower priority 444 * than the one currently being swapped in, if any. 445 */ 446 if (swapin_prp == NULL || swapin_pri > proc_pri) { 447 TRACE_2(TR_FAC_SCHED, TR_CHOOSE_SWAPOUT, 448 "hardswap: prp %p needs %lu", prp, needs); 449 450 if (swapout_prp == NULL || proc_pri < chosen_pri) { 451 swapout_prp = prp; 452 chosen_pri = proc_pri; 453 } 454 } 455 } 456 457 /* 458 * Acquire the "p_lock" before dropping "pidlock" 459 * to prevent the proc structure from being freed 460 * if the process exits before swapout completes. 461 */ 462 if (swapout_prp != NULL) 463 mutex_enter(&swapout_prp->p_lock); 464 mutex_exit(&pidlock); 465 466 if ((prp = swapout_prp) != NULL) { 467 uint_t swrss = 0; 468 int swapped; 469 470 swapped = swapout(prp, &swrss, HARDSWAP); 471 if (swapped) { 472 /* 473 * If desperate, we want to give the space obtained 474 * by swapping this process out to processes in core, 475 * so we give them a chance by increasing deficit. 476 */ 477 prp->p_swrss += swrss; 478 if (desperate) 479 deficit += MIN(prp->p_swrss, lotsfree); 480 hardswap++; 481 } 482 mutex_exit(&swapout_prp->p_lock); 483 484 if (swapped) 485 goto loop; 486 } 487 488 /* 489 * Delay for 1 second and look again later. 490 */ 491 TRACE_3(TR_FAC_SCHED, TR_RUNIN, 492 "schedrunin:runin nswapped: %d, avefree: %ld freemem: %ld", 493 nswapped, avefree, freemem); 494 495 block: 496 t = curthread; 497 thread_lock(t); 498 runin++; 499 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART); 500 t->t_whystop = PR_SUSPENDED; 501 t->t_whatstop = SUSPEND_NORMAL; 502 (void) new_mstate(t, LMS_SLEEP); 503 mutex_enter(&swap_cpr_lock); 504 CALLB_CPR_SAFE_BEGIN(&cprinfo); 505 mutex_exit(&swap_cpr_lock); 506 thread_stop(t); /* change to stop state and drop lock */ 507 swtch(); 508 mutex_enter(&swap_cpr_lock); 509 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock); 510 mutex_exit(&swap_cpr_lock); 511 goto loop; 512 } 513 514 /* 515 * Remove the specified thread from the swap queue. 516 */ 517 static void 518 swapdeq(kthread_id_t tp) 519 { 520 kthread_id_t *tpp; 521 522 ASSERT(THREAD_LOCK_HELD(tp)); 523 ASSERT(tp->t_schedflag & TS_ON_SWAPQ); 524 525 tpp = &tswap_queue; 526 for (;;) { 527 ASSERT(*tpp != NULL); 528 if (*tpp == tp) 529 break; 530 tpp = &(*tpp)->t_link; 531 } 532 *tpp = tp->t_link; 533 tp->t_schedflag &= ~TS_ON_SWAPQ; 534 } 535 536 /* 537 * Swap in lwps. Returns nonzero on success (i.e., if at least one lwp is 538 * swapped in) and 0 on failure. 539 */ 540 static int 541 swapin(proc_t *pp) 542 { 543 kthread_id_t tp; 544 int err; 545 int num_swapped_in = 0; 546 struct cpu *cpup = CPU; 547 pri_t thread_pri; 548 549 ASSERT(MUTEX_HELD(&pp->p_lock)); 550 ASSERT(pp->p_swapcnt); 551 552 top: 553 tp = pp->p_tlist; 554 do { 555 /* 556 * Only swapin eligible lwps (specified by the scheduling 557 * class) which are unloaded and ready to run. 558 */ 559 thread_lock(tp); 560 thread_pri = CL_SWAPIN(tp, 0); 561 if (thread_pri != -1 && tp->t_state == TS_RUN && 562 (tp->t_schedflag & TS_LOAD) == 0) { 563 size_t stack_size; 564 pgcnt_t stack_pages; 565 566 ASSERT((tp->t_schedflag & TS_ON_SWAPQ) == 0); 567 568 thread_unlock(tp); 569 /* 570 * Now drop the p_lock since the stack needs 571 * to brought in. 572 */ 573 mutex_exit(&pp->p_lock); 574 575 stack_size = swapsize(tp->t_swap); 576 stack_pages = btopr(stack_size); 577 /* Kernel probe */ 578 TNF_PROBE_4(swapin_lwp, "vm swap swapin", /* CSTYLED */, 579 tnf_pid, pid, pp->p_pid, 580 tnf_lwpid, lwpid, tp->t_tid, 581 tnf_kthread_id, tid, tp, 582 tnf_ulong, page_count, stack_pages); 583 584 rw_enter(&kas.a_lock, RW_READER); 585 err = segkp_fault(segkp->s_as->a_hat, segkp, 586 tp->t_swap, stack_size, F_SOFTLOCK, S_OTHER); 587 rw_exit(&kas.a_lock); 588 589 #ifdef __sparc 590 lwp_swapin(tp); 591 #endif /* __sparc */ 592 593 /* 594 * Re-acquire the p_lock. 595 */ 596 mutex_enter(&pp->p_lock); 597 if (err) { 598 num_swapped_in = 0; 599 break; 600 } else { 601 CPU_STATS_ADDQ(cpup, vm, swapin, 1); 602 CPU_STATS_ADDQ(cpup, vm, pgswapin, 603 stack_pages); 604 605 pp->p_swapcnt--; 606 pp->p_swrss -= stack_pages; 607 608 thread_lock(tp); 609 tp->t_schedflag |= TS_LOAD; 610 dq_sruninc(tp); 611 612 tp->t_stime = lbolt; /* set swapin time */ 613 thread_unlock(tp); 614 615 nswapped--; 616 tot_swapped_in++; 617 num_swapped_in++; 618 619 TRACE_2(TR_FAC_SCHED, TR_SWAPIN, 620 "swapin: pp %p stack_pages %lu", 621 pp, stack_pages); 622 goto top; 623 } 624 } 625 thread_unlock(tp); 626 } while ((tp = tp->t_forw) != pp->p_tlist); 627 return (num_swapped_in); 628 } 629 630 /* 631 * Swap out lwps. Returns nonzero on success (i.e., if at least one lwp is 632 * swapped out) and 0 on failure. 633 */ 634 static int 635 swapout(proc_t *pp, uint_t *swrss, int swapflags) 636 { 637 kthread_id_t tp; 638 pgcnt_t ws_pages = 0; 639 int err; 640 int swapped_lwps = 0; 641 struct as *as = pp->p_as; 642 struct cpu *cpup = CPU; 643 pri_t thread_pri; 644 645 ASSERT(MUTEX_HELD(&pp->p_lock)); 646 647 if (pp->p_lwpcnt == 0 || (pp->p_flag & SEXITLWPS)) 648 return (0); 649 650 top: 651 tp = pp->p_tlist; 652 do { 653 klwp_t *lwp = ttolwp(tp); 654 655 /* 656 * Swapout eligible lwps (specified by the scheduling 657 * class) which don't have TS_DONT_SWAP set. Set the 658 * "intent to swap" flag (TS_SWAPENQ) on threads 659 * which have TS_DONT_SWAP set so that they can be 660 * swapped if and when they reach a safe point. 661 */ 662 thread_lock(tp); 663 thread_pri = CL_SWAPOUT(tp, swapflags); 664 if (thread_pri != -1) { 665 if (tp->t_schedflag & TS_DONT_SWAP) { 666 tp->t_schedflag |= TS_SWAPENQ; 667 tp->t_trapret = 1; 668 aston(tp); 669 } else { 670 pgcnt_t stack_pages; 671 size_t stack_size; 672 673 ASSERT((tp->t_schedflag & 674 (TS_DONT_SWAP | TS_LOAD)) == TS_LOAD); 675 676 if (lock_try(&tp->t_lock)) { 677 /* 678 * Remove thread from the swap_queue. 679 */ 680 if (tp->t_schedflag & TS_ON_SWAPQ) { 681 ASSERT(!(tp->t_schedflag & 682 TS_SWAPENQ)); 683 swapdeq(tp); 684 } else if (tp->t_state == TS_RUN) 685 dq_srundec(tp); 686 687 tp->t_schedflag &= 688 ~(TS_LOAD | TS_SWAPENQ); 689 lock_clear(&tp->t_lock); 690 691 /* 692 * Set swapout time if the thread isn't 693 * sleeping. 694 */ 695 if (tp->t_state != TS_SLEEP) 696 tp->t_stime = lbolt; 697 thread_unlock(tp); 698 699 nswapped++; 700 tot_swapped_out++; 701 702 lwp->lwp_ru.nswap++; 703 704 /* 705 * Now drop the p_lock since the 706 * stack needs to pushed out. 707 */ 708 mutex_exit(&pp->p_lock); 709 710 stack_size = swapsize(tp->t_swap); 711 stack_pages = btopr(stack_size); 712 ws_pages += stack_pages; 713 /* Kernel probe */ 714 TNF_PROBE_4(swapout_lwp, 715 "vm swap swapout", 716 /* CSTYLED */, 717 tnf_pid, pid, pp->p_pid, 718 tnf_lwpid, lwpid, tp->t_tid, 719 tnf_kthread_id, tid, tp, 720 tnf_ulong, page_count, 721 stack_pages); 722 723 rw_enter(&kas.a_lock, RW_READER); 724 err = segkp_fault(segkp->s_as->a_hat, 725 segkp, tp->t_swap, stack_size, 726 F_SOFTUNLOCK, S_WRITE); 727 rw_exit(&kas.a_lock); 728 729 if (err) { 730 cmn_err(CE_PANIC, 731 "swapout: segkp_fault " 732 "failed err: %d", err); 733 } 734 CPU_STATS_ADDQ(cpup, 735 vm, pgswapout, stack_pages); 736 737 mutex_enter(&pp->p_lock); 738 pp->p_swapcnt++; 739 swapped_lwps++; 740 goto top; 741 } 742 } 743 } 744 thread_unlock(tp); 745 } while ((tp = tp->t_forw) != pp->p_tlist); 746 747 /* 748 * Unload address space when all lwps are swapped out. 749 */ 750 if (pp->p_swapcnt == pp->p_lwpcnt) { 751 size_t as_size = 0; 752 753 /* 754 * Avoid invoking as_swapout() if the process has 755 * no MMU resources since pageout will eventually 756 * steal pages belonging to this address space. This 757 * saves CPU cycles as the number of pages that are 758 * potentially freed or pushed out by the segment 759 * swapout operation is very small. 760 */ 761 if (rm_asrss(pp->p_as) != 0) 762 as_size = as_swapout(as); 763 764 CPU_STATS_ADDQ(cpup, vm, pgswapout, btop(as_size)); 765 CPU_STATS_ADDQ(cpup, vm, swapout, 1); 766 ws_pages += btop(as_size); 767 768 TRACE_2(TR_FAC_SCHED, TR_SWAPOUT, 769 "swapout: pp %p pages_pushed %lu", pp, ws_pages); 770 /* Kernel probe */ 771 TNF_PROBE_2(swapout_process, "vm swap swapout", /* CSTYLED */, 772 tnf_pid, pid, pp->p_pid, 773 tnf_ulong, page_count, ws_pages); 774 } 775 *swrss = ws_pages; 776 return (swapped_lwps); 777 } 778 779 void 780 swapout_lwp(klwp_t *lwp) 781 { 782 kthread_id_t tp = curthread; 783 784 ASSERT(curthread == lwptot(lwp)); 785 786 /* 787 * Don't insert the thread onto the swap queue if 788 * sufficient memory is available. 789 */ 790 if (avefree > desfree || avefree < desfree && freemem > desfree) { 791 thread_lock(tp); 792 tp->t_schedflag &= ~TS_SWAPENQ; 793 thread_unlock(tp); 794 return; 795 } 796 797 /* 798 * Lock the thread, then move it to the swapped queue from the 799 * onproc queue and set its state to be TS_RUN. 800 */ 801 thread_lock(tp); 802 ASSERT(tp->t_state == TS_ONPROC); 803 if (tp->t_schedflag & TS_SWAPENQ) { 804 tp->t_schedflag &= ~TS_SWAPENQ; 805 806 /* 807 * Set the state of this thread to be runnable 808 * and move it from the onproc queue to the swap queue. 809 */ 810 disp_swapped_enq(tp); 811 812 /* 813 * Insert the thread onto the swap queue. 814 */ 815 tp->t_link = tswap_queue; 816 tswap_queue = tp; 817 tp->t_schedflag |= TS_ON_SWAPQ; 818 819 thread_unlock_nopreempt(tp); 820 821 TRACE_1(TR_FAC_SCHED, TR_SWAPOUT_LWP, "swapout_lwp:%x", lwp); 822 823 swtch(); 824 } else { 825 thread_unlock(tp); 826 } 827 } 828 829 /* 830 * Swap all threads on the swap queue. 831 */ 832 static void 833 process_swap_queue(void) 834 { 835 kthread_id_t tp; 836 uint_t ws_pages; 837 proc_t *pp; 838 struct cpu *cpup = CPU; 839 klwp_t *lwp; 840 int err; 841 842 if (tswap_queue == NULL) 843 return; 844 845 /* 846 * Acquire the "swapped_lock" which locks the swap queue, 847 * and unload the stacks of all threads on it. 848 */ 849 disp_lock_enter(&swapped_lock); 850 while ((tp = tswap_queue) != NULL) { 851 pgcnt_t stack_pages; 852 size_t stack_size; 853 854 tswap_queue = tp->t_link; 855 tp->t_link = NULL; 856 857 /* 858 * Drop the "dispatcher lock" before acquiring "t_lock" 859 * to avoid spinning on it since the thread at the front 860 * of the swap queue could be pinned before giving up 861 * its "t_lock" in resume. 862 */ 863 disp_lock_exit(&swapped_lock); 864 lock_set(&tp->t_lock); 865 866 /* 867 * Now, re-acquire the "swapped_lock". Acquiring this lock 868 * results in locking the thread since its dispatcher lock 869 * (t_lockp) is the "swapped_lock". 870 */ 871 disp_lock_enter(&swapped_lock); 872 ASSERT(tp->t_state == TS_RUN); 873 ASSERT(tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)); 874 875 tp->t_schedflag &= ~(TS_LOAD | TS_ON_SWAPQ); 876 tp->t_stime = lbolt; /* swapout time */ 877 disp_lock_exit(&swapped_lock); 878 lock_clear(&tp->t_lock); 879 880 lwp = ttolwp(tp); 881 lwp->lwp_ru.nswap++; 882 883 pp = ttoproc(tp); 884 stack_size = swapsize(tp->t_swap); 885 stack_pages = btopr(stack_size); 886 887 /* Kernel probe */ 888 TNF_PROBE_4(swapout_lwp, "vm swap swapout", /* CSTYLED */, 889 tnf_pid, pid, pp->p_pid, 890 tnf_lwpid, lwpid, tp->t_tid, 891 tnf_kthread_id, tid, tp, 892 tnf_ulong, page_count, stack_pages); 893 894 rw_enter(&kas.a_lock, RW_READER); 895 err = segkp_fault(segkp->s_as->a_hat, segkp, tp->t_swap, 896 stack_size, F_SOFTUNLOCK, S_WRITE); 897 rw_exit(&kas.a_lock); 898 899 if (err) { 900 cmn_err(CE_PANIC, 901 "process_swap_list: segkp_fault failed err: %d", err); 902 } 903 CPU_STATS_ADDQ(cpup, vm, pgswapout, stack_pages); 904 905 nswapped++; 906 tot_swapped_out++; 907 swapqswap++; 908 909 /* 910 * Don't need p_lock since the swapper is the only 911 * thread which increments/decrements p_swapcnt and p_swrss. 912 */ 913 ws_pages = stack_pages; 914 pp->p_swapcnt++; 915 916 TRACE_1(TR_FAC_SCHED, TR_SWAPQ_LWP, "swaplist: pp %p", pp); 917 918 /* 919 * Unload address space when all lwps are swapped out. 920 */ 921 if (pp->p_swapcnt == pp->p_lwpcnt) { 922 size_t as_size = 0; 923 924 if (rm_asrss(pp->p_as) != 0) 925 as_size = as_swapout(pp->p_as); 926 927 CPU_STATS_ADDQ(cpup, vm, pgswapout, 928 btop(as_size)); 929 CPU_STATS_ADDQ(cpup, vm, swapout, 1); 930 931 ws_pages += btop(as_size); 932 933 TRACE_2(TR_FAC_SCHED, TR_SWAPQ_PROC, 934 "swaplist_proc: pp %p pages_pushed: %lu", 935 pp, ws_pages); 936 /* Kernel probe */ 937 TNF_PROBE_2(swapout_process, "vm swap swapout", 938 /* CSTYLED */, 939 tnf_pid, pid, pp->p_pid, 940 tnf_ulong, page_count, ws_pages); 941 } 942 pp->p_swrss += ws_pages; 943 disp_lock_enter(&swapped_lock); 944 } 945 disp_lock_exit(&swapped_lock); 946 } 947