1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 28 /* All Rights Reserved */ 29 30 31 #pragma ident "%Z%%M% %I% %E% SMI" 32 33 #include <sys/param.h> 34 #include <sys/types.h> 35 #include <sys/sysmacros.h> 36 #include <sys/systm.h> 37 #include <sys/proc.h> 38 #include <sys/cpuvar.h> 39 #include <sys/var.h> 40 #include <sys/tuneable.h> 41 #include <sys/cmn_err.h> 42 #include <sys/buf.h> 43 #include <sys/disp.h> 44 #include <sys/vmsystm.h> 45 #include <sys/vmparam.h> 46 #include <sys/class.h> 47 #include <sys/vtrace.h> 48 #include <sys/modctl.h> 49 #include <sys/debug.h> 50 #include <sys/tnf_probe.h> 51 #include <sys/procfs.h> 52 53 #include <vm/seg.h> 54 #include <vm/seg_kp.h> 55 #include <vm/as.h> 56 #include <vm/rm.h> 57 #include <vm/seg_kmem.h> 58 #include <sys/callb.h> 59 60 /* 61 * The swapper sleeps on runout when there is no one to swap in. 62 * It sleeps on runin when it could not find space to swap someone 63 * in or after swapping someone in. 64 */ 65 char runout; 66 char runin; 67 char wake_sched; /* flag tells clock to wake swapper on next tick */ 68 char wake_sched_sec; /* flag tells clock to wake swapper after a second */ 69 70 /* 71 * The swapper swaps processes to reduce memory demand and runs 72 * when avefree < desfree. The swapper resorts to SOFTSWAP when 73 * avefree < desfree which results in swapping out all processes 74 * sleeping for more than maxslp seconds. HARDSWAP occurs when the 75 * system is on the verge of thrashing and this results in swapping 76 * out runnable threads or threads sleeping for less than maxslp secs. 77 * 78 * The swapper runs through all the active processes in the system 79 * and invokes the scheduling class specific swapin/swapout routine 80 * for every thread in the process to obtain an effective priority 81 * for the process. A priority of -1 implies that the thread isn't 82 * swappable. This effective priority is used to find the most 83 * eligible process to swapout or swapin. 84 * 85 * NOTE: Threads which have been swapped are not linked on any 86 * queue and their dispatcher lock points at the "swapped_lock". 87 * 88 * Processes containing threads with the TS_DONT_SWAP flag set cannot be 89 * swapped out immediately by the swapper. This is due to the fact that 90 * such threads may be holding locks which may be needed by the swapper 91 * to push its pages out. The TS_SWAPENQ flag is set on such threads 92 * to prevent them running in user mode. When such threads reach a 93 * safe point (i.e., are not holding any locks - CL_TRAPRET), they 94 * queue themseleves onto the swap queue which is processed by the 95 * swapper. This results in reducing memory demand when the system 96 * is desparate for memory as the thread can't run in user mode. 97 * 98 * The swap queue consists of threads, linked via t_link, which are 99 * haven't been swapped, are runnable but not on the run queue. The 100 * swap queue is protected by the "swapped_lock". The dispatcher 101 * lock (t_lockp) of all threads on the swap queue points at the 102 * "swapped_lock". Thus, the entire queue and/or threads on the 103 * queue can be locked by acquiring "swapped_lock". 104 */ 105 static kthread_t *tswap_queue; 106 extern disp_lock_t swapped_lock; /* protects swap queue and threads on it */ 107 108 int maxslp = 0; 109 pgcnt_t avefree; /* 5 sec moving average of free memory */ 110 pgcnt_t avefree30; /* 30 sec moving average of free memory */ 111 112 /* 113 * Minimum size used to decide if sufficient memory is available 114 * before a process is swapped in. This is necessary since in most 115 * cases the actual size of a process (p_swrss) being swapped in 116 * is usually 2 pages (kernel stack pages). This is due to the fact 117 * almost all user pages of a process are stolen by pageout before 118 * the swapper decides to swapout it out. 119 */ 120 int min_procsize = 12; 121 122 static int swapin(proc_t *); 123 static int swapout(proc_t *, uint_t *, int); 124 static void process_swap_queue(); 125 126 #ifdef __sparc 127 extern void lwp_swapin(kthread_t *); 128 #endif /* __sparc */ 129 130 /* 131 * Counters to keep track of the number of swapins or swapouts. 132 */ 133 uint_t tot_swapped_in, tot_swapped_out; 134 uint_t softswap, hardswap, swapqswap; 135 136 /* 137 * Macro to determine if a process is eligble to be swapped. 138 */ 139 #define not_swappable(p) \ 140 (((p)->p_flag & SSYS) || (p)->p_stat == SIDL || \ 141 (p)->p_stat == SZOMB || (p)->p_as == NULL || \ 142 (p)->p_as == &kas) 143 144 /* 145 * Memory scheduler. 146 */ 147 void 148 sched() 149 { 150 kthread_id_t t; 151 pri_t proc_pri; 152 pri_t thread_pri; 153 pri_t swapin_pri; 154 int desperate; 155 pgcnt_t needs; 156 int divisor; 157 proc_t *prp; 158 proc_t *swapout_prp; 159 proc_t *swapin_prp; 160 spgcnt_t avail; 161 int chosen_pri; 162 time_t swapout_time; 163 time_t swapin_proc_time; 164 callb_cpr_t cprinfo; 165 kmutex_t swap_cpr_lock; 166 167 mutex_init(&swap_cpr_lock, NULL, MUTEX_DEFAULT, NULL); 168 CALLB_CPR_INIT(&cprinfo, &swap_cpr_lock, callb_generic_cpr, "sched"); 169 if (maxslp == 0) 170 maxslp = MAXSLP; 171 loop: 172 needs = 0; 173 desperate = 0; 174 175 swapin_pri = v.v_nglobpris; 176 swapin_prp = NULL; 177 chosen_pri = -1; 178 179 process_swap_queue(); 180 181 /* 182 * Set desperate if 183 * 1. At least 2 runnable processes (on average). 184 * 2. Short (5 sec) and longer (30 sec) average is less 185 * than minfree and desfree respectively. 186 * 3. Pagein + pageout rate is excessive. 187 */ 188 if (avenrun[0] >= 2 * FSCALE && 189 (MAX(avefree, avefree30) < desfree) && 190 (pginrate + pgoutrate > maxpgio || avefree < minfree)) { 191 TRACE_4(TR_FAC_SCHED, TR_DESPERATE, 192 "desp:avefree: %d, avefree30: %d, freemem: %d" 193 " pginrate: %d\n", avefree, avefree30, freemem, pginrate); 194 desperate = 1; 195 goto unload; 196 } 197 198 /* 199 * Search list of processes to swapin and swapout deadwood. 200 */ 201 swapin_proc_time = 0; 202 top: 203 mutex_enter(&pidlock); 204 for (prp = practive; prp != NULL; prp = prp->p_next) { 205 if (not_swappable(prp)) 206 continue; 207 208 /* 209 * Look at processes with at least one swapped lwp. 210 */ 211 if (prp->p_swapcnt) { 212 time_t proc_time; 213 214 /* 215 * Higher priority processes are good candidates 216 * to swapin. 217 */ 218 mutex_enter(&prp->p_lock); 219 proc_pri = -1; 220 t = prp->p_tlist; 221 proc_time = 0; 222 do { 223 if (t->t_schedflag & TS_LOAD) 224 continue; 225 226 thread_lock(t); 227 thread_pri = CL_SWAPIN(t, 0); 228 thread_unlock(t); 229 230 if (t->t_stime - proc_time > 0) 231 proc_time = t->t_stime; 232 if (thread_pri > proc_pri) 233 proc_pri = thread_pri; 234 } while ((t = t->t_forw) != prp->p_tlist); 235 mutex_exit(&prp->p_lock); 236 237 if (proc_pri == -1) 238 continue; 239 240 TRACE_3(TR_FAC_SCHED, TR_CHOOSE_SWAPIN, 241 "prp %p epri %d proc_time %d", 242 prp, proc_pri, proc_time); 243 244 /* 245 * Swapin processes with a high effective priority. 246 */ 247 if (swapin_prp == NULL || proc_pri > chosen_pri) { 248 swapin_prp = prp; 249 chosen_pri = proc_pri; 250 swapin_pri = proc_pri; 251 swapin_proc_time = proc_time; 252 } 253 } else { 254 /* 255 * No need to soft swap if we have sufficient 256 * memory. 257 */ 258 if (avefree > desfree || 259 avefree < desfree && freemem > desfree) 260 continue; 261 262 /* 263 * Skip processes that are exiting 264 * or whose address spaces are locked. 265 */ 266 mutex_enter(&prp->p_lock); 267 if ((prp->p_flag & SEXITING) || 268 (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) { 269 mutex_exit(&prp->p_lock); 270 continue; 271 } 272 273 /* 274 * Softswapping to kick out deadwood. 275 */ 276 proc_pri = -1; 277 t = prp->p_tlist; 278 do { 279 if ((t->t_schedflag & (TS_SWAPENQ | 280 TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD) 281 continue; 282 283 thread_lock(t); 284 thread_pri = CL_SWAPOUT(t, SOFTSWAP); 285 thread_unlock(t); 286 if (thread_pri > proc_pri) 287 proc_pri = thread_pri; 288 } while ((t = t->t_forw) != prp->p_tlist); 289 290 if (proc_pri != -1) { 291 uint_t swrss; 292 293 mutex_exit(&pidlock); 294 295 TRACE_1(TR_FAC_SCHED, TR_SOFTSWAP, 296 "softswap:prp %p", prp); 297 298 (void) swapout(prp, &swrss, SOFTSWAP); 299 softswap++; 300 prp->p_swrss += swrss; 301 mutex_exit(&prp->p_lock); 302 goto top; 303 } 304 mutex_exit(&prp->p_lock); 305 } 306 } 307 if (swapin_prp != NULL) 308 mutex_enter(&swapin_prp->p_lock); 309 mutex_exit(&pidlock); 310 311 if (swapin_prp == NULL) { 312 TRACE_3(TR_FAC_SCHED, TR_RUNOUT, 313 "schedrunout:runout nswapped: %d, avefree: %ld freemem: %ld", 314 nswapped, avefree, freemem); 315 316 t = curthread; 317 thread_lock(t); 318 runout++; 319 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART); 320 t->t_whystop = PR_SUSPENDED; 321 t->t_whatstop = SUSPEND_NORMAL; 322 (void) new_mstate(t, LMS_SLEEP); 323 mutex_enter(&swap_cpr_lock); 324 CALLB_CPR_SAFE_BEGIN(&cprinfo); 325 mutex_exit(&swap_cpr_lock); 326 thread_stop(t); /* change state and drop lock */ 327 swtch(); 328 mutex_enter(&swap_cpr_lock); 329 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock); 330 mutex_exit(&swap_cpr_lock); 331 goto loop; 332 } 333 334 /* 335 * Decide how deserving this process is to be brought in. 336 * Needs is an estimate of how much core the process will 337 * need. If the process has been out for a while, then we 338 * will bring it in with 1/2 the core needed, otherwise 339 * we are conservative. 340 */ 341 divisor = 1; 342 swapout_time = (lbolt - swapin_proc_time) / hz; 343 if (swapout_time > maxslp / 2) 344 divisor = 2; 345 346 needs = MIN(swapin_prp->p_swrss, lotsfree); 347 needs = MAX(needs, min_procsize); 348 needs = needs / divisor; 349 350 /* 351 * Use freemem, since we want processes to be swapped 352 * in quickly. 353 */ 354 avail = freemem - deficit; 355 if (avail > (spgcnt_t)needs) { 356 deficit += needs; 357 358 TRACE_2(TR_FAC_SCHED, TR_SWAPIN_VALUES, 359 "swapin_values: prp %p needs %lu", swapin_prp, needs); 360 361 if (swapin(swapin_prp)) { 362 mutex_exit(&swapin_prp->p_lock); 363 goto loop; 364 } 365 deficit -= MIN(needs, deficit); 366 mutex_exit(&swapin_prp->p_lock); 367 } else { 368 mutex_exit(&swapin_prp->p_lock); 369 /* 370 * If deficit is high, too many processes have been 371 * swapped in so wait a sec before attempting to 372 * swapin more. 373 */ 374 if (freemem > needs) { 375 TRACE_2(TR_FAC_SCHED, TR_HIGH_DEFICIT, 376 "deficit: prp %p needs %lu", swapin_prp, needs); 377 goto block; 378 } 379 } 380 381 TRACE_2(TR_FAC_SCHED, TR_UNLOAD, 382 "unload: prp %p needs %lu", swapin_prp, needs); 383 384 unload: 385 /* 386 * Unload all unloadable modules, free all other memory 387 * resources we can find, then look for a thread to hardswap. 388 */ 389 modreap(); 390 segkp_cache_free(); 391 392 swapout_prp = NULL; 393 mutex_enter(&pidlock); 394 for (prp = practive; prp != NULL; prp = prp->p_next) { 395 396 /* 397 * No need to soft swap if we have sufficient 398 * memory. 399 */ 400 if (not_swappable(prp)) 401 continue; 402 403 if (avefree > minfree || 404 avefree < minfree && freemem > desfree) { 405 swapout_prp = NULL; 406 break; 407 } 408 409 /* 410 * Skip processes that are exiting 411 * or whose address spaces are locked. 412 */ 413 mutex_enter(&prp->p_lock); 414 if ((prp->p_flag & SEXITING) || 415 (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) { 416 mutex_exit(&prp->p_lock); 417 continue; 418 } 419 420 proc_pri = -1; 421 t = prp->p_tlist; 422 do { 423 if ((t->t_schedflag & (TS_SWAPENQ | 424 TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD) 425 continue; 426 427 thread_lock(t); 428 thread_pri = CL_SWAPOUT(t, HARDSWAP); 429 thread_unlock(t); 430 if (thread_pri > proc_pri) 431 proc_pri = thread_pri; 432 } while ((t = t->t_forw) != prp->p_tlist); 433 434 mutex_exit(&prp->p_lock); 435 if (proc_pri == -1) 436 continue; 437 438 /* 439 * Swapout processes sleeping with a lower priority 440 * than the one currently being swapped in, if any. 441 */ 442 if (swapin_prp == NULL || swapin_pri > proc_pri) { 443 TRACE_2(TR_FAC_SCHED, TR_CHOOSE_SWAPOUT, 444 "hardswap: prp %p needs %lu", prp, needs); 445 446 if (swapout_prp == NULL || proc_pri < chosen_pri) { 447 swapout_prp = prp; 448 chosen_pri = proc_pri; 449 } 450 } 451 } 452 453 /* 454 * Acquire the "p_lock" before dropping "pidlock" 455 * to prevent the proc structure from being freed 456 * if the process exits before swapout completes. 457 */ 458 if (swapout_prp != NULL) 459 mutex_enter(&swapout_prp->p_lock); 460 mutex_exit(&pidlock); 461 462 if ((prp = swapout_prp) != NULL) { 463 uint_t swrss = 0; 464 int swapped; 465 466 swapped = swapout(prp, &swrss, HARDSWAP); 467 if (swapped) { 468 /* 469 * If desperate, we want to give the space obtained 470 * by swapping this process out to processes in core, 471 * so we give them a chance by increasing deficit. 472 */ 473 prp->p_swrss += swrss; 474 if (desperate) 475 deficit += MIN(prp->p_swrss, lotsfree); 476 hardswap++; 477 } 478 mutex_exit(&swapout_prp->p_lock); 479 480 if (swapped) 481 goto loop; 482 } 483 484 /* 485 * Delay for 1 second and look again later. 486 */ 487 TRACE_3(TR_FAC_SCHED, TR_RUNIN, 488 "schedrunin:runin nswapped: %d, avefree: %ld freemem: %ld", 489 nswapped, avefree, freemem); 490 491 block: 492 t = curthread; 493 thread_lock(t); 494 runin++; 495 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART); 496 t->t_whystop = PR_SUSPENDED; 497 t->t_whatstop = SUSPEND_NORMAL; 498 (void) new_mstate(t, LMS_SLEEP); 499 mutex_enter(&swap_cpr_lock); 500 CALLB_CPR_SAFE_BEGIN(&cprinfo); 501 mutex_exit(&swap_cpr_lock); 502 thread_stop(t); /* change to stop state and drop lock */ 503 swtch(); 504 mutex_enter(&swap_cpr_lock); 505 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock); 506 mutex_exit(&swap_cpr_lock); 507 goto loop; 508 } 509 510 /* 511 * Remove the specified thread from the swap queue. 512 */ 513 static void 514 swapdeq(kthread_id_t tp) 515 { 516 kthread_id_t *tpp; 517 518 ASSERT(THREAD_LOCK_HELD(tp)); 519 ASSERT(tp->t_schedflag & TS_ON_SWAPQ); 520 521 tpp = &tswap_queue; 522 for (;;) { 523 ASSERT(*tpp != NULL); 524 if (*tpp == tp) 525 break; 526 tpp = &(*tpp)->t_link; 527 } 528 *tpp = tp->t_link; 529 tp->t_schedflag &= ~TS_ON_SWAPQ; 530 } 531 532 /* 533 * Swap in lwps. Returns nonzero on success (i.e., if at least one lwp is 534 * swapped in) and 0 on failure. 535 */ 536 static int 537 swapin(proc_t *pp) 538 { 539 kthread_id_t tp; 540 int err; 541 int num_swapped_in = 0; 542 struct cpu *cpup = CPU; 543 pri_t thread_pri; 544 545 ASSERT(MUTEX_HELD(&pp->p_lock)); 546 ASSERT(pp->p_swapcnt); 547 548 top: 549 tp = pp->p_tlist; 550 do { 551 /* 552 * Only swapin eligible lwps (specified by the scheduling 553 * class) which are unloaded and ready to run. 554 */ 555 thread_lock(tp); 556 thread_pri = CL_SWAPIN(tp, 0); 557 if (thread_pri != -1 && tp->t_state == TS_RUN && 558 (tp->t_schedflag & TS_LOAD) == 0) { 559 size_t stack_size; 560 pgcnt_t stack_pages; 561 562 ASSERT((tp->t_schedflag & TS_ON_SWAPQ) == 0); 563 564 thread_unlock(tp); 565 /* 566 * Now drop the p_lock since the stack needs 567 * to brought in. 568 */ 569 mutex_exit(&pp->p_lock); 570 571 stack_size = swapsize(tp->t_swap); 572 stack_pages = btopr(stack_size); 573 /* Kernel probe */ 574 TNF_PROBE_4(swapin_lwp, "vm swap swapin", /* CSTYLED */, 575 tnf_pid, pid, pp->p_pid, 576 tnf_lwpid, lwpid, tp->t_tid, 577 tnf_kthread_id, tid, tp, 578 tnf_ulong, page_count, stack_pages); 579 580 rw_enter(&kas.a_lock, RW_READER); 581 err = segkp_fault(segkp->s_as->a_hat, segkp, 582 tp->t_swap, stack_size, F_SOFTLOCK, S_OTHER); 583 rw_exit(&kas.a_lock); 584 585 /* 586 * Re-acquire the p_lock. 587 */ 588 mutex_enter(&pp->p_lock); 589 if (err) { 590 num_swapped_in = 0; 591 break; 592 } else { 593 #ifdef __sparc 594 lwp_swapin(tp); 595 #endif /* __sparc */ 596 CPU_STATS_ADDQ(cpup, vm, swapin, 1); 597 CPU_STATS_ADDQ(cpup, vm, pgswapin, 598 stack_pages); 599 600 pp->p_swapcnt--; 601 pp->p_swrss -= stack_pages; 602 603 thread_lock(tp); 604 tp->t_schedflag |= TS_LOAD; 605 dq_sruninc(tp); 606 607 tp->t_stime = lbolt; /* set swapin time */ 608 thread_unlock(tp); 609 610 nswapped--; 611 tot_swapped_in++; 612 num_swapped_in++; 613 614 TRACE_2(TR_FAC_SCHED, TR_SWAPIN, 615 "swapin: pp %p stack_pages %lu", 616 pp, stack_pages); 617 goto top; 618 } 619 } 620 thread_unlock(tp); 621 } while ((tp = tp->t_forw) != pp->p_tlist); 622 return (num_swapped_in); 623 } 624 625 /* 626 * Swap out lwps. Returns nonzero on success (i.e., if at least one lwp is 627 * swapped out) and 0 on failure. 628 */ 629 static int 630 swapout(proc_t *pp, uint_t *swrss, int swapflags) 631 { 632 kthread_id_t tp; 633 pgcnt_t ws_pages = 0; 634 int err; 635 int swapped_lwps = 0; 636 struct as *as = pp->p_as; 637 struct cpu *cpup = CPU; 638 pri_t thread_pri; 639 640 ASSERT(MUTEX_HELD(&pp->p_lock)); 641 642 if (pp->p_flag & SEXITING) 643 return (0); 644 645 top: 646 tp = pp->p_tlist; 647 do { 648 klwp_t *lwp = ttolwp(tp); 649 650 /* 651 * Swapout eligible lwps (specified by the scheduling 652 * class) which don't have TS_DONT_SWAP set. Set the 653 * "intent to swap" flag (TS_SWAPENQ) on threads 654 * which have TS_DONT_SWAP set so that they can be 655 * swapped if and when they reach a safe point. 656 */ 657 thread_lock(tp); 658 thread_pri = CL_SWAPOUT(tp, swapflags); 659 if (thread_pri != -1) { 660 if (tp->t_schedflag & TS_DONT_SWAP) { 661 tp->t_schedflag |= TS_SWAPENQ; 662 tp->t_trapret = 1; 663 aston(tp); 664 } else { 665 pgcnt_t stack_pages; 666 size_t stack_size; 667 668 ASSERT((tp->t_schedflag & 669 (TS_DONT_SWAP | TS_LOAD)) == TS_LOAD); 670 671 if (lock_try(&tp->t_lock)) { 672 /* 673 * Remove thread from the swap_queue. 674 */ 675 if (tp->t_schedflag & TS_ON_SWAPQ) { 676 ASSERT(!(tp->t_schedflag & 677 TS_SWAPENQ)); 678 swapdeq(tp); 679 } else if (tp->t_state == TS_RUN) 680 dq_srundec(tp); 681 682 tp->t_schedflag &= 683 ~(TS_LOAD | TS_SWAPENQ); 684 lock_clear(&tp->t_lock); 685 686 /* 687 * Set swapout time if the thread isn't 688 * sleeping. 689 */ 690 if (tp->t_state != TS_SLEEP) 691 tp->t_stime = lbolt; 692 thread_unlock(tp); 693 694 nswapped++; 695 tot_swapped_out++; 696 697 lwp->lwp_ru.nswap++; 698 699 /* 700 * Now drop the p_lock since the 701 * stack needs to pushed out. 702 */ 703 mutex_exit(&pp->p_lock); 704 705 stack_size = swapsize(tp->t_swap); 706 stack_pages = btopr(stack_size); 707 ws_pages += stack_pages; 708 /* Kernel probe */ 709 TNF_PROBE_4(swapout_lwp, 710 "vm swap swapout", 711 /* CSTYLED */, 712 tnf_pid, pid, pp->p_pid, 713 tnf_lwpid, lwpid, tp->t_tid, 714 tnf_kthread_id, tid, tp, 715 tnf_ulong, page_count, 716 stack_pages); 717 718 rw_enter(&kas.a_lock, RW_READER); 719 err = segkp_fault(segkp->s_as->a_hat, 720 segkp, tp->t_swap, stack_size, 721 F_SOFTUNLOCK, S_WRITE); 722 rw_exit(&kas.a_lock); 723 724 if (err) { 725 cmn_err(CE_PANIC, 726 "swapout: segkp_fault " 727 "failed err: %d", err); 728 } 729 CPU_STATS_ADDQ(cpup, 730 vm, pgswapout, stack_pages); 731 732 mutex_enter(&pp->p_lock); 733 pp->p_swapcnt++; 734 swapped_lwps++; 735 goto top; 736 } 737 } 738 } 739 thread_unlock(tp); 740 } while ((tp = tp->t_forw) != pp->p_tlist); 741 742 /* 743 * Unload address space when all lwps are swapped out. 744 */ 745 if (pp->p_swapcnt == pp->p_lwpcnt) { 746 size_t as_size = 0; 747 748 /* 749 * Avoid invoking as_swapout() if the process has 750 * no MMU resources since pageout will eventually 751 * steal pages belonging to this address space. This 752 * saves CPU cycles as the number of pages that are 753 * potentially freed or pushed out by the segment 754 * swapout operation is very small. 755 */ 756 if (rm_asrss(pp->p_as) != 0) 757 as_size = as_swapout(as); 758 759 CPU_STATS_ADDQ(cpup, vm, pgswapout, btop(as_size)); 760 CPU_STATS_ADDQ(cpup, vm, swapout, 1); 761 ws_pages += btop(as_size); 762 763 TRACE_2(TR_FAC_SCHED, TR_SWAPOUT, 764 "swapout: pp %p pages_pushed %lu", pp, ws_pages); 765 /* Kernel probe */ 766 TNF_PROBE_2(swapout_process, "vm swap swapout", /* CSTYLED */, 767 tnf_pid, pid, pp->p_pid, 768 tnf_ulong, page_count, ws_pages); 769 } 770 *swrss = ws_pages; 771 return (swapped_lwps); 772 } 773 774 void 775 swapout_lwp(klwp_t *lwp) 776 { 777 kthread_id_t tp = curthread; 778 779 ASSERT(curthread == lwptot(lwp)); 780 781 /* 782 * Don't insert the thread onto the swap queue if 783 * sufficient memory is available. 784 */ 785 if (avefree > desfree || avefree < desfree && freemem > desfree) { 786 thread_lock(tp); 787 tp->t_schedflag &= ~TS_SWAPENQ; 788 thread_unlock(tp); 789 return; 790 } 791 792 /* 793 * Lock the thread, then move it to the swapped queue from the 794 * onproc queue and set its state to be TS_RUN. 795 */ 796 thread_lock(tp); 797 ASSERT(tp->t_state == TS_ONPROC); 798 if (tp->t_schedflag & TS_SWAPENQ) { 799 tp->t_schedflag &= ~TS_SWAPENQ; 800 801 /* 802 * Set the state of this thread to be runnable 803 * and move it from the onproc queue to the swap queue. 804 */ 805 disp_swapped_enq(tp); 806 807 /* 808 * Insert the thread onto the swap queue. 809 */ 810 tp->t_link = tswap_queue; 811 tswap_queue = tp; 812 tp->t_schedflag |= TS_ON_SWAPQ; 813 814 thread_unlock_nopreempt(tp); 815 816 TRACE_1(TR_FAC_SCHED, TR_SWAPOUT_LWP, "swapout_lwp:%x", lwp); 817 818 swtch(); 819 } else { 820 thread_unlock(tp); 821 } 822 } 823 824 /* 825 * Swap all threads on the swap queue. 826 */ 827 static void 828 process_swap_queue(void) 829 { 830 kthread_id_t tp; 831 uint_t ws_pages; 832 proc_t *pp; 833 struct cpu *cpup = CPU; 834 klwp_t *lwp; 835 int err; 836 837 if (tswap_queue == NULL) 838 return; 839 840 /* 841 * Acquire the "swapped_lock" which locks the swap queue, 842 * and unload the stacks of all threads on it. 843 */ 844 disp_lock_enter(&swapped_lock); 845 while ((tp = tswap_queue) != NULL) { 846 pgcnt_t stack_pages; 847 size_t stack_size; 848 849 tswap_queue = tp->t_link; 850 tp->t_link = NULL; 851 852 /* 853 * Drop the "dispatcher lock" before acquiring "t_lock" 854 * to avoid spinning on it since the thread at the front 855 * of the swap queue could be pinned before giving up 856 * its "t_lock" in resume. 857 */ 858 disp_lock_exit(&swapped_lock); 859 lock_set(&tp->t_lock); 860 861 /* 862 * Now, re-acquire the "swapped_lock". Acquiring this lock 863 * results in locking the thread since its dispatcher lock 864 * (t_lockp) is the "swapped_lock". 865 */ 866 disp_lock_enter(&swapped_lock); 867 ASSERT(tp->t_state == TS_RUN); 868 ASSERT(tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)); 869 870 tp->t_schedflag &= ~(TS_LOAD | TS_ON_SWAPQ); 871 tp->t_stime = lbolt; /* swapout time */ 872 disp_lock_exit(&swapped_lock); 873 lock_clear(&tp->t_lock); 874 875 lwp = ttolwp(tp); 876 lwp->lwp_ru.nswap++; 877 878 pp = ttoproc(tp); 879 stack_size = swapsize(tp->t_swap); 880 stack_pages = btopr(stack_size); 881 882 /* Kernel probe */ 883 TNF_PROBE_4(swapout_lwp, "vm swap swapout", /* CSTYLED */, 884 tnf_pid, pid, pp->p_pid, 885 tnf_lwpid, lwpid, tp->t_tid, 886 tnf_kthread_id, tid, tp, 887 tnf_ulong, page_count, stack_pages); 888 889 rw_enter(&kas.a_lock, RW_READER); 890 err = segkp_fault(segkp->s_as->a_hat, segkp, tp->t_swap, 891 stack_size, F_SOFTUNLOCK, S_WRITE); 892 rw_exit(&kas.a_lock); 893 894 if (err) { 895 cmn_err(CE_PANIC, 896 "process_swap_list: segkp_fault failed err: %d", err); 897 } 898 CPU_STATS_ADDQ(cpup, vm, pgswapout, stack_pages); 899 900 nswapped++; 901 tot_swapped_out++; 902 swapqswap++; 903 904 /* 905 * Don't need p_lock since the swapper is the only 906 * thread which increments/decrements p_swapcnt and p_swrss. 907 */ 908 ws_pages = stack_pages; 909 pp->p_swapcnt++; 910 911 TRACE_1(TR_FAC_SCHED, TR_SWAPQ_LWP, "swaplist: pp %p", pp); 912 913 /* 914 * Unload address space when all lwps are swapped out. 915 */ 916 if (pp->p_swapcnt == pp->p_lwpcnt) { 917 size_t as_size = 0; 918 919 if (rm_asrss(pp->p_as) != 0) 920 as_size = as_swapout(pp->p_as); 921 922 CPU_STATS_ADDQ(cpup, vm, pgswapout, 923 btop(as_size)); 924 CPU_STATS_ADDQ(cpup, vm, swapout, 1); 925 926 ws_pages += btop(as_size); 927 928 TRACE_2(TR_FAC_SCHED, TR_SWAPQ_PROC, 929 "swaplist_proc: pp %p pages_pushed: %lu", 930 pp, ws_pages); 931 /* Kernel probe */ 932 TNF_PROBE_2(swapout_process, "vm swap swapout", 933 /* CSTYLED */, 934 tnf_pid, pid, pp->p_pid, 935 tnf_ulong, page_count, ws_pages); 936 } 937 pp->p_swrss += ws_pages; 938 disp_lock_enter(&swapped_lock); 939 } 940 disp_lock_exit(&swapped_lock); 941 } 942