1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 28 /* All Rights Reserved */ 29 30 #include <sys/param.h> 31 #include <sys/types.h> 32 #include <sys/sysmacros.h> 33 #include <sys/systm.h> 34 #include <sys/proc.h> 35 #include <sys/cpuvar.h> 36 #include <sys/var.h> 37 #include <sys/tuneable.h> 38 #include <sys/cmn_err.h> 39 #include <sys/buf.h> 40 #include <sys/disp.h> 41 #include <sys/vmsystm.h> 42 #include <sys/vmparam.h> 43 #include <sys/class.h> 44 #include <sys/vtrace.h> 45 #include <sys/modctl.h> 46 #include <sys/debug.h> 47 #include <sys/sdt.h> 48 #include <sys/tnf_probe.h> 49 #include <sys/procfs.h> 50 51 #include <vm/seg.h> 52 #include <vm/seg_kp.h> 53 #include <vm/as.h> 54 #include <vm/rm.h> 55 #include <vm/seg_kmem.h> 56 #include <sys/callb.h> 57 58 /* 59 * The swapper sleeps on runout when there is no one to swap in. 60 * It sleeps on runin when it could not find space to swap someone 61 * in or after swapping someone in. 62 */ 63 char runout; 64 char runin; 65 char wake_sched; /* flag tells clock to wake swapper on next tick */ 66 char wake_sched_sec; /* flag tells clock to wake swapper after a second */ 67 68 /* 69 * The swapper swaps processes to reduce memory demand and runs 70 * when avefree < desfree. The swapper resorts to SOFTSWAP when 71 * avefree < desfree which results in swapping out all processes 72 * sleeping for more than maxslp seconds. HARDSWAP occurs when the 73 * system is on the verge of thrashing and this results in swapping 74 * out runnable threads or threads sleeping for less than maxslp secs. 75 * 76 * The swapper runs through all the active processes in the system 77 * and invokes the scheduling class specific swapin/swapout routine 78 * for every thread in the process to obtain an effective priority 79 * for the process. A priority of -1 implies that the thread isn't 80 * swappable. This effective priority is used to find the most 81 * eligible process to swapout or swapin. 82 * 83 * NOTE: Threads which have been swapped are not linked on any 84 * queue and their dispatcher lock points at the "swapped_lock". 85 * 86 * Processes containing threads with the TS_DONT_SWAP flag set cannot be 87 * swapped out immediately by the swapper. This is due to the fact that 88 * such threads may be holding locks which may be needed by the swapper 89 * to push its pages out. The TS_SWAPENQ flag is set on such threads 90 * to prevent them running in user mode. When such threads reach a 91 * safe point (i.e., are not holding any locks - CL_TRAPRET), they 92 * queue themseleves onto the swap queue which is processed by the 93 * swapper. This results in reducing memory demand when the system 94 * is desparate for memory as the thread can't run in user mode. 95 * 96 * The swap queue consists of threads, linked via t_link, which are 97 * haven't been swapped, are runnable but not on the run queue. The 98 * swap queue is protected by the "swapped_lock". The dispatcher 99 * lock (t_lockp) of all threads on the swap queue points at the 100 * "swapped_lock". Thus, the entire queue and/or threads on the 101 * queue can be locked by acquiring "swapped_lock". 102 */ 103 static kthread_t *tswap_queue; 104 extern disp_lock_t swapped_lock; /* protects swap queue and threads on it */ 105 106 int maxslp = 0; 107 pgcnt_t avefree; /* 5 sec moving average of free memory */ 108 pgcnt_t avefree30; /* 30 sec moving average of free memory */ 109 110 /* 111 * Minimum size used to decide if sufficient memory is available 112 * before a process is swapped in. This is necessary since in most 113 * cases the actual size of a process (p_swrss) being swapped in 114 * is usually 2 pages (kernel stack pages). This is due to the fact 115 * almost all user pages of a process are stolen by pageout before 116 * the swapper decides to swapout it out. 117 */ 118 int min_procsize = 12; 119 120 static int swapin(proc_t *); 121 static int swapout(proc_t *, uint_t *, int); 122 static void process_swap_queue(); 123 124 #ifdef __sparc 125 extern void lwp_swapin(kthread_t *); 126 #endif /* __sparc */ 127 128 /* 129 * Counters to keep track of the number of swapins or swapouts. 130 */ 131 uint_t tot_swapped_in, tot_swapped_out; 132 uint_t softswap, hardswap, swapqswap; 133 134 /* 135 * Macro to determine if a process is eligble to be swapped. 136 */ 137 #define not_swappable(p) \ 138 (((p)->p_flag & SSYS) || (p)->p_stat == SIDL || \ 139 (p)->p_stat == SZOMB || (p)->p_as == NULL || \ 140 (p)->p_as == &kas) 141 142 /* 143 * Memory scheduler. 144 */ 145 void 146 sched() 147 { 148 kthread_id_t t; 149 pri_t proc_pri; 150 pri_t thread_pri; 151 pri_t swapin_pri; 152 int desperate; 153 pgcnt_t needs; 154 int divisor; 155 proc_t *prp; 156 proc_t *swapout_prp; 157 proc_t *swapin_prp; 158 spgcnt_t avail; 159 int chosen_pri; 160 time_t swapout_time; 161 time_t swapin_proc_time; 162 callb_cpr_t cprinfo; 163 kmutex_t swap_cpr_lock; 164 165 mutex_init(&swap_cpr_lock, NULL, MUTEX_DEFAULT, NULL); 166 CALLB_CPR_INIT(&cprinfo, &swap_cpr_lock, callb_generic_cpr, "sched"); 167 if (maxslp == 0) 168 maxslp = MAXSLP; 169 loop: 170 needs = 0; 171 desperate = 0; 172 173 swapin_pri = v.v_nglobpris; 174 swapin_prp = NULL; 175 chosen_pri = -1; 176 177 process_swap_queue(); 178 179 /* 180 * Set desperate if 181 * 1. At least 2 runnable processes (on average). 182 * 2. Short (5 sec) and longer (30 sec) average is less 183 * than minfree and desfree respectively. 184 * 3. Pagein + pageout rate is excessive. 185 */ 186 if (avenrun[0] >= 2 * FSCALE && 187 (MAX(avefree, avefree30) < desfree) && 188 (pginrate + pgoutrate > maxpgio || avefree < minfree)) { 189 TRACE_4(TR_FAC_SCHED, TR_DESPERATE, 190 "desp:avefree: %d, avefree30: %d, freemem: %d" 191 " pginrate: %d\n", avefree, avefree30, freemem, pginrate); 192 desperate = 1; 193 goto unload; 194 } 195 196 /* 197 * Search list of processes to swapin and swapout deadwood. 198 */ 199 swapin_proc_time = 0; 200 top: 201 mutex_enter(&pidlock); 202 for (prp = practive; prp != NULL; prp = prp->p_next) { 203 if (not_swappable(prp)) 204 continue; 205 206 /* 207 * Look at processes with at least one swapped lwp. 208 */ 209 if (prp->p_swapcnt) { 210 time_t proc_time; 211 212 /* 213 * Higher priority processes are good candidates 214 * to swapin. 215 */ 216 mutex_enter(&prp->p_lock); 217 proc_pri = -1; 218 t = prp->p_tlist; 219 proc_time = 0; 220 do { 221 if (t->t_schedflag & TS_LOAD) 222 continue; 223 224 thread_lock(t); 225 thread_pri = CL_SWAPIN(t, 0); 226 thread_unlock(t); 227 228 if (t->t_stime - proc_time > 0) 229 proc_time = t->t_stime; 230 if (thread_pri > proc_pri) 231 proc_pri = thread_pri; 232 } while ((t = t->t_forw) != prp->p_tlist); 233 mutex_exit(&prp->p_lock); 234 235 if (proc_pri == -1) 236 continue; 237 238 TRACE_3(TR_FAC_SCHED, TR_CHOOSE_SWAPIN, 239 "prp %p epri %d proc_time %d", 240 prp, proc_pri, proc_time); 241 242 /* 243 * Swapin processes with a high effective priority. 244 */ 245 if (swapin_prp == NULL || proc_pri > chosen_pri) { 246 swapin_prp = prp; 247 chosen_pri = proc_pri; 248 swapin_pri = proc_pri; 249 swapin_proc_time = proc_time; 250 } 251 } else { 252 /* 253 * No need to soft swap if we have sufficient 254 * memory. 255 */ 256 if (avefree > desfree || 257 avefree < desfree && freemem > desfree) 258 continue; 259 260 /* 261 * Skip processes that are exiting 262 * or whose address spaces are locked. 263 */ 264 mutex_enter(&prp->p_lock); 265 if ((prp->p_flag & SEXITING) || 266 (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) { 267 mutex_exit(&prp->p_lock); 268 continue; 269 } 270 271 /* 272 * Softswapping to kick out deadwood. 273 */ 274 proc_pri = -1; 275 t = prp->p_tlist; 276 do { 277 if ((t->t_schedflag & (TS_SWAPENQ | 278 TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD) 279 continue; 280 281 thread_lock(t); 282 thread_pri = CL_SWAPOUT(t, SOFTSWAP); 283 thread_unlock(t); 284 if (thread_pri > proc_pri) 285 proc_pri = thread_pri; 286 } while ((t = t->t_forw) != prp->p_tlist); 287 288 if (proc_pri != -1) { 289 uint_t swrss; 290 291 mutex_exit(&pidlock); 292 293 TRACE_1(TR_FAC_SCHED, TR_SOFTSWAP, 294 "softswap:prp %p", prp); 295 296 (void) swapout(prp, &swrss, SOFTSWAP); 297 softswap++; 298 prp->p_swrss += swrss; 299 mutex_exit(&prp->p_lock); 300 goto top; 301 } 302 mutex_exit(&prp->p_lock); 303 } 304 } 305 if (swapin_prp != NULL) 306 mutex_enter(&swapin_prp->p_lock); 307 mutex_exit(&pidlock); 308 309 if (swapin_prp == NULL) { 310 TRACE_3(TR_FAC_SCHED, TR_RUNOUT, 311 "schedrunout:runout nswapped: %d, avefree: %ld freemem: %ld", 312 nswapped, avefree, freemem); 313 314 t = curthread; 315 thread_lock(t); 316 runout++; 317 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART); 318 t->t_whystop = PR_SUSPENDED; 319 t->t_whatstop = SUSPEND_NORMAL; 320 (void) new_mstate(t, LMS_SLEEP); 321 mutex_enter(&swap_cpr_lock); 322 CALLB_CPR_SAFE_BEGIN(&cprinfo); 323 mutex_exit(&swap_cpr_lock); 324 thread_stop(t); /* change state and drop lock */ 325 swtch(); 326 mutex_enter(&swap_cpr_lock); 327 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock); 328 mutex_exit(&swap_cpr_lock); 329 goto loop; 330 } 331 332 /* 333 * Decide how deserving this process is to be brought in. 334 * Needs is an estimate of how much core the process will 335 * need. If the process has been out for a while, then we 336 * will bring it in with 1/2 the core needed, otherwise 337 * we are conservative. 338 */ 339 divisor = 1; 340 swapout_time = (ddi_get_lbolt() - swapin_proc_time) / hz; 341 if (swapout_time > maxslp / 2) 342 divisor = 2; 343 344 needs = MIN(swapin_prp->p_swrss, lotsfree); 345 needs = MAX(needs, min_procsize); 346 needs = needs / divisor; 347 348 /* 349 * Use freemem, since we want processes to be swapped 350 * in quickly. 351 */ 352 avail = freemem - deficit; 353 if (avail > (spgcnt_t)needs) { 354 deficit += needs; 355 356 TRACE_2(TR_FAC_SCHED, TR_SWAPIN_VALUES, 357 "swapin_values: prp %p needs %lu", swapin_prp, needs); 358 359 if (swapin(swapin_prp)) { 360 mutex_exit(&swapin_prp->p_lock); 361 goto loop; 362 } 363 deficit -= MIN(needs, deficit); 364 mutex_exit(&swapin_prp->p_lock); 365 } else { 366 mutex_exit(&swapin_prp->p_lock); 367 /* 368 * If deficit is high, too many processes have been 369 * swapped in so wait a sec before attempting to 370 * swapin more. 371 */ 372 if (freemem > needs) { 373 TRACE_2(TR_FAC_SCHED, TR_HIGH_DEFICIT, 374 "deficit: prp %p needs %lu", swapin_prp, needs); 375 goto block; 376 } 377 } 378 379 TRACE_2(TR_FAC_SCHED, TR_UNLOAD, 380 "unload: prp %p needs %lu", swapin_prp, needs); 381 382 unload: 383 /* 384 * Unload all unloadable modules, free all other memory 385 * resources we can find, then look for a thread to hardswap. 386 */ 387 modreap(); 388 segkp_cache_free(); 389 390 swapout_prp = NULL; 391 mutex_enter(&pidlock); 392 for (prp = practive; prp != NULL; prp = prp->p_next) { 393 394 /* 395 * No need to soft swap if we have sufficient 396 * memory. 397 */ 398 if (not_swappable(prp)) 399 continue; 400 401 if (avefree > minfree || 402 avefree < minfree && freemem > desfree) { 403 swapout_prp = NULL; 404 break; 405 } 406 407 /* 408 * Skip processes that are exiting 409 * or whose address spaces are locked. 410 */ 411 mutex_enter(&prp->p_lock); 412 if ((prp->p_flag & SEXITING) || 413 (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) { 414 mutex_exit(&prp->p_lock); 415 continue; 416 } 417 418 proc_pri = -1; 419 t = prp->p_tlist; 420 do { 421 if ((t->t_schedflag & (TS_SWAPENQ | 422 TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD) 423 continue; 424 425 thread_lock(t); 426 thread_pri = CL_SWAPOUT(t, HARDSWAP); 427 thread_unlock(t); 428 if (thread_pri > proc_pri) 429 proc_pri = thread_pri; 430 } while ((t = t->t_forw) != prp->p_tlist); 431 432 mutex_exit(&prp->p_lock); 433 if (proc_pri == -1) 434 continue; 435 436 /* 437 * Swapout processes sleeping with a lower priority 438 * than the one currently being swapped in, if any. 439 */ 440 if (swapin_prp == NULL || swapin_pri > proc_pri) { 441 TRACE_2(TR_FAC_SCHED, TR_CHOOSE_SWAPOUT, 442 "hardswap: prp %p needs %lu", prp, needs); 443 444 if (swapout_prp == NULL || proc_pri < chosen_pri) { 445 swapout_prp = prp; 446 chosen_pri = proc_pri; 447 } 448 } 449 } 450 451 /* 452 * Acquire the "p_lock" before dropping "pidlock" 453 * to prevent the proc structure from being freed 454 * if the process exits before swapout completes. 455 */ 456 if (swapout_prp != NULL) 457 mutex_enter(&swapout_prp->p_lock); 458 mutex_exit(&pidlock); 459 460 if ((prp = swapout_prp) != NULL) { 461 uint_t swrss = 0; 462 int swapped; 463 464 swapped = swapout(prp, &swrss, HARDSWAP); 465 if (swapped) { 466 /* 467 * If desperate, we want to give the space obtained 468 * by swapping this process out to processes in core, 469 * so we give them a chance by increasing deficit. 470 */ 471 prp->p_swrss += swrss; 472 if (desperate) 473 deficit += MIN(prp->p_swrss, lotsfree); 474 hardswap++; 475 } 476 mutex_exit(&swapout_prp->p_lock); 477 478 if (swapped) 479 goto loop; 480 } 481 482 /* 483 * Delay for 1 second and look again later. 484 */ 485 TRACE_3(TR_FAC_SCHED, TR_RUNIN, 486 "schedrunin:runin nswapped: %d, avefree: %ld freemem: %ld", 487 nswapped, avefree, freemem); 488 489 block: 490 t = curthread; 491 thread_lock(t); 492 runin++; 493 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART); 494 t->t_whystop = PR_SUSPENDED; 495 t->t_whatstop = SUSPEND_NORMAL; 496 (void) new_mstate(t, LMS_SLEEP); 497 mutex_enter(&swap_cpr_lock); 498 CALLB_CPR_SAFE_BEGIN(&cprinfo); 499 mutex_exit(&swap_cpr_lock); 500 thread_stop(t); /* change to stop state and drop lock */ 501 swtch(); 502 mutex_enter(&swap_cpr_lock); 503 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock); 504 mutex_exit(&swap_cpr_lock); 505 goto loop; 506 } 507 508 /* 509 * Remove the specified thread from the swap queue. 510 */ 511 static void 512 swapdeq(kthread_id_t tp) 513 { 514 kthread_id_t *tpp; 515 516 ASSERT(THREAD_LOCK_HELD(tp)); 517 ASSERT(tp->t_schedflag & TS_ON_SWAPQ); 518 519 tpp = &tswap_queue; 520 for (;;) { 521 ASSERT(*tpp != NULL); 522 if (*tpp == tp) 523 break; 524 tpp = &(*tpp)->t_link; 525 } 526 *tpp = tp->t_link; 527 tp->t_schedflag &= ~TS_ON_SWAPQ; 528 } 529 530 /* 531 * Swap in lwps. Returns nonzero on success (i.e., if at least one lwp is 532 * swapped in) and 0 on failure. 533 */ 534 static int 535 swapin(proc_t *pp) 536 { 537 kthread_id_t tp; 538 int err; 539 int num_swapped_in = 0; 540 struct cpu *cpup = CPU; 541 pri_t thread_pri; 542 543 ASSERT(MUTEX_HELD(&pp->p_lock)); 544 ASSERT(pp->p_swapcnt); 545 546 top: 547 tp = pp->p_tlist; 548 do { 549 /* 550 * Only swapin eligible lwps (specified by the scheduling 551 * class) which are unloaded and ready to run. 552 */ 553 thread_lock(tp); 554 thread_pri = CL_SWAPIN(tp, 0); 555 if (thread_pri != -1 && tp->t_state == TS_RUN && 556 (tp->t_schedflag & TS_LOAD) == 0) { 557 size_t stack_size; 558 pgcnt_t stack_pages; 559 560 ASSERT((tp->t_schedflag & TS_ON_SWAPQ) == 0); 561 562 thread_unlock(tp); 563 /* 564 * Now drop the p_lock since the stack needs 565 * to brought in. 566 */ 567 mutex_exit(&pp->p_lock); 568 569 stack_size = swapsize(tp->t_swap); 570 stack_pages = btopr(stack_size); 571 572 /* Kernel probe */ 573 DTRACE_SCHED1(swapin__lwp, kthread_t *, tp); 574 TNF_PROBE_4(swapin_lwp, "vm swap swapin", /* CSTYLED */, 575 tnf_pid, pid, pp->p_pid, 576 tnf_lwpid, lwpid, tp->t_tid, 577 tnf_kthread_id, tid, tp, 578 tnf_ulong, page_count, stack_pages); 579 580 rw_enter(&kas.a_lock, RW_READER); 581 err = segkp_fault(segkp->s_as->a_hat, segkp, 582 tp->t_swap, stack_size, F_SOFTLOCK, S_OTHER); 583 rw_exit(&kas.a_lock); 584 585 /* 586 * Re-acquire the p_lock. 587 */ 588 mutex_enter(&pp->p_lock); 589 if (err) { 590 num_swapped_in = 0; 591 break; 592 } else { 593 #ifdef __sparc 594 lwp_swapin(tp); 595 #endif /* __sparc */ 596 CPU_STATS_ADDQ(cpup, vm, swapin, 1); 597 CPU_STATS_ADDQ(cpup, vm, pgswapin, 598 stack_pages); 599 600 pp->p_swapcnt--; 601 pp->p_swrss -= stack_pages; 602 603 thread_lock(tp); 604 tp->t_schedflag |= TS_LOAD; 605 dq_sruninc(tp); 606 607 /* set swapin time */ 608 tp->t_stime = ddi_get_lbolt(); 609 thread_unlock(tp); 610 611 nswapped--; 612 tot_swapped_in++; 613 num_swapped_in++; 614 615 TRACE_2(TR_FAC_SCHED, TR_SWAPIN, 616 "swapin: pp %p stack_pages %lu", 617 pp, stack_pages); 618 goto top; 619 } 620 } 621 thread_unlock(tp); 622 } while ((tp = tp->t_forw) != pp->p_tlist); 623 return (num_swapped_in); 624 } 625 626 /* 627 * Swap out lwps. Returns nonzero on success (i.e., if at least one lwp is 628 * swapped out) and 0 on failure. 629 */ 630 static int 631 swapout(proc_t *pp, uint_t *swrss, int swapflags) 632 { 633 kthread_id_t tp; 634 pgcnt_t ws_pages = 0; 635 int err; 636 int swapped_lwps = 0; 637 struct as *as = pp->p_as; 638 struct cpu *cpup = CPU; 639 pri_t thread_pri; 640 641 ASSERT(MUTEX_HELD(&pp->p_lock)); 642 643 if (pp->p_flag & SEXITING) 644 return (0); 645 646 top: 647 tp = pp->p_tlist; 648 do { 649 klwp_t *lwp = ttolwp(tp); 650 651 /* 652 * Swapout eligible lwps (specified by the scheduling 653 * class) which don't have TS_DONT_SWAP set. Set the 654 * "intent to swap" flag (TS_SWAPENQ) on threads 655 * which have TS_DONT_SWAP set so that they can be 656 * swapped if and when they reach a safe point. 657 */ 658 thread_lock(tp); 659 thread_pri = CL_SWAPOUT(tp, swapflags); 660 if (thread_pri != -1) { 661 if (tp->t_schedflag & TS_DONT_SWAP) { 662 tp->t_schedflag |= TS_SWAPENQ; 663 tp->t_trapret = 1; 664 aston(tp); 665 } else { 666 pgcnt_t stack_pages; 667 size_t stack_size; 668 669 ASSERT((tp->t_schedflag & 670 (TS_DONT_SWAP | TS_LOAD)) == TS_LOAD); 671 672 if (lock_try(&tp->t_lock)) { 673 /* 674 * Remove thread from the swap_queue. 675 */ 676 if (tp->t_schedflag & TS_ON_SWAPQ) { 677 ASSERT(!(tp->t_schedflag & 678 TS_SWAPENQ)); 679 swapdeq(tp); 680 } else if (tp->t_state == TS_RUN) 681 dq_srundec(tp); 682 683 tp->t_schedflag &= 684 ~(TS_LOAD | TS_SWAPENQ); 685 lock_clear(&tp->t_lock); 686 687 /* 688 * Set swapout time if the thread isn't 689 * sleeping. 690 */ 691 if (tp->t_state != TS_SLEEP) 692 tp->t_stime = ddi_get_lbolt(); 693 thread_unlock(tp); 694 695 nswapped++; 696 tot_swapped_out++; 697 698 lwp->lwp_ru.nswap++; 699 700 /* 701 * Now drop the p_lock since the 702 * stack needs to pushed out. 703 */ 704 mutex_exit(&pp->p_lock); 705 706 stack_size = swapsize(tp->t_swap); 707 stack_pages = btopr(stack_size); 708 ws_pages += stack_pages; 709 710 /* Kernel probe */ 711 DTRACE_SCHED1(swapout__lwp, 712 kthread_t *, tp); 713 TNF_PROBE_4(swapout_lwp, 714 "vm swap swapout", 715 /* CSTYLED */, 716 tnf_pid, pid, pp->p_pid, 717 tnf_lwpid, lwpid, tp->t_tid, 718 tnf_kthread_id, tid, tp, 719 tnf_ulong, page_count, 720 stack_pages); 721 722 rw_enter(&kas.a_lock, RW_READER); 723 err = segkp_fault(segkp->s_as->a_hat, 724 segkp, tp->t_swap, stack_size, 725 F_SOFTUNLOCK, S_WRITE); 726 rw_exit(&kas.a_lock); 727 728 if (err) { 729 cmn_err(CE_PANIC, 730 "swapout: segkp_fault " 731 "failed err: %d", err); 732 } 733 CPU_STATS_ADDQ(cpup, 734 vm, pgswapout, stack_pages); 735 736 mutex_enter(&pp->p_lock); 737 pp->p_swapcnt++; 738 swapped_lwps++; 739 goto top; 740 } 741 } 742 } 743 thread_unlock(tp); 744 } while ((tp = tp->t_forw) != pp->p_tlist); 745 746 /* 747 * Unload address space when all lwps are swapped out. 748 */ 749 if (pp->p_swapcnt == pp->p_lwpcnt) { 750 size_t as_size = 0; 751 752 /* 753 * Avoid invoking as_swapout() if the process has 754 * no MMU resources since pageout will eventually 755 * steal pages belonging to this address space. This 756 * saves CPU cycles as the number of pages that are 757 * potentially freed or pushed out by the segment 758 * swapout operation is very small. 759 */ 760 if (rm_asrss(pp->p_as) != 0) 761 as_size = as_swapout(as); 762 763 CPU_STATS_ADDQ(cpup, vm, pgswapout, btop(as_size)); 764 CPU_STATS_ADDQ(cpup, vm, swapout, 1); 765 ws_pages += btop(as_size); 766 767 TRACE_2(TR_FAC_SCHED, TR_SWAPOUT, 768 "swapout: pp %p pages_pushed %lu", pp, ws_pages); 769 /* Kernel probe */ 770 DTRACE_SCHED1(swapout__process, proc_t *, pp); 771 TNF_PROBE_2(swapout_process, "vm swap swapout", /* CSTYLED */, 772 tnf_pid, pid, pp->p_pid, 773 tnf_ulong, page_count, ws_pages); 774 } 775 *swrss = ws_pages; 776 return (swapped_lwps); 777 } 778 779 void 780 swapout_lwp(klwp_t *lwp) 781 { 782 kthread_id_t tp = curthread; 783 784 ASSERT(curthread == lwptot(lwp)); 785 786 /* 787 * Don't insert the thread onto the swap queue if 788 * sufficient memory is available. 789 */ 790 if (avefree > desfree || avefree < desfree && freemem > desfree) { 791 thread_lock(tp); 792 tp->t_schedflag &= ~TS_SWAPENQ; 793 thread_unlock(tp); 794 return; 795 } 796 797 /* 798 * Lock the thread, then move it to the swapped queue from the 799 * onproc queue and set its state to be TS_RUN. 800 */ 801 thread_lock(tp); 802 ASSERT(tp->t_state == TS_ONPROC); 803 if (tp->t_schedflag & TS_SWAPENQ) { 804 tp->t_schedflag &= ~TS_SWAPENQ; 805 806 /* 807 * Set the state of this thread to be runnable 808 * and move it from the onproc queue to the swap queue. 809 */ 810 disp_swapped_enq(tp); 811 812 /* 813 * Insert the thread onto the swap queue. 814 */ 815 tp->t_link = tswap_queue; 816 tswap_queue = tp; 817 tp->t_schedflag |= TS_ON_SWAPQ; 818 819 thread_unlock_nopreempt(tp); 820 821 TRACE_1(TR_FAC_SCHED, TR_SWAPOUT_LWP, "swapout_lwp:%x", lwp); 822 823 swtch(); 824 } else { 825 thread_unlock(tp); 826 } 827 } 828 829 /* 830 * Swap all threads on the swap queue. 831 */ 832 static void 833 process_swap_queue(void) 834 { 835 kthread_id_t tp; 836 uint_t ws_pages; 837 proc_t *pp; 838 struct cpu *cpup = CPU; 839 klwp_t *lwp; 840 int err; 841 842 if (tswap_queue == NULL) 843 return; 844 845 /* 846 * Acquire the "swapped_lock" which locks the swap queue, 847 * and unload the stacks of all threads on it. 848 */ 849 disp_lock_enter(&swapped_lock); 850 while ((tp = tswap_queue) != NULL) { 851 pgcnt_t stack_pages; 852 size_t stack_size; 853 854 tswap_queue = tp->t_link; 855 tp->t_link = NULL; 856 857 /* 858 * Drop the "dispatcher lock" before acquiring "t_lock" 859 * to avoid spinning on it since the thread at the front 860 * of the swap queue could be pinned before giving up 861 * its "t_lock" in resume. 862 */ 863 disp_lock_exit(&swapped_lock); 864 lock_set(&tp->t_lock); 865 866 /* 867 * Now, re-acquire the "swapped_lock". Acquiring this lock 868 * results in locking the thread since its dispatcher lock 869 * (t_lockp) is the "swapped_lock". 870 */ 871 disp_lock_enter(&swapped_lock); 872 ASSERT(tp->t_state == TS_RUN); 873 ASSERT(tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)); 874 875 tp->t_schedflag &= ~(TS_LOAD | TS_ON_SWAPQ); 876 tp->t_stime = ddi_get_lbolt(); /* swapout time */ 877 disp_lock_exit(&swapped_lock); 878 lock_clear(&tp->t_lock); 879 880 lwp = ttolwp(tp); 881 lwp->lwp_ru.nswap++; 882 883 pp = ttoproc(tp); 884 stack_size = swapsize(tp->t_swap); 885 stack_pages = btopr(stack_size); 886 887 /* Kernel probe */ 888 DTRACE_SCHED1(swapout__lwp, kthread_t *, tp); 889 TNF_PROBE_4(swapout_lwp, "vm swap swapout", /* CSTYLED */, 890 tnf_pid, pid, pp->p_pid, 891 tnf_lwpid, lwpid, tp->t_tid, 892 tnf_kthread_id, tid, tp, 893 tnf_ulong, page_count, stack_pages); 894 895 rw_enter(&kas.a_lock, RW_READER); 896 err = segkp_fault(segkp->s_as->a_hat, segkp, tp->t_swap, 897 stack_size, F_SOFTUNLOCK, S_WRITE); 898 rw_exit(&kas.a_lock); 899 900 if (err) { 901 cmn_err(CE_PANIC, 902 "process_swap_list: segkp_fault failed err: %d", err); 903 } 904 CPU_STATS_ADDQ(cpup, vm, pgswapout, stack_pages); 905 906 nswapped++; 907 tot_swapped_out++; 908 swapqswap++; 909 910 /* 911 * Don't need p_lock since the swapper is the only 912 * thread which increments/decrements p_swapcnt and p_swrss. 913 */ 914 ws_pages = stack_pages; 915 pp->p_swapcnt++; 916 917 TRACE_1(TR_FAC_SCHED, TR_SWAPQ_LWP, "swaplist: pp %p", pp); 918 919 /* 920 * Unload address space when all lwps are swapped out. 921 */ 922 if (pp->p_swapcnt == pp->p_lwpcnt) { 923 size_t as_size = 0; 924 925 if (rm_asrss(pp->p_as) != 0) 926 as_size = as_swapout(pp->p_as); 927 928 CPU_STATS_ADDQ(cpup, vm, pgswapout, 929 btop(as_size)); 930 CPU_STATS_ADDQ(cpup, vm, swapout, 1); 931 932 ws_pages += btop(as_size); 933 934 TRACE_2(TR_FAC_SCHED, TR_SWAPQ_PROC, 935 "swaplist_proc: pp %p pages_pushed: %lu", 936 pp, ws_pages); 937 /* Kernel probe */ 938 DTRACE_SCHED1(swapout__process, proc_t *, pp); 939 TNF_PROBE_2(swapout_process, "vm swap swapout", 940 /* CSTYLED */, 941 tnf_pid, pid, pp->p_pid, 942 tnf_ulong, page_count, ws_pages); 943 } 944 pp->p_swrss += ws_pages; 945 disp_lock_enter(&swapped_lock); 946 } 947 disp_lock_exit(&swapped_lock); 948 } 949