1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/param.h> 28 #include <sys/thread.h> 29 #include <sys/proc.h> 30 #include <sys/callb.h> 31 #include <sys/vnode.h> 32 #include <sys/debug.h> 33 #include <sys/systm.h> /* for bzero */ 34 #include <sys/memlist.h> 35 #include <sys/cmn_err.h> 36 #include <sys/sysmacros.h> 37 #include <sys/vmsystm.h> /* for NOMEMWAIT() */ 38 #include <sys/atomic.h> /* used to update kcage_freemem */ 39 #include <sys/kmem.h> /* for kmem_reap */ 40 #include <sys/errno.h> 41 #include <sys/mem_cage.h> 42 #include <vm/seg_kmem.h> 43 #include <vm/page.h> 44 #include <vm/hat.h> 45 #include <vm/vm_dep.h> 46 #include <sys/mem_config.h> 47 #include <sys/lgrp.h> 48 #include <sys/rwlock.h> 49 #include <sys/cpupart.h> 50 51 extern pri_t maxclsyspri; 52 53 #ifdef DEBUG 54 #define KCAGE_STATS 55 #endif 56 57 #ifdef KCAGE_STATS 58 59 #define KCAGE_STATS_VERSION 9 /* can help report generators */ 60 #define KCAGE_STATS_NSCANS 256 /* depth of scan statistics buffer */ 61 62 struct kcage_stats_scan { 63 /* managed by KCAGE_STAT_* macros */ 64 clock_t scan_lbolt; 65 uint_t scan_id; 66 67 /* set in kcage_cageout() */ 68 uint_t kt_passes; 69 clock_t kt_ticks; 70 pgcnt_t kt_kcage_freemem_start; 71 pgcnt_t kt_kcage_freemem_end; 72 pgcnt_t kt_freemem_start; 73 pgcnt_t kt_freemem_end; 74 uint_t kt_examined; 75 uint_t kt_cantlock; 76 uint_t kt_gotone; 77 uint_t kt_gotonefree; 78 uint_t kt_skiplevel; 79 uint_t kt_skipshared; 80 uint_t kt_skiprefd; 81 uint_t kt_destroy; 82 83 /* set in kcage_invalidate_page() */ 84 uint_t kip_reloclocked; 85 uint_t kip_relocmod; 86 uint_t kip_destroy; 87 uint_t kip_nomem; 88 uint_t kip_demotefailed; 89 90 /* set in kcage_expand() */ 91 uint_t ke_wanted; 92 uint_t ke_examined; 93 uint_t ke_lefthole; 94 uint_t ke_gotone; 95 uint_t ke_gotonefree; 96 }; 97 98 struct kcage_stats { 99 /* managed by KCAGE_STAT_* macros */ 100 uint_t version; 101 uint_t size; 102 103 /* set in kcage_cageout */ 104 uint_t kt_wakeups; 105 uint_t kt_scans; 106 uint_t kt_cageout_break; 107 108 /* set in kcage_expand */ 109 uint_t ke_calls; 110 uint_t ke_nopfn; 111 uint_t ke_nopaget; 112 uint_t ke_isnoreloc; 113 uint_t ke_deleting; 114 uint_t ke_lowfreemem; 115 uint_t ke_terminate; 116 117 /* set in kcage_freemem_add() */ 118 uint_t kfa_trottlewake; 119 120 /* set in kcage_freemem_sub() */ 121 uint_t kfs_cagewake; 122 123 /* set in kcage_create_throttle */ 124 uint_t kct_calls; 125 uint_t kct_cageout; 126 uint_t kct_critical; 127 uint_t kct_exempt; 128 uint_t kct_cagewake; 129 uint_t kct_wait; 130 uint_t kct_progress; 131 uint_t kct_noprogress; 132 uint_t kct_timeout; 133 134 /* set in kcage_cageout_wakeup */ 135 uint_t kcw_expandearly; 136 137 /* managed by KCAGE_STAT_* macros */ 138 uint_t scan_array_size; 139 uint_t scan_index; 140 struct kcage_stats_scan scans[KCAGE_STATS_NSCANS]; 141 }; 142 143 static struct kcage_stats kcage_stats; 144 static struct kcage_stats_scan kcage_stats_scan_zero; 145 146 /* 147 * No real need for atomics here. For the most part the incs and sets are 148 * done by the kernel cage thread. There are a few that are done by any 149 * number of other threads. Those cases are noted by comments. 150 */ 151 #define KCAGE_STAT_INCR(m) kcage_stats.m++ 152 153 #define KCAGE_STAT_NINCR(m, v) kcage_stats.m += (v) 154 155 #define KCAGE_STAT_INCR_SCAN(m) \ 156 KCAGE_STAT_INCR(scans[kcage_stats.scan_index].m) 157 158 #define KCAGE_STAT_NINCR_SCAN(m, v) \ 159 KCAGE_STAT_NINCR(scans[kcage_stats.scan_index].m, v) 160 161 #define KCAGE_STAT_SET(m, v) kcage_stats.m = (v) 162 163 #define KCAGE_STAT_SETZ(m, v) \ 164 if (kcage_stats.m == 0) kcage_stats.m = (v) 165 166 #define KCAGE_STAT_SET_SCAN(m, v) \ 167 KCAGE_STAT_SET(scans[kcage_stats.scan_index].m, v) 168 169 #define KCAGE_STAT_SETZ_SCAN(m, v) \ 170 KCAGE_STAT_SETZ(scans[kcage_stats.scan_index].m, v) 171 172 #define KCAGE_STAT_INC_SCAN_INDEX \ 173 KCAGE_STAT_SET_SCAN(scan_lbolt, lbolt); \ 174 KCAGE_STAT_SET_SCAN(scan_id, kcage_stats.scan_index); \ 175 kcage_stats.scan_index = \ 176 (kcage_stats.scan_index + 1) % KCAGE_STATS_NSCANS; \ 177 kcage_stats.scans[kcage_stats.scan_index] = kcage_stats_scan_zero 178 179 #define KCAGE_STAT_INIT_SCAN_INDEX \ 180 kcage_stats.version = KCAGE_STATS_VERSION; \ 181 kcage_stats.size = sizeof (kcage_stats); \ 182 kcage_stats.scan_array_size = KCAGE_STATS_NSCANS; \ 183 kcage_stats.scan_index = 0 184 185 #else /* KCAGE_STATS */ 186 187 #define KCAGE_STAT_INCR(v) 188 #define KCAGE_STAT_NINCR(m, v) 189 #define KCAGE_STAT_INCR_SCAN(v) 190 #define KCAGE_STAT_NINCR_SCAN(m, v) 191 #define KCAGE_STAT_SET(m, v) 192 #define KCAGE_STAT_SETZ(m, v) 193 #define KCAGE_STAT_SET_SCAN(m, v) 194 #define KCAGE_STAT_SETZ_SCAN(m, v) 195 #define KCAGE_STAT_INC_SCAN_INDEX 196 #define KCAGE_STAT_INIT_SCAN_INDEX 197 198 #endif /* KCAGE_STATS */ 199 200 static kmutex_t kcage_throttle_mutex; /* protects kcage_throttle_cv */ 201 static kcondvar_t kcage_throttle_cv; 202 203 static kmutex_t kcage_cageout_mutex; /* protects cv and ready flag */ 204 static kcondvar_t kcage_cageout_cv; /* cageout thread naps here */ 205 static int kcage_cageout_ready; /* nonzero when cageout thread ready */ 206 kthread_id_t kcage_cageout_thread; /* to aid debugging */ 207 208 static krwlock_t kcage_range_rwlock; /* protects kcage_glist elements */ 209 210 /* 211 * Cage expansion happens within a range. 212 */ 213 struct kcage_glist { 214 struct kcage_glist *next; 215 pfn_t base; 216 pfn_t lim; 217 pfn_t curr; 218 int decr; 219 }; 220 221 static struct kcage_glist *kcage_glist; 222 static struct kcage_glist *kcage_current_glist; 223 224 /* 225 * The firstfree element is provided so that kmem_alloc can be avoided 226 * until that cage has somewhere to go. This is not currently a problem 227 * as early kmem_alloc's use BOP_ALLOC instead of page_create_va. 228 */ 229 static vmem_t *kcage_arena; 230 static struct kcage_glist kcage_glist_firstfree; 231 static struct kcage_glist *kcage_glist_freelist = &kcage_glist_firstfree; 232 233 /* 234 * Miscellaneous forward references 235 */ 236 static struct kcage_glist *kcage_glist_alloc(void); 237 static int kcage_glist_delete(pfn_t, pfn_t, struct kcage_glist **); 238 static void kcage_cageout(void); 239 static int kcage_invalidate_page(page_t *, pgcnt_t *); 240 static int kcage_setnoreloc_pages(page_t *, se_t); 241 static int kcage_range_add_internal(pfn_t base, pgcnt_t npgs, kcage_dir_t); 242 static void kcage_init(pgcnt_t preferred_size); 243 static int kcage_range_delete_internal(pfn_t base, pgcnt_t npgs); 244 245 /* 246 * Kernel Memory Cage counters and thresholds. 247 */ 248 int kcage_on = 0; 249 pgcnt_t kcage_freemem; 250 pgcnt_t kcage_needfree; 251 pgcnt_t kcage_lotsfree; 252 pgcnt_t kcage_desfree; 253 pgcnt_t kcage_minfree; 254 pgcnt_t kcage_throttlefree; 255 pgcnt_t kcage_reserve; 256 int kcage_maxwait = 10; /* in seconds */ 257 258 /* when we use lp for kmem we start the cage at a higher initial value */ 259 pgcnt_t kcage_kmemlp_mincage; 260 261 #ifdef DEBUG 262 pgcnt_t kcage_pagets; 263 #define KCAGEPAGETS_INC() kcage_pagets++ 264 #else 265 #define KCAGEPAGETS_INC() 266 #endif 267 268 /* kstats to export what pages are currently caged */ 269 kmutex_t kcage_kstat_lock; 270 static int kcage_kstat_update(kstat_t *ksp, int rw); 271 static int kcage_kstat_snapshot(kstat_t *ksp, void *buf, int rw); 272 273 /* 274 * Startup and Dynamic Reconfiguration interfaces. 275 * kcage_range_add() 276 * kcage_range_del() 277 * kcage_range_delete_post_mem_del() 278 * kcage_range_init() 279 * kcage_set_thresholds() 280 */ 281 282 /* 283 * Called from page_get_contig_pages to get the approximate kcage pfn range 284 * for exclusion from search for contiguous pages. This routine is called 285 * without kcage_range lock (kcage routines can call page_get_contig_pages 286 * through page_relocate) and with the assumption, based on kcage_range_add, 287 * that kcage_current_glist always contain a valid pointer. 288 */ 289 290 int 291 kcage_current_pfn(pfn_t *pfncur) 292 { 293 struct kcage_glist *lp = kcage_current_glist; 294 295 ASSERT(kcage_on); 296 297 ASSERT(lp != NULL); 298 299 *pfncur = lp->curr; 300 301 return (lp->decr); 302 } 303 304 /* 305 * Called from vm_pagelist.c during coalesce to find kernel cage regions 306 * within an mnode. Looks for the lowest range between lo and hi. 307 * 308 * Kernel cage memory is defined between kcage_glist and kcage_current_glist. 309 * Non-cage memory is defined between kcage_current_glist and list end. 310 * 311 * If incage is set, returns the lowest kcage range. Otherwise returns lowest 312 * non-cage range. 313 * 314 * Returns zero on success and nlo, nhi: 315 * lo <= nlo < nhi <= hi 316 * Returns non-zero if no overlapping range is found. 317 */ 318 int 319 kcage_next_range(int incage, pfn_t lo, pfn_t hi, 320 pfn_t *nlo, pfn_t *nhi) 321 { 322 struct kcage_glist *lp; 323 pfn_t tlo = hi; 324 pfn_t thi = hi; 325 326 ASSERT(lo <= hi); 327 328 /* 329 * Reader lock protects the list, but kcage_get_pfn 330 * running concurrently may advance kcage_current_glist 331 * and also update kcage_current_glist->curr. Page 332 * coalesce can handle this race condition. 333 */ 334 rw_enter(&kcage_range_rwlock, RW_READER); 335 336 for (lp = incage ? kcage_glist : kcage_current_glist; 337 lp != NULL; lp = lp->next) { 338 339 pfn_t klo, khi; 340 341 /* find the range limits in this element */ 342 if ((incage && lp->decr) || (!incage && !lp->decr)) { 343 klo = lp->curr; 344 khi = lp->lim; 345 } else { 346 klo = lp->base; 347 khi = lp->curr; 348 } 349 350 /* handle overlap */ 351 if (klo < tlo && klo < khi && lo < khi && klo < hi) { 352 tlo = MAX(lo, klo); 353 thi = MIN(hi, khi); 354 if (tlo == lo) 355 break; 356 } 357 358 /* check end of kcage */ 359 if (incage && lp == kcage_current_glist) { 360 break; 361 } 362 } 363 364 rw_exit(&kcage_range_rwlock); 365 366 /* return non-zero if no overlapping range found */ 367 if (tlo == thi) 368 return (1); 369 370 ASSERT(lo <= tlo && tlo < thi && thi <= hi); 371 372 /* return overlapping range */ 373 *nlo = tlo; 374 *nhi = thi; 375 return (0); 376 } 377 378 void 379 kcage_range_init(struct memlist *ml, kcage_dir_t d, pgcnt_t preferred_size) 380 { 381 int ret = 0; 382 383 ASSERT(kcage_arena == NULL); 384 kcage_arena = vmem_create("kcage_arena", NULL, 0, sizeof (uint64_t), 385 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 386 ASSERT(kcage_arena != NULL); 387 388 if (d == KCAGE_DOWN) { 389 while (ml->next != NULL) 390 ml = ml->next; 391 } 392 393 rw_enter(&kcage_range_rwlock, RW_WRITER); 394 395 while (ml != NULL) { 396 ret = kcage_range_add_internal(btop(ml->address), 397 btop(ml->size), d); 398 if (ret) 399 panic("kcage_range_add_internal failed: " 400 "ml=%p, ret=0x%x\n", (void *)ml, ret); 401 402 ml = (d == KCAGE_DOWN ? ml->prev : ml->next); 403 } 404 405 rw_exit(&kcage_range_rwlock); 406 407 if (ret == 0) 408 kcage_init(preferred_size); 409 } 410 411 /* 412 * Third arg controls direction of growth: 0: increasing pfns, 413 * 1: decreasing. 414 */ 415 static int 416 kcage_range_add_internal(pfn_t base, pgcnt_t npgs, kcage_dir_t d) 417 { 418 struct kcage_glist *new, **lpp; 419 pfn_t lim; 420 421 ASSERT(rw_write_held(&kcage_range_rwlock)); 422 423 ASSERT(npgs != 0); 424 if (npgs == 0) 425 return (EINVAL); 426 427 lim = base + npgs; 428 429 ASSERT(lim > base); 430 if (lim <= base) 431 return (EINVAL); 432 433 new = kcage_glist_alloc(); 434 if (new == NULL) { 435 return (ENOMEM); 436 } 437 438 new->base = base; 439 new->lim = lim; 440 new->decr = (d == KCAGE_DOWN); 441 if (new->decr != 0) 442 new->curr = new->lim; 443 else 444 new->curr = new->base; 445 /* 446 * Any overlapping existing ranges are removed by deleting 447 * from the new list as we search for the tail. 448 */ 449 lpp = &kcage_glist; 450 while (*lpp != NULL) { 451 int ret; 452 ret = kcage_glist_delete((*lpp)->base, (*lpp)->lim, &new); 453 if (ret != 0) 454 return (ret); 455 lpp = &(*lpp)->next; 456 } 457 458 *lpp = new; 459 460 if (kcage_current_glist == NULL) { 461 kcage_current_glist = kcage_glist; 462 } 463 464 return (0); 465 } 466 467 int 468 kcage_range_add(pfn_t base, pgcnt_t npgs, kcage_dir_t d) 469 { 470 int ret; 471 472 rw_enter(&kcage_range_rwlock, RW_WRITER); 473 ret = kcage_range_add_internal(base, npgs, d); 474 rw_exit(&kcage_range_rwlock); 475 return (ret); 476 } 477 478 /* 479 * Calls to add and delete must be protected by kcage_range_rwlock 480 */ 481 static int 482 kcage_range_delete_internal(pfn_t base, pgcnt_t npgs) 483 { 484 struct kcage_glist *lp; 485 pfn_t lim; 486 487 ASSERT(rw_write_held(&kcage_range_rwlock)); 488 489 ASSERT(npgs != 0); 490 if (npgs == 0) 491 return (EINVAL); 492 493 lim = base + npgs; 494 495 ASSERT(lim > base); 496 if (lim <= base) 497 return (EINVAL); 498 499 /* 500 * Check if the delete is OK first as a number of elements 501 * might be involved and it will be difficult to go 502 * back and undo (can't just add the range back in). 503 */ 504 for (lp = kcage_glist; lp != NULL; lp = lp->next) { 505 /* 506 * If there have been no pages allocated from this 507 * element, we don't need to check it. 508 */ 509 if ((lp->decr == 0 && lp->curr == lp->base) || 510 (lp->decr != 0 && lp->curr == lp->lim)) 511 continue; 512 /* 513 * If the element does not overlap, its OK. 514 */ 515 if (base >= lp->lim || lim <= lp->base) 516 continue; 517 /* 518 * Overlapping element: Does the range to be deleted 519 * overlap the area already used? If so fail. 520 */ 521 if (lp->decr == 0 && base < lp->curr && lim >= lp->base) { 522 return (EBUSY); 523 } 524 if (lp->decr != 0 && base < lp->lim && lim >= lp->curr) { 525 return (EBUSY); 526 } 527 } 528 return (kcage_glist_delete(base, lim, &kcage_glist)); 529 } 530 531 int 532 kcage_range_delete(pfn_t base, pgcnt_t npgs) 533 { 534 int ret; 535 536 rw_enter(&kcage_range_rwlock, RW_WRITER); 537 ret = kcage_range_delete_internal(base, npgs); 538 rw_exit(&kcage_range_rwlock); 539 return (ret); 540 } 541 542 /* 543 * Calls to add and delete must be protected by kcage_range_rwlock. 544 * This routine gets called after successful Solaris memory 545 * delete operation from DR post memory delete routines. 546 */ 547 static int 548 kcage_range_delete_post_mem_del_internal(pfn_t base, pgcnt_t npgs) 549 { 550 pfn_t lim; 551 552 ASSERT(rw_write_held(&kcage_range_rwlock)); 553 554 ASSERT(npgs != 0); 555 if (npgs == 0) 556 return (EINVAL); 557 558 lim = base + npgs; 559 560 ASSERT(lim > base); 561 if (lim <= base) 562 return (EINVAL); 563 564 return (kcage_glist_delete(base, lim, &kcage_glist)); 565 } 566 567 int 568 kcage_range_delete_post_mem_del(pfn_t base, pgcnt_t npgs) 569 { 570 int ret; 571 572 rw_enter(&kcage_range_rwlock, RW_WRITER); 573 ret = kcage_range_delete_post_mem_del_internal(base, npgs); 574 rw_exit(&kcage_range_rwlock); 575 return (ret); 576 } 577 578 /* 579 * No locking is required here as the whole operation is covered 580 * by kcage_range_rwlock writer lock. 581 */ 582 static struct kcage_glist * 583 kcage_glist_alloc(void) 584 { 585 struct kcage_glist *new; 586 587 if ((new = kcage_glist_freelist) != NULL) { 588 kcage_glist_freelist = new->next; 589 } else { 590 new = vmem_alloc(kcage_arena, sizeof (*new), VM_NOSLEEP); 591 } 592 593 if (new != NULL) 594 bzero(new, sizeof (*new)); 595 596 return (new); 597 } 598 599 static void 600 kcage_glist_free(struct kcage_glist *lp) 601 { 602 lp->next = kcage_glist_freelist; 603 kcage_glist_freelist = lp; 604 } 605 606 static int 607 kcage_glist_delete(pfn_t base, pfn_t lim, struct kcage_glist **lpp) 608 { 609 struct kcage_glist *lp, *prev = *lpp; 610 611 while ((lp = *lpp) != NULL) { 612 if (lim > lp->base && base < lp->lim) { 613 /* The delete range overlaps this element. */ 614 if (base <= lp->base && lim >= lp->lim) { 615 /* Delete whole element. */ 616 *lpp = lp->next; 617 if (lp == kcage_current_glist) { 618 /* This can never happen. */ 619 ASSERT(kcage_current_glist != prev); 620 kcage_current_glist = prev; 621 } 622 kcage_glist_free(lp); 623 continue; 624 } 625 626 /* Partial delete. */ 627 if (base > lp->base && lim < lp->lim) { 628 struct kcage_glist *new; 629 630 /* 631 * Remove a section from the middle, 632 * need to allocate a new element. 633 */ 634 new = kcage_glist_alloc(); 635 if (new == NULL) { 636 return (ENOMEM); 637 } 638 639 /* 640 * Tranfser unused range to new. 641 * Edit lp in place to preserve 642 * kcage_current_glist. 643 */ 644 new->decr = lp->decr; 645 if (new->decr != 0) { 646 new->base = lp->base; 647 new->lim = base; 648 new->curr = base; 649 650 lp->base = lim; 651 } else { 652 new->base = lim; 653 new->lim = lp->lim; 654 new->curr = new->base; 655 656 lp->lim = base; 657 } 658 659 /* Insert new. */ 660 new->next = lp->next; 661 lp->next = new; 662 lpp = &lp->next; 663 } else { 664 /* Delete part of current block. */ 665 if (base > lp->base) { 666 ASSERT(lim >= lp->lim); 667 ASSERT(base < lp->lim); 668 if (lp->decr != 0 && 669 lp->curr == lp->lim) 670 lp->curr = base; 671 lp->lim = base; 672 } else { 673 ASSERT(base <= lp->base); 674 ASSERT(lim > lp->base); 675 if (lp->decr == 0 && 676 lp->curr == lp->base) 677 lp->curr = lim; 678 lp->base = lim; 679 } 680 } 681 } 682 prev = *lpp; 683 lpp = &(*lpp)->next; 684 } 685 686 return (0); 687 } 688 689 /* 690 * If lockit is 1, kcage_get_pfn holds the 691 * reader lock for kcage_range_rwlock. 692 * Changes to lp->curr can cause race conditions, but 693 * they are handled by higher level code (see kcage_next_range.) 694 */ 695 static pfn_t 696 kcage_get_pfn(int lockit) 697 { 698 struct kcage_glist *lp; 699 pfn_t pfn = PFN_INVALID; 700 701 if (lockit && !rw_tryenter(&kcage_range_rwlock, RW_READER)) 702 return (pfn); 703 704 lp = kcage_current_glist; 705 while (lp != NULL) { 706 if (lp->decr != 0) { 707 if (lp->curr != lp->base) { 708 pfn = --lp->curr; 709 break; 710 } 711 } else { 712 if (lp->curr != lp->lim) { 713 pfn = lp->curr++; 714 break; 715 } 716 } 717 718 lp = lp->next; 719 if (lp) 720 kcage_current_glist = lp; 721 } 722 723 if (lockit) 724 rw_exit(&kcage_range_rwlock); 725 return (pfn); 726 } 727 728 /* 729 * Walk the physical address space of the cage. 730 * This routine does not guarantee to return PFNs in the order 731 * in which they were allocated to the cage. Instead, it walks 732 * each range as they appear on the growth list returning the PFNs 733 * range in ascending order. 734 * 735 * To begin scanning at lower edge of cage, reset should be nonzero. 736 * To step through cage, reset should be zero. 737 * 738 * PFN_INVALID will be returned when the upper end of the cage is 739 * reached -- indicating a full scan of the cage has been completed since 740 * previous reset. PFN_INVALID will continue to be returned until 741 * kcage_walk_cage is reset. 742 * 743 * It is possible to receive a PFN_INVALID result on reset if a growth 744 * list is not installed or if none of the PFNs in the installed list have 745 * been allocated to the cage. In otherwords, there is no cage. 746 * 747 * Caller need not hold kcage_range_rwlock while calling this function 748 * as the front part of the list is static - pages never come out of 749 * the cage. 750 * 751 * The caller is expected to only be kcage_cageout(). 752 */ 753 static pfn_t 754 kcage_walk_cage(int reset) 755 { 756 static struct kcage_glist *lp = NULL; 757 static pfn_t pfn; 758 759 if (reset) 760 lp = NULL; 761 if (lp == NULL) { 762 lp = kcage_glist; 763 pfn = PFN_INVALID; 764 } 765 again: 766 if (pfn == PFN_INVALID) { 767 if (lp == NULL) 768 return (PFN_INVALID); 769 770 if (lp->decr != 0) { 771 /* 772 * In this range the cage grows from the highest 773 * address towards the lowest. 774 * Arrange to return pfns from curr to lim-1, 775 * inclusive, in ascending order. 776 */ 777 778 pfn = lp->curr; 779 } else { 780 /* 781 * In this range the cage grows from the lowest 782 * address towards the highest. 783 * Arrange to return pfns from base to curr, 784 * inclusive, in ascending order. 785 */ 786 787 pfn = lp->base; 788 } 789 } 790 791 if (lp->decr != 0) { /* decrementing pfn */ 792 if (pfn == lp->lim) { 793 /* Don't go beyond the static part of the glist. */ 794 if (lp == kcage_current_glist) 795 lp = NULL; 796 else 797 lp = lp->next; 798 pfn = PFN_INVALID; 799 goto again; 800 } 801 802 ASSERT(pfn >= lp->curr && pfn < lp->lim); 803 } else { /* incrementing pfn */ 804 if (pfn == lp->curr) { 805 /* Don't go beyond the static part of the glist. */ 806 if (lp == kcage_current_glist) 807 lp = NULL; 808 else 809 lp = lp->next; 810 pfn = PFN_INVALID; 811 goto again; 812 } 813 814 ASSERT(pfn >= lp->base && pfn < lp->curr); 815 } 816 817 return (pfn++); 818 } 819 820 /* 821 * Callback functions for to recalc cage thresholds after 822 * Kphysm memory add/delete operations. 823 */ 824 /*ARGSUSED*/ 825 static void 826 kcage_kphysm_postadd_cb(void *arg, pgcnt_t delta_pages) 827 { 828 kcage_recalc_thresholds(); 829 } 830 831 /*ARGSUSED*/ 832 static int 833 kcage_kphysm_predel_cb(void *arg, pgcnt_t delta_pages) 834 { 835 /* TODO: when should cage refuse memory delete requests? */ 836 return (0); 837 } 838 839 /*ARGSUSED*/ 840 static void 841 kcage_kphysm_postdel_cb(void *arg, pgcnt_t delta_pages, int cancelled) 842 { 843 kcage_recalc_thresholds(); 844 } 845 846 static kphysm_setup_vector_t kcage_kphysm_vectors = { 847 KPHYSM_SETUP_VECTOR_VERSION, 848 kcage_kphysm_postadd_cb, 849 kcage_kphysm_predel_cb, 850 kcage_kphysm_postdel_cb 851 }; 852 853 /* 854 * This is called before a CPR suspend and after a CPR resume. We have to 855 * turn off kcage_cageout_ready before a suspend, and turn it back on after a 856 * restart. 857 */ 858 /*ARGSUSED*/ 859 static boolean_t 860 kcage_cageout_cpr(void *arg, int code) 861 { 862 if (code == CB_CODE_CPR_CHKPT) { 863 ASSERT(kcage_cageout_ready); 864 kcage_cageout_ready = 0; 865 return (B_TRUE); 866 } else if (code == CB_CODE_CPR_RESUME) { 867 ASSERT(kcage_cageout_ready == 0); 868 kcage_cageout_ready = 1; 869 return (B_TRUE); 870 } 871 return (B_FALSE); 872 } 873 874 /* 875 * kcage_recalc_preferred_size() increases initial cage size to improve large 876 * page availability when lp for kmem is enabled and kpr is disabled 877 */ 878 static pgcnt_t 879 kcage_recalc_preferred_size(pgcnt_t preferred_size) 880 { 881 if (SEGKMEM_USE_LARGEPAGES && segkmem_reloc == 0) { 882 pgcnt_t lpmincage = kcage_kmemlp_mincage; 883 if (lpmincage == 0) { 884 lpmincage = MIN(P2ROUNDUP(((physmem * PAGESIZE) / 8), 885 segkmem_heaplp_quantum), 0x40000000UL) / PAGESIZE; 886 } 887 kcage_kmemlp_mincage = MIN(lpmincage, 888 (segkmem_kmemlp_max / PAGESIZE)); 889 preferred_size = MAX(kcage_kmemlp_mincage, preferred_size); 890 } 891 return (preferred_size); 892 } 893 894 /* 895 * Kcage_init() builds the cage and initializes the cage thresholds. 896 * The size of the cage is determined by the argument preferred_size. 897 * or the actual amount of memory, whichever is smaller. 898 */ 899 static void 900 kcage_init(pgcnt_t preferred_size) 901 { 902 pgcnt_t wanted; 903 pfn_t pfn; 904 page_t *pp; 905 kstat_t *ksp; 906 907 extern struct vnode kvp; 908 extern void page_list_noreloc_startup(page_t *); 909 910 ASSERT(!kcage_on); 911 912 /* increase preferred cage size for lp for kmem */ 913 preferred_size = kcage_recalc_preferred_size(preferred_size); 914 915 /* Debug note: initialize this now so early expansions can stat */ 916 KCAGE_STAT_INIT_SCAN_INDEX; 917 918 /* 919 * Initialize cage thresholds and install kphysm callback. 920 * If we can't arrange to have the thresholds track with 921 * available physical memory, then the cage thresholds may 922 * end up over time at levels that adversly effect system 923 * performance; so, bail out. 924 */ 925 kcage_recalc_thresholds(); 926 if (kphysm_setup_func_register(&kcage_kphysm_vectors, NULL)) { 927 ASSERT(0); /* Catch this in DEBUG kernels. */ 928 return; 929 } 930 931 /* 932 * Limit startup cage size within the range of kcage_minfree 933 * and availrmem, inclusively. 934 */ 935 wanted = MIN(MAX(preferred_size, kcage_minfree), availrmem); 936 937 /* 938 * Construct the cage. PFNs are allocated from the glist. It 939 * is assumed that the list has been properly ordered for the 940 * platform by the platform code. Typically, this is as simple 941 * as calling kcage_range_init(phys_avail, decr), where decr is 942 * 1 if the kernel has been loaded into upper end of physical 943 * memory, or 0 if the kernel has been loaded at the low end. 944 * 945 * Note: it is assumed that we are in the startup flow, so there 946 * is no reason to grab the page lock. 947 */ 948 kcage_freemem = 0; 949 pfn = PFN_INVALID; /* prime for alignment test */ 950 while (wanted != 0) { 951 if ((pfn = kcage_get_pfn(0)) == PFN_INVALID) 952 break; 953 954 if ((pp = page_numtopp_nolock(pfn)) != NULL) { 955 KCAGEPAGETS_INC(); 956 /* 957 * Set the noreloc state on the page. 958 * If the page is free and not already 959 * on the noreloc list then move it. 960 */ 961 if (PP_ISFREE(pp)) { 962 if (PP_ISNORELOC(pp) == 0) 963 page_list_noreloc_startup(pp); 964 } else { 965 ASSERT(pp->p_szc == 0); 966 PP_SETNORELOC(pp); 967 } 968 } 969 PLCNT_XFER_NORELOC(pp); 970 wanted -= 1; 971 } 972 973 /* 974 * Need to go through and find kernel allocated pages 975 * and capture them into the Cage. These will primarily 976 * be pages gotten through boot_alloc(). 977 */ 978 if (kvp.v_pages) { 979 980 pp = kvp.v_pages; 981 do { 982 ASSERT(!PP_ISFREE(pp)); 983 ASSERT(pp->p_szc == 0); 984 if (PP_ISNORELOC(pp) == 0) { 985 PP_SETNORELOC(pp); 986 PLCNT_XFER_NORELOC(pp); 987 } 988 } while ((pp = pp->p_vpnext) != kvp.v_pages); 989 990 } 991 992 kcage_on = 1; 993 994 /* 995 * CB_CL_CPR_POST_KERNEL is the class that executes from cpr_suspend() 996 * after the cageout thread is blocked, and executes from cpr_resume() 997 * before the cageout thread is restarted. By executing in this class, 998 * we are assured that the kernel cage thread won't miss wakeup calls 999 * and also CPR's larger kmem_alloc requests will not fail after 1000 * CPR shuts down the cageout kernel thread. 1001 */ 1002 (void) callb_add(kcage_cageout_cpr, NULL, CB_CL_CPR_POST_KERNEL, 1003 "cageout"); 1004 1005 /* 1006 * Coalesce pages to improve large page availability. A better fix 1007 * would to coalesce pages as they are included in the cage 1008 */ 1009 if (SEGKMEM_USE_LARGEPAGES) { 1010 extern void page_freelist_coalesce_all(int mnode); 1011 page_freelist_coalesce_all(-1); /* do all mnodes */ 1012 } 1013 1014 ksp = kstat_create("kcage", 0, "kcage_page_list", "misc", 1015 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL); 1016 if (ksp != NULL) { 1017 ksp->ks_update = kcage_kstat_update; 1018 ksp->ks_snapshot = kcage_kstat_snapshot; 1019 ksp->ks_lock = &kcage_kstat_lock; /* XXX - not really needed */ 1020 kstat_install(ksp); 1021 } 1022 } 1023 1024 static int 1025 kcage_kstat_update(kstat_t *ksp, int rw) 1026 { 1027 struct kcage_glist *lp; 1028 uint_t count; 1029 1030 if (rw == KSTAT_WRITE) 1031 return (EACCES); 1032 1033 count = 0; 1034 rw_enter(&kcage_range_rwlock, RW_WRITER); 1035 for (lp = kcage_glist; lp != NULL; lp = lp->next) { 1036 if (lp->decr) { 1037 if (lp->curr != lp->lim) { 1038 count++; 1039 } 1040 } else { 1041 if (lp->curr != lp->base) { 1042 count++; 1043 } 1044 } 1045 } 1046 rw_exit(&kcage_range_rwlock); 1047 1048 ksp->ks_ndata = count; 1049 ksp->ks_data_size = count * 2 * sizeof (uint64_t); 1050 1051 return (0); 1052 } 1053 1054 static int 1055 kcage_kstat_snapshot(kstat_t *ksp, void *buf, int rw) 1056 { 1057 struct kcage_glist *lp; 1058 struct memunit { 1059 uint64_t address; 1060 uint64_t size; 1061 } *kspmem; 1062 1063 if (rw == KSTAT_WRITE) 1064 return (EACCES); 1065 1066 ksp->ks_snaptime = gethrtime(); 1067 1068 kspmem = (struct memunit *)buf; 1069 rw_enter(&kcage_range_rwlock, RW_WRITER); 1070 for (lp = kcage_glist; lp != NULL; lp = lp->next, kspmem++) { 1071 if ((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size) 1072 break; 1073 1074 if (lp->decr) { 1075 if (lp->curr != lp->lim) { 1076 kspmem->address = ptob(lp->curr); 1077 kspmem->size = ptob(lp->lim - lp->curr); 1078 } 1079 } else { 1080 if (lp->curr != lp->base) { 1081 kspmem->address = ptob(lp->base); 1082 kspmem->size = ptob(lp->curr - lp->base); 1083 } 1084 } 1085 } 1086 rw_exit(&kcage_range_rwlock); 1087 1088 return (0); 1089 } 1090 1091 void 1092 kcage_recalc_thresholds() 1093 { 1094 static int first = 1; 1095 static pgcnt_t init_lotsfree; 1096 static pgcnt_t init_desfree; 1097 static pgcnt_t init_minfree; 1098 static pgcnt_t init_throttlefree; 1099 static pgcnt_t init_reserve; 1100 1101 /* TODO: any reason to take more care than this with live editing? */ 1102 mutex_enter(&kcage_cageout_mutex); 1103 mutex_enter(&freemem_lock); 1104 1105 if (first) { 1106 first = 0; 1107 init_lotsfree = kcage_lotsfree; 1108 init_desfree = kcage_desfree; 1109 init_minfree = kcage_minfree; 1110 init_throttlefree = kcage_throttlefree; 1111 init_reserve = kcage_reserve; 1112 } else { 1113 kcage_lotsfree = init_lotsfree; 1114 kcage_desfree = init_desfree; 1115 kcage_minfree = init_minfree; 1116 kcage_throttlefree = init_throttlefree; 1117 kcage_reserve = init_reserve; 1118 } 1119 1120 if (kcage_lotsfree == 0) 1121 kcage_lotsfree = MAX(32, total_pages / 256); 1122 1123 if (kcage_minfree == 0) 1124 kcage_minfree = MAX(32, kcage_lotsfree / 2); 1125 1126 if (kcage_desfree == 0) 1127 kcage_desfree = MAX(32, kcage_minfree); 1128 1129 if (kcage_throttlefree == 0) 1130 kcage_throttlefree = MAX(32, kcage_minfree / 2); 1131 1132 if (kcage_reserve == 0) 1133 kcage_reserve = MIN(32, kcage_throttlefree / 2); 1134 1135 mutex_exit(&freemem_lock); 1136 mutex_exit(&kcage_cageout_mutex); 1137 1138 if (kcage_cageout_ready) { 1139 if (kcage_freemem < kcage_desfree) 1140 kcage_cageout_wakeup(); 1141 1142 if (kcage_needfree) { 1143 mutex_enter(&kcage_throttle_mutex); 1144 cv_broadcast(&kcage_throttle_cv); 1145 mutex_exit(&kcage_throttle_mutex); 1146 } 1147 } 1148 } 1149 1150 /* 1151 * Pageout interface: 1152 * kcage_cageout_init() 1153 */ 1154 void 1155 kcage_cageout_init() 1156 { 1157 if (kcage_on) { 1158 1159 (void) thread_create(NULL, 0, kcage_cageout, 1160 NULL, 0, proc_pageout, TS_RUN, maxclsyspri - 1); 1161 } 1162 } 1163 1164 1165 /* 1166 * VM Interfaces: 1167 * kcage_create_throttle() 1168 * kcage_freemem_add() 1169 * kcage_freemem_sub() 1170 */ 1171 1172 /* 1173 * Wakeup cageout thread and throttle waiting for the number of pages 1174 * requested to become available. For non-critical requests, a 1175 * timeout is added, since freemem accounting is separate from cage 1176 * freemem accounting: it's possible for us to get stuck and not make 1177 * forward progress even though there was sufficient freemem before 1178 * arriving here. 1179 */ 1180 int 1181 kcage_create_throttle(pgcnt_t npages, int flags) 1182 { 1183 int niter = 0; 1184 pgcnt_t lastfree; 1185 int enough = kcage_freemem > kcage_throttlefree + npages; 1186 1187 KCAGE_STAT_INCR(kct_calls); /* unprotected incr. */ 1188 1189 kcage_cageout_wakeup(); /* just to be sure */ 1190 KCAGE_STAT_INCR(kct_cagewake); /* unprotected incr. */ 1191 1192 /* 1193 * Obviously, we can't throttle the cageout thread since 1194 * we depend on it. We also can't throttle the panic thread. 1195 */ 1196 if (curthread == kcage_cageout_thread || panicstr) { 1197 KCAGE_STAT_INCR(kct_cageout); /* unprotected incr. */ 1198 return (KCT_CRIT); 1199 } 1200 1201 /* 1202 * Don't throttle threads which are critical for proper 1203 * vm management if we're above kcage_throttlefree or 1204 * if freemem is very low. 1205 */ 1206 if (NOMEMWAIT()) { 1207 if (enough) { 1208 KCAGE_STAT_INCR(kct_exempt); /* unprotected incr. */ 1209 return (KCT_CRIT); 1210 } else if (freemem < minfree) { 1211 KCAGE_STAT_INCR(kct_critical); /* unprotected incr. */ 1212 return (KCT_CRIT); 1213 } 1214 } 1215 1216 /* 1217 * Don't throttle real-time threads if kcage_freemem > kcage_reserve. 1218 */ 1219 if (DISP_PRIO(curthread) > maxclsyspri && 1220 kcage_freemem > kcage_reserve) { 1221 KCAGE_STAT_INCR(kct_exempt); /* unprotected incr. */ 1222 return (KCT_CRIT); 1223 } 1224 1225 /* 1226 * Cause all other threads (which are assumed to not be 1227 * critical to cageout) to wait here until their request 1228 * can be satisfied. Be a little paranoid and wake the 1229 * kernel cage on each loop through this logic. 1230 */ 1231 while (kcage_freemem < kcage_throttlefree + npages) { 1232 ASSERT(kcage_on); 1233 1234 lastfree = kcage_freemem; 1235 1236 if (kcage_cageout_ready) { 1237 mutex_enter(&kcage_throttle_mutex); 1238 1239 kcage_needfree += npages; 1240 KCAGE_STAT_INCR(kct_wait); 1241 1242 kcage_cageout_wakeup(); 1243 KCAGE_STAT_INCR(kct_cagewake); 1244 1245 cv_wait(&kcage_throttle_cv, &kcage_throttle_mutex); 1246 1247 kcage_needfree -= npages; 1248 1249 mutex_exit(&kcage_throttle_mutex); 1250 } else { 1251 /* 1252 * NOTE: atomics are used just in case we enter 1253 * mp operation before the cageout thread is ready. 1254 */ 1255 atomic_add_long(&kcage_needfree, npages); 1256 1257 kcage_cageout_wakeup(); 1258 KCAGE_STAT_INCR(kct_cagewake); /* unprotected incr. */ 1259 1260 atomic_add_long(&kcage_needfree, -npages); 1261 } 1262 1263 if ((flags & PG_WAIT) == 0) { 1264 if (kcage_freemem > lastfree) { 1265 KCAGE_STAT_INCR(kct_progress); 1266 niter = 0; 1267 } else { 1268 KCAGE_STAT_INCR(kct_noprogress); 1269 if (++niter >= kcage_maxwait) { 1270 KCAGE_STAT_INCR(kct_timeout); 1271 return (KCT_FAILURE); 1272 } 1273 } 1274 } 1275 1276 if (NOMEMWAIT() && freemem < minfree) { 1277 return (KCT_CRIT); 1278 } 1279 1280 } 1281 return (KCT_NONCRIT); 1282 } 1283 1284 void 1285 kcage_freemem_add(pgcnt_t npages) 1286 { 1287 extern void wakeup_pcgs(void); 1288 1289 atomic_add_long(&kcage_freemem, npages); 1290 1291 wakeup_pcgs(); /* wakeup threads in pcgs() */ 1292 1293 if (kcage_needfree != 0 && 1294 kcage_freemem >= (kcage_throttlefree + kcage_needfree)) { 1295 1296 mutex_enter(&kcage_throttle_mutex); 1297 cv_broadcast(&kcage_throttle_cv); 1298 KCAGE_STAT_INCR(kfa_trottlewake); 1299 mutex_exit(&kcage_throttle_mutex); 1300 } 1301 } 1302 1303 void 1304 kcage_freemem_sub(pgcnt_t npages) 1305 { 1306 atomic_add_long(&kcage_freemem, -npages); 1307 1308 if (kcage_freemem < kcage_desfree) { 1309 kcage_cageout_wakeup(); 1310 KCAGE_STAT_INCR(kfs_cagewake); /* unprotected incr. */ 1311 } 1312 } 1313 1314 /* 1315 * return 0 on failure and 1 on success. 1316 */ 1317 static int 1318 kcage_setnoreloc_pages(page_t *rootpp, se_t se) 1319 { 1320 pgcnt_t npgs, i; 1321 page_t *pp; 1322 pfn_t rootpfn = page_pptonum(rootpp); 1323 uint_t szc; 1324 1325 ASSERT(!PP_ISFREE(rootpp)); 1326 ASSERT(PAGE_LOCKED_SE(rootpp, se)); 1327 if (!group_page_trylock(rootpp, se)) { 1328 return (0); 1329 } 1330 szc = rootpp->p_szc; 1331 if (szc == 0) { 1332 /* 1333 * The szc of a locked page can only change for pages that are 1334 * non-swapfs (i.e. anonymous memory) file system pages. 1335 */ 1336 ASSERT(rootpp->p_vnode != NULL && 1337 !PP_ISKAS(rootpp) && 1338 !IS_SWAPFSVP(rootpp->p_vnode)); 1339 PP_SETNORELOC(rootpp); 1340 return (1); 1341 } 1342 npgs = page_get_pagecnt(szc); 1343 ASSERT(IS_P2ALIGNED(rootpfn, npgs)); 1344 pp = rootpp; 1345 for (i = 0; i < npgs; i++, pp++) { 1346 ASSERT(PAGE_LOCKED_SE(pp, se)); 1347 ASSERT(!PP_ISFREE(pp)); 1348 ASSERT(pp->p_szc == szc); 1349 PP_SETNORELOC(pp); 1350 } 1351 group_page_unlock(rootpp); 1352 return (1); 1353 } 1354 1355 /* 1356 * Attempt to convert page to a caged page (set the P_NORELOC flag). 1357 * If successful and pages is free, move page to the tail of whichever 1358 * list it is on. 1359 * Returns: 1360 * EBUSY page already locked, assimilated but not free. 1361 * ENOMEM page assimilated, but memory too low to relocate. Page not free. 1362 * EAGAIN page not assimilated. Page not free. 1363 * ERANGE page assimilated. Page not root. 1364 * 0 page assimilated. Page free. 1365 * *nfreedp number of pages freed. 1366 * NOTE: With error codes ENOMEM, EBUSY, and 0 (zero), there is no way 1367 * to distinguish between a page that was already a NORELOC page from 1368 * those newly converted to NORELOC pages by this invocation of 1369 * kcage_assimilate_page. 1370 */ 1371 static int 1372 kcage_assimilate_page(page_t *pp, pgcnt_t *nfreedp) 1373 { 1374 if (page_trylock(pp, SE_EXCL)) { 1375 if (PP_ISNORELOC(pp)) { 1376 check_free_and_return: 1377 if (PP_ISFREE(pp)) { 1378 page_unlock(pp); 1379 *nfreedp = 0; 1380 return (0); 1381 } else { 1382 page_unlock(pp); 1383 return (EBUSY); 1384 } 1385 /*NOTREACHED*/ 1386 } 1387 } else { 1388 if (page_trylock(pp, SE_SHARED)) { 1389 if (PP_ISNORELOC(pp)) 1390 goto check_free_and_return; 1391 } else 1392 return (EAGAIN); 1393 1394 if (!PP_ISFREE(pp)) { 1395 page_unlock(pp); 1396 return (EAGAIN); 1397 } 1398 1399 /* 1400 * Need to upgrade the lock on it and set the NORELOC 1401 * bit. If it is free then remove it from the free 1402 * list so that the platform free list code can keep 1403 * NORELOC pages where they should be. 1404 */ 1405 /* 1406 * Before doing anything, get the exclusive lock. 1407 * This may fail (eg ISM pages are left shared locked). 1408 * If the page is free this will leave a hole in the 1409 * cage. There is no solution yet to this. 1410 */ 1411 if (!page_tryupgrade(pp)) { 1412 page_unlock(pp); 1413 return (EAGAIN); 1414 } 1415 } 1416 1417 ASSERT(PAGE_EXCL(pp)); 1418 1419 if (PP_ISFREE(pp)) { 1420 int which = PP_ISAGED(pp) ? PG_FREE_LIST : PG_CACHE_LIST; 1421 1422 page_list_sub(pp, which); 1423 ASSERT(pp->p_szc == 0); 1424 PP_SETNORELOC(pp); 1425 PLCNT_XFER_NORELOC(pp); 1426 page_list_add(pp, which | PG_LIST_TAIL); 1427 1428 page_unlock(pp); 1429 *nfreedp = 1; 1430 return (0); 1431 } else { 1432 if (pp->p_szc != 0) { 1433 if (!kcage_setnoreloc_pages(pp, SE_EXCL)) { 1434 page_unlock(pp); 1435 return (EAGAIN); 1436 } 1437 ASSERT(PP_ISNORELOC(pp)); 1438 } else { 1439 PP_SETNORELOC(pp); 1440 } 1441 PLCNT_XFER_NORELOC(pp); 1442 return (kcage_invalidate_page(pp, nfreedp)); 1443 } 1444 /*NOTREACHED*/ 1445 } 1446 1447 static int 1448 kcage_expand() 1449 { 1450 int did_something = 0; 1451 1452 spgcnt_t wanted; 1453 pfn_t pfn; 1454 page_t *pp; 1455 /* TODO: we don't really need n any more? */ 1456 pgcnt_t n; 1457 pgcnt_t nf, nfreed; 1458 1459 /* 1460 * Expand the cage if available cage memory is really low. Calculate 1461 * the amount required to return kcage_freemem to the level of 1462 * kcage_lotsfree, or to satisfy throttled requests, whichever is 1463 * more. It is rare for their sum to create an artificial threshold 1464 * above kcage_lotsfree, but it is possible. 1465 * 1466 * Exit early if expansion amount is equal to or less than zero. 1467 * (<0 is possible if kcage_freemem rises suddenly.) 1468 * 1469 * Exit early when the global page pool (apparently) does not 1470 * have enough free pages to page_relocate() even a single page. 1471 */ 1472 wanted = MAX(kcage_lotsfree, kcage_throttlefree + kcage_needfree) 1473 - kcage_freemem; 1474 if (wanted <= 0) 1475 return (0); 1476 else if (freemem < pageout_reserve + 1) { 1477 KCAGE_STAT_INCR(ke_lowfreemem); 1478 return (0); 1479 } 1480 1481 KCAGE_STAT_INCR(ke_calls); 1482 KCAGE_STAT_SET_SCAN(ke_wanted, (uint_t)wanted); 1483 1484 /* 1485 * Assimilate more pages from the global page pool into the cage. 1486 */ 1487 n = 0; /* number of pages PP_SETNORELOC'd */ 1488 nf = 0; /* number of those actually free */ 1489 while (kcage_on && nf < wanted) { 1490 pfn = kcage_get_pfn(1); 1491 if (pfn == PFN_INVALID) { /* eek! no where to grow */ 1492 KCAGE_STAT_INCR(ke_nopfn); 1493 goto terminate; 1494 } 1495 1496 KCAGE_STAT_INCR_SCAN(ke_examined); 1497 1498 if ((pp = page_numtopp_nolock(pfn)) == NULL) { 1499 KCAGE_STAT_INCR(ke_nopaget); 1500 continue; 1501 } 1502 KCAGEPAGETS_INC(); 1503 /* 1504 * Sanity check. Skip this pfn if it is 1505 * being deleted. 1506 */ 1507 if (pfn_is_being_deleted(pfn)) { 1508 KCAGE_STAT_INCR(ke_deleting); 1509 continue; 1510 } 1511 1512 if (PP_ISNORELOC(pp)) { 1513 KCAGE_STAT_INCR(ke_isnoreloc); 1514 continue; 1515 } 1516 1517 switch (kcage_assimilate_page(pp, &nfreed)) { 1518 case 0: /* assimilated, page is free */ 1519 KCAGE_STAT_NINCR_SCAN(ke_gotonefree, nfreed); 1520 did_something = 1; 1521 nf += nfreed; 1522 n++; 1523 break; 1524 1525 case EBUSY: /* assimilated, page not free */ 1526 case ERANGE: /* assimilated, page not root */ 1527 KCAGE_STAT_INCR_SCAN(ke_gotone); 1528 did_something = 1; 1529 n++; 1530 break; 1531 1532 case ENOMEM: /* assimilated, but no mem */ 1533 KCAGE_STAT_INCR(ke_terminate); 1534 did_something = 1; 1535 n++; 1536 goto terminate; 1537 1538 case EAGAIN: /* can't assimilate */ 1539 KCAGE_STAT_INCR_SCAN(ke_lefthole); 1540 break; 1541 1542 default: /* catch this with debug kernels */ 1543 ASSERT(0); 1544 break; 1545 } 1546 } 1547 1548 /* 1549 * Realign cage edge with the nearest physical address 1550 * boundry for big pages. This is done to give us a 1551 * better chance of actually getting usable big pages 1552 * in the cage. 1553 */ 1554 1555 terminate: 1556 1557 return (did_something); 1558 } 1559 1560 /* 1561 * Relocate page opp (Original Page Pointer) from cage pool to page rpp 1562 * (Replacement Page Pointer) in the global pool. Page opp will be freed 1563 * if relocation is successful, otherwise it is only unlocked. 1564 * On entry, page opp must be exclusively locked and not free. 1565 * *nfreedp: number of pages freed. 1566 */ 1567 static int 1568 kcage_relocate_page(page_t *pp, pgcnt_t *nfreedp) 1569 { 1570 page_t *opp = pp; 1571 page_t *rpp = NULL; 1572 spgcnt_t npgs; 1573 int result; 1574 1575 ASSERT(!PP_ISFREE(opp)); 1576 ASSERT(PAGE_EXCL(opp)); 1577 1578 result = page_relocate(&opp, &rpp, 1, 1, &npgs, NULL); 1579 *nfreedp = npgs; 1580 if (result == 0) { 1581 while (npgs-- > 0) { 1582 page_t *tpp; 1583 1584 ASSERT(rpp != NULL); 1585 tpp = rpp; 1586 page_sub(&rpp, tpp); 1587 page_unlock(tpp); 1588 } 1589 1590 ASSERT(rpp == NULL); 1591 1592 return (0); /* success */ 1593 } 1594 1595 page_unlock(opp); 1596 return (result); 1597 } 1598 1599 /* 1600 * Based on page_invalidate_pages() 1601 * 1602 * Kcage_invalidate_page() uses page_relocate() twice. Both instances 1603 * of use must be updated to match the new page_relocate() when it 1604 * becomes available. 1605 * 1606 * Return result of kcage_relocate_page or zero if page was directly freed. 1607 * *nfreedp: number of pages freed. 1608 */ 1609 static int 1610 kcage_invalidate_page(page_t *pp, pgcnt_t *nfreedp) 1611 { 1612 int result; 1613 1614 #if defined(__sparc) 1615 extern struct vnode prom_ppages; 1616 ASSERT(pp->p_vnode != &prom_ppages); 1617 #endif /* __sparc */ 1618 1619 ASSERT(!PP_ISFREE(pp)); 1620 ASSERT(PAGE_EXCL(pp)); 1621 1622 /* 1623 * Is this page involved in some I/O? shared? 1624 * The page_struct_lock need not be acquired to 1625 * examine these fields since the page has an 1626 * "exclusive" lock. 1627 */ 1628 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 1629 result = kcage_relocate_page(pp, nfreedp); 1630 #ifdef KCAGE_STATS 1631 if (result == 0) 1632 KCAGE_STAT_INCR_SCAN(kip_reloclocked); 1633 else if (result == ENOMEM) 1634 KCAGE_STAT_INCR_SCAN(kip_nomem); 1635 #endif 1636 return (result); 1637 } 1638 1639 ASSERT(pp->p_vnode->v_type != VCHR); 1640 1641 /* 1642 * Unload the mappings and check if mod bit is set. 1643 */ 1644 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1645 1646 if (hat_ismod(pp)) { 1647 result = kcage_relocate_page(pp, nfreedp); 1648 #ifdef KCAGE_STATS 1649 if (result == 0) 1650 KCAGE_STAT_INCR_SCAN(kip_relocmod); 1651 else if (result == ENOMEM) 1652 KCAGE_STAT_INCR_SCAN(kip_nomem); 1653 #endif 1654 return (result); 1655 } 1656 1657 if (!page_try_demote_pages(pp)) { 1658 KCAGE_STAT_INCR_SCAN(kip_demotefailed); 1659 page_unlock(pp); 1660 return (EAGAIN); 1661 } 1662 1663 /* LINTED: constant in conditional context */ 1664 VN_DISPOSE(pp, B_INVAL, 0, kcred); 1665 KCAGE_STAT_INCR_SCAN(kip_destroy); 1666 *nfreedp = 1; 1667 return (0); 1668 } 1669 1670 static void 1671 kcage_cageout() 1672 { 1673 pfn_t pfn; 1674 page_t *pp; 1675 callb_cpr_t cprinfo; 1676 int did_something; 1677 int scan_again; 1678 pfn_t start_pfn; 1679 int pass; 1680 int last_pass; 1681 int pages_skipped; 1682 int shared_skipped; 1683 ulong_t shared_level = 8; 1684 pgcnt_t nfreed; 1685 #ifdef KCAGE_STATS 1686 clock_t scan_start; 1687 #endif 1688 1689 CALLB_CPR_INIT(&cprinfo, &kcage_cageout_mutex, 1690 callb_generic_cpr, "cageout"); 1691 1692 mutex_enter(&kcage_cageout_mutex); 1693 kcage_cageout_thread = curthread; 1694 1695 pfn = PFN_INVALID; /* force scan reset */ 1696 start_pfn = PFN_INVALID; /* force init with 1st cage pfn */ 1697 kcage_cageout_ready = 1; /* switch kcage_cageout_wakeup mode */ 1698 1699 loop: 1700 /* 1701 * Wait here. Sooner or later, kcage_freemem_sub() will notice 1702 * that kcage_freemem is less than kcage_desfree. When it does 1703 * notice, kcage_freemem_sub() will wake us up via call to 1704 * kcage_cageout_wakeup(). 1705 */ 1706 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1707 cv_wait(&kcage_cageout_cv, &kcage_cageout_mutex); 1708 CALLB_CPR_SAFE_END(&cprinfo, &kcage_cageout_mutex); 1709 1710 KCAGE_STAT_INCR(kt_wakeups); 1711 KCAGE_STAT_SET_SCAN(kt_freemem_start, freemem); 1712 KCAGE_STAT_SET_SCAN(kt_kcage_freemem_start, kcage_freemem); 1713 pass = 0; 1714 last_pass = 0; 1715 1716 #ifdef KCAGE_STATS 1717 scan_start = lbolt; 1718 #endif 1719 1720 again: 1721 if (!kcage_on) 1722 goto loop; 1723 1724 KCAGE_STAT_INCR(kt_scans); 1725 KCAGE_STAT_INCR_SCAN(kt_passes); 1726 1727 did_something = 0; 1728 pages_skipped = 0; 1729 shared_skipped = 0; 1730 while ((kcage_freemem < kcage_lotsfree || kcage_needfree) && 1731 (pfn = kcage_walk_cage(pfn == PFN_INVALID)) != PFN_INVALID) { 1732 1733 if (start_pfn == PFN_INVALID) 1734 start_pfn = pfn; 1735 else if (start_pfn == pfn) { 1736 last_pass = pass; 1737 pass += 1; 1738 /* 1739 * Did a complete walk of kernel cage, but didn't free 1740 * any pages. If only one cpu is active then 1741 * stop kernel cage walk and try expanding. 1742 */ 1743 if (cp_default.cp_ncpus == 1 && did_something == 0) { 1744 KCAGE_STAT_INCR(kt_cageout_break); 1745 break; 1746 } 1747 } 1748 1749 pp = page_numtopp_nolock(pfn); 1750 if (pp == NULL) { 1751 continue; 1752 } 1753 1754 KCAGE_STAT_INCR_SCAN(kt_examined); 1755 1756 /* 1757 * Do a quick PP_ISNORELOC() and PP_ISFREE test outside 1758 * of the lock. If one is missed it will be seen next 1759 * time through. 1760 * 1761 * Skip non-caged-pages. These pages can exist in the cage 1762 * because, if during cage expansion, a page is 1763 * encountered that is long-term locked the lock prevents the 1764 * expansion logic from setting the P_NORELOC flag. Hence, 1765 * non-caged-pages surrounded by caged-pages. 1766 */ 1767 if (!PP_ISNORELOC(pp)) { 1768 switch (kcage_assimilate_page(pp, &nfreed)) { 1769 case 0: 1770 did_something = 1; 1771 KCAGE_STAT_NINCR_SCAN(kt_gotonefree, 1772 nfreed); 1773 break; 1774 1775 case EBUSY: 1776 case ERANGE: 1777 did_something = 1; 1778 KCAGE_STAT_INCR_SCAN(kt_gotone); 1779 break; 1780 1781 case EAGAIN: 1782 case ENOMEM: 1783 break; 1784 1785 default: 1786 /* catch this with debug kernels */ 1787 ASSERT(0); 1788 break; 1789 } 1790 1791 continue; 1792 } else { 1793 int prm; 1794 1795 if (PP_ISFREE(pp)) { 1796 continue; 1797 } 1798 1799 if ((PP_ISKAS(pp) && pp->p_lckcnt > 0) || 1800 !page_trylock(pp, SE_EXCL)) { 1801 KCAGE_STAT_INCR_SCAN(kt_cantlock); 1802 continue; 1803 } 1804 1805 /* P_NORELOC bit should not have gone away. */ 1806 ASSERT(PP_ISNORELOC(pp)); 1807 if (PP_ISFREE(pp) || (PP_ISKAS(pp) && 1808 pp->p_lckcnt > 0)) { 1809 page_unlock(pp); 1810 continue; 1811 } 1812 1813 KCAGE_STAT_SET_SCAN(kt_skiplevel, shared_level); 1814 if (hat_page_checkshare(pp, shared_level)) { 1815 page_unlock(pp); 1816 pages_skipped = 1; 1817 shared_skipped = 1; 1818 KCAGE_STAT_INCR_SCAN(kt_skipshared); 1819 continue; 1820 } 1821 1822 /* 1823 * In pass {0, 1}, skip page if ref bit is set. 1824 * In pass {0, 1, 2}, skip page if mod bit is set. 1825 */ 1826 prm = hat_pagesync(pp, 1827 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD); 1828 1829 /* On first pass ignore ref'd pages */ 1830 if (pass <= 1 && (prm & P_REF)) { 1831 KCAGE_STAT_INCR_SCAN(kt_skiprefd); 1832 pages_skipped = 1; 1833 page_unlock(pp); 1834 continue; 1835 } 1836 1837 /* On pass 2, VN_DISPOSE if mod bit is not set */ 1838 if (pass <= 2) { 1839 if (pp->p_szc != 0 || (prm & P_MOD) || 1840 pp->p_lckcnt || pp->p_cowcnt) { 1841 pages_skipped = 1; 1842 page_unlock(pp); 1843 } else { 1844 1845 /* 1846 * unload the mappings before 1847 * checking if mod bit is set 1848 */ 1849 (void) hat_pageunload(pp, 1850 HAT_FORCE_PGUNLOAD); 1851 1852 /* 1853 * skip this page if modified 1854 */ 1855 if (hat_ismod(pp)) { 1856 pages_skipped = 1; 1857 page_unlock(pp); 1858 continue; 1859 } 1860 1861 KCAGE_STAT_INCR_SCAN(kt_destroy); 1862 /* constant in conditional context */ 1863 /* LINTED */ 1864 VN_DISPOSE(pp, B_INVAL, 0, kcred); 1865 did_something = 1; 1866 } 1867 continue; 1868 } 1869 1870 if (kcage_invalidate_page(pp, &nfreed) == 0) { 1871 did_something = 1; 1872 KCAGE_STAT_NINCR_SCAN(kt_gotonefree, nfreed); 1873 } 1874 1875 /* 1876 * No need to drop the page lock here. 1877 * Kcage_invalidate_page has done that for us 1878 * either explicitly or through a page_free. 1879 */ 1880 } 1881 } 1882 1883 /* 1884 * Expand the cage only if available cage memory is really low. 1885 * This test is done only after a complete scan of the cage. 1886 * The reason for not checking and expanding more often is to 1887 * avoid rapid expansion of the cage. Naturally, scanning the 1888 * cage takes time. So by scanning first, we use that work as a 1889 * delay loop in between expand decisions. 1890 */ 1891 1892 scan_again = 0; 1893 if (kcage_freemem < kcage_minfree || kcage_needfree) { 1894 /* 1895 * Kcage_expand() will return a non-zero value if it was 1896 * able to expand the cage -- whether or not the new 1897 * pages are free and immediately usable. If non-zero, 1898 * we do another scan of the cage. The pages might be 1899 * freed during that scan or by time we get back here. 1900 * If not, we will attempt another expansion. 1901 * However, if kcage_expand() returns zero, then it was 1902 * unable to expand the cage. This is the case when the 1903 * the growth list is exausted, therefore no work was done 1904 * and there is no reason to scan the cage again. 1905 * Note: Kernel cage scan is not repeated when only one 1906 * cpu is active to avoid kernel cage thread hogging cpu. 1907 */ 1908 if (pass <= 3 && pages_skipped && cp_default.cp_ncpus > 1) 1909 scan_again = 1; 1910 else 1911 (void) kcage_expand(); /* don't scan again */ 1912 } else if (kcage_freemem < kcage_lotsfree) { 1913 /* 1914 * If available cage memory is less than abundant 1915 * and a full scan of the cage has not yet been completed, 1916 * or a scan has completed and some work was performed, 1917 * or pages were skipped because of sharing, 1918 * or we simply have not yet completed two passes, 1919 * then do another scan. 1920 */ 1921 if (pass <= 2 && pages_skipped) 1922 scan_again = 1; 1923 if (pass == last_pass || did_something) 1924 scan_again = 1; 1925 else if (shared_skipped && shared_level < (8<<24)) { 1926 shared_level <<= 1; 1927 scan_again = 1; 1928 } 1929 } 1930 1931 if (scan_again && cp_default.cp_ncpus > 1) 1932 goto again; 1933 else { 1934 if (shared_level > 8) 1935 shared_level >>= 1; 1936 1937 KCAGE_STAT_SET_SCAN(kt_freemem_end, freemem); 1938 KCAGE_STAT_SET_SCAN(kt_kcage_freemem_end, kcage_freemem); 1939 KCAGE_STAT_SET_SCAN(kt_ticks, lbolt - scan_start); 1940 KCAGE_STAT_INC_SCAN_INDEX; 1941 goto loop; 1942 } 1943 1944 /*NOTREACHED*/ 1945 } 1946 1947 void 1948 kcage_cageout_wakeup() 1949 { 1950 if (mutex_tryenter(&kcage_cageout_mutex)) { 1951 if (kcage_cageout_ready) { 1952 cv_signal(&kcage_cageout_cv); 1953 } else if (kcage_freemem < kcage_minfree || kcage_needfree) { 1954 /* 1955 * Available cage memory is really low. Time to 1956 * start expanding the cage. However, the 1957 * kernel cage thread is not yet ready to 1958 * do the work. Use *this* thread, which is 1959 * most likely to be t0, to do the work. 1960 */ 1961 KCAGE_STAT_INCR(kcw_expandearly); 1962 (void) kcage_expand(); 1963 KCAGE_STAT_INC_SCAN_INDEX; 1964 } 1965 1966 mutex_exit(&kcage_cageout_mutex); 1967 } 1968 /* else, kernel cage thread is already running */ 1969 } 1970 1971 void 1972 kcage_tick() 1973 { 1974 /* 1975 * Once per second we wake up all the threads throttled 1976 * waiting for cage memory, in case we've become stuck 1977 * and haven't made forward progress expanding the cage. 1978 */ 1979 if (kcage_on && kcage_cageout_ready) 1980 cv_broadcast(&kcage_throttle_cv); 1981 } 1982