1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/sysmacros.h> 31 #include <sys/kmem.h> 32 #include <sys/atomic.h> 33 #include <sys/bitmap.h> 34 #include <sys/machparam.h> 35 #include <sys/machsystm.h> 36 #include <sys/mman.h> 37 #include <sys/systm.h> 38 #include <sys/cpuvar.h> 39 #include <sys/thread.h> 40 #include <sys/proc.h> 41 #include <sys/cpu.h> 42 #include <sys/kmem.h> 43 #include <sys/disp.h> 44 #include <sys/vmem.h> 45 #include <sys/vmsystm.h> 46 #include <sys/promif.h> 47 #include <sys/var.h> 48 #include <sys/x86_archext.h> 49 #include <sys/archsystm.h> 50 #include <sys/bootconf.h> 51 #include <sys/dumphdr.h> 52 #include <vm/seg_kmem.h> 53 #include <vm/seg_kpm.h> 54 #include <vm/hat.h> 55 #include <vm/hat_i86.h> 56 #include <sys/cmn_err.h> 57 58 #include <sys/bootinfo.h> 59 #include <vm/kboot_mmu.h> 60 61 static void x86pte_zero(htable_t *dest, uint_t entry, uint_t count); 62 63 kmem_cache_t *htable_cache; 64 65 /* 66 * The variable htable_reserve_amount, rather than HTABLE_RESERVE_AMOUNT, 67 * is used in order to facilitate testing of the htable_steal() code. 68 * By resetting htable_reserve_amount to a lower value, we can force 69 * stealing to occur. The reserve amount is a guess to get us through boot. 70 */ 71 #define HTABLE_RESERVE_AMOUNT (200) 72 uint_t htable_reserve_amount = HTABLE_RESERVE_AMOUNT; 73 kmutex_t htable_reserve_mutex; 74 uint_t htable_reserve_cnt; 75 htable_t *htable_reserve_pool; 76 77 /* 78 * Used to hand test htable_steal(). 79 */ 80 #ifdef DEBUG 81 ulong_t force_steal = 0; 82 ulong_t ptable_cnt = 0; 83 #endif 84 85 /* 86 * This variable is so that we can tune this via /etc/system 87 * Any value works, but a power of two <= mmu.ptes_per_table is best. 88 */ 89 uint_t htable_steal_passes = 8; 90 91 /* 92 * mutex stuff for access to htable hash 93 */ 94 #define NUM_HTABLE_MUTEX 128 95 kmutex_t htable_mutex[NUM_HTABLE_MUTEX]; 96 #define HTABLE_MUTEX_HASH(h) ((h) & (NUM_HTABLE_MUTEX - 1)) 97 98 #define HTABLE_ENTER(h) mutex_enter(&htable_mutex[HTABLE_MUTEX_HASH(h)]); 99 #define HTABLE_EXIT(h) mutex_exit(&htable_mutex[HTABLE_MUTEX_HASH(h)]); 100 101 /* 102 * forward declarations 103 */ 104 static void link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr); 105 static void unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr); 106 static void htable_free(htable_t *ht); 107 static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index); 108 static void x86pte_release_pagetable(htable_t *ht); 109 static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, 110 x86pte_t new); 111 112 /* 113 * A counter to track if we are stealing or reaping htables. When non-zero 114 * htable_free() will directly free htables (either to the reserve or kmem) 115 * instead of putting them in a hat's htable cache. 116 */ 117 uint32_t htable_dont_cache = 0; 118 119 /* 120 * Track the number of active pagetables, so we can know how many to reap 121 */ 122 static uint32_t active_ptables = 0; 123 124 /* 125 * Allocate a memory page for a hardware page table. 126 * 127 * A wrapper around page_get_physical(), with some extra checks. 128 */ 129 static pfn_t 130 ptable_alloc(uintptr_t seed) 131 { 132 pfn_t pfn; 133 page_t *pp; 134 135 pfn = PFN_INVALID; 136 atomic_add_32(&active_ptables, 1); 137 138 /* 139 * The first check is to see if there is memory in the system. If we 140 * drop to throttlefree, then fail the ptable_alloc() and let the 141 * stealing code kick in. Note that we have to do this test here, 142 * since the test in page_create_throttle() would let the NOSLEEP 143 * allocation go through and deplete the page reserves. 144 * 145 * The !NOMEMWAIT() lets pageout, fsflush, etc. skip this check. 146 */ 147 if (!NOMEMWAIT() && freemem <= throttlefree + 1) 148 return (PFN_INVALID); 149 150 #ifdef DEBUG 151 /* 152 * This code makes htable_steal() easier to test. By setting 153 * force_steal we force pagetable allocations to fall 154 * into the stealing code. Roughly 1 in ever "force_steal" 155 * page table allocations will fail. 156 */ 157 if (proc_pageout != NULL && force_steal > 1 && 158 ++ptable_cnt > force_steal) { 159 ptable_cnt = 0; 160 return (PFN_INVALID); 161 } 162 #endif /* DEBUG */ 163 164 pp = page_get_physical(seed); 165 if (pp == NULL) 166 return (PFN_INVALID); 167 pfn = pp->p_pagenum; 168 page_downgrade(pp); 169 ASSERT(PAGE_SHARED(pp)); 170 171 if (pfn == PFN_INVALID) 172 panic("ptable_alloc(): Invalid PFN!!"); 173 HATSTAT_INC(hs_ptable_allocs); 174 return (pfn); 175 } 176 177 /* 178 * Free an htable's associated page table page. See the comments 179 * for ptable_alloc(). 180 */ 181 static void 182 ptable_free(pfn_t pfn) 183 { 184 page_t *pp = page_numtopp_nolock(pfn); 185 186 /* 187 * need to destroy the page used for the pagetable 188 */ 189 ASSERT(pfn != PFN_INVALID); 190 HATSTAT_INC(hs_ptable_frees); 191 atomic_add_32(&active_ptables, -1); 192 if (pp == NULL) 193 panic("ptable_free(): no page for pfn!"); 194 ASSERT(PAGE_SHARED(pp)); 195 ASSERT(pfn == pp->p_pagenum); 196 197 /* 198 * Get an exclusive lock, might have to wait for a kmem reader. 199 */ 200 if (!page_tryupgrade(pp)) { 201 page_unlock(pp); 202 /* 203 * RFE: we could change this to not loop forever 204 * George Cameron had some idea on how to do that. 205 * For now looping works - it's just like sfmmu. 206 */ 207 while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM)) 208 continue; 209 } 210 page_free(pp, 1); 211 page_unresv(1); 212 } 213 214 /* 215 * Put one htable on the reserve list. 216 */ 217 static void 218 htable_put_reserve(htable_t *ht) 219 { 220 ht->ht_hat = NULL; /* no longer tied to a hat */ 221 ASSERT(ht->ht_pfn == PFN_INVALID); 222 HATSTAT_INC(hs_htable_rputs); 223 mutex_enter(&htable_reserve_mutex); 224 ht->ht_next = htable_reserve_pool; 225 htable_reserve_pool = ht; 226 ++htable_reserve_cnt; 227 mutex_exit(&htable_reserve_mutex); 228 } 229 230 /* 231 * Take one htable from the reserve. 232 */ 233 static htable_t * 234 htable_get_reserve(void) 235 { 236 htable_t *ht = NULL; 237 238 mutex_enter(&htable_reserve_mutex); 239 if (htable_reserve_cnt != 0) { 240 ht = htable_reserve_pool; 241 ASSERT(ht != NULL); 242 ASSERT(ht->ht_pfn == PFN_INVALID); 243 htable_reserve_pool = ht->ht_next; 244 --htable_reserve_cnt; 245 HATSTAT_INC(hs_htable_rgets); 246 } 247 mutex_exit(&htable_reserve_mutex); 248 return (ht); 249 } 250 251 /* 252 * Allocate initial htables and put them on the reserve list 253 */ 254 void 255 htable_initial_reserve(uint_t count) 256 { 257 htable_t *ht; 258 259 count += HTABLE_RESERVE_AMOUNT; 260 while (count > 0) { 261 ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP); 262 ASSERT(ht != NULL); 263 264 ASSERT(use_boot_reserve); 265 ht->ht_pfn = PFN_INVALID; 266 htable_put_reserve(ht); 267 --count; 268 } 269 } 270 271 /* 272 * Readjust the reserves after a thread finishes using them. 273 */ 274 void 275 htable_adjust_reserve() 276 { 277 htable_t *ht; 278 279 /* 280 * Free any excess htables in the reserve list 281 */ 282 while (htable_reserve_cnt > htable_reserve_amount && 283 !USE_HAT_RESERVES()) { 284 ht = htable_get_reserve(); 285 if (ht == NULL) 286 return; 287 ASSERT(ht->ht_pfn == PFN_INVALID); 288 kmem_cache_free(htable_cache, ht); 289 } 290 } 291 292 293 /* 294 * This routine steals htables from user processes for htable_alloc() or 295 * for htable_reap(). 296 */ 297 static htable_t * 298 htable_steal(uint_t cnt) 299 { 300 hat_t *hat = kas.a_hat; /* list starts with khat */ 301 htable_t *list = NULL; 302 htable_t *ht; 303 htable_t *higher; 304 uint_t h; 305 uint_t h_start; 306 static uint_t h_seed = 0; 307 uint_t e; 308 uintptr_t va; 309 x86pte_t pte; 310 uint_t stolen = 0; 311 uint_t pass; 312 uint_t threshold; 313 314 /* 315 * Limit htable_steal_passes to something reasonable 316 */ 317 if (htable_steal_passes == 0) 318 htable_steal_passes = 1; 319 if (htable_steal_passes > mmu.ptes_per_table) 320 htable_steal_passes = mmu.ptes_per_table; 321 322 /* 323 * Loop through all user hats. The 1st pass takes cached htables that 324 * aren't in use. The later passes steal by removing mappings, too. 325 */ 326 atomic_add_32(&htable_dont_cache, 1); 327 for (pass = 0; pass <= htable_steal_passes && stolen < cnt; ++pass) { 328 threshold = pass * mmu.ptes_per_table / htable_steal_passes; 329 hat = kas.a_hat; 330 for (;;) { 331 332 /* 333 * Clear the victim flag and move to next hat 334 */ 335 mutex_enter(&hat_list_lock); 336 if (hat != kas.a_hat) { 337 hat->hat_flags &= ~HAT_VICTIM; 338 cv_broadcast(&hat_list_cv); 339 } 340 hat = hat->hat_next; 341 342 /* 343 * Skip any hat that is already being stolen from. 344 * 345 * We skip SHARED hats, as these are dummy 346 * hats that host ISM shared page tables. 347 * 348 * We also skip if HAT_FREEING because hat_pte_unmap() 349 * won't zero out the PTE's. That would lead to hitting 350 * stale PTEs either here or under hat_unload() when we 351 * steal and unload the same page table in competing 352 * threads. 353 */ 354 while (hat != NULL && 355 (hat->hat_flags & 356 (HAT_VICTIM | HAT_SHARED | HAT_FREEING)) != 0) 357 hat = hat->hat_next; 358 359 if (hat == NULL) { 360 mutex_exit(&hat_list_lock); 361 break; 362 } 363 364 /* 365 * Are we finished? 366 */ 367 if (stolen == cnt) { 368 /* 369 * Try to spread the pain of stealing, 370 * move victim HAT to the end of the HAT list. 371 */ 372 if (pass >= 1 && cnt == 1 && 373 kas.a_hat->hat_prev != hat) { 374 375 /* unlink victim hat */ 376 if (hat->hat_prev) 377 hat->hat_prev->hat_next = 378 hat->hat_next; 379 else 380 kas.a_hat->hat_next = 381 hat->hat_next; 382 if (hat->hat_next) 383 hat->hat_next->hat_prev = 384 hat->hat_prev; 385 else 386 kas.a_hat->hat_prev = 387 hat->hat_prev; 388 389 390 /* relink at end of hat list */ 391 hat->hat_next = NULL; 392 hat->hat_prev = kas.a_hat->hat_prev; 393 if (hat->hat_prev) 394 hat->hat_prev->hat_next = hat; 395 else 396 kas.a_hat->hat_next = hat; 397 kas.a_hat->hat_prev = hat; 398 399 } 400 401 mutex_exit(&hat_list_lock); 402 break; 403 } 404 405 /* 406 * Mark the HAT as a stealing victim. 407 */ 408 hat->hat_flags |= HAT_VICTIM; 409 mutex_exit(&hat_list_lock); 410 411 /* 412 * Take any htables from the hat's cached "free" list. 413 */ 414 hat_enter(hat); 415 while ((ht = hat->hat_ht_cached) != NULL && 416 stolen < cnt) { 417 hat->hat_ht_cached = ht->ht_next; 418 ht->ht_next = list; 419 list = ht; 420 ++stolen; 421 } 422 hat_exit(hat); 423 424 /* 425 * Don't steal on first pass. 426 */ 427 if (pass == 0 || stolen == cnt) 428 continue; 429 430 /* 431 * Search the active htables for one to steal. 432 * Start at a different hash bucket every time to 433 * help spread the pain of stealing. 434 */ 435 h = h_start = h_seed++ % hat->hat_num_hash; 436 do { 437 higher = NULL; 438 HTABLE_ENTER(h); 439 for (ht = hat->hat_ht_hash[h]; ht; 440 ht = ht->ht_next) { 441 442 /* 443 * Can we rule out reaping? 444 */ 445 if (ht->ht_busy != 0 || 446 (ht->ht_flags & HTABLE_SHARED_PFN)|| 447 ht->ht_level > 0 || 448 ht->ht_valid_cnt > threshold || 449 ht->ht_lock_cnt != 0) 450 continue; 451 452 /* 453 * Increment busy so the htable can't 454 * disappear. We drop the htable mutex 455 * to avoid deadlocks with 456 * hat_pageunload() and the hment mutex 457 * while we call hat_pte_unmap() 458 */ 459 ++ht->ht_busy; 460 HTABLE_EXIT(h); 461 462 /* 463 * Try stealing. 464 * - unload and invalidate all PTEs 465 */ 466 for (e = 0, va = ht->ht_vaddr; 467 e < HTABLE_NUM_PTES(ht) && 468 ht->ht_valid_cnt > 0 && 469 ht->ht_busy == 1 && 470 ht->ht_lock_cnt == 0; 471 ++e, va += MMU_PAGESIZE) { 472 pte = x86pte_get(ht, e); 473 if (!PTE_ISVALID(pte)) 474 continue; 475 hat_pte_unmap(ht, e, 476 HAT_UNLOAD, pte, NULL); 477 } 478 479 /* 480 * Reacquire htable lock. If we didn't 481 * remove all mappings in the table, 482 * or another thread added a new mapping 483 * behind us, give up on this table. 484 */ 485 HTABLE_ENTER(h); 486 if (ht->ht_busy != 1 || 487 ht->ht_valid_cnt != 0 || 488 ht->ht_lock_cnt != 0) { 489 --ht->ht_busy; 490 continue; 491 } 492 493 /* 494 * Steal it and unlink the page table. 495 */ 496 higher = ht->ht_parent; 497 unlink_ptp(higher, ht, ht->ht_vaddr); 498 499 /* 500 * remove from the hash list 501 */ 502 if (ht->ht_next) 503 ht->ht_next->ht_prev = 504 ht->ht_prev; 505 506 if (ht->ht_prev) { 507 ht->ht_prev->ht_next = 508 ht->ht_next; 509 } else { 510 ASSERT(hat->hat_ht_hash[h] == 511 ht); 512 hat->hat_ht_hash[h] = 513 ht->ht_next; 514 } 515 516 /* 517 * Break to outer loop to release the 518 * higher (ht_parent) pagetable. This 519 * spreads out the pain caused by 520 * pagefaults. 521 */ 522 ht->ht_next = list; 523 list = ht; 524 ++stolen; 525 break; 526 } 527 HTABLE_EXIT(h); 528 if (higher != NULL) 529 htable_release(higher); 530 if (++h == hat->hat_num_hash) 531 h = 0; 532 } while (stolen < cnt && h != h_start); 533 } 534 } 535 atomic_add_32(&htable_dont_cache, -1); 536 return (list); 537 } 538 539 540 /* 541 * This is invoked from kmem when the system is low on memory. We try 542 * to free hments, htables, and ptables to improve the memory situation. 543 */ 544 /*ARGSUSED*/ 545 static void 546 htable_reap(void *handle) 547 { 548 uint_t reap_cnt; 549 htable_t *list; 550 htable_t *ht; 551 552 HATSTAT_INC(hs_reap_attempts); 553 if (!can_steal_post_boot) 554 return; 555 556 /* 557 * Try to reap 5% of the page tables bounded by a maximum of 558 * 5% of physmem and a minimum of 10. 559 */ 560 reap_cnt = MIN(MAX(physmem / 20, active_ptables / 20), 10); 561 562 /* 563 * Let htable_steal() do the work, we just call htable_free() 564 */ 565 list = htable_steal(reap_cnt); 566 while ((ht = list) != NULL) { 567 list = ht->ht_next; 568 HATSTAT_INC(hs_reaped); 569 htable_free(ht); 570 } 571 572 /* 573 * Free up excess reserves 574 */ 575 htable_adjust_reserve(); 576 hment_adjust_reserve(); 577 } 578 579 /* 580 * Allocate an htable, stealing one or using the reserve if necessary 581 */ 582 static htable_t * 583 htable_alloc( 584 hat_t *hat, 585 uintptr_t vaddr, 586 level_t level, 587 htable_t *shared) 588 { 589 htable_t *ht = NULL; 590 uint_t is_vlp; 591 uint_t is_bare = 0; 592 uint_t need_to_zero = 1; 593 int kmflags = (can_steal_post_boot ? KM_NOSLEEP : KM_SLEEP); 594 595 if (level < 0 || level > TOP_LEVEL(hat)) 596 panic("htable_alloc(): level %d out of range\n", level); 597 598 is_vlp = (hat->hat_flags & HAT_VLP) && level == VLP_LEVEL; 599 if (is_vlp || shared != NULL) 600 is_bare = 1; 601 602 /* 603 * First reuse a cached htable from the hat_ht_cached field, this 604 * avoids unnecessary trips through kmem/page allocators. 605 */ 606 if (hat->hat_ht_cached != NULL && !is_bare) { 607 hat_enter(hat); 608 ht = hat->hat_ht_cached; 609 if (ht != NULL) { 610 hat->hat_ht_cached = ht->ht_next; 611 need_to_zero = 0; 612 /* XX64 ASSERT() they're all zero somehow */ 613 ASSERT(ht->ht_pfn != PFN_INVALID); 614 } 615 hat_exit(hat); 616 } 617 618 if (ht == NULL) { 619 /* 620 * Allocate an htable, possibly refilling the reserves. 621 */ 622 if (USE_HAT_RESERVES()) { 623 ht = htable_get_reserve(); 624 } else { 625 /* 626 * Donate successful htable allocations to the reserve. 627 */ 628 for (;;) { 629 ht = kmem_cache_alloc(htable_cache, kmflags); 630 if (ht == NULL) 631 break; 632 ht->ht_pfn = PFN_INVALID; 633 if (USE_HAT_RESERVES() || 634 htable_reserve_cnt >= htable_reserve_amount) 635 break; 636 htable_put_reserve(ht); 637 } 638 } 639 640 /* 641 * allocate a page for the hardware page table if needed 642 */ 643 if (ht != NULL && !is_bare) { 644 ht->ht_hat = hat; 645 ht->ht_pfn = ptable_alloc((uintptr_t)ht); 646 if (ht->ht_pfn == PFN_INVALID) { 647 if (USE_HAT_RESERVES()) 648 htable_put_reserve(ht); 649 else 650 kmem_cache_free(htable_cache, ht); 651 ht = NULL; 652 } 653 } 654 } 655 656 /* 657 * If allocations failed, kick off a kmem_reap() and resort to 658 * htable steal(). We may spin here if the system is very low on 659 * memory. If the kernel itself has consumed all memory and kmem_reap() 660 * can't free up anything, then we'll really get stuck here. 661 * That should only happen in a system where the administrator has 662 * misconfigured VM parameters via /etc/system. 663 */ 664 while (ht == NULL && can_steal_post_boot) { 665 kmem_reap(); 666 ht = htable_steal(1); 667 HATSTAT_INC(hs_steals); 668 669 /* 670 * If we stole for a bare htable, release the pagetable page. 671 */ 672 if (ht != NULL) { 673 if (is_bare) { 674 ptable_free(ht->ht_pfn); 675 ht->ht_pfn = PFN_INVALID; 676 } 677 } 678 } 679 680 /* 681 * All attempts to allocate or steal failed. This should only happen 682 * if we run out of memory during boot, due perhaps to a huge 683 * boot_archive. At this point there's no way to continue. 684 */ 685 if (ht == NULL) 686 panic("htable_alloc(): couldn't steal\n"); 687 688 /* 689 * Shared page tables have all entries locked and entries may not 690 * be added or deleted. 691 */ 692 ht->ht_flags = 0; 693 if (shared != NULL) { 694 ASSERT(level == 0); 695 ASSERT(shared->ht_valid_cnt > 0); 696 ht->ht_flags |= HTABLE_SHARED_PFN; 697 ht->ht_pfn = shared->ht_pfn; 698 ht->ht_lock_cnt = 0; 699 ht->ht_valid_cnt = 0; /* updated in hat_share() */ 700 ht->ht_shares = shared; 701 need_to_zero = 0; 702 } else { 703 ht->ht_shares = NULL; 704 ht->ht_lock_cnt = 0; 705 ht->ht_valid_cnt = 0; 706 } 707 708 /* 709 * setup flags, etc. for VLP htables 710 */ 711 if (is_vlp) { 712 ht->ht_flags |= HTABLE_VLP; 713 ASSERT(ht->ht_pfn == PFN_INVALID); 714 need_to_zero = 0; 715 } 716 717 /* 718 * fill in the htable 719 */ 720 ht->ht_hat = hat; 721 ht->ht_parent = NULL; 722 ht->ht_vaddr = vaddr; 723 ht->ht_level = level; 724 ht->ht_busy = 1; 725 ht->ht_next = NULL; 726 ht->ht_prev = NULL; 727 728 /* 729 * Zero out any freshly allocated page table 730 */ 731 if (need_to_zero) 732 x86pte_zero(ht, 0, mmu.ptes_per_table); 733 734 return (ht); 735 } 736 737 /* 738 * Free up an htable, either to a hat's cached list, the reserves or 739 * back to kmem. 740 */ 741 static void 742 htable_free(htable_t *ht) 743 { 744 hat_t *hat = ht->ht_hat; 745 746 /* 747 * If the process isn't exiting, cache the free htable in the hat 748 * structure. We always do this for the boot reserve. We don't 749 * do this if the hat is exiting or we are stealing/reaping htables. 750 */ 751 if (hat != NULL && 752 !(ht->ht_flags & HTABLE_SHARED_PFN) && 753 (use_boot_reserve || 754 (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) { 755 ASSERT((ht->ht_flags & HTABLE_VLP) == 0); 756 ASSERT(ht->ht_pfn != PFN_INVALID); 757 hat_enter(hat); 758 ht->ht_next = hat->hat_ht_cached; 759 hat->hat_ht_cached = ht; 760 hat_exit(hat); 761 return; 762 } 763 764 /* 765 * If we have a hardware page table, free it. 766 * We don't free page tables that are accessed by sharing. 767 */ 768 if (ht->ht_flags & HTABLE_SHARED_PFN) { 769 ASSERT(ht->ht_pfn != PFN_INVALID); 770 } else if (!(ht->ht_flags & HTABLE_VLP)) { 771 ptable_free(ht->ht_pfn); 772 } 773 ht->ht_pfn = PFN_INVALID; 774 775 /* 776 * Free htables or put into reserves. 777 */ 778 if (USE_HAT_RESERVES() || htable_reserve_cnt < htable_reserve_amount) { 779 htable_put_reserve(ht); 780 } else { 781 kmem_cache_free(htable_cache, ht); 782 htable_adjust_reserve(); 783 } 784 } 785 786 787 /* 788 * This is called when a hat is being destroyed or swapped out. We reap all 789 * the remaining htables in the hat cache. If destroying all left over 790 * htables are also destroyed. 791 * 792 * We also don't need to invalidate any of the PTPs nor do any demapping. 793 */ 794 void 795 htable_purge_hat(hat_t *hat) 796 { 797 htable_t *ht; 798 int h; 799 800 /* 801 * Purge the htable cache if just reaping. 802 */ 803 if (!(hat->hat_flags & HAT_FREEING)) { 804 atomic_add_32(&htable_dont_cache, 1); 805 for (;;) { 806 hat_enter(hat); 807 ht = hat->hat_ht_cached; 808 if (ht == NULL) { 809 hat_exit(hat); 810 break; 811 } 812 hat->hat_ht_cached = ht->ht_next; 813 hat_exit(hat); 814 htable_free(ht); 815 } 816 atomic_add_32(&htable_dont_cache, -1); 817 return; 818 } 819 820 /* 821 * if freeing, no locking is needed 822 */ 823 while ((ht = hat->hat_ht_cached) != NULL) { 824 hat->hat_ht_cached = ht->ht_next; 825 htable_free(ht); 826 } 827 828 /* 829 * walk thru the htable hash table and free all the htables in it. 830 */ 831 for (h = 0; h < hat->hat_num_hash; ++h) { 832 while ((ht = hat->hat_ht_hash[h]) != NULL) { 833 if (ht->ht_next) 834 ht->ht_next->ht_prev = ht->ht_prev; 835 836 if (ht->ht_prev) { 837 ht->ht_prev->ht_next = ht->ht_next; 838 } else { 839 ASSERT(hat->hat_ht_hash[h] == ht); 840 hat->hat_ht_hash[h] = ht->ht_next; 841 } 842 htable_free(ht); 843 } 844 } 845 } 846 847 /* 848 * Unlink an entry for a table at vaddr and level out of the existing table 849 * one level higher. We are always holding the HASH_ENTER() when doing this. 850 */ 851 static void 852 unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr) 853 { 854 uint_t entry = htable_va2entry(vaddr, higher); 855 x86pte_t expect = MAKEPTP(old->ht_pfn, old->ht_level); 856 x86pte_t found; 857 hat_t *hat = old->ht_hat; 858 859 ASSERT(higher->ht_busy > 0); 860 ASSERT(higher->ht_valid_cnt > 0); 861 ASSERT(old->ht_valid_cnt == 0); 862 found = x86pte_cas(higher, entry, expect, 0); 863 if (found != expect) 864 panic("Bad PTP found=" FMT_PTE ", expected=" FMT_PTE, 865 found, expect); 866 867 /* 868 * When any top level VLP page table entry changes, we must issue 869 * a reload of cr3 on all processors. Also some CPU types require 870 * invalidating when inner table entries are invalidated. 871 */ 872 if (!(hat->hat_flags & HAT_FREEING)) { 873 if (higher->ht_flags & HTABLE_VLP) 874 hat_tlb_inval(hat, DEMAP_ALL_ADDR); 875 else if (mmu.inval_nonleaf) 876 hat_tlb_inval(hat, old->ht_vaddr); 877 } 878 879 HTABLE_DEC(higher->ht_valid_cnt); 880 } 881 882 /* 883 * Link an entry for a new table at vaddr and level into the existing table 884 * one level higher. We are always holding the HASH_ENTER() when doing this. 885 */ 886 static void 887 link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr) 888 { 889 uint_t entry = htable_va2entry(vaddr, higher); 890 x86pte_t newptp = MAKEPTP(new->ht_pfn, new->ht_level); 891 x86pte_t found; 892 893 ASSERT(higher->ht_busy > 0); 894 895 ASSERT(new->ht_level != mmu.max_level); 896 897 HTABLE_INC(higher->ht_valid_cnt); 898 899 found = x86pte_cas(higher, entry, 0, newptp); 900 if ((found & ~PT_REF) != 0) 901 panic("HAT: ptp not 0, found=" FMT_PTE, found); 902 903 /* 904 * When any top level VLP page table entry changes, we must issue 905 * a reload of cr3 on all processors using it. 906 */ 907 if (higher->ht_flags & HTABLE_VLP) 908 hat_tlb_inval(higher->ht_hat, DEMAP_ALL_ADDR); 909 } 910 911 /* 912 * Release of hold on an htable. If this is the last use and the pagetable 913 * is empty we may want to free it, then recursively look at the pagetable 914 * above it. The recursion is handled by the outer while() loop. 915 */ 916 void 917 htable_release(htable_t *ht) 918 { 919 uint_t hashval; 920 htable_t *shared; 921 htable_t *higher; 922 hat_t *hat; 923 uintptr_t va; 924 level_t level; 925 926 while (ht != NULL) { 927 shared = NULL; 928 for (;;) { 929 hat = ht->ht_hat; 930 va = ht->ht_vaddr; 931 level = ht->ht_level; 932 hashval = HTABLE_HASH(hat, va, level); 933 934 /* 935 * The common case is that this isn't the last use of 936 * an htable so we don't want to free the htable. 937 */ 938 HTABLE_ENTER(hashval); 939 ASSERT(ht->ht_lock_cnt == 0 || ht->ht_valid_cnt > 0); 940 ASSERT(ht->ht_valid_cnt >= 0); 941 ASSERT(ht->ht_busy > 0); 942 if (ht->ht_valid_cnt > 0) 943 break; 944 if (ht->ht_busy > 1) 945 break; 946 947 /* 948 * we always release empty shared htables 949 */ 950 if (!(ht->ht_flags & HTABLE_SHARED_PFN)) { 951 952 /* 953 * don't release if in address space tear down 954 */ 955 if (hat->hat_flags & HAT_FREEING) 956 break; 957 958 /* 959 * At and above max_page_level, free if it's for 960 * a boot-time kernel mapping below kernelbase. 961 */ 962 if (level >= mmu.max_page_level && 963 (hat != kas.a_hat || va >= kernelbase)) 964 break; 965 } 966 967 /* 968 * Remember if we destroy an htable that shares its PFN 969 * from elsewhere. 970 */ 971 if (ht->ht_flags & HTABLE_SHARED_PFN) { 972 ASSERT(ht->ht_level == 0); 973 ASSERT(shared == NULL); 974 shared = ht->ht_shares; 975 HATSTAT_INC(hs_htable_unshared); 976 } 977 978 /* 979 * Handle release of a table and freeing the htable_t. 980 * Unlink it from the table higher (ie. ht_parent). 981 */ 982 ASSERT(ht->ht_lock_cnt == 0); 983 higher = ht->ht_parent; 984 ASSERT(higher != NULL); 985 986 /* 987 * Unlink the pagetable. 988 */ 989 unlink_ptp(higher, ht, va); 990 991 /* 992 * remove this htable from its hash list 993 */ 994 if (ht->ht_next) 995 ht->ht_next->ht_prev = ht->ht_prev; 996 997 if (ht->ht_prev) { 998 ht->ht_prev->ht_next = ht->ht_next; 999 } else { 1000 ASSERT(hat->hat_ht_hash[hashval] == ht); 1001 hat->hat_ht_hash[hashval] = ht->ht_next; 1002 } 1003 HTABLE_EXIT(hashval); 1004 htable_free(ht); 1005 ht = higher; 1006 } 1007 1008 ASSERT(ht->ht_busy >= 1); 1009 --ht->ht_busy; 1010 HTABLE_EXIT(hashval); 1011 1012 /* 1013 * If we released a shared htable, do a release on the htable 1014 * from which it shared 1015 */ 1016 ht = shared; 1017 } 1018 } 1019 1020 /* 1021 * Find the htable for the pagetable at the given level for the given address. 1022 * If found acquires a hold that eventually needs to be htable_release()d 1023 */ 1024 htable_t * 1025 htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level) 1026 { 1027 uintptr_t base; 1028 uint_t hashval; 1029 htable_t *ht = NULL; 1030 1031 ASSERT(level >= 0); 1032 ASSERT(level <= TOP_LEVEL(hat)); 1033 1034 if (level == TOP_LEVEL(hat)) 1035 base = 0; 1036 else 1037 base = vaddr & LEVEL_MASK(level + 1); 1038 1039 hashval = HTABLE_HASH(hat, base, level); 1040 HTABLE_ENTER(hashval); 1041 for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) { 1042 if (ht->ht_hat == hat && 1043 ht->ht_vaddr == base && 1044 ht->ht_level == level) 1045 break; 1046 } 1047 if (ht) 1048 ++ht->ht_busy; 1049 1050 HTABLE_EXIT(hashval); 1051 return (ht); 1052 } 1053 1054 /* 1055 * Acquires a hold on a known htable (from a locked hment entry). 1056 */ 1057 void 1058 htable_acquire(htable_t *ht) 1059 { 1060 hat_t *hat = ht->ht_hat; 1061 level_t level = ht->ht_level; 1062 uintptr_t base = ht->ht_vaddr; 1063 uint_t hashval = HTABLE_HASH(hat, base, level); 1064 1065 HTABLE_ENTER(hashval); 1066 #ifdef DEBUG 1067 /* 1068 * make sure the htable is there 1069 */ 1070 { 1071 htable_t *h; 1072 1073 for (h = hat->hat_ht_hash[hashval]; 1074 h && h != ht; 1075 h = h->ht_next) 1076 ; 1077 ASSERT(h == ht); 1078 } 1079 #endif /* DEBUG */ 1080 ++ht->ht_busy; 1081 HTABLE_EXIT(hashval); 1082 } 1083 1084 /* 1085 * Find the htable for the pagetable at the given level for the given address. 1086 * If found acquires a hold that eventually needs to be htable_release()d 1087 * If not found the table is created. 1088 * 1089 * Since we can't hold a hash table mutex during allocation, we have to 1090 * drop it and redo the search on a create. Then we may have to free the newly 1091 * allocated htable if another thread raced in and created it ahead of us. 1092 */ 1093 htable_t * 1094 htable_create( 1095 hat_t *hat, 1096 uintptr_t vaddr, 1097 level_t level, 1098 htable_t *shared) 1099 { 1100 uint_t h; 1101 level_t l; 1102 uintptr_t base; 1103 htable_t *ht; 1104 htable_t *higher = NULL; 1105 htable_t *new = NULL; 1106 1107 if (level < 0 || level > TOP_LEVEL(hat)) 1108 panic("htable_create(): level %d out of range\n", level); 1109 1110 /* 1111 * Create the page tables in top down order. 1112 */ 1113 for (l = TOP_LEVEL(hat); l >= level; --l) { 1114 new = NULL; 1115 if (l == TOP_LEVEL(hat)) 1116 base = 0; 1117 else 1118 base = vaddr & LEVEL_MASK(l + 1); 1119 1120 h = HTABLE_HASH(hat, base, l); 1121 try_again: 1122 /* 1123 * look up the htable at this level 1124 */ 1125 HTABLE_ENTER(h); 1126 if (l == TOP_LEVEL(hat)) { 1127 ht = hat->hat_htable; 1128 } else { 1129 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { 1130 ASSERT(ht->ht_hat == hat); 1131 if (ht->ht_vaddr == base && 1132 ht->ht_level == l) 1133 break; 1134 } 1135 } 1136 1137 /* 1138 * if we found the htable, increment its busy cnt 1139 * and if we had allocated a new htable, free it. 1140 */ 1141 if (ht != NULL) { 1142 /* 1143 * If we find a pre-existing shared table, it must 1144 * share from the same place. 1145 */ 1146 if (l == level && shared && ht->ht_shares && 1147 ht->ht_shares != shared) { 1148 panic("htable shared from wrong place " 1149 "found htable=%p shared=%p", ht, shared); 1150 } 1151 ++ht->ht_busy; 1152 HTABLE_EXIT(h); 1153 if (new) 1154 htable_free(new); 1155 if (higher != NULL) 1156 htable_release(higher); 1157 higher = ht; 1158 1159 /* 1160 * if we didn't find it on the first search 1161 * allocate a new one and search again 1162 */ 1163 } else if (new == NULL) { 1164 HTABLE_EXIT(h); 1165 new = htable_alloc(hat, base, l, 1166 l == level ? shared : NULL); 1167 goto try_again; 1168 1169 /* 1170 * 2nd search and still not there, use "new" table 1171 * Link new table into higher, when not at top level. 1172 */ 1173 } else { 1174 ht = new; 1175 if (higher != NULL) { 1176 link_ptp(higher, ht, base); 1177 ht->ht_parent = higher; 1178 } 1179 ht->ht_next = hat->hat_ht_hash[h]; 1180 ASSERT(ht->ht_prev == NULL); 1181 if (hat->hat_ht_hash[h]) 1182 hat->hat_ht_hash[h]->ht_prev = ht; 1183 hat->hat_ht_hash[h] = ht; 1184 HTABLE_EXIT(h); 1185 1186 /* 1187 * Note we don't do htable_release(higher). 1188 * That happens recursively when "new" is removed by 1189 * htable_release() or htable_steal(). 1190 */ 1191 higher = ht; 1192 1193 /* 1194 * If we just created a new shared page table we 1195 * increment the shared htable's busy count, so that 1196 * it can't be the victim of a steal even if it's empty. 1197 */ 1198 if (l == level && shared) { 1199 (void) htable_lookup(shared->ht_hat, 1200 shared->ht_vaddr, shared->ht_level); 1201 HATSTAT_INC(hs_htable_shared); 1202 } 1203 } 1204 } 1205 1206 return (ht); 1207 } 1208 1209 /* 1210 * Inherit initial pagetables from the boot program. 1211 */ 1212 void 1213 htable_attach( 1214 hat_t *hat, 1215 uintptr_t base, 1216 level_t level, 1217 htable_t *parent, 1218 pfn_t pfn) 1219 { 1220 htable_t *ht; 1221 uint_t h; 1222 uint_t i; 1223 x86pte_t pte; 1224 x86pte_t *ptep; 1225 page_t *pp; 1226 extern page_t *boot_claim_page(pfn_t); 1227 1228 ht = htable_get_reserve(); 1229 if (level == mmu.max_level) 1230 kas.a_hat->hat_htable = ht; 1231 ht->ht_hat = hat; 1232 ht->ht_parent = parent; 1233 ht->ht_vaddr = base; 1234 ht->ht_level = level; 1235 ht->ht_busy = 1; 1236 ht->ht_next = NULL; 1237 ht->ht_prev = NULL; 1238 ht->ht_flags = 0; 1239 ht->ht_pfn = pfn; 1240 ht->ht_lock_cnt = 0; 1241 ht->ht_valid_cnt = 0; 1242 if (parent != NULL) 1243 ++parent->ht_busy; 1244 1245 h = HTABLE_HASH(hat, base, level); 1246 HTABLE_ENTER(h); 1247 ht->ht_next = hat->hat_ht_hash[h]; 1248 ASSERT(ht->ht_prev == NULL); 1249 if (hat->hat_ht_hash[h]) 1250 hat->hat_ht_hash[h]->ht_prev = ht; 1251 hat->hat_ht_hash[h] = ht; 1252 HTABLE_EXIT(h); 1253 1254 /* 1255 * make sure the page table physical page is not FREE 1256 */ 1257 if (page_resv(1, KM_NOSLEEP) == 0) 1258 panic("page_resv() failed in ptable alloc"); 1259 1260 pp = boot_claim_page(pfn); 1261 ASSERT(pp != NULL); 1262 page_downgrade(pp); 1263 /* 1264 * Record in the page_t that is a pagetable for segkpm setup. 1265 */ 1266 if (kpm_vbase) 1267 pp->p_index = 1; 1268 1269 /* 1270 * Count valid mappings and recursively attach lower level pagetables. 1271 */ 1272 ptep = kbm_remap_window(pfn_to_pa(pfn), 0); 1273 for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) { 1274 if (mmu.pae_hat) 1275 pte = ptep[i]; 1276 else 1277 pte = ((x86pte32_t *)ptep)[i]; 1278 if (!IN_HYPERVISOR_VA(base) && PTE_ISVALID(pte)) { 1279 ++ht->ht_valid_cnt; 1280 if (!PTE_ISPAGE(pte, level)) { 1281 htable_attach(hat, base, level - 1, 1282 ht, PTE2PFN(pte, level)); 1283 ptep = kbm_remap_window(pfn_to_pa(pfn), 0); 1284 } 1285 } 1286 base += LEVEL_SIZE(level); 1287 if (base == mmu.hole_start) 1288 base = (mmu.hole_end + MMU_PAGEOFFSET) & MMU_PAGEMASK; 1289 } 1290 1291 /* 1292 * As long as all the mappings we had were below kernel base 1293 * we can release the htable. 1294 */ 1295 if (base < kernelbase) 1296 htable_release(ht); 1297 } 1298 1299 /* 1300 * Walk through a given htable looking for the first valid entry. This 1301 * routine takes both a starting and ending address. The starting address 1302 * is required to be within the htable provided by the caller, but there is 1303 * no such restriction on the ending address. 1304 * 1305 * If the routine finds a valid entry in the htable (at or beyond the 1306 * starting address), the PTE (and its address) will be returned. 1307 * This PTE may correspond to either a page or a pagetable - it is the 1308 * caller's responsibility to determine which. If no valid entry is 1309 * found, 0 (and invalid PTE) and the next unexamined address will be 1310 * returned. 1311 * 1312 * The loop has been carefully coded for optimization. 1313 */ 1314 static x86pte_t 1315 htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr) 1316 { 1317 uint_t e; 1318 x86pte_t found_pte = (x86pte_t)0; 1319 caddr_t pte_ptr; 1320 caddr_t end_pte_ptr; 1321 int l = ht->ht_level; 1322 uintptr_t va = *vap & LEVEL_MASK(l); 1323 size_t pgsize = LEVEL_SIZE(l); 1324 1325 ASSERT(va >= ht->ht_vaddr); 1326 ASSERT(va <= HTABLE_LAST_PAGE(ht)); 1327 1328 /* 1329 * Compute the starting index and ending virtual address 1330 */ 1331 e = htable_va2entry(va, ht); 1332 1333 /* 1334 * The following page table scan code knows that the valid 1335 * bit of a PTE is in the lowest byte AND that x86 is little endian!! 1336 */ 1337 pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0); 1338 end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht)); 1339 pte_ptr = (caddr_t)PT_INDEX_PTR((x86pte_t *)pte_ptr, e); 1340 while (!PTE_ISVALID(*pte_ptr)) { 1341 va += pgsize; 1342 if (va >= eaddr) 1343 break; 1344 pte_ptr += mmu.pte_size; 1345 ASSERT(pte_ptr <= end_pte_ptr); 1346 if (pte_ptr == end_pte_ptr) 1347 break; 1348 } 1349 1350 /* 1351 * if we found a valid PTE, load the entire PTE 1352 */ 1353 if (va < eaddr && pte_ptr != end_pte_ptr) 1354 found_pte = GET_PTE((x86pte_t *)pte_ptr); 1355 x86pte_release_pagetable(ht); 1356 1357 #if defined(__amd64) 1358 /* 1359 * deal with VA hole on amd64 1360 */ 1361 if (l == mmu.max_level && va >= mmu.hole_start && va <= mmu.hole_end) 1362 va = mmu.hole_end + va - mmu.hole_start; 1363 #endif /* __amd64 */ 1364 1365 *vap = va; 1366 return (found_pte); 1367 } 1368 1369 /* 1370 * Find the address and htable for the first populated translation at or 1371 * above the given virtual address. The caller may also specify an upper 1372 * limit to the address range to search. Uses level information to quickly 1373 * skip unpopulated sections of virtual address spaces. 1374 * 1375 * If not found returns NULL. When found, returns the htable and virt addr 1376 * and has a hold on the htable. 1377 */ 1378 x86pte_t 1379 htable_walk( 1380 struct hat *hat, 1381 htable_t **htp, 1382 uintptr_t *vaddr, 1383 uintptr_t eaddr) 1384 { 1385 uintptr_t va = *vaddr; 1386 htable_t *ht; 1387 htable_t *prev = *htp; 1388 level_t l; 1389 level_t max_mapped_level; 1390 x86pte_t pte; 1391 1392 ASSERT(eaddr > va); 1393 1394 /* 1395 * If this is a user address, then we know we need not look beyond 1396 * kernelbase. 1397 */ 1398 ASSERT(hat == kas.a_hat || eaddr <= kernelbase || 1399 eaddr == HTABLE_WALK_TO_END); 1400 if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END) 1401 eaddr = kernelbase; 1402 1403 /* 1404 * If we're coming in with a previous page table, search it first 1405 * without doing an htable_lookup(), this should be frequent. 1406 */ 1407 if (prev) { 1408 ASSERT(prev->ht_busy > 0); 1409 ASSERT(prev->ht_vaddr <= va); 1410 l = prev->ht_level; 1411 if (va <= HTABLE_LAST_PAGE(prev)) { 1412 pte = htable_scan(prev, &va, eaddr); 1413 1414 if (PTE_ISPAGE(pte, l)) { 1415 *vaddr = va; 1416 *htp = prev; 1417 return (pte); 1418 } 1419 } 1420 1421 /* 1422 * We found nothing in the htable provided by the caller, 1423 * so fall through and do the full search 1424 */ 1425 htable_release(prev); 1426 } 1427 1428 /* 1429 * Find the level of the largest pagesize used by this HAT. 1430 */ 1431 max_mapped_level = 0; 1432 for (l = 1; l <= mmu.max_page_level; ++l) 1433 if (hat->hat_pages_mapped[l] != 0) 1434 max_mapped_level = l; 1435 1436 while (va < eaddr && va >= *vaddr) { 1437 ASSERT(!IN_VA_HOLE(va)); 1438 1439 /* 1440 * Find lowest table with any entry for given address. 1441 */ 1442 for (l = 0; l <= TOP_LEVEL(hat); ++l) { 1443 ht = htable_lookup(hat, va, l); 1444 if (ht != NULL) { 1445 pte = htable_scan(ht, &va, eaddr); 1446 if (PTE_ISPAGE(pte, l)) { 1447 *vaddr = va; 1448 *htp = ht; 1449 return (pte); 1450 } 1451 htable_release(ht); 1452 break; 1453 } 1454 1455 /* 1456 * The ht is never NULL at the top level since 1457 * the top level htable is created in hat_alloc(). 1458 */ 1459 ASSERT(l < TOP_LEVEL(hat)); 1460 1461 /* 1462 * No htable covers the address. If there is no 1463 * larger page size that could cover it, we 1464 * skip to the start of the next page table. 1465 */ 1466 if (l >= max_mapped_level) { 1467 va = NEXT_ENTRY_VA(va, l + 1); 1468 break; 1469 } 1470 } 1471 } 1472 1473 *vaddr = 0; 1474 *htp = NULL; 1475 return (0); 1476 } 1477 1478 /* 1479 * Find the htable and page table entry index of the given virtual address 1480 * with pagesize at or below given level. 1481 * If not found returns NULL. When found, returns the htable, sets 1482 * entry, and has a hold on the htable. 1483 */ 1484 htable_t * 1485 htable_getpte( 1486 struct hat *hat, 1487 uintptr_t vaddr, 1488 uint_t *entry, 1489 x86pte_t *pte, 1490 level_t level) 1491 { 1492 htable_t *ht; 1493 level_t l; 1494 uint_t e; 1495 1496 ASSERT(level <= mmu.max_page_level); 1497 1498 for (l = 0; l <= level; ++l) { 1499 ht = htable_lookup(hat, vaddr, l); 1500 if (ht == NULL) 1501 continue; 1502 e = htable_va2entry(vaddr, ht); 1503 if (entry != NULL) 1504 *entry = e; 1505 if (pte != NULL) 1506 *pte = x86pte_get(ht, e); 1507 return (ht); 1508 } 1509 return (NULL); 1510 } 1511 1512 /* 1513 * Find the htable and page table entry index of the given virtual address. 1514 * There must be a valid page mapped at the given address. 1515 * If not found returns NULL. When found, returns the htable, sets 1516 * entry, and has a hold on the htable. 1517 */ 1518 htable_t * 1519 htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry) 1520 { 1521 htable_t *ht; 1522 uint_t e; 1523 x86pte_t pte; 1524 1525 ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level); 1526 if (ht == NULL) 1527 return (NULL); 1528 1529 if (entry) 1530 *entry = e; 1531 1532 if (PTE_ISPAGE(pte, ht->ht_level)) 1533 return (ht); 1534 htable_release(ht); 1535 return (NULL); 1536 } 1537 1538 1539 void 1540 htable_init() 1541 { 1542 /* 1543 * To save on kernel VA usage, we avoid debug information in 32 bit 1544 * kernels. 1545 */ 1546 #if defined(__amd64) 1547 int kmem_flags = KMC_NOHASH; 1548 #elif defined(__i386) 1549 int kmem_flags = KMC_NOHASH | KMC_NODEBUG; 1550 #endif 1551 1552 /* 1553 * initialize kmem caches 1554 */ 1555 htable_cache = kmem_cache_create("htable_t", 1556 sizeof (htable_t), 0, NULL, NULL, 1557 htable_reap, NULL, hat_memload_arena, kmem_flags); 1558 } 1559 1560 /* 1561 * get the pte index for the virtual address in the given htable's pagetable 1562 */ 1563 uint_t 1564 htable_va2entry(uintptr_t va, htable_t *ht) 1565 { 1566 level_t l = ht->ht_level; 1567 1568 ASSERT(va >= ht->ht_vaddr); 1569 ASSERT(va <= HTABLE_LAST_PAGE(ht)); 1570 return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1)); 1571 } 1572 1573 /* 1574 * Given an htable and the index of a pte in it, return the virtual address 1575 * of the page. 1576 */ 1577 uintptr_t 1578 htable_e2va(htable_t *ht, uint_t entry) 1579 { 1580 level_t l = ht->ht_level; 1581 uintptr_t va; 1582 1583 ASSERT(entry < HTABLE_NUM_PTES(ht)); 1584 va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l)); 1585 1586 /* 1587 * Need to skip over any VA hole in top level table 1588 */ 1589 #if defined(__amd64) 1590 if (ht->ht_level == mmu.max_level && va >= mmu.hole_start) 1591 va += ((mmu.hole_end - mmu.hole_start) + 1); 1592 #endif 1593 1594 return (va); 1595 } 1596 1597 /* 1598 * The code uses compare and swap instructions to read/write PTE's to 1599 * avoid atomicity problems, since PTEs can be 8 bytes on 32 bit systems. 1600 * will naturally be atomic. 1601 * 1602 * The combination of using kpreempt_disable()/_enable() and the hci_mutex 1603 * are used to ensure that an interrupt won't overwrite a temporary mapping 1604 * while it's in use. If an interrupt thread tries to access a PTE, it will 1605 * yield briefly back to the pinned thread which holds the cpu's hci_mutex. 1606 */ 1607 void 1608 x86pte_cpu_init(cpu_t *cpu) 1609 { 1610 struct hat_cpu_info *hci; 1611 1612 hci = kmem_zalloc(sizeof (*hci), KM_SLEEP); 1613 mutex_init(&hci->hci_mutex, NULL, MUTEX_DEFAULT, NULL); 1614 cpu->cpu_hat_info = hci; 1615 } 1616 1617 void 1618 x86pte_cpu_fini(cpu_t *cpu) 1619 { 1620 struct hat_cpu_info *hci = cpu->cpu_hat_info; 1621 1622 kmem_free(hci, sizeof (*hci)); 1623 cpu->cpu_hat_info = NULL; 1624 } 1625 1626 #ifdef __i386 1627 /* 1628 * On 32 bit kernels, loading a 64 bit PTE is a little tricky 1629 */ 1630 x86pte_t 1631 get_pte64(x86pte_t *ptr) 1632 { 1633 volatile uint32_t *p = (uint32_t *)ptr; 1634 x86pte_t t; 1635 1636 ASSERT(mmu.pae_hat != 0); 1637 for (;;) { 1638 t = p[0]; 1639 t |= (uint64_t)p[1] << 32; 1640 if ((t & 0xffffffff) == p[0]) 1641 return (t); 1642 } 1643 } 1644 #endif /* __i386 */ 1645 1646 /* 1647 * Disable preemption and establish a mapping to the pagetable with the 1648 * given pfn. This is optimized for there case where it's the same 1649 * pfn as we last used referenced from this CPU. 1650 */ 1651 static x86pte_t * 1652 x86pte_access_pagetable(htable_t *ht, uint_t index) 1653 { 1654 /* 1655 * VLP pagetables are contained in the hat_t 1656 */ 1657 if (ht->ht_flags & HTABLE_VLP) 1658 return (PT_INDEX_PTR(ht->ht_hat->hat_vlp_ptes, index)); 1659 return (x86pte_mapin(ht->ht_pfn, index, ht)); 1660 } 1661 1662 /* 1663 * map the given pfn into the page table window. 1664 */ 1665 /*ARGSUSED*/ 1666 x86pte_t * 1667 x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht) 1668 { 1669 x86pte_t *pteptr; 1670 x86pte_t pte; 1671 x86pte_t newpte; 1672 int x; 1673 1674 ASSERT(pfn != PFN_INVALID); 1675 1676 if (!khat_running) { 1677 caddr_t va = kbm_remap_window(pfn_to_pa(pfn), 1); 1678 return (PT_INDEX_PTR(va, index)); 1679 } 1680 1681 /* 1682 * If kpm is available, use it. 1683 */ 1684 if (kpm_vbase) 1685 return (PT_INDEX_PTR(hat_kpm_pfn2va(pfn), index)); 1686 1687 /* 1688 * Disable preemption and grab the CPU's hci_mutex 1689 */ 1690 kpreempt_disable(); 1691 ASSERT(CPU->cpu_hat_info != NULL); 1692 mutex_enter(&CPU->cpu_hat_info->hci_mutex); 1693 x = PWIN_TABLE(CPU->cpu_id); 1694 pteptr = (x86pte_t *)PWIN_PTE_VA(x); 1695 if (mmu.pae_hat) 1696 pte = *pteptr; 1697 else 1698 pte = *(x86pte32_t *)pteptr; 1699 1700 newpte = MAKEPTE(pfn, 0) | mmu.pt_global | mmu.pt_nx; 1701 newpte |= PT_WRITABLE; 1702 1703 if (!PTE_EQUIV(newpte, pte)) { 1704 if (mmu.pae_hat) 1705 *pteptr = newpte; 1706 else 1707 *(x86pte32_t *)pteptr = newpte; 1708 mmu_tlbflush_entry((caddr_t)(PWIN_VA(x))); 1709 } 1710 return (PT_INDEX_PTR(PWIN_VA(x), index)); 1711 } 1712 1713 /* 1714 * Release access to a page table. 1715 */ 1716 static void 1717 x86pte_release_pagetable(htable_t *ht) 1718 { 1719 /* 1720 * nothing to do for VLP htables 1721 */ 1722 if (ht->ht_flags & HTABLE_VLP) 1723 return; 1724 1725 x86pte_mapout(); 1726 } 1727 1728 void 1729 x86pte_mapout(void) 1730 { 1731 if (mmu.pwin_base == NULL || !khat_running) 1732 return; 1733 1734 /* 1735 * Drop the CPU's hci_mutex and restore preemption. 1736 */ 1737 mutex_exit(&CPU->cpu_hat_info->hci_mutex); 1738 kpreempt_enable(); 1739 } 1740 1741 /* 1742 * Atomic retrieval of a pagetable entry 1743 */ 1744 x86pte_t 1745 x86pte_get(htable_t *ht, uint_t entry) 1746 { 1747 x86pte_t pte; 1748 x86pte_t *ptep; 1749 1750 /* 1751 * Be careful that loading PAE entries in 32 bit kernel is atomic. 1752 */ 1753 ASSERT(entry < mmu.ptes_per_table); 1754 ptep = x86pte_access_pagetable(ht, entry); 1755 pte = GET_PTE(ptep); 1756 x86pte_release_pagetable(ht); 1757 return (pte); 1758 } 1759 1760 /* 1761 * Atomic unconditional set of a page table entry, it returns the previous 1762 * value. For pre-existing mappings if the PFN changes, then we don't care 1763 * about the old pte's REF / MOD bits. If the PFN remains the same, we leave 1764 * the MOD/REF bits unchanged. 1765 * 1766 * If asked to overwrite a link to a lower page table with a large page 1767 * mapping, this routine returns the special value of LPAGE_ERROR. This 1768 * allows the upper HAT layers to retry with a smaller mapping size. 1769 */ 1770 x86pte_t 1771 x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr) 1772 { 1773 x86pte_t old; 1774 x86pte_t prev; 1775 x86pte_t *ptep; 1776 level_t l = ht->ht_level; 1777 x86pte_t pfn_mask = (l != 0) ? PT_PADDR_LGPG : PT_PADDR; 1778 x86pte_t n; 1779 uintptr_t addr = htable_e2va(ht, entry); 1780 hat_t *hat = ht->ht_hat; 1781 1782 ASSERT(new != 0); /* don't use to invalidate a PTE, see x86pte_update */ 1783 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); 1784 if (ptr == NULL) 1785 ptep = x86pte_access_pagetable(ht, entry); 1786 else 1787 ptep = ptr; 1788 1789 /* 1790 * Install the new PTE. If remapping the same PFN, then 1791 * copy existing REF/MOD bits to new mapping. 1792 */ 1793 do { 1794 prev = GET_PTE(ptep); 1795 n = new; 1796 if (PTE_ISVALID(n) && (prev & pfn_mask) == (new & pfn_mask)) 1797 n |= prev & (PT_REF | PT_MOD); 1798 1799 /* 1800 * Another thread may have installed this mapping already, 1801 * flush the local TLB and be done. 1802 */ 1803 if (prev == n) { 1804 old = new; 1805 mmu_tlbflush_entry((caddr_t)addr); 1806 goto done; 1807 } 1808 1809 /* 1810 * Detect if we have a collision of installing a large 1811 * page mapping where there already is a lower page table. 1812 */ 1813 if (l > 0 && (prev & PT_VALID) && !(prev & PT_PAGESIZE)) { 1814 old = LPAGE_ERROR; 1815 goto done; 1816 } 1817 1818 old = CAS_PTE(ptep, prev, n); 1819 } while (old != prev); 1820 1821 /* 1822 * Do a TLB demap if needed, ie. the old pte was valid. 1823 * 1824 * Note that a stale TLB writeback to the PTE here either can't happen 1825 * or doesn't matter. The PFN can only change for NOSYNC|NOCONSIST 1826 * mappings, but they were created with REF and MOD already set, so 1827 * no stale writeback will happen. 1828 * 1829 * Segmap is the only place where remaps happen on the same pfn and for 1830 * that we want to preserve the stale REF/MOD bits. 1831 */ 1832 if (old & PT_REF) 1833 hat_tlb_inval(hat, addr); 1834 1835 done: 1836 if (ptr == NULL) 1837 x86pte_release_pagetable(ht); 1838 return (old); 1839 } 1840 1841 /* 1842 * Atomic compare and swap of a page table entry. No TLB invalidates are done. 1843 * This is used for links between pagetables of different levels. 1844 * Note we always create these links with dirty/access set, so they should 1845 * never change. 1846 */ 1847 x86pte_t 1848 x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new) 1849 { 1850 x86pte_t pte; 1851 x86pte_t *ptep; 1852 1853 ptep = x86pte_access_pagetable(ht, entry); 1854 pte = CAS_PTE(ptep, old, new); 1855 x86pte_release_pagetable(ht); 1856 return (pte); 1857 } 1858 1859 /* 1860 * Invalidate a page table entry as long as it currently maps something that 1861 * matches the value determined by expect. 1862 * 1863 * Also invalidates any TLB entries and returns the previous value of the PTE. 1864 */ 1865 x86pte_t 1866 x86pte_inval( 1867 htable_t *ht, 1868 uint_t entry, 1869 x86pte_t expect, 1870 x86pte_t *pte_ptr) 1871 { 1872 x86pte_t *ptep; 1873 x86pte_t oldpte; 1874 x86pte_t found; 1875 1876 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); 1877 ASSERT(ht->ht_level != VLP_LEVEL); 1878 1879 if (pte_ptr != NULL) 1880 ptep = pte_ptr; 1881 else 1882 ptep = x86pte_access_pagetable(ht, entry); 1883 1884 /* 1885 * Note that the loop is needed to handle changes due to h/w updating 1886 * of PT_MOD/PT_REF. 1887 */ 1888 do { 1889 oldpte = GET_PTE(ptep); 1890 if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR)) 1891 goto done; 1892 found = CAS_PTE(ptep, oldpte, 0); 1893 } while (found != oldpte); 1894 if (oldpte & (PT_REF | PT_MOD)) 1895 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry)); 1896 1897 done: 1898 if (pte_ptr == NULL) 1899 x86pte_release_pagetable(ht); 1900 return (oldpte); 1901 } 1902 1903 /* 1904 * Change a page table entry af it currently matches the value in expect. 1905 */ 1906 x86pte_t 1907 x86pte_update( 1908 htable_t *ht, 1909 uint_t entry, 1910 x86pte_t expect, 1911 x86pte_t new) 1912 { 1913 x86pte_t *ptep; 1914 x86pte_t found; 1915 1916 ASSERT(new != 0); 1917 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); 1918 ASSERT(ht->ht_level != VLP_LEVEL); 1919 1920 ptep = x86pte_access_pagetable(ht, entry); 1921 found = CAS_PTE(ptep, expect, new); 1922 if (found == expect) { 1923 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry)); 1924 1925 /* 1926 * When removing write permission *and* clearing the 1927 * MOD bit, check if a write happened via a stale 1928 * TLB entry before the TLB shootdown finished. 1929 * 1930 * If it did happen, simply re-enable write permission and 1931 * act like the original CAS failed. 1932 */ 1933 if ((expect & (PT_WRITABLE | PT_MOD)) == PT_WRITABLE && 1934 (new & (PT_WRITABLE | PT_MOD)) == 0 && 1935 (GET_PTE(ptep) & PT_MOD) != 0) { 1936 do { 1937 found = GET_PTE(ptep); 1938 found = 1939 CAS_PTE(ptep, found, found | PT_WRITABLE); 1940 } while ((found & PT_WRITABLE) == 0); 1941 } 1942 } 1943 x86pte_release_pagetable(ht); 1944 return (found); 1945 } 1946 1947 /* 1948 * Copy page tables - this is just a little more complicated than the 1949 * previous routines. Note that it's also not atomic! It also is never 1950 * used for VLP pagetables. 1951 */ 1952 void 1953 x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count) 1954 { 1955 caddr_t src_va; 1956 caddr_t dst_va; 1957 size_t size; 1958 x86pte_t *pteptr; 1959 x86pte_t pte; 1960 1961 ASSERT(khat_running); 1962 ASSERT(!(dest->ht_flags & HTABLE_VLP)); 1963 ASSERT(!(src->ht_flags & HTABLE_VLP)); 1964 ASSERT(!(src->ht_flags & HTABLE_SHARED_PFN)); 1965 ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN)); 1966 1967 /* 1968 * Acquire access to the CPU pagetable windows for the dest and source. 1969 */ 1970 dst_va = (caddr_t)x86pte_access_pagetable(dest, entry); 1971 if (kpm_vbase) { 1972 src_va = (caddr_t) 1973 PT_INDEX_PTR(hat_kpm_pfn2va(src->ht_pfn), entry); 1974 } else { 1975 uint_t x = PWIN_SRC(CPU->cpu_id); 1976 1977 /* 1978 * Finish defining the src pagetable mapping 1979 */ 1980 src_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry); 1981 pte = MAKEPTE(src->ht_pfn, 0) | mmu.pt_global | mmu.pt_nx; 1982 pteptr = (x86pte_t *)PWIN_PTE_VA(x); 1983 if (mmu.pae_hat) 1984 *pteptr = pte; 1985 else 1986 *(x86pte32_t *)pteptr = pte; 1987 mmu_tlbflush_entry((caddr_t)(PWIN_VA(x))); 1988 } 1989 1990 /* 1991 * now do the copy 1992 */ 1993 size = count << mmu.pte_size_shift; 1994 bcopy(src_va, dst_va, size); 1995 1996 x86pte_release_pagetable(dest); 1997 } 1998 1999 /* 2000 * Zero page table entries - Note this doesn't use atomic stores! 2001 */ 2002 static void 2003 x86pte_zero(htable_t *dest, uint_t entry, uint_t count) 2004 { 2005 caddr_t dst_va; 2006 size_t size; 2007 2008 /* 2009 * Map in the page table to be zeroed. 2010 */ 2011 ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN)); 2012 ASSERT(!(dest->ht_flags & HTABLE_VLP)); 2013 2014 dst_va = (caddr_t)x86pte_access_pagetable(dest, entry); 2015 2016 size = count << mmu.pte_size_shift; 2017 ASSERT(size > BLOCKZEROALIGN); 2018 #ifdef __i386 2019 if ((x86_feature & X86_SSE2) == 0) 2020 bzero(dst_va, size); 2021 else 2022 #endif 2023 block_zero_no_xmm(dst_va, size); 2024 2025 x86pte_release_pagetable(dest); 2026 } 2027 2028 /* 2029 * Called to ensure that all pagetables are in the system dump 2030 */ 2031 void 2032 hat_dump(void) 2033 { 2034 hat_t *hat; 2035 uint_t h; 2036 htable_t *ht; 2037 2038 /* 2039 * Dump all page tables 2040 */ 2041 for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) { 2042 for (h = 0; h < hat->hat_num_hash; ++h) { 2043 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { 2044 if ((ht->ht_flags & HTABLE_VLP) == 0) 2045 dump_page(ht->ht_pfn); 2046 } 2047 } 2048 } 2049 } 2050