1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 27 /* 28 * VM - Hardware Address Translation management for i386 and amd64 29 * 30 * Implementation of the interfaces described in <common/vm/hat.h> 31 * 32 * Nearly all the details of how the hardware is managed should not be 33 * visible outside this layer except for misc. machine specific functions 34 * that work in conjunction with this code. 35 * 36 * Routines used only inside of i86pc/vm start with hati_ for HAT Internal. 37 */ 38 39 #include <sys/machparam.h> 40 #include <sys/machsystm.h> 41 #include <sys/mman.h> 42 #include <sys/types.h> 43 #include <sys/systm.h> 44 #include <sys/cpuvar.h> 45 #include <sys/thread.h> 46 #include <sys/proc.h> 47 #include <sys/cpu.h> 48 #include <sys/kmem.h> 49 #include <sys/disp.h> 50 #include <sys/shm.h> 51 #include <sys/sysmacros.h> 52 #include <sys/machparam.h> 53 #include <sys/vmem.h> 54 #include <sys/vmsystm.h> 55 #include <sys/promif.h> 56 #include <sys/var.h> 57 #include <sys/x86_archext.h> 58 #include <sys/atomic.h> 59 #include <sys/bitmap.h> 60 #include <sys/controlregs.h> 61 #include <sys/bootconf.h> 62 #include <sys/bootsvcs.h> 63 #include <sys/bootinfo.h> 64 #include <sys/archsystm.h> 65 66 #include <vm/seg_kmem.h> 67 #include <vm/hat_i86.h> 68 #include <vm/as.h> 69 #include <vm/seg.h> 70 #include <vm/page.h> 71 #include <vm/seg_kp.h> 72 #include <vm/seg_kpm.h> 73 #include <vm/vm_dep.h> 74 #ifdef __xpv 75 #include <sys/hypervisor.h> 76 #endif 77 #include <vm/kboot_mmu.h> 78 #include <vm/seg_spt.h> 79 80 #include <sys/cmn_err.h> 81 82 /* 83 * Basic parameters for hat operation. 84 */ 85 struct hat_mmu_info mmu; 86 87 /* 88 * The page that is the kernel's top level pagetable. 89 * 90 * For 32 bit PAE support on i86pc, the kernel hat will use the 1st 4 entries 91 * on this 4K page for its top level page table. The remaining groups of 92 * 4 entries are used for per processor copies of user VLP pagetables for 93 * running threads. See hat_switch() and reload_pae32() for details. 94 * 95 * vlp_page[0..3] - level==2 PTEs for kernel HAT 96 * vlp_page[4..7] - level==2 PTEs for user thread on cpu 0 97 * vlp_page[8..11] - level==2 PTE for user thread on cpu 1 98 * etc... 99 */ 100 static x86pte_t *vlp_page; 101 102 /* 103 * forward declaration of internal utility routines 104 */ 105 static x86pte_t hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, 106 x86pte_t new); 107 108 /* 109 * The kernel address space exists in all HATs. To implement this the 110 * kernel reserves a fixed number of entries in the topmost level(s) of page 111 * tables. The values are setup during startup and then copied to every user 112 * hat created by hat_alloc(). This means that kernelbase must be: 113 * 114 * 4Meg aligned for 32 bit kernels 115 * 512Gig aligned for x86_64 64 bit kernel 116 * 117 * The hat_kernel_range_ts describe what needs to be copied from kernel hat 118 * to each user hat. 119 */ 120 typedef struct hat_kernel_range { 121 level_t hkr_level; 122 uintptr_t hkr_start_va; 123 uintptr_t hkr_end_va; /* zero means to end of memory */ 124 } hat_kernel_range_t; 125 #define NUM_KERNEL_RANGE 2 126 static hat_kernel_range_t kernel_ranges[NUM_KERNEL_RANGE]; 127 static int num_kernel_ranges; 128 129 uint_t use_boot_reserve = 1; /* cleared after early boot process */ 130 uint_t can_steal_post_boot = 0; /* set late in boot to enable stealing */ 131 132 /* 133 * enable_1gpg: controls 1g page support for user applications. 134 * By default, 1g pages are exported to user applications. enable_1gpg can 135 * be set to 0 to not export. 136 */ 137 int enable_1gpg = 1; 138 139 /* 140 * AMD shanghai processors provide better management of 1gb ptes in its tlb. 141 * By default, 1g page suppport will be disabled for pre-shanghai AMD 142 * processors that don't have optimal tlb support for the 1g page size. 143 * chk_optimal_1gtlb can be set to 0 to force 1g page support on sub-optimal 144 * processors. 145 */ 146 int chk_optimal_1gtlb = 1; 147 148 149 #ifdef DEBUG 150 uint_t map1gcnt; 151 #endif 152 153 154 /* 155 * A cpuset for all cpus. This is used for kernel address cross calls, since 156 * the kernel addresses apply to all cpus. 157 */ 158 cpuset_t khat_cpuset; 159 160 /* 161 * management stuff for hat structures 162 */ 163 kmutex_t hat_list_lock; 164 kcondvar_t hat_list_cv; 165 kmem_cache_t *hat_cache; 166 kmem_cache_t *hat_hash_cache; 167 kmem_cache_t *vlp_hash_cache; 168 169 /* 170 * Simple statistics 171 */ 172 struct hatstats hatstat; 173 174 /* 175 * Some earlier hypervisor versions do not emulate cmpxchg of PTEs 176 * correctly. For such hypervisors we must set PT_USER for kernel 177 * entries ourselves (normally the emulation would set PT_USER for 178 * kernel entries and PT_USER|PT_GLOBAL for user entries). pt_kern is 179 * thus set appropriately. Note that dboot/kbm is OK, as only the full 180 * HAT uses cmpxchg() and the other paths (hypercall etc.) were never 181 * incorrect. 182 */ 183 int pt_kern; 184 185 /* 186 * useful stuff for atomic access/clearing/setting REF/MOD/RO bits in page_t's. 187 */ 188 extern void atomic_orb(uchar_t *addr, uchar_t val); 189 extern void atomic_andb(uchar_t *addr, uchar_t val); 190 191 #define PP_GETRM(pp, rmmask) (pp->p_nrm & rmmask) 192 #define PP_ISMOD(pp) PP_GETRM(pp, P_MOD) 193 #define PP_ISREF(pp) PP_GETRM(pp, P_REF) 194 #define PP_ISRO(pp) PP_GETRM(pp, P_RO) 195 196 #define PP_SETRM(pp, rm) atomic_orb(&(pp->p_nrm), rm) 197 #define PP_SETMOD(pp) PP_SETRM(pp, P_MOD) 198 #define PP_SETREF(pp) PP_SETRM(pp, P_REF) 199 #define PP_SETRO(pp) PP_SETRM(pp, P_RO) 200 201 #define PP_CLRRM(pp, rm) atomic_andb(&(pp->p_nrm), ~(rm)) 202 #define PP_CLRMOD(pp) PP_CLRRM(pp, P_MOD) 203 #define PP_CLRREF(pp) PP_CLRRM(pp, P_REF) 204 #define PP_CLRRO(pp) PP_CLRRM(pp, P_RO) 205 #define PP_CLRALL(pp) PP_CLRRM(pp, P_MOD | P_REF | P_RO) 206 207 /* 208 * kmem cache constructor for struct hat 209 */ 210 /*ARGSUSED*/ 211 static int 212 hati_constructor(void *buf, void *handle, int kmflags) 213 { 214 hat_t *hat = buf; 215 216 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); 217 bzero(hat->hat_pages_mapped, 218 sizeof (pgcnt_t) * (mmu.max_page_level + 1)); 219 hat->hat_ism_pgcnt = 0; 220 hat->hat_stats = 0; 221 hat->hat_flags = 0; 222 CPUSET_ZERO(hat->hat_cpus); 223 hat->hat_htable = NULL; 224 hat->hat_ht_hash = NULL; 225 return (0); 226 } 227 228 /* 229 * Allocate a hat structure for as. We also create the top level 230 * htable and initialize it to contain the kernel hat entries. 231 */ 232 hat_t * 233 hat_alloc(struct as *as) 234 { 235 hat_t *hat; 236 htable_t *ht; /* top level htable */ 237 uint_t use_vlp; 238 uint_t r; 239 hat_kernel_range_t *rp; 240 uintptr_t va; 241 uintptr_t eva; 242 uint_t start; 243 uint_t cnt; 244 htable_t *src; 245 246 /* 247 * Once we start creating user process HATs we can enable 248 * the htable_steal() code. 249 */ 250 if (can_steal_post_boot == 0) 251 can_steal_post_boot = 1; 252 253 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 254 hat = kmem_cache_alloc(hat_cache, KM_SLEEP); 255 hat->hat_as = as; 256 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); 257 ASSERT(hat->hat_flags == 0); 258 259 #if defined(__xpv) 260 /* 261 * No VLP stuff on the hypervisor due to the 64-bit split top level 262 * page tables. On 32-bit it's not needed as the hypervisor takes 263 * care of copying the top level PTEs to a below 4Gig page. 264 */ 265 use_vlp = 0; 266 #else /* __xpv */ 267 /* 32 bit processes uses a VLP style hat when running with PAE */ 268 #if defined(__amd64) 269 use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32); 270 #elif defined(__i386) 271 use_vlp = mmu.pae_hat; 272 #endif 273 #endif /* __xpv */ 274 if (use_vlp) { 275 hat->hat_flags = HAT_VLP; 276 bzero(hat->hat_vlp_ptes, VLP_SIZE); 277 } 278 279 /* 280 * Allocate the htable hash 281 */ 282 if ((hat->hat_flags & HAT_VLP)) { 283 hat->hat_num_hash = mmu.vlp_hash_cnt; 284 hat->hat_ht_hash = kmem_cache_alloc(vlp_hash_cache, KM_SLEEP); 285 } else { 286 hat->hat_num_hash = mmu.hash_cnt; 287 hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_SLEEP); 288 } 289 bzero(hat->hat_ht_hash, hat->hat_num_hash * sizeof (htable_t *)); 290 291 /* 292 * Initialize Kernel HAT entries at the top of the top level page 293 * tables for the new hat. 294 */ 295 hat->hat_htable = NULL; 296 hat->hat_ht_cached = NULL; 297 XPV_DISALLOW_MIGRATE(); 298 ht = htable_create(hat, (uintptr_t)0, TOP_LEVEL(hat), NULL); 299 hat->hat_htable = ht; 300 301 #if defined(__amd64) 302 if (hat->hat_flags & HAT_VLP) 303 goto init_done; 304 #endif 305 306 for (r = 0; r < num_kernel_ranges; ++r) { 307 rp = &kernel_ranges[r]; 308 for (va = rp->hkr_start_va; va != rp->hkr_end_va; 309 va += cnt * LEVEL_SIZE(rp->hkr_level)) { 310 311 if (rp->hkr_level == TOP_LEVEL(hat)) 312 ht = hat->hat_htable; 313 else 314 ht = htable_create(hat, va, rp->hkr_level, 315 NULL); 316 317 start = htable_va2entry(va, ht); 318 cnt = HTABLE_NUM_PTES(ht) - start; 319 eva = va + 320 ((uintptr_t)cnt << LEVEL_SHIFT(rp->hkr_level)); 321 if (rp->hkr_end_va != 0 && 322 (eva > rp->hkr_end_va || eva == 0)) 323 cnt = htable_va2entry(rp->hkr_end_va, ht) - 324 start; 325 326 #if defined(__i386) && !defined(__xpv) 327 if (ht->ht_flags & HTABLE_VLP) { 328 bcopy(&vlp_page[start], 329 &hat->hat_vlp_ptes[start], 330 cnt * sizeof (x86pte_t)); 331 continue; 332 } 333 #endif 334 src = htable_lookup(kas.a_hat, va, rp->hkr_level); 335 ASSERT(src != NULL); 336 x86pte_copy(src, ht, start, cnt); 337 htable_release(src); 338 } 339 } 340 341 init_done: 342 343 #if defined(__xpv) 344 /* 345 * Pin top level page tables after initializing them 346 */ 347 xen_pin(hat->hat_htable->ht_pfn, mmu.max_level); 348 #if defined(__amd64) 349 xen_pin(hat->hat_user_ptable, mmu.max_level); 350 #endif 351 #endif 352 XPV_ALLOW_MIGRATE(); 353 354 /* 355 * Put it at the start of the global list of all hats (used by stealing) 356 * 357 * kas.a_hat is not in the list but is instead used to find the 358 * first and last items in the list. 359 * 360 * - kas.a_hat->hat_next points to the start of the user hats. 361 * The list ends where hat->hat_next == NULL 362 * 363 * - kas.a_hat->hat_prev points to the last of the user hats. 364 * The list begins where hat->hat_prev == NULL 365 */ 366 mutex_enter(&hat_list_lock); 367 hat->hat_prev = NULL; 368 hat->hat_next = kas.a_hat->hat_next; 369 if (hat->hat_next) 370 hat->hat_next->hat_prev = hat; 371 else 372 kas.a_hat->hat_prev = hat; 373 kas.a_hat->hat_next = hat; 374 mutex_exit(&hat_list_lock); 375 376 return (hat); 377 } 378 379 /* 380 * process has finished executing but as has not been cleaned up yet. 381 */ 382 /*ARGSUSED*/ 383 void 384 hat_free_start(hat_t *hat) 385 { 386 ASSERT(AS_WRITE_HELD(hat->hat_as, &hat->hat_as->a_lock)); 387 388 /* 389 * If the hat is currently a stealing victim, wait for the stealing 390 * to finish. Once we mark it as HAT_FREEING, htable_steal() 391 * won't look at its pagetables anymore. 392 */ 393 mutex_enter(&hat_list_lock); 394 while (hat->hat_flags & HAT_VICTIM) 395 cv_wait(&hat_list_cv, &hat_list_lock); 396 hat->hat_flags |= HAT_FREEING; 397 mutex_exit(&hat_list_lock); 398 } 399 400 /* 401 * An address space is being destroyed, so we destroy the associated hat. 402 */ 403 void 404 hat_free_end(hat_t *hat) 405 { 406 kmem_cache_t *cache; 407 408 ASSERT(hat->hat_flags & HAT_FREEING); 409 410 /* 411 * must not be running on the given hat 412 */ 413 ASSERT(CPU->cpu_current_hat != hat); 414 415 /* 416 * Remove it from the list of HATs 417 */ 418 mutex_enter(&hat_list_lock); 419 if (hat->hat_prev) 420 hat->hat_prev->hat_next = hat->hat_next; 421 else 422 kas.a_hat->hat_next = hat->hat_next; 423 if (hat->hat_next) 424 hat->hat_next->hat_prev = hat->hat_prev; 425 else 426 kas.a_hat->hat_prev = hat->hat_prev; 427 mutex_exit(&hat_list_lock); 428 hat->hat_next = hat->hat_prev = NULL; 429 430 #if defined(__xpv) 431 /* 432 * On the hypervisor, unpin top level page table(s) 433 */ 434 xen_unpin(hat->hat_htable->ht_pfn); 435 #if defined(__amd64) 436 xen_unpin(hat->hat_user_ptable); 437 #endif 438 #endif 439 440 /* 441 * Make a pass through the htables freeing them all up. 442 */ 443 htable_purge_hat(hat); 444 445 /* 446 * Decide which kmem cache the hash table came from, then free it. 447 */ 448 if (hat->hat_flags & HAT_VLP) 449 cache = vlp_hash_cache; 450 else 451 cache = hat_hash_cache; 452 kmem_cache_free(cache, hat->hat_ht_hash); 453 hat->hat_ht_hash = NULL; 454 455 hat->hat_flags = 0; 456 kmem_cache_free(hat_cache, hat); 457 } 458 459 /* 460 * round kernelbase down to a supported value to use for _userlimit 461 * 462 * userlimit must be aligned down to an entry in the top level htable. 463 * The one exception is for 32 bit HAT's running PAE. 464 */ 465 uintptr_t 466 hat_kernelbase(uintptr_t va) 467 { 468 #if defined(__i386) 469 va &= LEVEL_MASK(1); 470 #endif 471 if (IN_VA_HOLE(va)) 472 panic("_userlimit %p will fall in VA hole\n", (void *)va); 473 return (va); 474 } 475 476 /* 477 * 478 */ 479 static void 480 set_max_page_level() 481 { 482 level_t lvl; 483 484 if (!kbm_largepage_support) { 485 lvl = 0; 486 } else { 487 if (x86_feature & X86_1GPG) { 488 lvl = 2; 489 if (chk_optimal_1gtlb && 490 cpuid_opteron_erratum(CPU, 6671130)) { 491 lvl = 1; 492 } 493 if (plat_mnode_xcheck(LEVEL_SIZE(2) >> 494 LEVEL_SHIFT(0))) { 495 lvl = 1; 496 } 497 } else { 498 lvl = 1; 499 } 500 } 501 mmu.max_page_level = lvl; 502 503 if ((lvl == 2) && (enable_1gpg == 0)) 504 mmu.umax_page_level = 1; 505 else 506 mmu.umax_page_level = lvl; 507 } 508 509 /* 510 * Initialize hat data structures based on processor MMU information. 511 */ 512 void 513 mmu_init(void) 514 { 515 uint_t max_htables; 516 uint_t pa_bits; 517 uint_t va_bits; 518 int i; 519 520 /* 521 * If CPU enabled the page table global bit, use it for the kernel 522 * This is bit 7 in CR4 (PGE - Page Global Enable). 523 */ 524 if ((x86_feature & X86_PGE) != 0 && (getcr4() & CR4_PGE) != 0) 525 mmu.pt_global = PT_GLOBAL; 526 527 /* 528 * Detect NX and PAE usage. 529 */ 530 mmu.pae_hat = kbm_pae_support; 531 if (kbm_nx_support) 532 mmu.pt_nx = PT_NX; 533 else 534 mmu.pt_nx = 0; 535 536 /* 537 * Use CPU info to set various MMU parameters 538 */ 539 cpuid_get_addrsize(CPU, &pa_bits, &va_bits); 540 541 if (va_bits < sizeof (void *) * NBBY) { 542 mmu.hole_start = (1ul << (va_bits - 1)); 543 mmu.hole_end = 0ul - mmu.hole_start - 1; 544 } else { 545 mmu.hole_end = 0; 546 mmu.hole_start = mmu.hole_end - 1; 547 } 548 #if defined(OPTERON_ERRATUM_121) 549 /* 550 * If erratum 121 has already been detected at this time, hole_start 551 * contains the value to be subtracted from mmu.hole_start. 552 */ 553 ASSERT(hole_start == 0 || opteron_erratum_121 != 0); 554 hole_start = mmu.hole_start - hole_start; 555 #else 556 hole_start = mmu.hole_start; 557 #endif 558 hole_end = mmu.hole_end; 559 560 mmu.highest_pfn = mmu_btop((1ull << pa_bits) - 1); 561 if (mmu.pae_hat == 0 && pa_bits > 32) 562 mmu.highest_pfn = PFN_4G - 1; 563 564 if (mmu.pae_hat) { 565 mmu.pte_size = 8; /* 8 byte PTEs */ 566 mmu.pte_size_shift = 3; 567 } else { 568 mmu.pte_size = 4; /* 4 byte PTEs */ 569 mmu.pte_size_shift = 2; 570 } 571 572 if (mmu.pae_hat && (x86_feature & X86_PAE) == 0) 573 panic("Processor does not support PAE"); 574 575 if ((x86_feature & X86_CX8) == 0) 576 panic("Processor does not support cmpxchg8b instruction"); 577 578 #if defined(__amd64) 579 580 mmu.num_level = 4; 581 mmu.max_level = 3; 582 mmu.ptes_per_table = 512; 583 mmu.top_level_count = 512; 584 585 mmu.level_shift[0] = 12; 586 mmu.level_shift[1] = 21; 587 mmu.level_shift[2] = 30; 588 mmu.level_shift[3] = 39; 589 590 #elif defined(__i386) 591 592 if (mmu.pae_hat) { 593 mmu.num_level = 3; 594 mmu.max_level = 2; 595 mmu.ptes_per_table = 512; 596 mmu.top_level_count = 4; 597 598 mmu.level_shift[0] = 12; 599 mmu.level_shift[1] = 21; 600 mmu.level_shift[2] = 30; 601 602 } else { 603 mmu.num_level = 2; 604 mmu.max_level = 1; 605 mmu.ptes_per_table = 1024; 606 mmu.top_level_count = 1024; 607 608 mmu.level_shift[0] = 12; 609 mmu.level_shift[1] = 22; 610 } 611 612 #endif /* __i386 */ 613 614 for (i = 0; i < mmu.num_level; ++i) { 615 mmu.level_size[i] = 1UL << mmu.level_shift[i]; 616 mmu.level_offset[i] = mmu.level_size[i] - 1; 617 mmu.level_mask[i] = ~mmu.level_offset[i]; 618 } 619 620 set_max_page_level(); 621 622 mmu_page_sizes = mmu.max_page_level + 1; 623 mmu_exported_page_sizes = mmu.umax_page_level + 1; 624 625 /* restrict legacy applications from using pagesizes 1g and above */ 626 mmu_legacy_page_sizes = 627 (mmu_exported_page_sizes > 2) ? 2 : mmu_exported_page_sizes; 628 629 630 for (i = 0; i <= mmu.max_page_level; ++i) { 631 mmu.pte_bits[i] = PT_VALID | pt_kern; 632 if (i > 0) 633 mmu.pte_bits[i] |= PT_PAGESIZE; 634 } 635 636 /* 637 * NOTE Legacy 32 bit PAE mode only has the P_VALID bit at top level. 638 */ 639 for (i = 1; i < mmu.num_level; ++i) 640 mmu.ptp_bits[i] = PT_PTPBITS; 641 642 #if defined(__i386) 643 mmu.ptp_bits[2] = PT_VALID; 644 #endif 645 646 /* 647 * Compute how many hash table entries to have per process for htables. 648 * We start with 1 page's worth of entries. 649 * 650 * If physical memory is small, reduce the amount need to cover it. 651 */ 652 max_htables = physmax / mmu.ptes_per_table; 653 mmu.hash_cnt = MMU_PAGESIZE / sizeof (htable_t *); 654 while (mmu.hash_cnt > 16 && mmu.hash_cnt >= max_htables) 655 mmu.hash_cnt >>= 1; 656 mmu.vlp_hash_cnt = mmu.hash_cnt; 657 658 #if defined(__amd64) 659 /* 660 * If running in 64 bits and physical memory is large, 661 * increase the size of the cache to cover all of memory for 662 * a 64 bit process. 663 */ 664 #define HASH_MAX_LENGTH 4 665 while (mmu.hash_cnt * HASH_MAX_LENGTH < max_htables) 666 mmu.hash_cnt <<= 1; 667 #endif 668 } 669 670 671 /* 672 * initialize hat data structures 673 */ 674 void 675 hat_init() 676 { 677 #if defined(__i386) 678 /* 679 * _userlimit must be aligned correctly 680 */ 681 if ((_userlimit & LEVEL_MASK(1)) != _userlimit) { 682 prom_printf("hat_init(): _userlimit=%p, not aligned at %p\n", 683 (void *)_userlimit, (void *)LEVEL_SIZE(1)); 684 halt("hat_init(): Unable to continue"); 685 } 686 #endif 687 688 cv_init(&hat_list_cv, NULL, CV_DEFAULT, NULL); 689 690 /* 691 * initialize kmem caches 692 */ 693 htable_init(); 694 hment_init(); 695 696 hat_cache = kmem_cache_create("hat_t", 697 sizeof (hat_t), 0, hati_constructor, NULL, NULL, 698 NULL, 0, 0); 699 700 hat_hash_cache = kmem_cache_create("HatHash", 701 mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL, 702 NULL, 0, 0); 703 704 /* 705 * VLP hats can use a smaller hash table size on large memroy machines 706 */ 707 if (mmu.hash_cnt == mmu.vlp_hash_cnt) { 708 vlp_hash_cache = hat_hash_cache; 709 } else { 710 vlp_hash_cache = kmem_cache_create("HatVlpHash", 711 mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL, 712 NULL, 0, 0); 713 } 714 715 /* 716 * Set up the kernel's hat 717 */ 718 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 719 kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP); 720 mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); 721 kas.a_hat->hat_as = &kas; 722 kas.a_hat->hat_flags = 0; 723 AS_LOCK_EXIT(&kas, &kas.a_lock); 724 725 CPUSET_ZERO(khat_cpuset); 726 CPUSET_ADD(khat_cpuset, CPU->cpu_id); 727 728 /* 729 * The kernel hat's next pointer serves as the head of the hat list . 730 * The kernel hat's prev pointer tracks the last hat on the list for 731 * htable_steal() to use. 732 */ 733 kas.a_hat->hat_next = NULL; 734 kas.a_hat->hat_prev = NULL; 735 736 /* 737 * Allocate an htable hash bucket for the kernel 738 * XX64 - tune for 64 bit procs 739 */ 740 kas.a_hat->hat_num_hash = mmu.hash_cnt; 741 kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP); 742 bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *)); 743 744 /* 745 * zero out the top level and cached htable pointers 746 */ 747 kas.a_hat->hat_ht_cached = NULL; 748 kas.a_hat->hat_htable = NULL; 749 750 /* 751 * Pre-allocate hrm_hashtab before enabling the collection of 752 * refmod statistics. Allocating on the fly would mean us 753 * running the risk of suffering recursive mutex enters or 754 * deadlocks. 755 */ 756 hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *), 757 KM_SLEEP); 758 } 759 760 /* 761 * Prepare CPU specific pagetables for VLP processes on 64 bit kernels. 762 * 763 * Each CPU has a set of 2 pagetables that are reused for any 32 bit 764 * process it runs. They are the top level pagetable, hci_vlp_l3ptes, and 765 * the next to top level table for the bottom 512 Gig, hci_vlp_l2ptes. 766 */ 767 /*ARGSUSED*/ 768 static void 769 hat_vlp_setup(struct cpu *cpu) 770 { 771 #if defined(__amd64) && !defined(__xpv) 772 struct hat_cpu_info *hci = cpu->cpu_hat_info; 773 pfn_t pfn; 774 775 /* 776 * allocate the level==2 page table for the bottom most 777 * 512Gig of address space (this is where 32 bit apps live) 778 */ 779 ASSERT(hci != NULL); 780 hci->hci_vlp_l2ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP); 781 782 /* 783 * Allocate a top level pagetable and copy the kernel's 784 * entries into it. Then link in hci_vlp_l2ptes in the 1st entry. 785 */ 786 hci->hci_vlp_l3ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP); 787 hci->hci_vlp_pfn = 788 hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l3ptes); 789 ASSERT(hci->hci_vlp_pfn != PFN_INVALID); 790 bcopy(vlp_page, hci->hci_vlp_l3ptes, MMU_PAGESIZE); 791 792 pfn = hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l2ptes); 793 ASSERT(pfn != PFN_INVALID); 794 hci->hci_vlp_l3ptes[0] = MAKEPTP(pfn, 2); 795 #endif /* __amd64 && !__xpv */ 796 } 797 798 /*ARGSUSED*/ 799 static void 800 hat_vlp_teardown(cpu_t *cpu) 801 { 802 #if defined(__amd64) && !defined(__xpv) 803 struct hat_cpu_info *hci; 804 805 if ((hci = cpu->cpu_hat_info) == NULL) 806 return; 807 if (hci->hci_vlp_l2ptes) 808 kmem_free(hci->hci_vlp_l2ptes, MMU_PAGESIZE); 809 if (hci->hci_vlp_l3ptes) 810 kmem_free(hci->hci_vlp_l3ptes, MMU_PAGESIZE); 811 #endif 812 } 813 814 #define NEXT_HKR(r, l, s, e) { \ 815 kernel_ranges[r].hkr_level = l; \ 816 kernel_ranges[r].hkr_start_va = s; \ 817 kernel_ranges[r].hkr_end_va = e; \ 818 ++r; \ 819 } 820 821 /* 822 * Finish filling in the kernel hat. 823 * Pre fill in all top level kernel page table entries for the kernel's 824 * part of the address range. From this point on we can't use any new 825 * kernel large pages if they need PTE's at max_level 826 * 827 * create the kmap mappings. 828 */ 829 void 830 hat_init_finish(void) 831 { 832 size_t size; 833 uint_t r = 0; 834 uintptr_t va; 835 hat_kernel_range_t *rp; 836 837 838 /* 839 * We are now effectively running on the kernel hat. 840 * Clearing use_boot_reserve shuts off using the pre-allocated boot 841 * reserve for all HAT allocations. From here on, the reserves are 842 * only used when avoiding recursion in kmem_alloc(). 843 */ 844 use_boot_reserve = 0; 845 htable_adjust_reserve(); 846 847 /* 848 * User HATs are initialized with copies of all kernel mappings in 849 * higher level page tables. Ensure that those entries exist. 850 */ 851 #if defined(__amd64) 852 853 NEXT_HKR(r, 3, kernelbase, 0); 854 #if defined(__xpv) 855 NEXT_HKR(r, 3, HYPERVISOR_VIRT_START, HYPERVISOR_VIRT_END); 856 #endif 857 858 #elif defined(__i386) 859 860 #if !defined(__xpv) 861 if (mmu.pae_hat) { 862 va = kernelbase; 863 if ((va & LEVEL_MASK(2)) != va) { 864 va = P2ROUNDUP(va, LEVEL_SIZE(2)); 865 NEXT_HKR(r, 1, kernelbase, va); 866 } 867 if (va != 0) 868 NEXT_HKR(r, 2, va, 0); 869 } else 870 #endif /* __xpv */ 871 NEXT_HKR(r, 1, kernelbase, 0); 872 873 #endif /* __i386 */ 874 875 num_kernel_ranges = r; 876 877 /* 878 * Create all the kernel pagetables that will have entries 879 * shared to user HATs. 880 */ 881 for (r = 0; r < num_kernel_ranges; ++r) { 882 rp = &kernel_ranges[r]; 883 for (va = rp->hkr_start_va; va != rp->hkr_end_va; 884 va += LEVEL_SIZE(rp->hkr_level)) { 885 htable_t *ht; 886 887 if (IN_HYPERVISOR_VA(va)) 888 continue; 889 890 /* can/must skip if a page mapping already exists */ 891 if (rp->hkr_level <= mmu.max_page_level && 892 (ht = htable_getpage(kas.a_hat, va, NULL)) != 893 NULL) { 894 htable_release(ht); 895 continue; 896 } 897 898 (void) htable_create(kas.a_hat, va, rp->hkr_level - 1, 899 NULL); 900 } 901 } 902 903 /* 904 * 32 bit PAE metal kernels use only 4 of the 512 entries in the 905 * page holding the top level pagetable. We use the remainder for 906 * the "per CPU" page tables for VLP processes. 907 * Map the top level kernel pagetable into the kernel to make 908 * it easy to use bcopy access these tables. 909 */ 910 if (mmu.pae_hat) { 911 vlp_page = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP); 912 hat_devload(kas.a_hat, (caddr_t)vlp_page, MMU_PAGESIZE, 913 kas.a_hat->hat_htable->ht_pfn, 914 #if !defined(__xpv) 915 PROT_WRITE | 916 #endif 917 PROT_READ | HAT_NOSYNC | HAT_UNORDERED_OK, 918 HAT_LOAD | HAT_LOAD_NOCONSIST); 919 } 920 hat_vlp_setup(CPU); 921 922 /* 923 * Create kmap (cached mappings of kernel PTEs) 924 * for 32 bit we map from segmap_start .. ekernelheap 925 * for 64 bit we map from segmap_start .. segmap_start + segmapsize; 926 */ 927 #if defined(__i386) 928 size = (uintptr_t)ekernelheap - segmap_start; 929 #elif defined(__amd64) 930 size = segmapsize; 931 #endif 932 hat_kmap_init((uintptr_t)segmap_start, size); 933 } 934 935 /* 936 * On 32 bit PAE mode, PTE's are 64 bits, but ordinary atomic memory references 937 * are 32 bit, so for safety we must use cas64() to install these. 938 */ 939 #ifdef __i386 940 static void 941 reload_pae32(hat_t *hat, cpu_t *cpu) 942 { 943 x86pte_t *src; 944 x86pte_t *dest; 945 x86pte_t pte; 946 int i; 947 948 /* 949 * Load the 4 entries of the level 2 page table into this 950 * cpu's range of the vlp_page and point cr3 at them. 951 */ 952 ASSERT(mmu.pae_hat); 953 src = hat->hat_vlp_ptes; 954 dest = vlp_page + (cpu->cpu_id + 1) * VLP_NUM_PTES; 955 for (i = 0; i < VLP_NUM_PTES; ++i) { 956 for (;;) { 957 pte = dest[i]; 958 if (pte == src[i]) 959 break; 960 if (cas64(dest + i, pte, src[i]) != src[i]) 961 break; 962 } 963 } 964 } 965 #endif 966 967 /* 968 * Switch to a new active hat, maintaining bit masks to track active CPUs. 969 * 970 * On the 32-bit PAE hypervisor, %cr3 is a 64-bit value, on metal it 971 * remains a 32-bit value. 972 */ 973 void 974 hat_switch(hat_t *hat) 975 { 976 uint64_t newcr3; 977 cpu_t *cpu = CPU; 978 hat_t *old = cpu->cpu_current_hat; 979 980 /* 981 * set up this information first, so we don't miss any cross calls 982 */ 983 if (old != NULL) { 984 if (old == hat) 985 return; 986 if (old != kas.a_hat) 987 CPUSET_ATOMIC_DEL(old->hat_cpus, cpu->cpu_id); 988 } 989 990 /* 991 * Add this CPU to the active set for this HAT. 992 */ 993 if (hat != kas.a_hat) { 994 CPUSET_ATOMIC_ADD(hat->hat_cpus, cpu->cpu_id); 995 } 996 cpu->cpu_current_hat = hat; 997 998 /* 999 * now go ahead and load cr3 1000 */ 1001 if (hat->hat_flags & HAT_VLP) { 1002 #if defined(__amd64) 1003 x86pte_t *vlpptep = cpu->cpu_hat_info->hci_vlp_l2ptes; 1004 1005 VLP_COPY(hat->hat_vlp_ptes, vlpptep); 1006 newcr3 = MAKECR3(cpu->cpu_hat_info->hci_vlp_pfn); 1007 #elif defined(__i386) 1008 reload_pae32(hat, cpu); 1009 newcr3 = MAKECR3(kas.a_hat->hat_htable->ht_pfn) + 1010 (cpu->cpu_id + 1) * VLP_SIZE; 1011 #endif 1012 } else { 1013 newcr3 = MAKECR3((uint64_t)hat->hat_htable->ht_pfn); 1014 } 1015 #ifdef __xpv 1016 { 1017 struct mmuext_op t[2]; 1018 uint_t retcnt; 1019 uint_t opcnt = 1; 1020 1021 t[0].cmd = MMUEXT_NEW_BASEPTR; 1022 t[0].arg1.mfn = mmu_btop(pa_to_ma(newcr3)); 1023 #if defined(__amd64) 1024 /* 1025 * There's an interesting problem here, as to what to 1026 * actually specify when switching to the kernel hat. 1027 * For now we'll reuse the kernel hat again. 1028 */ 1029 t[1].cmd = MMUEXT_NEW_USER_BASEPTR; 1030 if (hat == kas.a_hat) 1031 t[1].arg1.mfn = mmu_btop(pa_to_ma(newcr3)); 1032 else 1033 t[1].arg1.mfn = pfn_to_mfn(hat->hat_user_ptable); 1034 ++opcnt; 1035 #endif /* __amd64 */ 1036 if (HYPERVISOR_mmuext_op(t, opcnt, &retcnt, DOMID_SELF) < 0) 1037 panic("HYPERVISOR_mmu_update() failed"); 1038 ASSERT(retcnt == opcnt); 1039 1040 } 1041 #else 1042 setcr3(newcr3); 1043 #endif 1044 ASSERT(cpu == CPU); 1045 } 1046 1047 /* 1048 * Utility to return a valid x86pte_t from protections, pfn, and level number 1049 */ 1050 static x86pte_t 1051 hati_mkpte(pfn_t pfn, uint_t attr, level_t level, uint_t flags) 1052 { 1053 x86pte_t pte; 1054 uint_t cache_attr = attr & HAT_ORDER_MASK; 1055 1056 pte = MAKEPTE(pfn, level); 1057 1058 if (attr & PROT_WRITE) 1059 PTE_SET(pte, PT_WRITABLE); 1060 1061 if (attr & PROT_USER) 1062 PTE_SET(pte, PT_USER); 1063 1064 if (!(attr & PROT_EXEC)) 1065 PTE_SET(pte, mmu.pt_nx); 1066 1067 /* 1068 * Set the software bits used track ref/mod sync's and hments. 1069 * If not using REF/MOD, set them to avoid h/w rewriting PTEs. 1070 */ 1071 if (flags & HAT_LOAD_NOCONSIST) 1072 PTE_SET(pte, PT_NOCONSIST | PT_REF | PT_MOD); 1073 else if (attr & HAT_NOSYNC) 1074 PTE_SET(pte, PT_NOSYNC | PT_REF | PT_MOD); 1075 1076 /* 1077 * Set the caching attributes in the PTE. The combination 1078 * of attributes are poorly defined, so we pay attention 1079 * to them in the given order. 1080 * 1081 * The test for HAT_STRICTORDER is different because it's defined 1082 * as "0" - which was a stupid thing to do, but is too late to change! 1083 */ 1084 if (cache_attr == HAT_STRICTORDER) { 1085 PTE_SET(pte, PT_NOCACHE); 1086 /*LINTED [Lint hates empty ifs, but it's the obvious way to do this] */ 1087 } else if (cache_attr & (HAT_UNORDERED_OK | HAT_STORECACHING_OK)) { 1088 /* nothing to set */; 1089 } else if (cache_attr & (HAT_MERGING_OK | HAT_LOADCACHING_OK)) { 1090 PTE_SET(pte, PT_NOCACHE); 1091 if (x86_feature & X86_PAT) 1092 PTE_SET(pte, (level == 0) ? PT_PAT_4K : PT_PAT_LARGE); 1093 else 1094 PTE_SET(pte, PT_WRITETHRU); 1095 } else { 1096 panic("hati_mkpte(): bad caching attributes: %x\n", cache_attr); 1097 } 1098 1099 return (pte); 1100 } 1101 1102 /* 1103 * Duplicate address translations of the parent to the child. 1104 * This function really isn't used anymore. 1105 */ 1106 /*ARGSUSED*/ 1107 int 1108 hat_dup(hat_t *old, hat_t *new, caddr_t addr, size_t len, uint_t flag) 1109 { 1110 ASSERT((uintptr_t)addr < kernelbase); 1111 ASSERT(new != kas.a_hat); 1112 ASSERT(old != kas.a_hat); 1113 return (0); 1114 } 1115 1116 /* 1117 * Allocate any hat resources required for a process being swapped in. 1118 */ 1119 /*ARGSUSED*/ 1120 void 1121 hat_swapin(hat_t *hat) 1122 { 1123 /* do nothing - we let everything fault back in */ 1124 } 1125 1126 /* 1127 * Unload all translations associated with an address space of a process 1128 * that is being swapped out. 1129 */ 1130 void 1131 hat_swapout(hat_t *hat) 1132 { 1133 uintptr_t vaddr = (uintptr_t)0; 1134 uintptr_t eaddr = _userlimit; 1135 htable_t *ht = NULL; 1136 level_t l; 1137 1138 XPV_DISALLOW_MIGRATE(); 1139 /* 1140 * We can't just call hat_unload(hat, 0, _userlimit...) here, because 1141 * seg_spt and shared pagetables can't be swapped out. 1142 * Take a look at segspt_shmswapout() - it's a big no-op. 1143 * 1144 * Instead we'll walk through all the address space and unload 1145 * any mappings which we are sure are not shared, not locked. 1146 */ 1147 ASSERT(IS_PAGEALIGNED(vaddr)); 1148 ASSERT(IS_PAGEALIGNED(eaddr)); 1149 ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1150 if ((uintptr_t)hat->hat_as->a_userlimit < eaddr) 1151 eaddr = (uintptr_t)hat->hat_as->a_userlimit; 1152 1153 while (vaddr < eaddr) { 1154 (void) htable_walk(hat, &ht, &vaddr, eaddr); 1155 if (ht == NULL) 1156 break; 1157 1158 ASSERT(!IN_VA_HOLE(vaddr)); 1159 1160 /* 1161 * If the page table is shared skip its entire range. 1162 */ 1163 l = ht->ht_level; 1164 if (ht->ht_flags & HTABLE_SHARED_PFN) { 1165 vaddr = ht->ht_vaddr + LEVEL_SIZE(l + 1); 1166 htable_release(ht); 1167 ht = NULL; 1168 continue; 1169 } 1170 1171 /* 1172 * If the page table has no locked entries, unload this one. 1173 */ 1174 if (ht->ht_lock_cnt == 0) 1175 hat_unload(hat, (caddr_t)vaddr, LEVEL_SIZE(l), 1176 HAT_UNLOAD_UNMAP); 1177 1178 /* 1179 * If we have a level 0 page table with locked entries, 1180 * skip the entire page table, otherwise skip just one entry. 1181 */ 1182 if (ht->ht_lock_cnt > 0 && l == 0) 1183 vaddr = ht->ht_vaddr + LEVEL_SIZE(1); 1184 else 1185 vaddr += LEVEL_SIZE(l); 1186 } 1187 if (ht) 1188 htable_release(ht); 1189 1190 /* 1191 * We're in swapout because the system is low on memory, so 1192 * go back and flush all the htables off the cached list. 1193 */ 1194 htable_purge_hat(hat); 1195 XPV_ALLOW_MIGRATE(); 1196 } 1197 1198 /* 1199 * returns number of bytes that have valid mappings in hat. 1200 */ 1201 size_t 1202 hat_get_mapped_size(hat_t *hat) 1203 { 1204 size_t total = 0; 1205 int l; 1206 1207 for (l = 0; l <= mmu.max_page_level; l++) 1208 total += (hat->hat_pages_mapped[l] << LEVEL_SHIFT(l)); 1209 total += hat->hat_ism_pgcnt; 1210 1211 return (total); 1212 } 1213 1214 /* 1215 * enable/disable collection of stats for hat. 1216 */ 1217 int 1218 hat_stats_enable(hat_t *hat) 1219 { 1220 atomic_add_32(&hat->hat_stats, 1); 1221 return (1); 1222 } 1223 1224 void 1225 hat_stats_disable(hat_t *hat) 1226 { 1227 atomic_add_32(&hat->hat_stats, -1); 1228 } 1229 1230 /* 1231 * Utility to sync the ref/mod bits from a page table entry to the page_t 1232 * We must be holding the mapping list lock when this is called. 1233 */ 1234 static void 1235 hati_sync_pte_to_page(page_t *pp, x86pte_t pte, level_t level) 1236 { 1237 uint_t rm = 0; 1238 pgcnt_t pgcnt; 1239 1240 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC) 1241 return; 1242 1243 if (PTE_GET(pte, PT_REF)) 1244 rm |= P_REF; 1245 1246 if (PTE_GET(pte, PT_MOD)) 1247 rm |= P_MOD; 1248 1249 if (rm == 0) 1250 return; 1251 1252 /* 1253 * sync to all constituent pages of a large page 1254 */ 1255 ASSERT(x86_hm_held(pp)); 1256 pgcnt = page_get_pagecnt(level); 1257 ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt)); 1258 for (; pgcnt > 0; --pgcnt) { 1259 /* 1260 * hat_page_demote() can't decrease 1261 * pszc below this mapping size 1262 * since this large mapping existed after we 1263 * took mlist lock. 1264 */ 1265 ASSERT(pp->p_szc >= level); 1266 hat_page_setattr(pp, rm); 1267 ++pp; 1268 } 1269 } 1270 1271 /* 1272 * This the set of PTE bits for PFN, permissions and caching 1273 * that are allowed to change on a HAT_LOAD_REMAP 1274 */ 1275 #define PT_REMAP_BITS \ 1276 (PT_PADDR | PT_NX | PT_WRITABLE | PT_WRITETHRU | \ 1277 PT_NOCACHE | PT_PAT_4K | PT_PAT_LARGE | PT_IGNORE | PT_REF | PT_MOD) 1278 1279 #define REMAPASSERT(EX) if (!(EX)) panic("hati_pte_map: " #EX) 1280 /* 1281 * Do the low-level work to get a mapping entered into a HAT's pagetables 1282 * and in the mapping list of the associated page_t. 1283 */ 1284 static int 1285 hati_pte_map( 1286 htable_t *ht, 1287 uint_t entry, 1288 page_t *pp, 1289 x86pte_t pte, 1290 int flags, 1291 void *pte_ptr) 1292 { 1293 hat_t *hat = ht->ht_hat; 1294 x86pte_t old_pte; 1295 level_t l = ht->ht_level; 1296 hment_t *hm; 1297 uint_t is_consist; 1298 uint_t is_locked; 1299 int rv = 0; 1300 1301 /* 1302 * Is this a consistant (ie. need mapping list lock) mapping? 1303 */ 1304 is_consist = (pp != NULL && (flags & HAT_LOAD_NOCONSIST) == 0); 1305 1306 /* 1307 * Track locked mapping count in the htable. Do this first, 1308 * as we track locking even if there already is a mapping present. 1309 */ 1310 is_locked = (flags & HAT_LOAD_LOCK) != 0 && hat != kas.a_hat; 1311 if (is_locked) 1312 HTABLE_LOCK_INC(ht); 1313 1314 /* 1315 * Acquire the page's mapping list lock and get an hment to use. 1316 * Note that hment_prepare() might return NULL. 1317 */ 1318 if (is_consist) { 1319 x86_hm_enter(pp); 1320 hm = hment_prepare(ht, entry, pp); 1321 } 1322 1323 /* 1324 * Set the new pte, retrieving the old one at the same time. 1325 */ 1326 old_pte = x86pte_set(ht, entry, pte, pte_ptr); 1327 1328 /* 1329 * Did we get a large page / page table collision? 1330 */ 1331 if (old_pte == LPAGE_ERROR) { 1332 if (is_locked) 1333 HTABLE_LOCK_DEC(ht); 1334 rv = -1; 1335 goto done; 1336 } 1337 1338 /* 1339 * If the mapping didn't change there is nothing more to do. 1340 */ 1341 if (PTE_EQUIV(pte, old_pte)) 1342 goto done; 1343 1344 /* 1345 * Install a new mapping in the page's mapping list 1346 */ 1347 if (!PTE_ISVALID(old_pte)) { 1348 if (is_consist) { 1349 hment_assign(ht, entry, pp, hm); 1350 x86_hm_exit(pp); 1351 } else { 1352 ASSERT(flags & HAT_LOAD_NOCONSIST); 1353 } 1354 #if defined(__amd64) 1355 if (ht->ht_flags & HTABLE_VLP) { 1356 cpu_t *cpu = CPU; 1357 x86pte_t *vlpptep = cpu->cpu_hat_info->hci_vlp_l2ptes; 1358 VLP_COPY(hat->hat_vlp_ptes, vlpptep); 1359 } 1360 #endif 1361 HTABLE_INC(ht->ht_valid_cnt); 1362 PGCNT_INC(hat, l); 1363 return (rv); 1364 } 1365 1366 /* 1367 * Remap's are more complicated: 1368 * - HAT_LOAD_REMAP must be specified if changing the pfn. 1369 * We also require that NOCONSIST be specified. 1370 * - Otherwise only permission or caching bits may change. 1371 */ 1372 if (!PTE_ISPAGE(old_pte, l)) 1373 panic("non-null/page mapping pte=" FMT_PTE, old_pte); 1374 1375 if (PTE2PFN(old_pte, l) != PTE2PFN(pte, l)) { 1376 REMAPASSERT(flags & HAT_LOAD_REMAP); 1377 REMAPASSERT(flags & HAT_LOAD_NOCONSIST); 1378 REMAPASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST); 1379 REMAPASSERT(pf_is_memory(PTE2PFN(old_pte, l)) == 1380 pf_is_memory(PTE2PFN(pte, l))); 1381 REMAPASSERT(!is_consist); 1382 } 1383 1384 /* 1385 * We only let remaps change the certain bits in the PTE. 1386 */ 1387 if (PTE_GET(old_pte, ~PT_REMAP_BITS) != PTE_GET(pte, ~PT_REMAP_BITS)) 1388 panic("remap bits changed: old_pte="FMT_PTE", pte="FMT_PTE"\n", 1389 old_pte, pte); 1390 1391 /* 1392 * We don't create any mapping list entries on a remap, so release 1393 * any allocated hment after we drop the mapping list lock. 1394 */ 1395 done: 1396 if (is_consist) { 1397 x86_hm_exit(pp); 1398 if (hm != NULL) 1399 hment_free(hm); 1400 } 1401 return (rv); 1402 } 1403 1404 /* 1405 * Internal routine to load a single page table entry. This only fails if 1406 * we attempt to overwrite a page table link with a large page. 1407 */ 1408 static int 1409 hati_load_common( 1410 hat_t *hat, 1411 uintptr_t va, 1412 page_t *pp, 1413 uint_t attr, 1414 uint_t flags, 1415 level_t level, 1416 pfn_t pfn) 1417 { 1418 htable_t *ht; 1419 uint_t entry; 1420 x86pte_t pte; 1421 int rv = 0; 1422 1423 /* 1424 * The number 16 is arbitrary and here to catch a recursion problem 1425 * early before we blow out the kernel stack. 1426 */ 1427 ++curthread->t_hatdepth; 1428 ASSERT(curthread->t_hatdepth < 16); 1429 1430 ASSERT(hat == kas.a_hat || 1431 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1432 1433 if (flags & HAT_LOAD_SHARE) 1434 hat->hat_flags |= HAT_SHARED; 1435 1436 /* 1437 * Find the page table that maps this page if it already exists. 1438 */ 1439 ht = htable_lookup(hat, va, level); 1440 1441 /* 1442 * We must have HAT_LOAD_NOCONSIST if page_t is NULL. 1443 */ 1444 if (pp == NULL) 1445 flags |= HAT_LOAD_NOCONSIST; 1446 1447 if (ht == NULL) { 1448 ht = htable_create(hat, va, level, NULL); 1449 ASSERT(ht != NULL); 1450 } 1451 entry = htable_va2entry(va, ht); 1452 1453 /* 1454 * a bunch of paranoid error checking 1455 */ 1456 ASSERT(ht->ht_busy > 0); 1457 if (ht->ht_vaddr > va || va > HTABLE_LAST_PAGE(ht)) 1458 panic("hati_load_common: bad htable %p, va %p", 1459 (void *)ht, (void *)va); 1460 ASSERT(ht->ht_level == level); 1461 1462 /* 1463 * construct the new PTE 1464 */ 1465 if (hat == kas.a_hat) 1466 attr &= ~PROT_USER; 1467 pte = hati_mkpte(pfn, attr, level, flags); 1468 if (hat == kas.a_hat && va >= kernelbase) 1469 PTE_SET(pte, mmu.pt_global); 1470 1471 /* 1472 * establish the mapping 1473 */ 1474 rv = hati_pte_map(ht, entry, pp, pte, flags, NULL); 1475 1476 /* 1477 * release the htable and any reserves 1478 */ 1479 htable_release(ht); 1480 --curthread->t_hatdepth; 1481 return (rv); 1482 } 1483 1484 /* 1485 * special case of hat_memload to deal with some kernel addrs for performance 1486 */ 1487 static void 1488 hat_kmap_load( 1489 caddr_t addr, 1490 page_t *pp, 1491 uint_t attr, 1492 uint_t flags) 1493 { 1494 uintptr_t va = (uintptr_t)addr; 1495 x86pte_t pte; 1496 pfn_t pfn = page_pptonum(pp); 1497 pgcnt_t pg_off = mmu_btop(va - mmu.kmap_addr); 1498 htable_t *ht; 1499 uint_t entry; 1500 void *pte_ptr; 1501 1502 /* 1503 * construct the requested PTE 1504 */ 1505 attr &= ~PROT_USER; 1506 attr |= HAT_STORECACHING_OK; 1507 pte = hati_mkpte(pfn, attr, 0, flags); 1508 PTE_SET(pte, mmu.pt_global); 1509 1510 /* 1511 * Figure out the pte_ptr and htable and use common code to finish up 1512 */ 1513 if (mmu.pae_hat) 1514 pte_ptr = mmu.kmap_ptes + pg_off; 1515 else 1516 pte_ptr = (x86pte32_t *)mmu.kmap_ptes + pg_off; 1517 ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) >> 1518 LEVEL_SHIFT(1)]; 1519 entry = htable_va2entry(va, ht); 1520 ++curthread->t_hatdepth; 1521 ASSERT(curthread->t_hatdepth < 16); 1522 (void) hati_pte_map(ht, entry, pp, pte, flags, pte_ptr); 1523 --curthread->t_hatdepth; 1524 } 1525 1526 /* 1527 * hat_memload() - load a translation to the given page struct 1528 * 1529 * Flags for hat_memload/hat_devload/hat_*attr. 1530 * 1531 * HAT_LOAD Default flags to load a translation to the page. 1532 * 1533 * HAT_LOAD_LOCK Lock down mapping resources; hat_map(), hat_memload(), 1534 * and hat_devload(). 1535 * 1536 * HAT_LOAD_NOCONSIST Do not add mapping to page_t mapping list. 1537 * sets PT_NOCONSIST 1538 * 1539 * HAT_LOAD_SHARE A flag to hat_memload() to indicate h/w page tables 1540 * that map some user pages (not kas) is shared by more 1541 * than one process (eg. ISM). 1542 * 1543 * HAT_LOAD_REMAP Reload a valid pte with a different page frame. 1544 * 1545 * HAT_NO_KALLOC Do not kmem_alloc while creating the mapping; at this 1546 * point, it's setting up mapping to allocate internal 1547 * hat layer data structures. This flag forces hat layer 1548 * to tap its reserves in order to prevent infinite 1549 * recursion. 1550 * 1551 * The following is a protection attribute (like PROT_READ, etc.) 1552 * 1553 * HAT_NOSYNC set PT_NOSYNC - this mapping's ref/mod bits 1554 * are never cleared. 1555 * 1556 * Installing new valid PTE's and creation of the mapping list 1557 * entry are controlled under the same lock. It's derived from the 1558 * page_t being mapped. 1559 */ 1560 static uint_t supported_memload_flags = 1561 HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST | 1562 HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT; 1563 1564 void 1565 hat_memload( 1566 hat_t *hat, 1567 caddr_t addr, 1568 page_t *pp, 1569 uint_t attr, 1570 uint_t flags) 1571 { 1572 uintptr_t va = (uintptr_t)addr; 1573 level_t level = 0; 1574 pfn_t pfn = page_pptonum(pp); 1575 1576 XPV_DISALLOW_MIGRATE(); 1577 ASSERT(IS_PAGEALIGNED(va)); 1578 ASSERT(hat == kas.a_hat || va < _userlimit); 1579 ASSERT(hat == kas.a_hat || 1580 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1581 ASSERT((flags & supported_memload_flags) == flags); 1582 1583 ASSERT(!IN_VA_HOLE(va)); 1584 ASSERT(!PP_ISFREE(pp)); 1585 1586 /* 1587 * kernel address special case for performance. 1588 */ 1589 if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) { 1590 ASSERT(hat == kas.a_hat); 1591 hat_kmap_load(addr, pp, attr, flags); 1592 XPV_ALLOW_MIGRATE(); 1593 return; 1594 } 1595 1596 /* 1597 * This is used for memory with normal caching enabled, so 1598 * always set HAT_STORECACHING_OK. 1599 */ 1600 attr |= HAT_STORECACHING_OK; 1601 if (hati_load_common(hat, va, pp, attr, flags, level, pfn) != 0) 1602 panic("unexpected hati_load_common() failure"); 1603 XPV_ALLOW_MIGRATE(); 1604 } 1605 1606 /* ARGSUSED */ 1607 void 1608 hat_memload_region(struct hat *hat, caddr_t addr, struct page *pp, 1609 uint_t attr, uint_t flags, hat_region_cookie_t rcookie) 1610 { 1611 hat_memload(hat, addr, pp, attr, flags); 1612 } 1613 1614 /* 1615 * Load the given array of page structs using large pages when possible 1616 */ 1617 void 1618 hat_memload_array( 1619 hat_t *hat, 1620 caddr_t addr, 1621 size_t len, 1622 page_t **pages, 1623 uint_t attr, 1624 uint_t flags) 1625 { 1626 uintptr_t va = (uintptr_t)addr; 1627 uintptr_t eaddr = va + len; 1628 level_t level; 1629 size_t pgsize; 1630 pgcnt_t pgindx = 0; 1631 pfn_t pfn; 1632 pgcnt_t i; 1633 1634 XPV_DISALLOW_MIGRATE(); 1635 ASSERT(IS_PAGEALIGNED(va)); 1636 ASSERT(hat == kas.a_hat || va + len <= _userlimit); 1637 ASSERT(hat == kas.a_hat || 1638 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1639 ASSERT((flags & supported_memload_flags) == flags); 1640 1641 /* 1642 * memload is used for memory with full caching enabled, so 1643 * set HAT_STORECACHING_OK. 1644 */ 1645 attr |= HAT_STORECACHING_OK; 1646 1647 /* 1648 * handle all pages using largest possible pagesize 1649 */ 1650 while (va < eaddr) { 1651 /* 1652 * decide what level mapping to use (ie. pagesize) 1653 */ 1654 pfn = page_pptonum(pages[pgindx]); 1655 for (level = mmu.max_page_level; ; --level) { 1656 pgsize = LEVEL_SIZE(level); 1657 if (level == 0) 1658 break; 1659 1660 if (!IS_P2ALIGNED(va, pgsize) || 1661 (eaddr - va) < pgsize || 1662 !IS_P2ALIGNED(pfn_to_pa(pfn), pgsize)) 1663 continue; 1664 1665 /* 1666 * To use a large mapping of this size, all the 1667 * pages we are passed must be sequential subpages 1668 * of the large page. 1669 * hat_page_demote() can't change p_szc because 1670 * all pages are locked. 1671 */ 1672 if (pages[pgindx]->p_szc >= level) { 1673 for (i = 0; i < mmu_btop(pgsize); ++i) { 1674 if (pfn + i != 1675 page_pptonum(pages[pgindx + i])) 1676 break; 1677 ASSERT(pages[pgindx + i]->p_szc >= 1678 level); 1679 ASSERT(pages[pgindx] + i == 1680 pages[pgindx + i]); 1681 } 1682 if (i == mmu_btop(pgsize)) { 1683 #ifdef DEBUG 1684 if (level == 2) 1685 map1gcnt++; 1686 #endif 1687 break; 1688 } 1689 } 1690 } 1691 1692 /* 1693 * Load this page mapping. If the load fails, try a smaller 1694 * pagesize. 1695 */ 1696 ASSERT(!IN_VA_HOLE(va)); 1697 while (hati_load_common(hat, va, pages[pgindx], attr, 1698 flags, level, pfn) != 0) { 1699 if (level == 0) 1700 panic("unexpected hati_load_common() failure"); 1701 --level; 1702 pgsize = LEVEL_SIZE(level); 1703 } 1704 1705 /* 1706 * move to next page 1707 */ 1708 va += pgsize; 1709 pgindx += mmu_btop(pgsize); 1710 } 1711 XPV_ALLOW_MIGRATE(); 1712 } 1713 1714 /* ARGSUSED */ 1715 void 1716 hat_memload_array_region(struct hat *hat, caddr_t addr, size_t len, 1717 struct page **pps, uint_t attr, uint_t flags, 1718 hat_region_cookie_t rcookie) 1719 { 1720 hat_memload_array(hat, addr, len, pps, attr, flags); 1721 } 1722 1723 /* 1724 * void hat_devload(hat, addr, len, pf, attr, flags) 1725 * load/lock the given page frame number 1726 * 1727 * Advisory ordering attributes. Apply only to device mappings. 1728 * 1729 * HAT_STRICTORDER: the CPU must issue the references in order, as the 1730 * programmer specified. This is the default. 1731 * HAT_UNORDERED_OK: the CPU may reorder the references (this is all kinds 1732 * of reordering; store or load with store or load). 1733 * HAT_MERGING_OK: merging and batching: the CPU may merge individual stores 1734 * to consecutive locations (for example, turn two consecutive byte 1735 * stores into one halfword store), and it may batch individual loads 1736 * (for example, turn two consecutive byte loads into one halfword load). 1737 * This also implies re-ordering. 1738 * HAT_LOADCACHING_OK: the CPU may cache the data it fetches and reuse it 1739 * until another store occurs. The default is to fetch new data 1740 * on every load. This also implies merging. 1741 * HAT_STORECACHING_OK: the CPU may keep the data in the cache and push it to 1742 * the device (perhaps with other data) at a later time. The default is 1743 * to push the data right away. This also implies load caching. 1744 * 1745 * Equivalent of hat_memload(), but can be used for device memory where 1746 * there are no page_t's and we support additional flags (write merging, etc). 1747 * Note that we can have large page mappings with this interface. 1748 */ 1749 int supported_devload_flags = HAT_LOAD | HAT_LOAD_LOCK | 1750 HAT_LOAD_NOCONSIST | HAT_STRICTORDER | HAT_UNORDERED_OK | 1751 HAT_MERGING_OK | HAT_LOADCACHING_OK | HAT_STORECACHING_OK; 1752 1753 void 1754 hat_devload( 1755 hat_t *hat, 1756 caddr_t addr, 1757 size_t len, 1758 pfn_t pfn, 1759 uint_t attr, 1760 int flags) 1761 { 1762 uintptr_t va = ALIGN2PAGE(addr); 1763 uintptr_t eva = va + len; 1764 level_t level; 1765 size_t pgsize; 1766 page_t *pp; 1767 int f; /* per PTE copy of flags - maybe modified */ 1768 uint_t a; /* per PTE copy of attr */ 1769 1770 XPV_DISALLOW_MIGRATE(); 1771 ASSERT(IS_PAGEALIGNED(va)); 1772 ASSERT(hat == kas.a_hat || eva <= _userlimit); 1773 ASSERT(hat == kas.a_hat || 1774 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1775 ASSERT((flags & supported_devload_flags) == flags); 1776 1777 /* 1778 * handle all pages 1779 */ 1780 while (va < eva) { 1781 1782 /* 1783 * decide what level mapping to use (ie. pagesize) 1784 */ 1785 for (level = mmu.max_page_level; ; --level) { 1786 pgsize = LEVEL_SIZE(level); 1787 if (level == 0) 1788 break; 1789 if (IS_P2ALIGNED(va, pgsize) && 1790 (eva - va) >= pgsize && 1791 IS_P2ALIGNED(pfn, mmu_btop(pgsize))) { 1792 #ifdef DEBUG 1793 if (level == 2) 1794 map1gcnt++; 1795 #endif 1796 break; 1797 } 1798 } 1799 1800 /* 1801 * If this is just memory then allow caching (this happens 1802 * for the nucleus pages) - though HAT_PLAT_NOCACHE can be used 1803 * to override that. If we don't have a page_t then make sure 1804 * NOCONSIST is set. 1805 */ 1806 a = attr; 1807 f = flags; 1808 if (!pf_is_memory(pfn)) 1809 f |= HAT_LOAD_NOCONSIST; 1810 else if (!(a & HAT_PLAT_NOCACHE)) 1811 a |= HAT_STORECACHING_OK; 1812 1813 if (f & HAT_LOAD_NOCONSIST) 1814 pp = NULL; 1815 else 1816 pp = page_numtopp_nolock(pfn); 1817 1818 /* 1819 * load this page mapping 1820 */ 1821 ASSERT(!IN_VA_HOLE(va)); 1822 while (hati_load_common(hat, va, pp, a, f, level, pfn) != 0) { 1823 if (level == 0) 1824 panic("unexpected hati_load_common() failure"); 1825 --level; 1826 pgsize = LEVEL_SIZE(level); 1827 } 1828 1829 /* 1830 * move to next page 1831 */ 1832 va += pgsize; 1833 pfn += mmu_btop(pgsize); 1834 } 1835 XPV_ALLOW_MIGRATE(); 1836 } 1837 1838 /* 1839 * void hat_unlock(hat, addr, len) 1840 * unlock the mappings to a given range of addresses 1841 * 1842 * Locks are tracked by ht_lock_cnt in the htable. 1843 */ 1844 void 1845 hat_unlock(hat_t *hat, caddr_t addr, size_t len) 1846 { 1847 uintptr_t vaddr = (uintptr_t)addr; 1848 uintptr_t eaddr = vaddr + len; 1849 htable_t *ht = NULL; 1850 1851 /* 1852 * kernel entries are always locked, we don't track lock counts 1853 */ 1854 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); 1855 ASSERT(IS_PAGEALIGNED(vaddr)); 1856 ASSERT(IS_PAGEALIGNED(eaddr)); 1857 if (hat == kas.a_hat) 1858 return; 1859 if (eaddr > _userlimit) 1860 panic("hat_unlock() address out of range - above _userlimit"); 1861 1862 XPV_DISALLOW_MIGRATE(); 1863 ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1864 while (vaddr < eaddr) { 1865 (void) htable_walk(hat, &ht, &vaddr, eaddr); 1866 if (ht == NULL) 1867 break; 1868 1869 ASSERT(!IN_VA_HOLE(vaddr)); 1870 1871 if (ht->ht_lock_cnt < 1) 1872 panic("hat_unlock(): lock_cnt < 1, " 1873 "htable=%p, vaddr=%p\n", (void *)ht, (void *)vaddr); 1874 HTABLE_LOCK_DEC(ht); 1875 1876 vaddr += LEVEL_SIZE(ht->ht_level); 1877 } 1878 if (ht) 1879 htable_release(ht); 1880 XPV_ALLOW_MIGRATE(); 1881 } 1882 1883 /* ARGSUSED */ 1884 void 1885 hat_unlock_region(struct hat *hat, caddr_t addr, size_t len, 1886 hat_region_cookie_t rcookie) 1887 { 1888 panic("No shared region support on x86"); 1889 } 1890 1891 #if !defined(__xpv) 1892 /* 1893 * Cross call service routine to demap a virtual page on 1894 * the current CPU or flush all mappings in TLB. 1895 */ 1896 /*ARGSUSED*/ 1897 static int 1898 hati_demap_func(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3) 1899 { 1900 hat_t *hat = (hat_t *)a1; 1901 caddr_t addr = (caddr_t)a2; 1902 1903 /* 1904 * If the target hat isn't the kernel and this CPU isn't operating 1905 * in the target hat, we can ignore the cross call. 1906 */ 1907 if (hat != kas.a_hat && hat != CPU->cpu_current_hat) 1908 return (0); 1909 1910 /* 1911 * For a normal address, we just flush one page mapping 1912 */ 1913 if ((uintptr_t)addr != DEMAP_ALL_ADDR) { 1914 mmu_tlbflush_entry(addr); 1915 return (0); 1916 } 1917 1918 /* 1919 * Otherwise we reload cr3 to effect a complete TLB flush. 1920 * 1921 * A reload of cr3 on a VLP process also means we must also recopy in 1922 * the pte values from the struct hat 1923 */ 1924 if (hat->hat_flags & HAT_VLP) { 1925 #if defined(__amd64) 1926 x86pte_t *vlpptep = CPU->cpu_hat_info->hci_vlp_l2ptes; 1927 1928 VLP_COPY(hat->hat_vlp_ptes, vlpptep); 1929 #elif defined(__i386) 1930 reload_pae32(hat, CPU); 1931 #endif 1932 } 1933 reload_cr3(); 1934 return (0); 1935 } 1936 1937 /* 1938 * Flush all TLB entries, including global (ie. kernel) ones. 1939 */ 1940 static void 1941 flush_all_tlb_entries(void) 1942 { 1943 ulong_t cr4 = getcr4(); 1944 1945 if (cr4 & CR4_PGE) { 1946 setcr4(cr4 & ~(ulong_t)CR4_PGE); 1947 setcr4(cr4); 1948 1949 /* 1950 * 32 bit PAE also needs to always reload_cr3() 1951 */ 1952 if (mmu.max_level == 2) 1953 reload_cr3(); 1954 } else { 1955 reload_cr3(); 1956 } 1957 } 1958 1959 #define TLB_CPU_HALTED (01ul) 1960 #define TLB_INVAL_ALL (02ul) 1961 #define CAS_TLB_INFO(cpu, old, new) \ 1962 caslong((ulong_t *)&(cpu)->cpu_m.mcpu_tlb_info, (old), (new)) 1963 1964 /* 1965 * Record that a CPU is going idle 1966 */ 1967 void 1968 tlb_going_idle(void) 1969 { 1970 atomic_or_long((ulong_t *)&CPU->cpu_m.mcpu_tlb_info, TLB_CPU_HALTED); 1971 } 1972 1973 /* 1974 * Service a delayed TLB flush if coming out of being idle. 1975 */ 1976 void 1977 tlb_service(void) 1978 { 1979 ulong_t flags = getflags(); 1980 ulong_t tlb_info; 1981 ulong_t found; 1982 1983 /* 1984 * Be sure interrupts are off while doing this so that 1985 * higher level interrupts correctly wait for flushes to finish. 1986 */ 1987 if (flags & PS_IE) 1988 flags = intr_clear(); 1989 1990 /* 1991 * We only have to do something if coming out of being idle. 1992 */ 1993 tlb_info = CPU->cpu_m.mcpu_tlb_info; 1994 if (tlb_info & TLB_CPU_HALTED) { 1995 ASSERT(CPU->cpu_current_hat == kas.a_hat); 1996 1997 /* 1998 * Atomic clear and fetch of old state. 1999 */ 2000 while ((found = CAS_TLB_INFO(CPU, tlb_info, 0)) != tlb_info) { 2001 ASSERT(found & TLB_CPU_HALTED); 2002 tlb_info = found; 2003 SMT_PAUSE(); 2004 } 2005 if (tlb_info & TLB_INVAL_ALL) 2006 flush_all_tlb_entries(); 2007 } 2008 2009 /* 2010 * Restore interrupt enable control bit. 2011 */ 2012 if (flags & PS_IE) 2013 sti(); 2014 } 2015 #endif /* !__xpv */ 2016 2017 /* 2018 * Internal routine to do cross calls to invalidate a range of pages on 2019 * all CPUs using a given hat. 2020 */ 2021 void 2022 hat_tlb_inval(hat_t *hat, uintptr_t va) 2023 { 2024 extern int flushes_require_xcalls; /* from mp_startup.c */ 2025 cpuset_t justme; 2026 cpuset_t cpus_to_shootdown; 2027 #ifndef __xpv 2028 cpuset_t check_cpus; 2029 cpu_t *cpup; 2030 int c; 2031 #endif 2032 2033 /* 2034 * If the hat is being destroyed, there are no more users, so 2035 * demap need not do anything. 2036 */ 2037 if (hat->hat_flags & HAT_FREEING) 2038 return; 2039 2040 /* 2041 * If demapping from a shared pagetable, we best demap the 2042 * entire set of user TLBs, since we don't know what addresses 2043 * these were shared at. 2044 */ 2045 if (hat->hat_flags & HAT_SHARED) { 2046 hat = kas.a_hat; 2047 va = DEMAP_ALL_ADDR; 2048 } 2049 2050 /* 2051 * if not running with multiple CPUs, don't use cross calls 2052 */ 2053 if (panicstr || !flushes_require_xcalls) { 2054 #ifdef __xpv 2055 if (va == DEMAP_ALL_ADDR) 2056 xen_flush_tlb(); 2057 else 2058 xen_flush_va((caddr_t)va); 2059 #else 2060 (void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL); 2061 #endif 2062 return; 2063 } 2064 2065 2066 /* 2067 * Determine CPUs to shootdown. Kernel changes always do all CPUs. 2068 * Otherwise it's just CPUs currently executing in this hat. 2069 */ 2070 kpreempt_disable(); 2071 CPUSET_ONLY(justme, CPU->cpu_id); 2072 if (hat == kas.a_hat) 2073 cpus_to_shootdown = khat_cpuset; 2074 else 2075 cpus_to_shootdown = hat->hat_cpus; 2076 2077 #ifndef __xpv 2078 /* 2079 * If any CPUs in the set are idle, just request a delayed flush 2080 * and avoid waking them up. 2081 */ 2082 check_cpus = cpus_to_shootdown; 2083 for (c = 0; c < NCPU && !CPUSET_ISNULL(check_cpus); ++c) { 2084 ulong_t tlb_info; 2085 2086 if (!CPU_IN_SET(check_cpus, c)) 2087 continue; 2088 CPUSET_DEL(check_cpus, c); 2089 cpup = cpu[c]; 2090 if (cpup == NULL) 2091 continue; 2092 2093 tlb_info = cpup->cpu_m.mcpu_tlb_info; 2094 while (tlb_info == TLB_CPU_HALTED) { 2095 (void) CAS_TLB_INFO(cpup, TLB_CPU_HALTED, 2096 TLB_CPU_HALTED | TLB_INVAL_ALL); 2097 SMT_PAUSE(); 2098 tlb_info = cpup->cpu_m.mcpu_tlb_info; 2099 } 2100 if (tlb_info == (TLB_CPU_HALTED | TLB_INVAL_ALL)) { 2101 HATSTAT_INC(hs_tlb_inval_delayed); 2102 CPUSET_DEL(cpus_to_shootdown, c); 2103 } 2104 } 2105 #endif 2106 2107 if (CPUSET_ISNULL(cpus_to_shootdown) || 2108 CPUSET_ISEQUAL(cpus_to_shootdown, justme)) { 2109 2110 #ifdef __xpv 2111 if (va == DEMAP_ALL_ADDR) 2112 xen_flush_tlb(); 2113 else 2114 xen_flush_va((caddr_t)va); 2115 #else 2116 (void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL); 2117 #endif 2118 2119 } else { 2120 2121 CPUSET_ADD(cpus_to_shootdown, CPU->cpu_id); 2122 #ifdef __xpv 2123 if (va == DEMAP_ALL_ADDR) 2124 xen_gflush_tlb(cpus_to_shootdown); 2125 else 2126 xen_gflush_va((caddr_t)va, cpus_to_shootdown); 2127 #else 2128 xc_call((xc_arg_t)hat, (xc_arg_t)va, NULL, X_CALL_HIPRI, 2129 cpus_to_shootdown, hati_demap_func); 2130 #endif 2131 2132 } 2133 kpreempt_enable(); 2134 } 2135 2136 /* 2137 * Interior routine for HAT_UNLOADs from hat_unload_callback(), 2138 * hat_kmap_unload() OR from hat_steal() code. This routine doesn't 2139 * handle releasing of the htables. 2140 */ 2141 void 2142 hat_pte_unmap( 2143 htable_t *ht, 2144 uint_t entry, 2145 uint_t flags, 2146 x86pte_t old_pte, 2147 void *pte_ptr) 2148 { 2149 hat_t *hat = ht->ht_hat; 2150 hment_t *hm = NULL; 2151 page_t *pp = NULL; 2152 level_t l = ht->ht_level; 2153 pfn_t pfn; 2154 2155 /* 2156 * We always track the locking counts, even if nothing is unmapped 2157 */ 2158 if ((flags & HAT_UNLOAD_UNLOCK) != 0 && hat != kas.a_hat) { 2159 ASSERT(ht->ht_lock_cnt > 0); 2160 HTABLE_LOCK_DEC(ht); 2161 } 2162 2163 /* 2164 * Figure out which page's mapping list lock to acquire using the PFN 2165 * passed in "old" PTE. We then attempt to invalidate the PTE. 2166 * If another thread, probably a hat_pageunload, has asynchronously 2167 * unmapped/remapped this address we'll loop here. 2168 */ 2169 ASSERT(ht->ht_busy > 0); 2170 while (PTE_ISVALID(old_pte)) { 2171 pfn = PTE2PFN(old_pte, l); 2172 if (PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST) { 2173 pp = NULL; 2174 } else { 2175 #ifdef __xpv 2176 if (pfn == PFN_INVALID) 2177 panic("Invalid PFN, but not PT_NOCONSIST"); 2178 #endif 2179 pp = page_numtopp_nolock(pfn); 2180 if (pp == NULL) { 2181 panic("no page_t, not NOCONSIST: old_pte=" 2182 FMT_PTE " ht=%lx entry=0x%x pte_ptr=%lx", 2183 old_pte, (uintptr_t)ht, entry, 2184 (uintptr_t)pte_ptr); 2185 } 2186 x86_hm_enter(pp); 2187 } 2188 2189 /* 2190 * If freeing the address space, check that the PTE 2191 * hasn't changed, as the mappings are no longer in use by 2192 * any thread, invalidation is unnecessary. 2193 * If not freeing, do a full invalidate. 2194 * 2195 * On the hypervisor we must always remove mappings, as a 2196 * writable mapping left behind could cause a page table 2197 * allocation to fail. 2198 */ 2199 #if !defined(__xpv) 2200 if (hat->hat_flags & HAT_FREEING) 2201 old_pte = x86pte_get(ht, entry); 2202 else 2203 #endif 2204 old_pte = x86pte_inval(ht, entry, old_pte, pte_ptr); 2205 2206 /* 2207 * If the page hadn't changed we've unmapped it and can proceed 2208 */ 2209 if (PTE_ISVALID(old_pte) && PTE2PFN(old_pte, l) == pfn) 2210 break; 2211 2212 /* 2213 * Otherwise, we'll have to retry with the current old_pte. 2214 * Drop the hment lock, since the pfn may have changed. 2215 */ 2216 if (pp != NULL) { 2217 x86_hm_exit(pp); 2218 pp = NULL; 2219 } else { 2220 ASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST); 2221 } 2222 } 2223 2224 /* 2225 * If the old mapping wasn't valid, there's nothing more to do 2226 */ 2227 if (!PTE_ISVALID(old_pte)) { 2228 if (pp != NULL) 2229 x86_hm_exit(pp); 2230 return; 2231 } 2232 2233 /* 2234 * Take care of syncing any MOD/REF bits and removing the hment. 2235 */ 2236 if (pp != NULL) { 2237 if (!(flags & HAT_UNLOAD_NOSYNC)) 2238 hati_sync_pte_to_page(pp, old_pte, l); 2239 hm = hment_remove(pp, ht, entry); 2240 x86_hm_exit(pp); 2241 if (hm != NULL) 2242 hment_free(hm); 2243 } 2244 2245 /* 2246 * Handle book keeping in the htable and hat 2247 */ 2248 ASSERT(ht->ht_valid_cnt > 0); 2249 HTABLE_DEC(ht->ht_valid_cnt); 2250 PGCNT_DEC(hat, l); 2251 } 2252 2253 /* 2254 * very cheap unload implementation to special case some kernel addresses 2255 */ 2256 static void 2257 hat_kmap_unload(caddr_t addr, size_t len, uint_t flags) 2258 { 2259 uintptr_t va = (uintptr_t)addr; 2260 uintptr_t eva = va + len; 2261 pgcnt_t pg_index; 2262 htable_t *ht; 2263 uint_t entry; 2264 x86pte_t *pte_ptr; 2265 x86pte_t old_pte; 2266 2267 for (; va < eva; va += MMU_PAGESIZE) { 2268 /* 2269 * Get the PTE 2270 */ 2271 pg_index = mmu_btop(va - mmu.kmap_addr); 2272 pte_ptr = PT_INDEX_PTR(mmu.kmap_ptes, pg_index); 2273 old_pte = GET_PTE(pte_ptr); 2274 2275 /* 2276 * get the htable / entry 2277 */ 2278 ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) 2279 >> LEVEL_SHIFT(1)]; 2280 entry = htable_va2entry(va, ht); 2281 2282 /* 2283 * use mostly common code to unmap it. 2284 */ 2285 hat_pte_unmap(ht, entry, flags, old_pte, pte_ptr); 2286 } 2287 } 2288 2289 2290 /* 2291 * unload a range of virtual address space (no callback) 2292 */ 2293 void 2294 hat_unload(hat_t *hat, caddr_t addr, size_t len, uint_t flags) 2295 { 2296 uintptr_t va = (uintptr_t)addr; 2297 2298 XPV_DISALLOW_MIGRATE(); 2299 ASSERT(hat == kas.a_hat || va + len <= _userlimit); 2300 2301 /* 2302 * special case for performance. 2303 */ 2304 if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) { 2305 ASSERT(hat == kas.a_hat); 2306 hat_kmap_unload(addr, len, flags); 2307 } else { 2308 hat_unload_callback(hat, addr, len, flags, NULL); 2309 } 2310 XPV_ALLOW_MIGRATE(); 2311 } 2312 2313 /* 2314 * Do the callbacks for ranges being unloaded. 2315 */ 2316 typedef struct range_info { 2317 uintptr_t rng_va; 2318 ulong_t rng_cnt; 2319 level_t rng_level; 2320 } range_info_t; 2321 2322 static void 2323 handle_ranges(hat_callback_t *cb, uint_t cnt, range_info_t *range) 2324 { 2325 /* 2326 * do callbacks to upper level VM system 2327 */ 2328 while (cb != NULL && cnt > 0) { 2329 --cnt; 2330 cb->hcb_start_addr = (caddr_t)range[cnt].rng_va; 2331 cb->hcb_end_addr = cb->hcb_start_addr; 2332 cb->hcb_end_addr += 2333 range[cnt].rng_cnt << LEVEL_SIZE(range[cnt].rng_level); 2334 cb->hcb_function(cb); 2335 } 2336 } 2337 2338 /* 2339 * Unload a given range of addresses (has optional callback) 2340 * 2341 * Flags: 2342 * define HAT_UNLOAD 0x00 2343 * define HAT_UNLOAD_NOSYNC 0x02 2344 * define HAT_UNLOAD_UNLOCK 0x04 2345 * define HAT_UNLOAD_OTHER 0x08 - not used 2346 * define HAT_UNLOAD_UNMAP 0x10 - same as HAT_UNLOAD 2347 */ 2348 #define MAX_UNLOAD_CNT (8) 2349 void 2350 hat_unload_callback( 2351 hat_t *hat, 2352 caddr_t addr, 2353 size_t len, 2354 uint_t flags, 2355 hat_callback_t *cb) 2356 { 2357 uintptr_t vaddr = (uintptr_t)addr; 2358 uintptr_t eaddr = vaddr + len; 2359 htable_t *ht = NULL; 2360 uint_t entry; 2361 uintptr_t contig_va = (uintptr_t)-1L; 2362 range_info_t r[MAX_UNLOAD_CNT]; 2363 uint_t r_cnt = 0; 2364 x86pte_t old_pte; 2365 2366 XPV_DISALLOW_MIGRATE(); 2367 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); 2368 ASSERT(IS_PAGEALIGNED(vaddr)); 2369 ASSERT(IS_PAGEALIGNED(eaddr)); 2370 2371 /* 2372 * Special case a single page being unloaded for speed. This happens 2373 * quite frequently, COW faults after a fork() for example. 2374 */ 2375 if (cb == NULL && len == MMU_PAGESIZE) { 2376 ht = htable_getpte(hat, vaddr, &entry, &old_pte, 0); 2377 if (ht != NULL) { 2378 if (PTE_ISVALID(old_pte)) 2379 hat_pte_unmap(ht, entry, flags, old_pte, NULL); 2380 htable_release(ht); 2381 } 2382 XPV_ALLOW_MIGRATE(); 2383 return; 2384 } 2385 2386 while (vaddr < eaddr) { 2387 old_pte = htable_walk(hat, &ht, &vaddr, eaddr); 2388 if (ht == NULL) 2389 break; 2390 2391 ASSERT(!IN_VA_HOLE(vaddr)); 2392 2393 if (vaddr < (uintptr_t)addr) 2394 panic("hat_unload_callback(): unmap inside large page"); 2395 2396 /* 2397 * We'll do the call backs for contiguous ranges 2398 */ 2399 if (vaddr != contig_va || 2400 (r_cnt > 0 && r[r_cnt - 1].rng_level != ht->ht_level)) { 2401 if (r_cnt == MAX_UNLOAD_CNT) { 2402 handle_ranges(cb, r_cnt, r); 2403 r_cnt = 0; 2404 } 2405 r[r_cnt].rng_va = vaddr; 2406 r[r_cnt].rng_cnt = 0; 2407 r[r_cnt].rng_level = ht->ht_level; 2408 ++r_cnt; 2409 } 2410 2411 /* 2412 * Unload one mapping from the page tables. 2413 */ 2414 entry = htable_va2entry(vaddr, ht); 2415 hat_pte_unmap(ht, entry, flags, old_pte, NULL); 2416 ASSERT(ht->ht_level <= mmu.max_page_level); 2417 vaddr += LEVEL_SIZE(ht->ht_level); 2418 contig_va = vaddr; 2419 ++r[r_cnt - 1].rng_cnt; 2420 } 2421 if (ht) 2422 htable_release(ht); 2423 2424 /* 2425 * handle last range for callbacks 2426 */ 2427 if (r_cnt > 0) 2428 handle_ranges(cb, r_cnt, r); 2429 XPV_ALLOW_MIGRATE(); 2430 } 2431 2432 /* 2433 * synchronize mapping with software data structures 2434 * 2435 * This interface is currently only used by the working set monitor 2436 * driver. 2437 */ 2438 /*ARGSUSED*/ 2439 void 2440 hat_sync(hat_t *hat, caddr_t addr, size_t len, uint_t flags) 2441 { 2442 uintptr_t vaddr = (uintptr_t)addr; 2443 uintptr_t eaddr = vaddr + len; 2444 htable_t *ht = NULL; 2445 uint_t entry; 2446 x86pte_t pte; 2447 x86pte_t save_pte; 2448 x86pte_t new; 2449 page_t *pp; 2450 2451 ASSERT(!IN_VA_HOLE(vaddr)); 2452 ASSERT(IS_PAGEALIGNED(vaddr)); 2453 ASSERT(IS_PAGEALIGNED(eaddr)); 2454 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); 2455 2456 XPV_DISALLOW_MIGRATE(); 2457 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) { 2458 try_again: 2459 pte = htable_walk(hat, &ht, &vaddr, eaddr); 2460 if (ht == NULL) 2461 break; 2462 entry = htable_va2entry(vaddr, ht); 2463 2464 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC || 2465 PTE_GET(pte, PT_REF | PT_MOD) == 0) 2466 continue; 2467 2468 /* 2469 * We need to acquire the mapping list lock to protect 2470 * against hat_pageunload(), hat_unload(), etc. 2471 */ 2472 pp = page_numtopp_nolock(PTE2PFN(pte, ht->ht_level)); 2473 if (pp == NULL) 2474 break; 2475 x86_hm_enter(pp); 2476 save_pte = pte; 2477 pte = x86pte_get(ht, entry); 2478 if (pte != save_pte) { 2479 x86_hm_exit(pp); 2480 goto try_again; 2481 } 2482 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC || 2483 PTE_GET(pte, PT_REF | PT_MOD) == 0) { 2484 x86_hm_exit(pp); 2485 continue; 2486 } 2487 2488 /* 2489 * Need to clear ref or mod bits. We may compete with 2490 * hardware updating the R/M bits and have to try again. 2491 */ 2492 if (flags == HAT_SYNC_ZERORM) { 2493 new = pte; 2494 PTE_CLR(new, PT_REF | PT_MOD); 2495 pte = hati_update_pte(ht, entry, pte, new); 2496 if (pte != 0) { 2497 x86_hm_exit(pp); 2498 goto try_again; 2499 } 2500 } else { 2501 /* 2502 * sync the PTE to the page_t 2503 */ 2504 hati_sync_pte_to_page(pp, save_pte, ht->ht_level); 2505 } 2506 x86_hm_exit(pp); 2507 } 2508 if (ht) 2509 htable_release(ht); 2510 XPV_ALLOW_MIGRATE(); 2511 } 2512 2513 /* 2514 * void hat_map(hat, addr, len, flags) 2515 */ 2516 /*ARGSUSED*/ 2517 void 2518 hat_map(hat_t *hat, caddr_t addr, size_t len, uint_t flags) 2519 { 2520 /* does nothing */ 2521 } 2522 2523 /* 2524 * uint_t hat_getattr(hat, addr, *attr) 2525 * returns attr for <hat,addr> in *attr. returns 0 if there was a 2526 * mapping and *attr is valid, nonzero if there was no mapping and 2527 * *attr is not valid. 2528 */ 2529 uint_t 2530 hat_getattr(hat_t *hat, caddr_t addr, uint_t *attr) 2531 { 2532 uintptr_t vaddr = ALIGN2PAGE(addr); 2533 htable_t *ht = NULL; 2534 x86pte_t pte; 2535 2536 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2537 2538 if (IN_VA_HOLE(vaddr)) 2539 return ((uint_t)-1); 2540 2541 ht = htable_getpte(hat, vaddr, NULL, &pte, mmu.max_page_level); 2542 if (ht == NULL) 2543 return ((uint_t)-1); 2544 2545 if (!PTE_ISVALID(pte) || !PTE_ISPAGE(pte, ht->ht_level)) { 2546 htable_release(ht); 2547 return ((uint_t)-1); 2548 } 2549 2550 *attr = PROT_READ; 2551 if (PTE_GET(pte, PT_WRITABLE)) 2552 *attr |= PROT_WRITE; 2553 if (PTE_GET(pte, PT_USER)) 2554 *attr |= PROT_USER; 2555 if (!PTE_GET(pte, mmu.pt_nx)) 2556 *attr |= PROT_EXEC; 2557 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC) 2558 *attr |= HAT_NOSYNC; 2559 htable_release(ht); 2560 return (0); 2561 } 2562 2563 /* 2564 * hat_updateattr() applies the given attribute change to an existing mapping 2565 */ 2566 #define HAT_LOAD_ATTR 1 2567 #define HAT_SET_ATTR 2 2568 #define HAT_CLR_ATTR 3 2569 2570 static void 2571 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what) 2572 { 2573 uintptr_t vaddr = (uintptr_t)addr; 2574 uintptr_t eaddr = (uintptr_t)addr + len; 2575 htable_t *ht = NULL; 2576 uint_t entry; 2577 x86pte_t oldpte, newpte; 2578 page_t *pp; 2579 2580 XPV_DISALLOW_MIGRATE(); 2581 ASSERT(IS_PAGEALIGNED(vaddr)); 2582 ASSERT(IS_PAGEALIGNED(eaddr)); 2583 ASSERT(hat == kas.a_hat || 2584 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 2585 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) { 2586 try_again: 2587 oldpte = htable_walk(hat, &ht, &vaddr, eaddr); 2588 if (ht == NULL) 2589 break; 2590 if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST) 2591 continue; 2592 2593 pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level)); 2594 if (pp == NULL) 2595 continue; 2596 x86_hm_enter(pp); 2597 2598 newpte = oldpte; 2599 /* 2600 * We found a page table entry in the desired range, 2601 * figure out the new attributes. 2602 */ 2603 if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) { 2604 if ((attr & PROT_WRITE) && 2605 !PTE_GET(oldpte, PT_WRITABLE)) 2606 newpte |= PT_WRITABLE; 2607 2608 if ((attr & HAT_NOSYNC) && 2609 PTE_GET(oldpte, PT_SOFTWARE) < PT_NOSYNC) 2610 newpte |= PT_NOSYNC; 2611 2612 if ((attr & PROT_EXEC) && PTE_GET(oldpte, mmu.pt_nx)) 2613 newpte &= ~mmu.pt_nx; 2614 } 2615 2616 if (what == HAT_LOAD_ATTR) { 2617 if (!(attr & PROT_WRITE) && 2618 PTE_GET(oldpte, PT_WRITABLE)) 2619 newpte &= ~PT_WRITABLE; 2620 2621 if (!(attr & HAT_NOSYNC) && 2622 PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC) 2623 newpte &= ~PT_SOFTWARE; 2624 2625 if (!(attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx)) 2626 newpte |= mmu.pt_nx; 2627 } 2628 2629 if (what == HAT_CLR_ATTR) { 2630 if ((attr & PROT_WRITE) && PTE_GET(oldpte, PT_WRITABLE)) 2631 newpte &= ~PT_WRITABLE; 2632 2633 if ((attr & HAT_NOSYNC) && 2634 PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC) 2635 newpte &= ~PT_SOFTWARE; 2636 2637 if ((attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx)) 2638 newpte |= mmu.pt_nx; 2639 } 2640 2641 /* 2642 * Ensure NOSYNC/NOCONSIST mappings have REF and MOD set. 2643 * x86pte_set() depends on this. 2644 */ 2645 if (PTE_GET(newpte, PT_SOFTWARE) >= PT_NOSYNC) 2646 newpte |= PT_REF | PT_MOD; 2647 2648 /* 2649 * what about PROT_READ or others? this code only handles: 2650 * EXEC, WRITE, NOSYNC 2651 */ 2652 2653 /* 2654 * If new PTE really changed, update the table. 2655 */ 2656 if (newpte != oldpte) { 2657 entry = htable_va2entry(vaddr, ht); 2658 oldpte = hati_update_pte(ht, entry, oldpte, newpte); 2659 if (oldpte != 0) { 2660 x86_hm_exit(pp); 2661 goto try_again; 2662 } 2663 } 2664 x86_hm_exit(pp); 2665 } 2666 if (ht) 2667 htable_release(ht); 2668 XPV_ALLOW_MIGRATE(); 2669 } 2670 2671 /* 2672 * Various wrappers for hat_updateattr() 2673 */ 2674 void 2675 hat_setattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) 2676 { 2677 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2678 hat_updateattr(hat, addr, len, attr, HAT_SET_ATTR); 2679 } 2680 2681 void 2682 hat_clrattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) 2683 { 2684 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2685 hat_updateattr(hat, addr, len, attr, HAT_CLR_ATTR); 2686 } 2687 2688 void 2689 hat_chgattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) 2690 { 2691 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2692 hat_updateattr(hat, addr, len, attr, HAT_LOAD_ATTR); 2693 } 2694 2695 void 2696 hat_chgprot(hat_t *hat, caddr_t addr, size_t len, uint_t vprot) 2697 { 2698 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2699 hat_updateattr(hat, addr, len, vprot & HAT_PROT_MASK, HAT_LOAD_ATTR); 2700 } 2701 2702 /* 2703 * size_t hat_getpagesize(hat, addr) 2704 * returns pagesize in bytes for <hat, addr>. returns -1 of there is 2705 * no mapping. This is an advisory call. 2706 */ 2707 ssize_t 2708 hat_getpagesize(hat_t *hat, caddr_t addr) 2709 { 2710 uintptr_t vaddr = ALIGN2PAGE(addr); 2711 htable_t *ht; 2712 size_t pagesize; 2713 2714 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2715 if (IN_VA_HOLE(vaddr)) 2716 return (-1); 2717 ht = htable_getpage(hat, vaddr, NULL); 2718 if (ht == NULL) 2719 return (-1); 2720 pagesize = LEVEL_SIZE(ht->ht_level); 2721 htable_release(ht); 2722 return (pagesize); 2723 } 2724 2725 2726 2727 /* 2728 * pfn_t hat_getpfnum(hat, addr) 2729 * returns pfn for <hat, addr> or PFN_INVALID if mapping is invalid. 2730 */ 2731 pfn_t 2732 hat_getpfnum(hat_t *hat, caddr_t addr) 2733 { 2734 uintptr_t vaddr = ALIGN2PAGE(addr); 2735 htable_t *ht; 2736 uint_t entry; 2737 pfn_t pfn = PFN_INVALID; 2738 2739 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2740 if (khat_running == 0) 2741 return (PFN_INVALID); 2742 2743 if (IN_VA_HOLE(vaddr)) 2744 return (PFN_INVALID); 2745 2746 XPV_DISALLOW_MIGRATE(); 2747 /* 2748 * A very common use of hat_getpfnum() is from the DDI for kernel pages. 2749 * Use the kmap_ptes (which also covers the 32 bit heap) to speed 2750 * this up. 2751 */ 2752 if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) { 2753 x86pte_t pte; 2754 pgcnt_t pg_index; 2755 2756 pg_index = mmu_btop(vaddr - mmu.kmap_addr); 2757 pte = GET_PTE(PT_INDEX_PTR(mmu.kmap_ptes, pg_index)); 2758 if (PTE_ISVALID(pte)) 2759 /*LINTED [use of constant 0 causes a lint warning] */ 2760 pfn = PTE2PFN(pte, 0); 2761 XPV_ALLOW_MIGRATE(); 2762 return (pfn); 2763 } 2764 2765 ht = htable_getpage(hat, vaddr, &entry); 2766 if (ht == NULL) { 2767 XPV_ALLOW_MIGRATE(); 2768 return (PFN_INVALID); 2769 } 2770 ASSERT(vaddr >= ht->ht_vaddr); 2771 ASSERT(vaddr <= HTABLE_LAST_PAGE(ht)); 2772 pfn = PTE2PFN(x86pte_get(ht, entry), ht->ht_level); 2773 if (ht->ht_level > 0) 2774 pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level)); 2775 htable_release(ht); 2776 XPV_ALLOW_MIGRATE(); 2777 return (pfn); 2778 } 2779 2780 /* 2781 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 2782 * Use hat_getpfnum(kas.a_hat, ...) instead. 2783 * 2784 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 2785 * but can't right now due to the fact that some software has grown to use 2786 * this interface incorrectly. So for now when the interface is misused, 2787 * return a warning to the user that in the future it won't work in the 2788 * way they're abusing it, and carry on. 2789 * 2790 * Note that hat_getkpfnum() is never supported on amd64. 2791 */ 2792 #if !defined(__amd64) 2793 pfn_t 2794 hat_getkpfnum(caddr_t addr) 2795 { 2796 pfn_t pfn; 2797 int badcaller = 0; 2798 2799 if (khat_running == 0) 2800 panic("hat_getkpfnum(): called too early\n"); 2801 if ((uintptr_t)addr < kernelbase) 2802 return (PFN_INVALID); 2803 2804 XPV_DISALLOW_MIGRATE(); 2805 if (segkpm && IS_KPM_ADDR(addr)) { 2806 badcaller = 1; 2807 pfn = hat_kpm_va2pfn(addr); 2808 } else { 2809 pfn = hat_getpfnum(kas.a_hat, addr); 2810 badcaller = pf_is_memory(pfn); 2811 } 2812 2813 if (badcaller) 2814 hat_getkpfnum_badcall(caller()); 2815 XPV_ALLOW_MIGRATE(); 2816 return (pfn); 2817 } 2818 #endif /* __amd64 */ 2819 2820 /* 2821 * int hat_probe(hat, addr) 2822 * return 0 if no valid mapping is present. Faster version 2823 * of hat_getattr in certain architectures. 2824 */ 2825 int 2826 hat_probe(hat_t *hat, caddr_t addr) 2827 { 2828 uintptr_t vaddr = ALIGN2PAGE(addr); 2829 uint_t entry; 2830 htable_t *ht; 2831 pgcnt_t pg_off; 2832 2833 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2834 ASSERT(hat == kas.a_hat || 2835 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 2836 if (IN_VA_HOLE(vaddr)) 2837 return (0); 2838 2839 /* 2840 * Most common use of hat_probe is from segmap. We special case it 2841 * for performance. 2842 */ 2843 if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) { 2844 pg_off = mmu_btop(vaddr - mmu.kmap_addr); 2845 if (mmu.pae_hat) 2846 return (PTE_ISVALID(mmu.kmap_ptes[pg_off])); 2847 else 2848 return (PTE_ISVALID( 2849 ((x86pte32_t *)mmu.kmap_ptes)[pg_off])); 2850 } 2851 2852 ht = htable_getpage(hat, vaddr, &entry); 2853 htable_release(ht); 2854 return (ht != NULL); 2855 } 2856 2857 /* 2858 * Find out if the segment for hat_share()/hat_unshare() is DISM or locked ISM. 2859 */ 2860 static int 2861 is_it_dism(hat_t *hat, caddr_t va) 2862 { 2863 struct seg *seg; 2864 struct shm_data *shmd; 2865 struct spt_data *sptd; 2866 2867 seg = as_findseg(hat->hat_as, va, 0); 2868 ASSERT(seg != NULL); 2869 ASSERT(seg->s_base <= va); 2870 shmd = (struct shm_data *)seg->s_data; 2871 ASSERT(shmd != NULL); 2872 sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2873 ASSERT(sptd != NULL); 2874 if (sptd->spt_flags & SHM_PAGEABLE) 2875 return (1); 2876 return (0); 2877 } 2878 2879 /* 2880 * Simple implementation of ISM. hat_share() is similar to hat_memload_array(), 2881 * except that we use the ism_hat's existing mappings to determine the pages 2882 * and protections to use for this hat. If we find a full properly aligned 2883 * and sized pagetable, we will attempt to share the pagetable itself. 2884 */ 2885 /*ARGSUSED*/ 2886 int 2887 hat_share( 2888 hat_t *hat, 2889 caddr_t addr, 2890 hat_t *ism_hat, 2891 caddr_t src_addr, 2892 size_t len, /* almost useless value, see below.. */ 2893 uint_t ismszc) 2894 { 2895 uintptr_t vaddr_start = (uintptr_t)addr; 2896 uintptr_t vaddr; 2897 uintptr_t eaddr = vaddr_start + len; 2898 uintptr_t ism_addr_start = (uintptr_t)src_addr; 2899 uintptr_t ism_addr = ism_addr_start; 2900 uintptr_t e_ism_addr = ism_addr + len; 2901 htable_t *ism_ht = NULL; 2902 htable_t *ht; 2903 x86pte_t pte; 2904 page_t *pp; 2905 pfn_t pfn; 2906 level_t l; 2907 pgcnt_t pgcnt; 2908 uint_t prot; 2909 int is_dism; 2910 int flags; 2911 2912 /* 2913 * We might be asked to share an empty DISM hat by as_dup() 2914 */ 2915 ASSERT(hat != kas.a_hat); 2916 ASSERT(eaddr <= _userlimit); 2917 if (!(ism_hat->hat_flags & HAT_SHARED)) { 2918 ASSERT(hat_get_mapped_size(ism_hat) == 0); 2919 return (0); 2920 } 2921 XPV_DISALLOW_MIGRATE(); 2922 2923 /* 2924 * The SPT segment driver often passes us a size larger than there are 2925 * valid mappings. That's because it rounds the segment size up to a 2926 * large pagesize, even if the actual memory mapped by ism_hat is less. 2927 */ 2928 ASSERT(IS_PAGEALIGNED(vaddr_start)); 2929 ASSERT(IS_PAGEALIGNED(ism_addr_start)); 2930 ASSERT(ism_hat->hat_flags & HAT_SHARED); 2931 is_dism = is_it_dism(hat, addr); 2932 while (ism_addr < e_ism_addr) { 2933 /* 2934 * use htable_walk to get the next valid ISM mapping 2935 */ 2936 pte = htable_walk(ism_hat, &ism_ht, &ism_addr, e_ism_addr); 2937 if (ism_ht == NULL) 2938 break; 2939 2940 /* 2941 * First check to see if we already share the page table. 2942 */ 2943 l = ism_ht->ht_level; 2944 vaddr = vaddr_start + (ism_addr - ism_addr_start); 2945 ht = htable_lookup(hat, vaddr, l); 2946 if (ht != NULL) { 2947 if (ht->ht_flags & HTABLE_SHARED_PFN) 2948 goto shared; 2949 htable_release(ht); 2950 goto not_shared; 2951 } 2952 2953 /* 2954 * Can't ever share top table. 2955 */ 2956 if (l == mmu.max_level) 2957 goto not_shared; 2958 2959 /* 2960 * Avoid level mismatches later due to DISM faults. 2961 */ 2962 if (is_dism && l > 0) 2963 goto not_shared; 2964 2965 /* 2966 * addresses and lengths must align 2967 * table must be fully populated 2968 * no lower level page tables 2969 */ 2970 if (ism_addr != ism_ht->ht_vaddr || 2971 (vaddr & LEVEL_OFFSET(l + 1)) != 0) 2972 goto not_shared; 2973 2974 /* 2975 * The range of address space must cover a full table. 2976 */ 2977 if (e_ism_addr - ism_addr < LEVEL_SIZE(l + 1)) 2978 goto not_shared; 2979 2980 /* 2981 * All entries in the ISM page table must be leaf PTEs. 2982 */ 2983 if (l > 0) { 2984 int e; 2985 2986 /* 2987 * We know the 0th is from htable_walk() above. 2988 */ 2989 for (e = 1; e < HTABLE_NUM_PTES(ism_ht); ++e) { 2990 x86pte_t pte; 2991 pte = x86pte_get(ism_ht, e); 2992 if (!PTE_ISPAGE(pte, l)) 2993 goto not_shared; 2994 } 2995 } 2996 2997 /* 2998 * share the page table 2999 */ 3000 ht = htable_create(hat, vaddr, l, ism_ht); 3001 shared: 3002 ASSERT(ht->ht_flags & HTABLE_SHARED_PFN); 3003 ASSERT(ht->ht_shares == ism_ht); 3004 hat->hat_ism_pgcnt += 3005 (ism_ht->ht_valid_cnt - ht->ht_valid_cnt) << 3006 (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT); 3007 ht->ht_valid_cnt = ism_ht->ht_valid_cnt; 3008 htable_release(ht); 3009 ism_addr = ism_ht->ht_vaddr + LEVEL_SIZE(l + 1); 3010 htable_release(ism_ht); 3011 ism_ht = NULL; 3012 continue; 3013 3014 not_shared: 3015 /* 3016 * Unable to share the page table. Instead we will 3017 * create new mappings from the values in the ISM mappings. 3018 * Figure out what level size mappings to use; 3019 */ 3020 for (l = ism_ht->ht_level; l > 0; --l) { 3021 if (LEVEL_SIZE(l) <= eaddr - vaddr && 3022 (vaddr & LEVEL_OFFSET(l)) == 0) 3023 break; 3024 } 3025 3026 /* 3027 * The ISM mapping might be larger than the share area, 3028 * be careful to truncate it if needed. 3029 */ 3030 if (eaddr - vaddr >= LEVEL_SIZE(ism_ht->ht_level)) { 3031 pgcnt = mmu_btop(LEVEL_SIZE(ism_ht->ht_level)); 3032 } else { 3033 pgcnt = mmu_btop(eaddr - vaddr); 3034 l = 0; 3035 } 3036 3037 pfn = PTE2PFN(pte, ism_ht->ht_level); 3038 ASSERT(pfn != PFN_INVALID); 3039 while (pgcnt > 0) { 3040 /* 3041 * Make a new pte for the PFN for this level. 3042 * Copy protections for the pte from the ISM pte. 3043 */ 3044 pp = page_numtopp_nolock(pfn); 3045 ASSERT(pp != NULL); 3046 3047 prot = PROT_USER | PROT_READ | HAT_UNORDERED_OK; 3048 if (PTE_GET(pte, PT_WRITABLE)) 3049 prot |= PROT_WRITE; 3050 if (!PTE_GET(pte, PT_NX)) 3051 prot |= PROT_EXEC; 3052 3053 flags = HAT_LOAD; 3054 if (!is_dism) 3055 flags |= HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST; 3056 while (hati_load_common(hat, vaddr, pp, prot, flags, 3057 l, pfn) != 0) { 3058 if (l == 0) 3059 panic("hati_load_common() failure"); 3060 --l; 3061 } 3062 3063 vaddr += LEVEL_SIZE(l); 3064 ism_addr += LEVEL_SIZE(l); 3065 pfn += mmu_btop(LEVEL_SIZE(l)); 3066 pgcnt -= mmu_btop(LEVEL_SIZE(l)); 3067 } 3068 } 3069 if (ism_ht != NULL) 3070 htable_release(ism_ht); 3071 XPV_ALLOW_MIGRATE(); 3072 return (0); 3073 } 3074 3075 3076 /* 3077 * hat_unshare() is similar to hat_unload_callback(), but 3078 * we have to look for empty shared pagetables. Note that 3079 * hat_unshare() is always invoked against an entire segment. 3080 */ 3081 /*ARGSUSED*/ 3082 void 3083 hat_unshare(hat_t *hat, caddr_t addr, size_t len, uint_t ismszc) 3084 { 3085 uint64_t vaddr = (uintptr_t)addr; 3086 uintptr_t eaddr = vaddr + len; 3087 htable_t *ht = NULL; 3088 uint_t need_demaps = 0; 3089 int flags = HAT_UNLOAD_UNMAP; 3090 level_t l; 3091 3092 ASSERT(hat != kas.a_hat); 3093 ASSERT(eaddr <= _userlimit); 3094 ASSERT(IS_PAGEALIGNED(vaddr)); 3095 ASSERT(IS_PAGEALIGNED(eaddr)); 3096 XPV_DISALLOW_MIGRATE(); 3097 3098 /* 3099 * First go through and remove any shared pagetables. 3100 * 3101 * Note that it's ok to delay the TLB shootdown till the entire range is 3102 * finished, because if hat_pageunload() were to unload a shared 3103 * pagetable page, its hat_tlb_inval() will do a global TLB invalidate. 3104 */ 3105 l = mmu.max_page_level; 3106 if (l == mmu.max_level) 3107 --l; 3108 for (; l >= 0; --l) { 3109 for (vaddr = (uintptr_t)addr; vaddr < eaddr; 3110 vaddr = (vaddr & LEVEL_MASK(l + 1)) + LEVEL_SIZE(l + 1)) { 3111 ASSERT(!IN_VA_HOLE(vaddr)); 3112 /* 3113 * find a pagetable that maps the current address 3114 */ 3115 ht = htable_lookup(hat, vaddr, l); 3116 if (ht == NULL) 3117 continue; 3118 if (ht->ht_flags & HTABLE_SHARED_PFN) { 3119 /* 3120 * clear page count, set valid_cnt to 0, 3121 * let htable_release() finish the job 3122 */ 3123 hat->hat_ism_pgcnt -= ht->ht_valid_cnt << 3124 (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT); 3125 ht->ht_valid_cnt = 0; 3126 need_demaps = 1; 3127 } 3128 htable_release(ht); 3129 } 3130 } 3131 3132 /* 3133 * flush the TLBs - since we're probably dealing with MANY mappings 3134 * we do just one CR3 reload. 3135 */ 3136 if (!(hat->hat_flags & HAT_FREEING) && need_demaps) 3137 hat_tlb_inval(hat, DEMAP_ALL_ADDR); 3138 3139 /* 3140 * Now go back and clean up any unaligned mappings that 3141 * couldn't share pagetables. 3142 */ 3143 if (!is_it_dism(hat, addr)) 3144 flags |= HAT_UNLOAD_UNLOCK; 3145 hat_unload(hat, addr, len, flags); 3146 XPV_ALLOW_MIGRATE(); 3147 } 3148 3149 3150 /* 3151 * hat_reserve() does nothing 3152 */ 3153 /*ARGSUSED*/ 3154 void 3155 hat_reserve(struct as *as, caddr_t addr, size_t len) 3156 { 3157 } 3158 3159 3160 /* 3161 * Called when all mappings to a page should have write permission removed. 3162 * Mostly stolem from hat_pagesync() 3163 */ 3164 static void 3165 hati_page_clrwrt(struct page *pp) 3166 { 3167 hment_t *hm = NULL; 3168 htable_t *ht; 3169 uint_t entry; 3170 x86pte_t old; 3171 x86pte_t new; 3172 uint_t pszc = 0; 3173 3174 XPV_DISALLOW_MIGRATE(); 3175 next_size: 3176 /* 3177 * walk thru the mapping list clearing write permission 3178 */ 3179 x86_hm_enter(pp); 3180 while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) { 3181 if (ht->ht_level < pszc) 3182 continue; 3183 old = x86pte_get(ht, entry); 3184 3185 for (;;) { 3186 /* 3187 * Is this mapping of interest? 3188 */ 3189 if (PTE2PFN(old, ht->ht_level) != pp->p_pagenum || 3190 PTE_GET(old, PT_WRITABLE) == 0) 3191 break; 3192 3193 /* 3194 * Clear ref/mod writable bits. This requires cross 3195 * calls to ensure any executing TLBs see cleared bits. 3196 */ 3197 new = old; 3198 PTE_CLR(new, PT_REF | PT_MOD | PT_WRITABLE); 3199 old = hati_update_pte(ht, entry, old, new); 3200 if (old != 0) 3201 continue; 3202 3203 break; 3204 } 3205 } 3206 x86_hm_exit(pp); 3207 while (pszc < pp->p_szc) { 3208 page_t *tpp; 3209 pszc++; 3210 tpp = PP_GROUPLEADER(pp, pszc); 3211 if (pp != tpp) { 3212 pp = tpp; 3213 goto next_size; 3214 } 3215 } 3216 XPV_ALLOW_MIGRATE(); 3217 } 3218 3219 /* 3220 * void hat_page_setattr(pp, flag) 3221 * void hat_page_clrattr(pp, flag) 3222 * used to set/clr ref/mod bits. 3223 */ 3224 void 3225 hat_page_setattr(struct page *pp, uint_t flag) 3226 { 3227 vnode_t *vp = pp->p_vnode; 3228 kmutex_t *vphm = NULL; 3229 page_t **listp; 3230 int noshuffle; 3231 3232 noshuffle = flag & P_NSH; 3233 flag &= ~P_NSH; 3234 3235 if (PP_GETRM(pp, flag) == flag) 3236 return; 3237 3238 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp) && 3239 !noshuffle) { 3240 vphm = page_vnode_mutex(vp); 3241 mutex_enter(vphm); 3242 } 3243 3244 PP_SETRM(pp, flag); 3245 3246 if (vphm != NULL) { 3247 3248 /* 3249 * Some File Systems examine v_pages for NULL w/o 3250 * grabbing the vphm mutex. Must not let it become NULL when 3251 * pp is the only page on the list. 3252 */ 3253 if (pp->p_vpnext != pp) { 3254 page_vpsub(&vp->v_pages, pp); 3255 if (vp->v_pages != NULL) 3256 listp = &vp->v_pages->p_vpprev->p_vpnext; 3257 else 3258 listp = &vp->v_pages; 3259 page_vpadd(listp, pp); 3260 } 3261 mutex_exit(vphm); 3262 } 3263 } 3264 3265 void 3266 hat_page_clrattr(struct page *pp, uint_t flag) 3267 { 3268 vnode_t *vp = pp->p_vnode; 3269 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 3270 3271 /* 3272 * Caller is expected to hold page's io lock for VMODSORT to work 3273 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod 3274 * bit is cleared. 3275 * We don't have assert to avoid tripping some existing third party 3276 * code. The dirty page is moved back to top of the v_page list 3277 * after IO is done in pvn_write_done(). 3278 */ 3279 PP_CLRRM(pp, flag); 3280 3281 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 3282 3283 /* 3284 * VMODSORT works by removing write permissions and getting 3285 * a fault when a page is made dirty. At this point 3286 * we need to remove write permission from all mappings 3287 * to this page. 3288 */ 3289 hati_page_clrwrt(pp); 3290 } 3291 } 3292 3293 /* 3294 * If flag is specified, returns 0 if attribute is disabled 3295 * and non zero if enabled. If flag specifes multiple attributs 3296 * then returns 0 if ALL atriibutes are disabled. This is an advisory 3297 * call. 3298 */ 3299 uint_t 3300 hat_page_getattr(struct page *pp, uint_t flag) 3301 { 3302 return (PP_GETRM(pp, flag)); 3303 } 3304 3305 3306 /* 3307 * common code used by hat_pageunload() and hment_steal() 3308 */ 3309 hment_t * 3310 hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry) 3311 { 3312 x86pte_t old_pte; 3313 pfn_t pfn = pp->p_pagenum; 3314 hment_t *hm; 3315 3316 /* 3317 * We need to acquire a hold on the htable in order to 3318 * do the invalidate. We know the htable must exist, since 3319 * unmap's don't release the htable until after removing any 3320 * hment. Having x86_hm_enter() keeps that from proceeding. 3321 */ 3322 htable_acquire(ht); 3323 3324 /* 3325 * Invalidate the PTE and remove the hment. 3326 */ 3327 old_pte = x86pte_inval(ht, entry, 0, NULL); 3328 if (PTE2PFN(old_pte, ht->ht_level) != pfn) { 3329 panic("x86pte_inval() failure found PTE = " FMT_PTE 3330 " pfn being unmapped is %lx ht=0x%lx entry=0x%x", 3331 old_pte, pfn, (uintptr_t)ht, entry); 3332 } 3333 3334 /* 3335 * Clean up all the htable information for this mapping 3336 */ 3337 ASSERT(ht->ht_valid_cnt > 0); 3338 HTABLE_DEC(ht->ht_valid_cnt); 3339 PGCNT_DEC(ht->ht_hat, ht->ht_level); 3340 3341 /* 3342 * sync ref/mod bits to the page_t 3343 */ 3344 if (PTE_GET(old_pte, PT_SOFTWARE) < PT_NOSYNC) 3345 hati_sync_pte_to_page(pp, old_pte, ht->ht_level); 3346 3347 /* 3348 * Remove the mapping list entry for this page. 3349 */ 3350 hm = hment_remove(pp, ht, entry); 3351 3352 /* 3353 * drop the mapping list lock so that we might free the 3354 * hment and htable. 3355 */ 3356 x86_hm_exit(pp); 3357 htable_release(ht); 3358 return (hm); 3359 } 3360 3361 extern int vpm_enable; 3362 /* 3363 * Unload all translations to a page. If the page is a subpage of a large 3364 * page, the large page mappings are also removed. 3365 * 3366 * The forceflags are unused. 3367 */ 3368 3369 /*ARGSUSED*/ 3370 static int 3371 hati_pageunload(struct page *pp, uint_t pg_szcd, uint_t forceflag) 3372 { 3373 page_t *cur_pp = pp; 3374 hment_t *hm; 3375 hment_t *prev; 3376 htable_t *ht; 3377 uint_t entry; 3378 level_t level; 3379 3380 XPV_DISALLOW_MIGRATE(); 3381 #if defined(__amd64) 3382 /* 3383 * clear the vpm ref. 3384 */ 3385 if (vpm_enable) { 3386 pp->p_vpmref = 0; 3387 } 3388 #endif 3389 /* 3390 * The loop with next_size handles pages with multiple pagesize mappings 3391 */ 3392 next_size: 3393 for (;;) { 3394 3395 /* 3396 * Get a mapping list entry 3397 */ 3398 x86_hm_enter(cur_pp); 3399 for (prev = NULL; ; prev = hm) { 3400 hm = hment_walk(cur_pp, &ht, &entry, prev); 3401 if (hm == NULL) { 3402 x86_hm_exit(cur_pp); 3403 3404 /* 3405 * If not part of a larger page, we're done. 3406 */ 3407 if (cur_pp->p_szc <= pg_szcd) { 3408 XPV_ALLOW_MIGRATE(); 3409 return (0); 3410 } 3411 3412 /* 3413 * Else check the next larger page size. 3414 * hat_page_demote() may decrease p_szc 3415 * but that's ok we'll just take an extra 3416 * trip discover there're no larger mappings 3417 * and return. 3418 */ 3419 ++pg_szcd; 3420 cur_pp = PP_GROUPLEADER(cur_pp, pg_szcd); 3421 goto next_size; 3422 } 3423 3424 /* 3425 * If this mapping size matches, remove it. 3426 */ 3427 level = ht->ht_level; 3428 if (level == pg_szcd) 3429 break; 3430 } 3431 3432 /* 3433 * Remove the mapping list entry for this page. 3434 * Note this does the x86_hm_exit() for us. 3435 */ 3436 hm = hati_page_unmap(cur_pp, ht, entry); 3437 if (hm != NULL) 3438 hment_free(hm); 3439 } 3440 } 3441 3442 int 3443 hat_pageunload(struct page *pp, uint_t forceflag) 3444 { 3445 ASSERT(PAGE_EXCL(pp)); 3446 return (hati_pageunload(pp, 0, forceflag)); 3447 } 3448 3449 /* 3450 * Unload all large mappings to pp and reduce by 1 p_szc field of every large 3451 * page level that included pp. 3452 * 3453 * pp must be locked EXCL. Even though no other constituent pages are locked 3454 * it's legal to unload large mappings to pp because all constituent pages of 3455 * large locked mappings have to be locked SHARED. therefore if we have EXCL 3456 * lock on one of constituent pages none of the large mappings to pp are 3457 * locked. 3458 * 3459 * Change (always decrease) p_szc field starting from the last constituent 3460 * page and ending with root constituent page so that root's pszc always shows 3461 * the area where hat_page_demote() may be active. 3462 * 3463 * This mechanism is only used for file system pages where it's not always 3464 * possible to get EXCL locks on all constituent pages to demote the size code 3465 * (as is done for anonymous or kernel large pages). 3466 */ 3467 void 3468 hat_page_demote(page_t *pp) 3469 { 3470 uint_t pszc; 3471 uint_t rszc; 3472 uint_t szc; 3473 page_t *rootpp; 3474 page_t *firstpp; 3475 page_t *lastpp; 3476 pgcnt_t pgcnt; 3477 3478 ASSERT(PAGE_EXCL(pp)); 3479 ASSERT(!PP_ISFREE(pp)); 3480 ASSERT(page_szc_lock_assert(pp)); 3481 3482 if (pp->p_szc == 0) 3483 return; 3484 3485 rootpp = PP_GROUPLEADER(pp, 1); 3486 (void) hati_pageunload(rootpp, 1, HAT_FORCE_PGUNLOAD); 3487 3488 /* 3489 * all large mappings to pp are gone 3490 * and no new can be setup since pp is locked exclusively. 3491 * 3492 * Lock the root to make sure there's only one hat_page_demote() 3493 * outstanding within the area of this root's pszc. 3494 * 3495 * Second potential hat_page_demote() is already eliminated by upper 3496 * VM layer via page_szc_lock() but we don't rely on it and use our 3497 * own locking (so that upper layer locking can be changed without 3498 * assumptions that hat depends on upper layer VM to prevent multiple 3499 * hat_page_demote() to be issued simultaneously to the same large 3500 * page). 3501 */ 3502 again: 3503 pszc = pp->p_szc; 3504 if (pszc == 0) 3505 return; 3506 rootpp = PP_GROUPLEADER(pp, pszc); 3507 x86_hm_enter(rootpp); 3508 /* 3509 * If root's p_szc is different from pszc we raced with another 3510 * hat_page_demote(). Drop the lock and try to find the root again. 3511 * If root's p_szc is greater than pszc previous hat_page_demote() is 3512 * not done yet. Take and release mlist lock of root's root to wait 3513 * for previous hat_page_demote() to complete. 3514 */ 3515 if ((rszc = rootpp->p_szc) != pszc) { 3516 x86_hm_exit(rootpp); 3517 if (rszc > pszc) { 3518 /* p_szc of a locked non free page can't increase */ 3519 ASSERT(pp != rootpp); 3520 3521 rootpp = PP_GROUPLEADER(rootpp, rszc); 3522 x86_hm_enter(rootpp); 3523 x86_hm_exit(rootpp); 3524 } 3525 goto again; 3526 } 3527 ASSERT(pp->p_szc == pszc); 3528 3529 /* 3530 * Decrement by 1 p_szc of every constituent page of a region that 3531 * covered pp. For example if original szc is 3 it gets changed to 2 3532 * everywhere except in region 2 that covered pp. Region 2 that 3533 * covered pp gets demoted to 1 everywhere except in region 1 that 3534 * covered pp. The region 1 that covered pp is demoted to region 3535 * 0. It's done this way because from region 3 we removed level 3 3536 * mappings, from region 2 that covered pp we removed level 2 mappings 3537 * and from region 1 that covered pp we removed level 1 mappings. All 3538 * changes are done from from high pfn's to low pfn's so that roots 3539 * are changed last allowing one to know the largest region where 3540 * hat_page_demote() is stil active by only looking at the root page. 3541 * 3542 * This algorithm is implemented in 2 while loops. First loop changes 3543 * p_szc of pages to the right of pp's level 1 region and second 3544 * loop changes p_szc of pages of level 1 region that covers pp 3545 * and all pages to the left of level 1 region that covers pp. 3546 * In the first loop p_szc keeps dropping with every iteration 3547 * and in the second loop it keeps increasing with every iteration. 3548 * 3549 * First loop description: Demote pages to the right of pp outside of 3550 * level 1 region that covers pp. In every iteration of the while 3551 * loop below find the last page of szc region and the first page of 3552 * (szc - 1) region that is immediately to the right of (szc - 1) 3553 * region that covers pp. From last such page to first such page 3554 * change every page's szc to szc - 1. Decrement szc and continue 3555 * looping until szc is 1. If pp belongs to the last (szc - 1) region 3556 * of szc region skip to the next iteration. 3557 */ 3558 szc = pszc; 3559 while (szc > 1) { 3560 lastpp = PP_GROUPLEADER(pp, szc); 3561 pgcnt = page_get_pagecnt(szc); 3562 lastpp += pgcnt - 1; 3563 firstpp = PP_GROUPLEADER(pp, (szc - 1)); 3564 pgcnt = page_get_pagecnt(szc - 1); 3565 if (lastpp - firstpp < pgcnt) { 3566 szc--; 3567 continue; 3568 } 3569 firstpp += pgcnt; 3570 while (lastpp != firstpp) { 3571 ASSERT(lastpp->p_szc == pszc); 3572 lastpp->p_szc = szc - 1; 3573 lastpp--; 3574 } 3575 firstpp->p_szc = szc - 1; 3576 szc--; 3577 } 3578 3579 /* 3580 * Second loop description: 3581 * First iteration changes p_szc to 0 of every 3582 * page of level 1 region that covers pp. 3583 * Subsequent iterations find last page of szc region 3584 * immediately to the left of szc region that covered pp 3585 * and first page of (szc + 1) region that covers pp. 3586 * From last to first page change p_szc of every page to szc. 3587 * Increment szc and continue looping until szc is pszc. 3588 * If pp belongs to the fist szc region of (szc + 1) region 3589 * skip to the next iteration. 3590 * 3591 */ 3592 szc = 0; 3593 while (szc < pszc) { 3594 firstpp = PP_GROUPLEADER(pp, (szc + 1)); 3595 if (szc == 0) { 3596 pgcnt = page_get_pagecnt(1); 3597 lastpp = firstpp + (pgcnt - 1); 3598 } else { 3599 lastpp = PP_GROUPLEADER(pp, szc); 3600 if (firstpp == lastpp) { 3601 szc++; 3602 continue; 3603 } 3604 lastpp--; 3605 pgcnt = page_get_pagecnt(szc); 3606 } 3607 while (lastpp != firstpp) { 3608 ASSERT(lastpp->p_szc == pszc); 3609 lastpp->p_szc = szc; 3610 lastpp--; 3611 } 3612 firstpp->p_szc = szc; 3613 if (firstpp == rootpp) 3614 break; 3615 szc++; 3616 } 3617 x86_hm_exit(rootpp); 3618 } 3619 3620 /* 3621 * get hw stats from hardware into page struct and reset hw stats 3622 * returns attributes of page 3623 * Flags for hat_pagesync, hat_getstat, hat_sync 3624 * 3625 * define HAT_SYNC_ZERORM 0x01 3626 * 3627 * Additional flags for hat_pagesync 3628 * 3629 * define HAT_SYNC_STOPON_REF 0x02 3630 * define HAT_SYNC_STOPON_MOD 0x04 3631 * define HAT_SYNC_STOPON_RM 0x06 3632 * define HAT_SYNC_STOPON_SHARED 0x08 3633 */ 3634 uint_t 3635 hat_pagesync(struct page *pp, uint_t flags) 3636 { 3637 hment_t *hm = NULL; 3638 htable_t *ht; 3639 uint_t entry; 3640 x86pte_t old, save_old; 3641 x86pte_t new; 3642 uchar_t nrmbits = P_REF|P_MOD|P_RO; 3643 extern ulong_t po_share; 3644 page_t *save_pp = pp; 3645 uint_t pszc = 0; 3646 3647 ASSERT(PAGE_LOCKED(pp) || panicstr); 3648 3649 if (PP_ISRO(pp) && (flags & HAT_SYNC_STOPON_MOD)) 3650 return (pp->p_nrm & nrmbits); 3651 3652 if ((flags & HAT_SYNC_ZERORM) == 0) { 3653 3654 if ((flags & HAT_SYNC_STOPON_REF) != 0 && PP_ISREF(pp)) 3655 return (pp->p_nrm & nrmbits); 3656 3657 if ((flags & HAT_SYNC_STOPON_MOD) != 0 && PP_ISMOD(pp)) 3658 return (pp->p_nrm & nrmbits); 3659 3660 if ((flags & HAT_SYNC_STOPON_SHARED) != 0 && 3661 hat_page_getshare(pp) > po_share) { 3662 if (PP_ISRO(pp)) 3663 PP_SETREF(pp); 3664 return (pp->p_nrm & nrmbits); 3665 } 3666 } 3667 3668 XPV_DISALLOW_MIGRATE(); 3669 next_size: 3670 /* 3671 * walk thru the mapping list syncing (and clearing) ref/mod bits. 3672 */ 3673 x86_hm_enter(pp); 3674 while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) { 3675 if (ht->ht_level < pszc) 3676 continue; 3677 old = x86pte_get(ht, entry); 3678 try_again: 3679 3680 ASSERT(PTE2PFN(old, ht->ht_level) == pp->p_pagenum); 3681 3682 if (PTE_GET(old, PT_REF | PT_MOD) == 0) 3683 continue; 3684 3685 save_old = old; 3686 if ((flags & HAT_SYNC_ZERORM) != 0) { 3687 3688 /* 3689 * Need to clear ref or mod bits. Need to demap 3690 * to make sure any executing TLBs see cleared bits. 3691 */ 3692 new = old; 3693 PTE_CLR(new, PT_REF | PT_MOD); 3694 old = hati_update_pte(ht, entry, old, new); 3695 if (old != 0) 3696 goto try_again; 3697 3698 old = save_old; 3699 } 3700 3701 /* 3702 * Sync the PTE 3703 */ 3704 if (!(flags & HAT_SYNC_ZERORM) && 3705 PTE_GET(old, PT_SOFTWARE) <= PT_NOSYNC) 3706 hati_sync_pte_to_page(pp, old, ht->ht_level); 3707 3708 /* 3709 * can stop short if we found a ref'd or mod'd page 3710 */ 3711 if ((flags & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp) || 3712 (flags & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp)) { 3713 x86_hm_exit(pp); 3714 goto done; 3715 } 3716 } 3717 x86_hm_exit(pp); 3718 while (pszc < pp->p_szc) { 3719 page_t *tpp; 3720 pszc++; 3721 tpp = PP_GROUPLEADER(pp, pszc); 3722 if (pp != tpp) { 3723 pp = tpp; 3724 goto next_size; 3725 } 3726 } 3727 done: 3728 XPV_ALLOW_MIGRATE(); 3729 return (save_pp->p_nrm & nrmbits); 3730 } 3731 3732 /* 3733 * returns approx number of mappings to this pp. A return of 0 implies 3734 * there are no mappings to the page. 3735 */ 3736 ulong_t 3737 hat_page_getshare(page_t *pp) 3738 { 3739 uint_t cnt; 3740 cnt = hment_mapcnt(pp); 3741 #if defined(__amd64) 3742 if (vpm_enable && pp->p_vpmref) { 3743 cnt += 1; 3744 } 3745 #endif 3746 return (cnt); 3747 } 3748 3749 /* 3750 * Return 1 the number of mappings exceeds sh_thresh. Return 0 3751 * otherwise. 3752 */ 3753 int 3754 hat_page_checkshare(page_t *pp, ulong_t sh_thresh) 3755 { 3756 return (hat_page_getshare(pp) > sh_thresh); 3757 } 3758 3759 /* 3760 * hat_softlock isn't supported anymore 3761 */ 3762 /*ARGSUSED*/ 3763 faultcode_t 3764 hat_softlock( 3765 hat_t *hat, 3766 caddr_t addr, 3767 size_t *len, 3768 struct page **page_array, 3769 uint_t flags) 3770 { 3771 return (FC_NOSUPPORT); 3772 } 3773 3774 3775 3776 /* 3777 * Routine to expose supported HAT features to platform independent code. 3778 */ 3779 /*ARGSUSED*/ 3780 int 3781 hat_supported(enum hat_features feature, void *arg) 3782 { 3783 switch (feature) { 3784 3785 case HAT_SHARED_PT: /* this is really ISM */ 3786 return (1); 3787 3788 case HAT_DYNAMIC_ISM_UNMAP: 3789 return (0); 3790 3791 case HAT_VMODSORT: 3792 return (1); 3793 3794 case HAT_SHARED_REGIONS: 3795 return (0); 3796 3797 default: 3798 panic("hat_supported() - unknown feature"); 3799 } 3800 return (0); 3801 } 3802 3803 /* 3804 * Called when a thread is exiting and has been switched to the kernel AS 3805 */ 3806 void 3807 hat_thread_exit(kthread_t *thd) 3808 { 3809 ASSERT(thd->t_procp->p_as == &kas); 3810 XPV_DISALLOW_MIGRATE(); 3811 hat_switch(thd->t_procp->p_as->a_hat); 3812 XPV_ALLOW_MIGRATE(); 3813 } 3814 3815 /* 3816 * Setup the given brand new hat structure as the new HAT on this cpu's mmu. 3817 */ 3818 /*ARGSUSED*/ 3819 void 3820 hat_setup(hat_t *hat, int flags) 3821 { 3822 XPV_DISALLOW_MIGRATE(); 3823 kpreempt_disable(); 3824 3825 hat_switch(hat); 3826 3827 kpreempt_enable(); 3828 XPV_ALLOW_MIGRATE(); 3829 } 3830 3831 /* 3832 * Prepare for a CPU private mapping for the given address. 3833 * 3834 * The address can only be used from a single CPU and can be remapped 3835 * using hat_mempte_remap(). Return the address of the PTE. 3836 * 3837 * We do the htable_create() if necessary and increment the valid count so 3838 * the htable can't disappear. We also hat_devload() the page table into 3839 * kernel so that the PTE is quickly accessed. 3840 */ 3841 hat_mempte_t 3842 hat_mempte_setup(caddr_t addr) 3843 { 3844 uintptr_t va = (uintptr_t)addr; 3845 htable_t *ht; 3846 uint_t entry; 3847 x86pte_t oldpte; 3848 hat_mempte_t p; 3849 3850 ASSERT(IS_PAGEALIGNED(va)); 3851 ASSERT(!IN_VA_HOLE(va)); 3852 ++curthread->t_hatdepth; 3853 XPV_DISALLOW_MIGRATE(); 3854 ht = htable_getpte(kas.a_hat, va, &entry, &oldpte, 0); 3855 if (ht == NULL) { 3856 ht = htable_create(kas.a_hat, va, 0, NULL); 3857 entry = htable_va2entry(va, ht); 3858 ASSERT(ht->ht_level == 0); 3859 oldpte = x86pte_get(ht, entry); 3860 } 3861 if (PTE_ISVALID(oldpte)) 3862 panic("hat_mempte_setup(): address already mapped" 3863 "ht=%p, entry=%d, pte=" FMT_PTE, (void *)ht, entry, oldpte); 3864 3865 /* 3866 * increment ht_valid_cnt so that the pagetable can't disappear 3867 */ 3868 HTABLE_INC(ht->ht_valid_cnt); 3869 3870 /* 3871 * return the PTE physical address to the caller. 3872 */ 3873 htable_release(ht); 3874 XPV_ALLOW_MIGRATE(); 3875 p = PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry); 3876 --curthread->t_hatdepth; 3877 return (p); 3878 } 3879 3880 /* 3881 * Release a CPU private mapping for the given address. 3882 * We decrement the htable valid count so it might be destroyed. 3883 */ 3884 /*ARGSUSED1*/ 3885 void 3886 hat_mempte_release(caddr_t addr, hat_mempte_t pte_pa) 3887 { 3888 htable_t *ht; 3889 3890 XPV_DISALLOW_MIGRATE(); 3891 /* 3892 * invalidate any left over mapping and decrement the htable valid count 3893 */ 3894 #ifdef __xpv 3895 if (HYPERVISOR_update_va_mapping((uintptr_t)addr, 0, 3896 UVMF_INVLPG | UVMF_LOCAL)) 3897 panic("HYPERVISOR_update_va_mapping() failed"); 3898 #else 3899 { 3900 x86pte_t *pteptr; 3901 3902 pteptr = x86pte_mapin(mmu_btop(pte_pa), 3903 (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL); 3904 if (mmu.pae_hat) 3905 *pteptr = 0; 3906 else 3907 *(x86pte32_t *)pteptr = 0; 3908 mmu_tlbflush_entry(addr); 3909 x86pte_mapout(); 3910 } 3911 #endif 3912 3913 ht = htable_getpte(kas.a_hat, ALIGN2PAGE(addr), NULL, NULL, 0); 3914 if (ht == NULL) 3915 panic("hat_mempte_release(): invalid address"); 3916 ASSERT(ht->ht_level == 0); 3917 HTABLE_DEC(ht->ht_valid_cnt); 3918 htable_release(ht); 3919 XPV_ALLOW_MIGRATE(); 3920 } 3921 3922 /* 3923 * Apply a temporary CPU private mapping to a page. We flush the TLB only 3924 * on this CPU, so this ought to have been called with preemption disabled. 3925 */ 3926 void 3927 hat_mempte_remap( 3928 pfn_t pfn, 3929 caddr_t addr, 3930 hat_mempte_t pte_pa, 3931 uint_t attr, 3932 uint_t flags) 3933 { 3934 uintptr_t va = (uintptr_t)addr; 3935 x86pte_t pte; 3936 3937 /* 3938 * Remap the given PTE to the new page's PFN. Invalidate only 3939 * on this CPU. 3940 */ 3941 #ifdef DEBUG 3942 htable_t *ht; 3943 uint_t entry; 3944 3945 ASSERT(IS_PAGEALIGNED(va)); 3946 ASSERT(!IN_VA_HOLE(va)); 3947 ht = htable_getpte(kas.a_hat, va, &entry, NULL, 0); 3948 ASSERT(ht != NULL); 3949 ASSERT(ht->ht_level == 0); 3950 ASSERT(ht->ht_valid_cnt > 0); 3951 ASSERT(ht->ht_pfn == mmu_btop(pte_pa)); 3952 htable_release(ht); 3953 #endif 3954 XPV_DISALLOW_MIGRATE(); 3955 pte = hati_mkpte(pfn, attr, 0, flags); 3956 #ifdef __xpv 3957 if (HYPERVISOR_update_va_mapping(va, pte, UVMF_INVLPG | UVMF_LOCAL)) 3958 panic("HYPERVISOR_update_va_mapping() failed"); 3959 #else 3960 { 3961 x86pte_t *pteptr; 3962 3963 pteptr = x86pte_mapin(mmu_btop(pte_pa), 3964 (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL); 3965 if (mmu.pae_hat) 3966 *(x86pte_t *)pteptr = pte; 3967 else 3968 *(x86pte32_t *)pteptr = (x86pte32_t)pte; 3969 mmu_tlbflush_entry(addr); 3970 x86pte_mapout(); 3971 } 3972 #endif 3973 XPV_ALLOW_MIGRATE(); 3974 } 3975 3976 3977 3978 /* 3979 * Hat locking functions 3980 * XXX - these two functions are currently being used by hatstats 3981 * they can be removed by using a per-as mutex for hatstats. 3982 */ 3983 void 3984 hat_enter(hat_t *hat) 3985 { 3986 mutex_enter(&hat->hat_mutex); 3987 } 3988 3989 void 3990 hat_exit(hat_t *hat) 3991 { 3992 mutex_exit(&hat->hat_mutex); 3993 } 3994 3995 /* 3996 * HAT part of cpu initialization. 3997 */ 3998 void 3999 hat_cpu_online(struct cpu *cpup) 4000 { 4001 if (cpup != CPU) { 4002 x86pte_cpu_init(cpup); 4003 hat_vlp_setup(cpup); 4004 } 4005 CPUSET_ATOMIC_ADD(khat_cpuset, cpup->cpu_id); 4006 } 4007 4008 /* 4009 * HAT part of cpu deletion. 4010 * (currently, we only call this after the cpu is safely passivated.) 4011 */ 4012 void 4013 hat_cpu_offline(struct cpu *cpup) 4014 { 4015 ASSERT(cpup != CPU); 4016 4017 CPUSET_ATOMIC_DEL(khat_cpuset, cpup->cpu_id); 4018 x86pte_cpu_fini(cpup); 4019 hat_vlp_teardown(cpup); 4020 } 4021 4022 /* 4023 * Function called after all CPUs are brought online. 4024 * Used to remove low address boot mappings. 4025 */ 4026 void 4027 clear_boot_mappings(uintptr_t low, uintptr_t high) 4028 { 4029 uintptr_t vaddr = low; 4030 htable_t *ht = NULL; 4031 level_t level; 4032 uint_t entry; 4033 x86pte_t pte; 4034 4035 /* 4036 * On 1st CPU we can unload the prom mappings, basically we blow away 4037 * all virtual mappings under _userlimit. 4038 */ 4039 while (vaddr < high) { 4040 pte = htable_walk(kas.a_hat, &ht, &vaddr, high); 4041 if (ht == NULL) 4042 break; 4043 4044 level = ht->ht_level; 4045 entry = htable_va2entry(vaddr, ht); 4046 ASSERT(level <= mmu.max_page_level); 4047 ASSERT(PTE_ISPAGE(pte, level)); 4048 4049 /* 4050 * Unload the mapping from the page tables. 4051 */ 4052 (void) x86pte_inval(ht, entry, 0, NULL); 4053 ASSERT(ht->ht_valid_cnt > 0); 4054 HTABLE_DEC(ht->ht_valid_cnt); 4055 PGCNT_DEC(ht->ht_hat, ht->ht_level); 4056 4057 vaddr += LEVEL_SIZE(ht->ht_level); 4058 } 4059 if (ht) 4060 htable_release(ht); 4061 } 4062 4063 /* 4064 * Atomically update a new translation for a single page. If the 4065 * currently installed PTE doesn't match the value we expect to find, 4066 * it's not updated and we return the PTE we found. 4067 * 4068 * If activating nosync or NOWRITE and the page was modified we need to sync 4069 * with the page_t. Also sync with page_t if clearing ref/mod bits. 4070 */ 4071 static x86pte_t 4072 hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, x86pte_t new) 4073 { 4074 page_t *pp; 4075 uint_t rm = 0; 4076 x86pte_t replaced; 4077 4078 if (PTE_GET(expected, PT_SOFTWARE) < PT_NOSYNC && 4079 PTE_GET(expected, PT_MOD | PT_REF) && 4080 (PTE_GET(new, PT_NOSYNC) || !PTE_GET(new, PT_WRITABLE) || 4081 !PTE_GET(new, PT_MOD | PT_REF))) { 4082 4083 ASSERT(!pfn_is_foreign(PTE2PFN(expected, ht->ht_level))); 4084 pp = page_numtopp_nolock(PTE2PFN(expected, ht->ht_level)); 4085 ASSERT(pp != NULL); 4086 if (PTE_GET(expected, PT_MOD)) 4087 rm |= P_MOD; 4088 if (PTE_GET(expected, PT_REF)) 4089 rm |= P_REF; 4090 PTE_CLR(new, PT_MOD | PT_REF); 4091 } 4092 4093 replaced = x86pte_update(ht, entry, expected, new); 4094 if (replaced != expected) 4095 return (replaced); 4096 4097 if (rm) { 4098 /* 4099 * sync to all constituent pages of a large page 4100 */ 4101 pgcnt_t pgcnt = page_get_pagecnt(ht->ht_level); 4102 ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt)); 4103 while (pgcnt-- > 0) { 4104 /* 4105 * hat_page_demote() can't decrease 4106 * pszc below this mapping size 4107 * since large mapping existed after we 4108 * took mlist lock. 4109 */ 4110 ASSERT(pp->p_szc >= ht->ht_level); 4111 hat_page_setattr(pp, rm); 4112 ++pp; 4113 } 4114 } 4115 4116 return (0); 4117 } 4118 4119 /* ARGSUSED */ 4120 void 4121 hat_join_srd(struct hat *hat, vnode_t *evp) 4122 { 4123 } 4124 4125 /* ARGSUSED */ 4126 hat_region_cookie_t 4127 hat_join_region(struct hat *hat, 4128 caddr_t r_saddr, 4129 size_t r_size, 4130 void *r_obj, 4131 u_offset_t r_objoff, 4132 uchar_t r_perm, 4133 uchar_t r_pgszc, 4134 hat_rgn_cb_func_t r_cb_function, 4135 uint_t flags) 4136 { 4137 panic("No shared region support on x86"); 4138 return (HAT_INVALID_REGION_COOKIE); 4139 } 4140 4141 /* ARGSUSED */ 4142 void 4143 hat_leave_region(struct hat *hat, hat_region_cookie_t rcookie, uint_t flags) 4144 { 4145 panic("No shared region support on x86"); 4146 } 4147 4148 /* ARGSUSED */ 4149 void 4150 hat_dup_region(struct hat *hat, hat_region_cookie_t rcookie) 4151 { 4152 panic("No shared region support on x86"); 4153 } 4154 4155 4156 /* 4157 * Kernel Physical Mapping (kpm) facility 4158 * 4159 * Most of the routines needed to support segkpm are almost no-ops on the 4160 * x86 platform. We map in the entire segment when it is created and leave 4161 * it mapped in, so there is no additional work required to set up and tear 4162 * down individual mappings. All of these routines were created to support 4163 * SPARC platforms that have to avoid aliasing in their virtually indexed 4164 * caches. 4165 * 4166 * Most of the routines have sanity checks in them (e.g. verifying that the 4167 * passed-in page is locked). We don't actually care about most of these 4168 * checks on x86, but we leave them in place to identify problems in the 4169 * upper levels. 4170 */ 4171 4172 /* 4173 * Map in a locked page and return the vaddr. 4174 */ 4175 /*ARGSUSED*/ 4176 caddr_t 4177 hat_kpm_mapin(struct page *pp, struct kpme *kpme) 4178 { 4179 caddr_t vaddr; 4180 4181 #ifdef DEBUG 4182 if (kpm_enable == 0) { 4183 cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set\n"); 4184 return ((caddr_t)NULL); 4185 } 4186 4187 if (pp == NULL || PAGE_LOCKED(pp) == 0) { 4188 cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked\n"); 4189 return ((caddr_t)NULL); 4190 } 4191 #endif 4192 4193 vaddr = hat_kpm_page2va(pp, 1); 4194 4195 return (vaddr); 4196 } 4197 4198 /* 4199 * Mapout a locked page. 4200 */ 4201 /*ARGSUSED*/ 4202 void 4203 hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr) 4204 { 4205 #ifdef DEBUG 4206 if (kpm_enable == 0) { 4207 cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set\n"); 4208 return; 4209 } 4210 4211 if (IS_KPM_ADDR(vaddr) == 0) { 4212 cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address\n"); 4213 return; 4214 } 4215 4216 if (pp == NULL || PAGE_LOCKED(pp) == 0) { 4217 cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked\n"); 4218 return; 4219 } 4220 #endif 4221 } 4222 4223 /* 4224 * Return the kpm virtual address for a specific pfn 4225 */ 4226 caddr_t 4227 hat_kpm_pfn2va(pfn_t pfn) 4228 { 4229 uintptr_t vaddr = (uintptr_t)kpm_vbase + mmu_ptob(pfn); 4230 4231 ASSERT(!pfn_is_foreign(pfn)); 4232 return ((caddr_t)vaddr); 4233 } 4234 4235 /* 4236 * Return the kpm virtual address for the page at pp. 4237 */ 4238 /*ARGSUSED*/ 4239 caddr_t 4240 hat_kpm_page2va(struct page *pp, int checkswap) 4241 { 4242 return (hat_kpm_pfn2va(pp->p_pagenum)); 4243 } 4244 4245 /* 4246 * Return the page frame number for the kpm virtual address vaddr. 4247 */ 4248 pfn_t 4249 hat_kpm_va2pfn(caddr_t vaddr) 4250 { 4251 pfn_t pfn; 4252 4253 ASSERT(IS_KPM_ADDR(vaddr)); 4254 4255 pfn = (pfn_t)btop(vaddr - kpm_vbase); 4256 4257 return (pfn); 4258 } 4259 4260 4261 /* 4262 * Return the page for the kpm virtual address vaddr. 4263 */ 4264 page_t * 4265 hat_kpm_vaddr2page(caddr_t vaddr) 4266 { 4267 pfn_t pfn; 4268 4269 ASSERT(IS_KPM_ADDR(vaddr)); 4270 4271 pfn = hat_kpm_va2pfn(vaddr); 4272 4273 return (page_numtopp_nolock(pfn)); 4274 } 4275 4276 /* 4277 * hat_kpm_fault is called from segkpm_fault when we take a page fault on a 4278 * KPM page. This should never happen on x86 4279 */ 4280 int 4281 hat_kpm_fault(hat_t *hat, caddr_t vaddr) 4282 { 4283 panic("pagefault in seg_kpm. hat: 0x%p vaddr: 0x%p", 4284 (void *)hat, (void *)vaddr); 4285 4286 return (0); 4287 } 4288 4289 /*ARGSUSED*/ 4290 void 4291 hat_kpm_mseghash_clear(int nentries) 4292 {} 4293 4294 /*ARGSUSED*/ 4295 void 4296 hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp) 4297 {} 4298 4299 #ifdef __xpv 4300 /* 4301 * There are specific Hypervisor calls to establish and remove mappings 4302 * to grant table references and the privcmd driver. We have to ensure 4303 * that a page table actually exists. 4304 */ 4305 void 4306 hat_prepare_mapping(hat_t *hat, caddr_t addr, uint64_t *pte_ma) 4307 { 4308 maddr_t base_ma; 4309 htable_t *ht; 4310 uint_t entry; 4311 4312 ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE)); 4313 XPV_DISALLOW_MIGRATE(); 4314 ht = htable_create(hat, (uintptr_t)addr, 0, NULL); 4315 4316 /* 4317 * if an address for pte_ma is passed in, return the MA of the pte 4318 * for this specific address. This address is only valid as long 4319 * as the htable stays locked. 4320 */ 4321 if (pte_ma != NULL) { 4322 entry = htable_va2entry((uintptr_t)addr, ht); 4323 base_ma = pa_to_ma(ptob(ht->ht_pfn)); 4324 *pte_ma = base_ma + (entry << mmu.pte_size_shift); 4325 } 4326 XPV_ALLOW_MIGRATE(); 4327 } 4328 4329 void 4330 hat_release_mapping(hat_t *hat, caddr_t addr) 4331 { 4332 htable_t *ht; 4333 4334 ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE)); 4335 XPV_DISALLOW_MIGRATE(); 4336 ht = htable_lookup(hat, (uintptr_t)addr, 0); 4337 ASSERT(ht != NULL); 4338 ASSERT(ht->ht_busy >= 2); 4339 htable_release(ht); 4340 htable_release(ht); 4341 XPV_ALLOW_MIGRATE(); 4342 } 4343 #endif 4344