1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 27 /* 28 * VM - Hardware Address Translation management for i386 and amd64 29 * 30 * Implementation of the interfaces described in <common/vm/hat.h> 31 * 32 * Nearly all the details of how the hardware is managed should not be 33 * visible outside this layer except for misc. machine specific functions 34 * that work in conjunction with this code. 35 * 36 * Routines used only inside of i86pc/vm start with hati_ for HAT Internal. 37 */ 38 39 #include <sys/machparam.h> 40 #include <sys/machsystm.h> 41 #include <sys/mman.h> 42 #include <sys/types.h> 43 #include <sys/systm.h> 44 #include <sys/cpuvar.h> 45 #include <sys/thread.h> 46 #include <sys/proc.h> 47 #include <sys/cpu.h> 48 #include <sys/kmem.h> 49 #include <sys/disp.h> 50 #include <sys/shm.h> 51 #include <sys/sysmacros.h> 52 #include <sys/machparam.h> 53 #include <sys/vmem.h> 54 #include <sys/vmsystm.h> 55 #include <sys/promif.h> 56 #include <sys/var.h> 57 #include <sys/x86_archext.h> 58 #include <sys/atomic.h> 59 #include <sys/bitmap.h> 60 #include <sys/controlregs.h> 61 #include <sys/bootconf.h> 62 #include <sys/bootsvcs.h> 63 #include <sys/bootinfo.h> 64 #include <sys/archsystm.h> 65 66 #include <vm/seg_kmem.h> 67 #include <vm/hat_i86.h> 68 #include <vm/as.h> 69 #include <vm/seg.h> 70 #include <vm/page.h> 71 #include <vm/seg_kp.h> 72 #include <vm/seg_kpm.h> 73 #include <vm/vm_dep.h> 74 #ifdef __xpv 75 #include <sys/hypervisor.h> 76 #endif 77 #include <vm/kboot_mmu.h> 78 #include <vm/seg_spt.h> 79 80 #include <sys/cmn_err.h> 81 82 /* 83 * Basic parameters for hat operation. 84 */ 85 struct hat_mmu_info mmu; 86 87 /* 88 * The page that is the kernel's top level pagetable. 89 * 90 * For 32 bit PAE support on i86pc, the kernel hat will use the 1st 4 entries 91 * on this 4K page for its top level page table. The remaining groups of 92 * 4 entries are used for per processor copies of user VLP pagetables for 93 * running threads. See hat_switch() and reload_pae32() for details. 94 * 95 * vlp_page[0..3] - level==2 PTEs for kernel HAT 96 * vlp_page[4..7] - level==2 PTEs for user thread on cpu 0 97 * vlp_page[8..11] - level==2 PTE for user thread on cpu 1 98 * etc... 99 */ 100 static x86pte_t *vlp_page; 101 102 /* 103 * forward declaration of internal utility routines 104 */ 105 static x86pte_t hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, 106 x86pte_t new); 107 108 /* 109 * The kernel address space exists in all HATs. To implement this the 110 * kernel reserves a fixed number of entries in the topmost level(s) of page 111 * tables. The values are setup during startup and then copied to every user 112 * hat created by hat_alloc(). This means that kernelbase must be: 113 * 114 * 4Meg aligned for 32 bit kernels 115 * 512Gig aligned for x86_64 64 bit kernel 116 * 117 * The hat_kernel_range_ts describe what needs to be copied from kernel hat 118 * to each user hat. 119 */ 120 typedef struct hat_kernel_range { 121 level_t hkr_level; 122 uintptr_t hkr_start_va; 123 uintptr_t hkr_end_va; /* zero means to end of memory */ 124 } hat_kernel_range_t; 125 #define NUM_KERNEL_RANGE 2 126 static hat_kernel_range_t kernel_ranges[NUM_KERNEL_RANGE]; 127 static int num_kernel_ranges; 128 129 uint_t use_boot_reserve = 1; /* cleared after early boot process */ 130 uint_t can_steal_post_boot = 0; /* set late in boot to enable stealing */ 131 132 /* 133 * enable_1gpg: controls 1g page support for user applications. 134 * By default, 1g pages are exported to user applications. enable_1gpg can 135 * be set to 0 to not export. 136 */ 137 int enable_1gpg = 1; 138 139 /* 140 * AMD shanghai processors provide better management of 1gb ptes in its tlb. 141 * By default, 1g page suppport will be disabled for pre-shanghai AMD 142 * processors that don't have optimal tlb support for the 1g page size. 143 * chk_optimal_1gtlb can be set to 0 to force 1g page support on sub-optimal 144 * processors. 145 */ 146 int chk_optimal_1gtlb = 1; 147 148 149 #ifdef DEBUG 150 uint_t map1gcnt; 151 #endif 152 153 154 /* 155 * A cpuset for all cpus. This is used for kernel address cross calls, since 156 * the kernel addresses apply to all cpus. 157 */ 158 cpuset_t khat_cpuset; 159 160 /* 161 * management stuff for hat structures 162 */ 163 kmutex_t hat_list_lock; 164 kcondvar_t hat_list_cv; 165 kmem_cache_t *hat_cache; 166 kmem_cache_t *hat_hash_cache; 167 kmem_cache_t *vlp_hash_cache; 168 169 /* 170 * Simple statistics 171 */ 172 struct hatstats hatstat; 173 174 /* 175 * Some earlier hypervisor versions do not emulate cmpxchg of PTEs 176 * correctly. For such hypervisors we must set PT_USER for kernel 177 * entries ourselves (normally the emulation would set PT_USER for 178 * kernel entries and PT_USER|PT_GLOBAL for user entries). pt_kern is 179 * thus set appropriately. Note that dboot/kbm is OK, as only the full 180 * HAT uses cmpxchg() and the other paths (hypercall etc.) were never 181 * incorrect. 182 */ 183 int pt_kern; 184 185 /* 186 * useful stuff for atomic access/clearing/setting REF/MOD/RO bits in page_t's. 187 */ 188 extern void atomic_orb(uchar_t *addr, uchar_t val); 189 extern void atomic_andb(uchar_t *addr, uchar_t val); 190 191 #define PP_GETRM(pp, rmmask) (pp->p_nrm & rmmask) 192 #define PP_ISMOD(pp) PP_GETRM(pp, P_MOD) 193 #define PP_ISREF(pp) PP_GETRM(pp, P_REF) 194 #define PP_ISRO(pp) PP_GETRM(pp, P_RO) 195 196 #define PP_SETRM(pp, rm) atomic_orb(&(pp->p_nrm), rm) 197 #define PP_SETMOD(pp) PP_SETRM(pp, P_MOD) 198 #define PP_SETREF(pp) PP_SETRM(pp, P_REF) 199 #define PP_SETRO(pp) PP_SETRM(pp, P_RO) 200 201 #define PP_CLRRM(pp, rm) atomic_andb(&(pp->p_nrm), ~(rm)) 202 #define PP_CLRMOD(pp) PP_CLRRM(pp, P_MOD) 203 #define PP_CLRREF(pp) PP_CLRRM(pp, P_REF) 204 #define PP_CLRRO(pp) PP_CLRRM(pp, P_RO) 205 #define PP_CLRALL(pp) PP_CLRRM(pp, P_MOD | P_REF | P_RO) 206 207 /* 208 * kmem cache constructor for struct hat 209 */ 210 /*ARGSUSED*/ 211 static int 212 hati_constructor(void *buf, void *handle, int kmflags) 213 { 214 hat_t *hat = buf; 215 216 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); 217 bzero(hat->hat_pages_mapped, 218 sizeof (pgcnt_t) * (mmu.max_page_level + 1)); 219 hat->hat_ism_pgcnt = 0; 220 hat->hat_stats = 0; 221 hat->hat_flags = 0; 222 CPUSET_ZERO(hat->hat_cpus); 223 hat->hat_htable = NULL; 224 hat->hat_ht_hash = NULL; 225 return (0); 226 } 227 228 /* 229 * Allocate a hat structure for as. We also create the top level 230 * htable and initialize it to contain the kernel hat entries. 231 */ 232 hat_t * 233 hat_alloc(struct as *as) 234 { 235 hat_t *hat; 236 htable_t *ht; /* top level htable */ 237 uint_t use_vlp; 238 uint_t r; 239 hat_kernel_range_t *rp; 240 uintptr_t va; 241 uintptr_t eva; 242 uint_t start; 243 uint_t cnt; 244 htable_t *src; 245 246 /* 247 * Once we start creating user process HATs we can enable 248 * the htable_steal() code. 249 */ 250 if (can_steal_post_boot == 0) 251 can_steal_post_boot = 1; 252 253 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 254 hat = kmem_cache_alloc(hat_cache, KM_SLEEP); 255 hat->hat_as = as; 256 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); 257 ASSERT(hat->hat_flags == 0); 258 259 #if defined(__xpv) 260 /* 261 * No VLP stuff on the hypervisor due to the 64-bit split top level 262 * page tables. On 32-bit it's not needed as the hypervisor takes 263 * care of copying the top level PTEs to a below 4Gig page. 264 */ 265 use_vlp = 0; 266 #else /* __xpv */ 267 /* 32 bit processes uses a VLP style hat when running with PAE */ 268 #if defined(__amd64) 269 use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32); 270 #elif defined(__i386) 271 use_vlp = mmu.pae_hat; 272 #endif 273 #endif /* __xpv */ 274 if (use_vlp) { 275 hat->hat_flags = HAT_VLP; 276 bzero(hat->hat_vlp_ptes, VLP_SIZE); 277 } 278 279 /* 280 * Allocate the htable hash 281 */ 282 if ((hat->hat_flags & HAT_VLP)) { 283 hat->hat_num_hash = mmu.vlp_hash_cnt; 284 hat->hat_ht_hash = kmem_cache_alloc(vlp_hash_cache, KM_SLEEP); 285 } else { 286 hat->hat_num_hash = mmu.hash_cnt; 287 hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_SLEEP); 288 } 289 bzero(hat->hat_ht_hash, hat->hat_num_hash * sizeof (htable_t *)); 290 291 /* 292 * Initialize Kernel HAT entries at the top of the top level page 293 * tables for the new hat. 294 */ 295 hat->hat_htable = NULL; 296 hat->hat_ht_cached = NULL; 297 XPV_DISALLOW_MIGRATE(); 298 ht = htable_create(hat, (uintptr_t)0, TOP_LEVEL(hat), NULL); 299 hat->hat_htable = ht; 300 301 #if defined(__amd64) 302 if (hat->hat_flags & HAT_VLP) 303 goto init_done; 304 #endif 305 306 for (r = 0; r < num_kernel_ranges; ++r) { 307 rp = &kernel_ranges[r]; 308 for (va = rp->hkr_start_va; va != rp->hkr_end_va; 309 va += cnt * LEVEL_SIZE(rp->hkr_level)) { 310 311 if (rp->hkr_level == TOP_LEVEL(hat)) 312 ht = hat->hat_htable; 313 else 314 ht = htable_create(hat, va, rp->hkr_level, 315 NULL); 316 317 start = htable_va2entry(va, ht); 318 cnt = HTABLE_NUM_PTES(ht) - start; 319 eva = va + 320 ((uintptr_t)cnt << LEVEL_SHIFT(rp->hkr_level)); 321 if (rp->hkr_end_va != 0 && 322 (eva > rp->hkr_end_va || eva == 0)) 323 cnt = htable_va2entry(rp->hkr_end_va, ht) - 324 start; 325 326 #if defined(__i386) && !defined(__xpv) 327 if (ht->ht_flags & HTABLE_VLP) { 328 bcopy(&vlp_page[start], 329 &hat->hat_vlp_ptes[start], 330 cnt * sizeof (x86pte_t)); 331 continue; 332 } 333 #endif 334 src = htable_lookup(kas.a_hat, va, rp->hkr_level); 335 ASSERT(src != NULL); 336 x86pte_copy(src, ht, start, cnt); 337 htable_release(src); 338 } 339 } 340 341 init_done: 342 343 #if defined(__xpv) 344 /* 345 * Pin top level page tables after initializing them 346 */ 347 xen_pin(hat->hat_htable->ht_pfn, mmu.max_level); 348 #if defined(__amd64) 349 xen_pin(hat->hat_user_ptable, mmu.max_level); 350 #endif 351 #endif 352 XPV_ALLOW_MIGRATE(); 353 354 /* 355 * Put it at the start of the global list of all hats (used by stealing) 356 * 357 * kas.a_hat is not in the list but is instead used to find the 358 * first and last items in the list. 359 * 360 * - kas.a_hat->hat_next points to the start of the user hats. 361 * The list ends where hat->hat_next == NULL 362 * 363 * - kas.a_hat->hat_prev points to the last of the user hats. 364 * The list begins where hat->hat_prev == NULL 365 */ 366 mutex_enter(&hat_list_lock); 367 hat->hat_prev = NULL; 368 hat->hat_next = kas.a_hat->hat_next; 369 if (hat->hat_next) 370 hat->hat_next->hat_prev = hat; 371 else 372 kas.a_hat->hat_prev = hat; 373 kas.a_hat->hat_next = hat; 374 mutex_exit(&hat_list_lock); 375 376 return (hat); 377 } 378 379 /* 380 * process has finished executing but as has not been cleaned up yet. 381 */ 382 /*ARGSUSED*/ 383 void 384 hat_free_start(hat_t *hat) 385 { 386 ASSERT(AS_WRITE_HELD(hat->hat_as, &hat->hat_as->a_lock)); 387 388 /* 389 * If the hat is currently a stealing victim, wait for the stealing 390 * to finish. Once we mark it as HAT_FREEING, htable_steal() 391 * won't look at its pagetables anymore. 392 */ 393 mutex_enter(&hat_list_lock); 394 while (hat->hat_flags & HAT_VICTIM) 395 cv_wait(&hat_list_cv, &hat_list_lock); 396 hat->hat_flags |= HAT_FREEING; 397 mutex_exit(&hat_list_lock); 398 } 399 400 /* 401 * An address space is being destroyed, so we destroy the associated hat. 402 */ 403 void 404 hat_free_end(hat_t *hat) 405 { 406 kmem_cache_t *cache; 407 408 ASSERT(hat->hat_flags & HAT_FREEING); 409 410 /* 411 * must not be running on the given hat 412 */ 413 ASSERT(CPU->cpu_current_hat != hat); 414 415 /* 416 * Remove it from the list of HATs 417 */ 418 mutex_enter(&hat_list_lock); 419 if (hat->hat_prev) 420 hat->hat_prev->hat_next = hat->hat_next; 421 else 422 kas.a_hat->hat_next = hat->hat_next; 423 if (hat->hat_next) 424 hat->hat_next->hat_prev = hat->hat_prev; 425 else 426 kas.a_hat->hat_prev = hat->hat_prev; 427 mutex_exit(&hat_list_lock); 428 hat->hat_next = hat->hat_prev = NULL; 429 430 #if defined(__xpv) 431 /* 432 * On the hypervisor, unpin top level page table(s) 433 */ 434 xen_unpin(hat->hat_htable->ht_pfn); 435 #if defined(__amd64) 436 xen_unpin(hat->hat_user_ptable); 437 #endif 438 #endif 439 440 /* 441 * Make a pass through the htables freeing them all up. 442 */ 443 htable_purge_hat(hat); 444 445 /* 446 * Decide which kmem cache the hash table came from, then free it. 447 */ 448 if (hat->hat_flags & HAT_VLP) 449 cache = vlp_hash_cache; 450 else 451 cache = hat_hash_cache; 452 kmem_cache_free(cache, hat->hat_ht_hash); 453 hat->hat_ht_hash = NULL; 454 455 hat->hat_flags = 0; 456 kmem_cache_free(hat_cache, hat); 457 } 458 459 /* 460 * round kernelbase down to a supported value to use for _userlimit 461 * 462 * userlimit must be aligned down to an entry in the top level htable. 463 * The one exception is for 32 bit HAT's running PAE. 464 */ 465 uintptr_t 466 hat_kernelbase(uintptr_t va) 467 { 468 #if defined(__i386) 469 va &= LEVEL_MASK(1); 470 #endif 471 if (IN_VA_HOLE(va)) 472 panic("_userlimit %p will fall in VA hole\n", (void *)va); 473 return (va); 474 } 475 476 /* 477 * 478 */ 479 static void 480 set_max_page_level() 481 { 482 level_t lvl; 483 484 if (!kbm_largepage_support) { 485 lvl = 0; 486 } else { 487 if (x86_feature & X86_1GPG) { 488 lvl = 2; 489 if (chk_optimal_1gtlb && 490 cpuid_opteron_erratum(CPU, 6671130)) { 491 lvl = 1; 492 } 493 if (plat_mnode_xcheck(LEVEL_SIZE(2) >> 494 LEVEL_SHIFT(0))) { 495 lvl = 1; 496 } 497 } else { 498 lvl = 1; 499 } 500 } 501 mmu.max_page_level = lvl; 502 503 if ((lvl == 2) && (enable_1gpg == 0)) 504 mmu.umax_page_level = 1; 505 else 506 mmu.umax_page_level = lvl; 507 } 508 509 /* 510 * Initialize hat data structures based on processor MMU information. 511 */ 512 void 513 mmu_init(void) 514 { 515 uint_t max_htables; 516 uint_t pa_bits; 517 uint_t va_bits; 518 int i; 519 520 /* 521 * If CPU enabled the page table global bit, use it for the kernel 522 * This is bit 7 in CR4 (PGE - Page Global Enable). 523 */ 524 if ((x86_feature & X86_PGE) != 0 && (getcr4() & CR4_PGE) != 0) 525 mmu.pt_global = PT_GLOBAL; 526 527 /* 528 * Detect NX and PAE usage. 529 */ 530 mmu.pae_hat = kbm_pae_support; 531 if (kbm_nx_support) 532 mmu.pt_nx = PT_NX; 533 else 534 mmu.pt_nx = 0; 535 536 /* 537 * Use CPU info to set various MMU parameters 538 */ 539 cpuid_get_addrsize(CPU, &pa_bits, &va_bits); 540 541 if (va_bits < sizeof (void *) * NBBY) { 542 mmu.hole_start = (1ul << (va_bits - 1)); 543 mmu.hole_end = 0ul - mmu.hole_start - 1; 544 } else { 545 mmu.hole_end = 0; 546 mmu.hole_start = mmu.hole_end - 1; 547 } 548 #if defined(OPTERON_ERRATUM_121) 549 /* 550 * If erratum 121 has already been detected at this time, hole_start 551 * contains the value to be subtracted from mmu.hole_start. 552 */ 553 ASSERT(hole_start == 0 || opteron_erratum_121 != 0); 554 hole_start = mmu.hole_start - hole_start; 555 #else 556 hole_start = mmu.hole_start; 557 #endif 558 hole_end = mmu.hole_end; 559 560 mmu.highest_pfn = mmu_btop((1ull << pa_bits) - 1); 561 if (mmu.pae_hat == 0 && pa_bits > 32) 562 mmu.highest_pfn = PFN_4G - 1; 563 564 if (mmu.pae_hat) { 565 mmu.pte_size = 8; /* 8 byte PTEs */ 566 mmu.pte_size_shift = 3; 567 } else { 568 mmu.pte_size = 4; /* 4 byte PTEs */ 569 mmu.pte_size_shift = 2; 570 } 571 572 if (mmu.pae_hat && (x86_feature & X86_PAE) == 0) 573 panic("Processor does not support PAE"); 574 575 if ((x86_feature & X86_CX8) == 0) 576 panic("Processor does not support cmpxchg8b instruction"); 577 578 #if defined(__amd64) 579 580 mmu.num_level = 4; 581 mmu.max_level = 3; 582 mmu.ptes_per_table = 512; 583 mmu.top_level_count = 512; 584 585 mmu.level_shift[0] = 12; 586 mmu.level_shift[1] = 21; 587 mmu.level_shift[2] = 30; 588 mmu.level_shift[3] = 39; 589 590 #elif defined(__i386) 591 592 if (mmu.pae_hat) { 593 mmu.num_level = 3; 594 mmu.max_level = 2; 595 mmu.ptes_per_table = 512; 596 mmu.top_level_count = 4; 597 598 mmu.level_shift[0] = 12; 599 mmu.level_shift[1] = 21; 600 mmu.level_shift[2] = 30; 601 602 } else { 603 mmu.num_level = 2; 604 mmu.max_level = 1; 605 mmu.ptes_per_table = 1024; 606 mmu.top_level_count = 1024; 607 608 mmu.level_shift[0] = 12; 609 mmu.level_shift[1] = 22; 610 } 611 612 #endif /* __i386 */ 613 614 for (i = 0; i < mmu.num_level; ++i) { 615 mmu.level_size[i] = 1UL << mmu.level_shift[i]; 616 mmu.level_offset[i] = mmu.level_size[i] - 1; 617 mmu.level_mask[i] = ~mmu.level_offset[i]; 618 } 619 620 set_max_page_level(); 621 622 mmu_page_sizes = mmu.max_page_level + 1; 623 mmu_exported_page_sizes = mmu.umax_page_level + 1; 624 625 /* restrict legacy applications from using pagesizes 1g and above */ 626 mmu_legacy_page_sizes = 627 (mmu_exported_page_sizes > 2) ? 2 : mmu_exported_page_sizes; 628 629 630 for (i = 0; i <= mmu.max_page_level; ++i) { 631 mmu.pte_bits[i] = PT_VALID | pt_kern; 632 if (i > 0) 633 mmu.pte_bits[i] |= PT_PAGESIZE; 634 } 635 636 /* 637 * NOTE Legacy 32 bit PAE mode only has the P_VALID bit at top level. 638 */ 639 for (i = 1; i < mmu.num_level; ++i) 640 mmu.ptp_bits[i] = PT_PTPBITS; 641 642 #if defined(__i386) 643 mmu.ptp_bits[2] = PT_VALID; 644 #endif 645 646 /* 647 * Compute how many hash table entries to have per process for htables. 648 * We start with 1 page's worth of entries. 649 * 650 * If physical memory is small, reduce the amount need to cover it. 651 */ 652 max_htables = physmax / mmu.ptes_per_table; 653 mmu.hash_cnt = MMU_PAGESIZE / sizeof (htable_t *); 654 while (mmu.hash_cnt > 16 && mmu.hash_cnt >= max_htables) 655 mmu.hash_cnt >>= 1; 656 mmu.vlp_hash_cnt = mmu.hash_cnt; 657 658 #if defined(__amd64) 659 /* 660 * If running in 64 bits and physical memory is large, 661 * increase the size of the cache to cover all of memory for 662 * a 64 bit process. 663 */ 664 #define HASH_MAX_LENGTH 4 665 while (mmu.hash_cnt * HASH_MAX_LENGTH < max_htables) 666 mmu.hash_cnt <<= 1; 667 #endif 668 } 669 670 671 /* 672 * initialize hat data structures 673 */ 674 void 675 hat_init() 676 { 677 #if defined(__i386) 678 /* 679 * _userlimit must be aligned correctly 680 */ 681 if ((_userlimit & LEVEL_MASK(1)) != _userlimit) { 682 prom_printf("hat_init(): _userlimit=%p, not aligned at %p\n", 683 (void *)_userlimit, (void *)LEVEL_SIZE(1)); 684 halt("hat_init(): Unable to continue"); 685 } 686 #endif 687 688 cv_init(&hat_list_cv, NULL, CV_DEFAULT, NULL); 689 690 /* 691 * initialize kmem caches 692 */ 693 htable_init(); 694 hment_init(); 695 696 hat_cache = kmem_cache_create("hat_t", 697 sizeof (hat_t), 0, hati_constructor, NULL, NULL, 698 NULL, 0, 0); 699 700 hat_hash_cache = kmem_cache_create("HatHash", 701 mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL, 702 NULL, 0, 0); 703 704 /* 705 * VLP hats can use a smaller hash table size on large memroy machines 706 */ 707 if (mmu.hash_cnt == mmu.vlp_hash_cnt) { 708 vlp_hash_cache = hat_hash_cache; 709 } else { 710 vlp_hash_cache = kmem_cache_create("HatVlpHash", 711 mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL, 712 NULL, 0, 0); 713 } 714 715 /* 716 * Set up the kernel's hat 717 */ 718 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 719 kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP); 720 mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); 721 kas.a_hat->hat_as = &kas; 722 kas.a_hat->hat_flags = 0; 723 AS_LOCK_EXIT(&kas, &kas.a_lock); 724 725 CPUSET_ZERO(khat_cpuset); 726 CPUSET_ADD(khat_cpuset, CPU->cpu_id); 727 728 /* 729 * The kernel hat's next pointer serves as the head of the hat list . 730 * The kernel hat's prev pointer tracks the last hat on the list for 731 * htable_steal() to use. 732 */ 733 kas.a_hat->hat_next = NULL; 734 kas.a_hat->hat_prev = NULL; 735 736 /* 737 * Allocate an htable hash bucket for the kernel 738 * XX64 - tune for 64 bit procs 739 */ 740 kas.a_hat->hat_num_hash = mmu.hash_cnt; 741 kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP); 742 bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *)); 743 744 /* 745 * zero out the top level and cached htable pointers 746 */ 747 kas.a_hat->hat_ht_cached = NULL; 748 kas.a_hat->hat_htable = NULL; 749 750 /* 751 * Pre-allocate hrm_hashtab before enabling the collection of 752 * refmod statistics. Allocating on the fly would mean us 753 * running the risk of suffering recursive mutex enters or 754 * deadlocks. 755 */ 756 hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *), 757 KM_SLEEP); 758 } 759 760 /* 761 * Prepare CPU specific pagetables for VLP processes on 64 bit kernels. 762 * 763 * Each CPU has a set of 2 pagetables that are reused for any 32 bit 764 * process it runs. They are the top level pagetable, hci_vlp_l3ptes, and 765 * the next to top level table for the bottom 512 Gig, hci_vlp_l2ptes. 766 */ 767 /*ARGSUSED*/ 768 static void 769 hat_vlp_setup(struct cpu *cpu) 770 { 771 #if defined(__amd64) && !defined(__xpv) 772 struct hat_cpu_info *hci = cpu->cpu_hat_info; 773 pfn_t pfn; 774 775 /* 776 * allocate the level==2 page table for the bottom most 777 * 512Gig of address space (this is where 32 bit apps live) 778 */ 779 ASSERT(hci != NULL); 780 hci->hci_vlp_l2ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP); 781 782 /* 783 * Allocate a top level pagetable and copy the kernel's 784 * entries into it. Then link in hci_vlp_l2ptes in the 1st entry. 785 */ 786 hci->hci_vlp_l3ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP); 787 hci->hci_vlp_pfn = 788 hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l3ptes); 789 ASSERT(hci->hci_vlp_pfn != PFN_INVALID); 790 bcopy(vlp_page, hci->hci_vlp_l3ptes, MMU_PAGESIZE); 791 792 pfn = hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l2ptes); 793 ASSERT(pfn != PFN_INVALID); 794 hci->hci_vlp_l3ptes[0] = MAKEPTP(pfn, 2); 795 #endif /* __amd64 && !__xpv */ 796 } 797 798 /*ARGSUSED*/ 799 static void 800 hat_vlp_teardown(cpu_t *cpu) 801 { 802 #if defined(__amd64) && !defined(__xpv) 803 struct hat_cpu_info *hci; 804 805 if ((hci = cpu->cpu_hat_info) == NULL) 806 return; 807 if (hci->hci_vlp_l2ptes) 808 kmem_free(hci->hci_vlp_l2ptes, MMU_PAGESIZE); 809 if (hci->hci_vlp_l3ptes) 810 kmem_free(hci->hci_vlp_l3ptes, MMU_PAGESIZE); 811 #endif 812 } 813 814 #define NEXT_HKR(r, l, s, e) { \ 815 kernel_ranges[r].hkr_level = l; \ 816 kernel_ranges[r].hkr_start_va = s; \ 817 kernel_ranges[r].hkr_end_va = e; \ 818 ++r; \ 819 } 820 821 /* 822 * Finish filling in the kernel hat. 823 * Pre fill in all top level kernel page table entries for the kernel's 824 * part of the address range. From this point on we can't use any new 825 * kernel large pages if they need PTE's at max_level 826 * 827 * create the kmap mappings. 828 */ 829 void 830 hat_init_finish(void) 831 { 832 size_t size; 833 uint_t r = 0; 834 uintptr_t va; 835 hat_kernel_range_t *rp; 836 837 838 /* 839 * We are now effectively running on the kernel hat. 840 * Clearing use_boot_reserve shuts off using the pre-allocated boot 841 * reserve for all HAT allocations. From here on, the reserves are 842 * only used when avoiding recursion in kmem_alloc(). 843 */ 844 use_boot_reserve = 0; 845 htable_adjust_reserve(); 846 847 /* 848 * User HATs are initialized with copies of all kernel mappings in 849 * higher level page tables. Ensure that those entries exist. 850 */ 851 #if defined(__amd64) 852 853 NEXT_HKR(r, 3, kernelbase, 0); 854 #if defined(__xpv) 855 NEXT_HKR(r, 3, HYPERVISOR_VIRT_START, HYPERVISOR_VIRT_END); 856 #endif 857 858 #elif defined(__i386) 859 860 #if !defined(__xpv) 861 if (mmu.pae_hat) { 862 va = kernelbase; 863 if ((va & LEVEL_MASK(2)) != va) { 864 va = P2ROUNDUP(va, LEVEL_SIZE(2)); 865 NEXT_HKR(r, 1, kernelbase, va); 866 } 867 if (va != 0) 868 NEXT_HKR(r, 2, va, 0); 869 } else 870 #endif /* __xpv */ 871 NEXT_HKR(r, 1, kernelbase, 0); 872 873 #endif /* __i386 */ 874 875 num_kernel_ranges = r; 876 877 /* 878 * Create all the kernel pagetables that will have entries 879 * shared to user HATs. 880 */ 881 for (r = 0; r < num_kernel_ranges; ++r) { 882 rp = &kernel_ranges[r]; 883 for (va = rp->hkr_start_va; va != rp->hkr_end_va; 884 va += LEVEL_SIZE(rp->hkr_level)) { 885 htable_t *ht; 886 887 if (IN_HYPERVISOR_VA(va)) 888 continue; 889 890 /* can/must skip if a page mapping already exists */ 891 if (rp->hkr_level <= mmu.max_page_level && 892 (ht = htable_getpage(kas.a_hat, va, NULL)) != 893 NULL) { 894 htable_release(ht); 895 continue; 896 } 897 898 (void) htable_create(kas.a_hat, va, rp->hkr_level - 1, 899 NULL); 900 } 901 } 902 903 /* 904 * 32 bit PAE metal kernels use only 4 of the 512 entries in the 905 * page holding the top level pagetable. We use the remainder for 906 * the "per CPU" page tables for VLP processes. 907 * Map the top level kernel pagetable into the kernel to make 908 * it easy to use bcopy access these tables. 909 */ 910 if (mmu.pae_hat) { 911 vlp_page = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP); 912 hat_devload(kas.a_hat, (caddr_t)vlp_page, MMU_PAGESIZE, 913 kas.a_hat->hat_htable->ht_pfn, 914 #if !defined(__xpv) 915 PROT_WRITE | 916 #endif 917 PROT_READ | HAT_NOSYNC | HAT_UNORDERED_OK, 918 HAT_LOAD | HAT_LOAD_NOCONSIST); 919 } 920 hat_vlp_setup(CPU); 921 922 /* 923 * Create kmap (cached mappings of kernel PTEs) 924 * for 32 bit we map from segmap_start .. ekernelheap 925 * for 64 bit we map from segmap_start .. segmap_start + segmapsize; 926 */ 927 #if defined(__i386) 928 size = (uintptr_t)ekernelheap - segmap_start; 929 #elif defined(__amd64) 930 size = segmapsize; 931 #endif 932 hat_kmap_init((uintptr_t)segmap_start, size); 933 } 934 935 /* 936 * On 32 bit PAE mode, PTE's are 64 bits, but ordinary atomic memory references 937 * are 32 bit, so for safety we must use cas64() to install these. 938 */ 939 #ifdef __i386 940 static void 941 reload_pae32(hat_t *hat, cpu_t *cpu) 942 { 943 x86pte_t *src; 944 x86pte_t *dest; 945 x86pte_t pte; 946 int i; 947 948 /* 949 * Load the 4 entries of the level 2 page table into this 950 * cpu's range of the vlp_page and point cr3 at them. 951 */ 952 ASSERT(mmu.pae_hat); 953 src = hat->hat_vlp_ptes; 954 dest = vlp_page + (cpu->cpu_id + 1) * VLP_NUM_PTES; 955 for (i = 0; i < VLP_NUM_PTES; ++i) { 956 for (;;) { 957 pte = dest[i]; 958 if (pte == src[i]) 959 break; 960 if (cas64(dest + i, pte, src[i]) != src[i]) 961 break; 962 } 963 } 964 } 965 #endif 966 967 /* 968 * Switch to a new active hat, maintaining bit masks to track active CPUs. 969 * 970 * On the 32-bit PAE hypervisor, %cr3 is a 64-bit value, on metal it 971 * remains a 32-bit value. 972 */ 973 void 974 hat_switch(hat_t *hat) 975 { 976 uint64_t newcr3; 977 cpu_t *cpu = CPU; 978 hat_t *old = cpu->cpu_current_hat; 979 980 /* 981 * set up this information first, so we don't miss any cross calls 982 */ 983 if (old != NULL) { 984 if (old == hat) 985 return; 986 if (old != kas.a_hat) 987 CPUSET_ATOMIC_DEL(old->hat_cpus, cpu->cpu_id); 988 } 989 990 /* 991 * Add this CPU to the active set for this HAT. 992 */ 993 if (hat != kas.a_hat) { 994 CPUSET_ATOMIC_ADD(hat->hat_cpus, cpu->cpu_id); 995 } 996 cpu->cpu_current_hat = hat; 997 998 /* 999 * now go ahead and load cr3 1000 */ 1001 if (hat->hat_flags & HAT_VLP) { 1002 #if defined(__amd64) 1003 x86pte_t *vlpptep = cpu->cpu_hat_info->hci_vlp_l2ptes; 1004 1005 VLP_COPY(hat->hat_vlp_ptes, vlpptep); 1006 newcr3 = MAKECR3(cpu->cpu_hat_info->hci_vlp_pfn); 1007 #elif defined(__i386) 1008 reload_pae32(hat, cpu); 1009 newcr3 = MAKECR3(kas.a_hat->hat_htable->ht_pfn) + 1010 (cpu->cpu_id + 1) * VLP_SIZE; 1011 #endif 1012 } else { 1013 newcr3 = MAKECR3((uint64_t)hat->hat_htable->ht_pfn); 1014 } 1015 #ifdef __xpv 1016 { 1017 struct mmuext_op t[2]; 1018 uint_t retcnt; 1019 uint_t opcnt = 1; 1020 1021 t[0].cmd = MMUEXT_NEW_BASEPTR; 1022 t[0].arg1.mfn = mmu_btop(pa_to_ma(newcr3)); 1023 #if defined(__amd64) 1024 /* 1025 * There's an interesting problem here, as to what to 1026 * actually specify when switching to the kernel hat. 1027 * For now we'll reuse the kernel hat again. 1028 */ 1029 t[1].cmd = MMUEXT_NEW_USER_BASEPTR; 1030 if (hat == kas.a_hat) 1031 t[1].arg1.mfn = mmu_btop(pa_to_ma(newcr3)); 1032 else 1033 t[1].arg1.mfn = pfn_to_mfn(hat->hat_user_ptable); 1034 ++opcnt; 1035 #endif /* __amd64 */ 1036 if (HYPERVISOR_mmuext_op(t, opcnt, &retcnt, DOMID_SELF) < 0) 1037 panic("HYPERVISOR_mmu_update() failed"); 1038 ASSERT(retcnt == opcnt); 1039 1040 } 1041 #else 1042 setcr3(newcr3); 1043 #endif 1044 ASSERT(cpu == CPU); 1045 } 1046 1047 /* 1048 * Utility to return a valid x86pte_t from protections, pfn, and level number 1049 */ 1050 static x86pte_t 1051 hati_mkpte(pfn_t pfn, uint_t attr, level_t level, uint_t flags) 1052 { 1053 x86pte_t pte; 1054 uint_t cache_attr = attr & HAT_ORDER_MASK; 1055 1056 pte = MAKEPTE(pfn, level); 1057 1058 if (attr & PROT_WRITE) 1059 PTE_SET(pte, PT_WRITABLE); 1060 1061 if (attr & PROT_USER) 1062 PTE_SET(pte, PT_USER); 1063 1064 if (!(attr & PROT_EXEC)) 1065 PTE_SET(pte, mmu.pt_nx); 1066 1067 /* 1068 * Set the software bits used track ref/mod sync's and hments. 1069 * If not using REF/MOD, set them to avoid h/w rewriting PTEs. 1070 */ 1071 if (flags & HAT_LOAD_NOCONSIST) 1072 PTE_SET(pte, PT_NOCONSIST | PT_REF | PT_MOD); 1073 else if (attr & HAT_NOSYNC) 1074 PTE_SET(pte, PT_NOSYNC | PT_REF | PT_MOD); 1075 1076 /* 1077 * Set the caching attributes in the PTE. The combination 1078 * of attributes are poorly defined, so we pay attention 1079 * to them in the given order. 1080 * 1081 * The test for HAT_STRICTORDER is different because it's defined 1082 * as "0" - which was a stupid thing to do, but is too late to change! 1083 */ 1084 if (cache_attr == HAT_STRICTORDER) { 1085 PTE_SET(pte, PT_NOCACHE); 1086 /*LINTED [Lint hates empty ifs, but it's the obvious way to do this] */ 1087 } else if (cache_attr & (HAT_UNORDERED_OK | HAT_STORECACHING_OK)) { 1088 /* nothing to set */; 1089 } else if (cache_attr & (HAT_MERGING_OK | HAT_LOADCACHING_OK)) { 1090 PTE_SET(pte, PT_NOCACHE); 1091 if (x86_feature & X86_PAT) 1092 PTE_SET(pte, (level == 0) ? PT_PAT_4K : PT_PAT_LARGE); 1093 else 1094 PTE_SET(pte, PT_WRITETHRU); 1095 } else { 1096 panic("hati_mkpte(): bad caching attributes: %x\n", cache_attr); 1097 } 1098 1099 return (pte); 1100 } 1101 1102 /* 1103 * Duplicate address translations of the parent to the child. 1104 * This function really isn't used anymore. 1105 */ 1106 /*ARGSUSED*/ 1107 int 1108 hat_dup(hat_t *old, hat_t *new, caddr_t addr, size_t len, uint_t flag) 1109 { 1110 ASSERT((uintptr_t)addr < kernelbase); 1111 ASSERT(new != kas.a_hat); 1112 ASSERT(old != kas.a_hat); 1113 return (0); 1114 } 1115 1116 /* 1117 * Allocate any hat resources required for a process being swapped in. 1118 */ 1119 /*ARGSUSED*/ 1120 void 1121 hat_swapin(hat_t *hat) 1122 { 1123 /* do nothing - we let everything fault back in */ 1124 } 1125 1126 /* 1127 * Unload all translations associated with an address space of a process 1128 * that is being swapped out. 1129 */ 1130 void 1131 hat_swapout(hat_t *hat) 1132 { 1133 uintptr_t vaddr = (uintptr_t)0; 1134 uintptr_t eaddr = _userlimit; 1135 htable_t *ht = NULL; 1136 level_t l; 1137 1138 XPV_DISALLOW_MIGRATE(); 1139 /* 1140 * We can't just call hat_unload(hat, 0, _userlimit...) here, because 1141 * seg_spt and shared pagetables can't be swapped out. 1142 * Take a look at segspt_shmswapout() - it's a big no-op. 1143 * 1144 * Instead we'll walk through all the address space and unload 1145 * any mappings which we are sure are not shared, not locked. 1146 */ 1147 ASSERT(IS_PAGEALIGNED(vaddr)); 1148 ASSERT(IS_PAGEALIGNED(eaddr)); 1149 ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1150 if ((uintptr_t)hat->hat_as->a_userlimit < eaddr) 1151 eaddr = (uintptr_t)hat->hat_as->a_userlimit; 1152 1153 while (vaddr < eaddr) { 1154 (void) htable_walk(hat, &ht, &vaddr, eaddr); 1155 if (ht == NULL) 1156 break; 1157 1158 ASSERT(!IN_VA_HOLE(vaddr)); 1159 1160 /* 1161 * If the page table is shared skip its entire range. 1162 */ 1163 l = ht->ht_level; 1164 if (ht->ht_flags & HTABLE_SHARED_PFN) { 1165 vaddr = ht->ht_vaddr + LEVEL_SIZE(l + 1); 1166 htable_release(ht); 1167 ht = NULL; 1168 continue; 1169 } 1170 1171 /* 1172 * If the page table has no locked entries, unload this one. 1173 */ 1174 if (ht->ht_lock_cnt == 0) 1175 hat_unload(hat, (caddr_t)vaddr, LEVEL_SIZE(l), 1176 HAT_UNLOAD_UNMAP); 1177 1178 /* 1179 * If we have a level 0 page table with locked entries, 1180 * skip the entire page table, otherwise skip just one entry. 1181 */ 1182 if (ht->ht_lock_cnt > 0 && l == 0) 1183 vaddr = ht->ht_vaddr + LEVEL_SIZE(1); 1184 else 1185 vaddr += LEVEL_SIZE(l); 1186 } 1187 if (ht) 1188 htable_release(ht); 1189 1190 /* 1191 * We're in swapout because the system is low on memory, so 1192 * go back and flush all the htables off the cached list. 1193 */ 1194 htable_purge_hat(hat); 1195 XPV_ALLOW_MIGRATE(); 1196 } 1197 1198 /* 1199 * returns number of bytes that have valid mappings in hat. 1200 */ 1201 size_t 1202 hat_get_mapped_size(hat_t *hat) 1203 { 1204 size_t total = 0; 1205 int l; 1206 1207 for (l = 0; l <= mmu.max_page_level; l++) 1208 total += (hat->hat_pages_mapped[l] << LEVEL_SHIFT(l)); 1209 total += hat->hat_ism_pgcnt; 1210 1211 return (total); 1212 } 1213 1214 /* 1215 * enable/disable collection of stats for hat. 1216 */ 1217 int 1218 hat_stats_enable(hat_t *hat) 1219 { 1220 atomic_add_32(&hat->hat_stats, 1); 1221 return (1); 1222 } 1223 1224 void 1225 hat_stats_disable(hat_t *hat) 1226 { 1227 atomic_add_32(&hat->hat_stats, -1); 1228 } 1229 1230 /* 1231 * Utility to sync the ref/mod bits from a page table entry to the page_t 1232 * We must be holding the mapping list lock when this is called. 1233 */ 1234 static void 1235 hati_sync_pte_to_page(page_t *pp, x86pte_t pte, level_t level) 1236 { 1237 uint_t rm = 0; 1238 pgcnt_t pgcnt; 1239 1240 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC) 1241 return; 1242 1243 if (PTE_GET(pte, PT_REF)) 1244 rm |= P_REF; 1245 1246 if (PTE_GET(pte, PT_MOD)) 1247 rm |= P_MOD; 1248 1249 if (rm == 0) 1250 return; 1251 1252 /* 1253 * sync to all constituent pages of a large page 1254 */ 1255 ASSERT(x86_hm_held(pp)); 1256 pgcnt = page_get_pagecnt(level); 1257 ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt)); 1258 for (; pgcnt > 0; --pgcnt) { 1259 /* 1260 * hat_page_demote() can't decrease 1261 * pszc below this mapping size 1262 * since this large mapping existed after we 1263 * took mlist lock. 1264 */ 1265 ASSERT(pp->p_szc >= level); 1266 hat_page_setattr(pp, rm); 1267 ++pp; 1268 } 1269 } 1270 1271 /* 1272 * This the set of PTE bits for PFN, permissions and caching 1273 * that are allowed to change on a HAT_LOAD_REMAP 1274 */ 1275 #define PT_REMAP_BITS \ 1276 (PT_PADDR | PT_NX | PT_WRITABLE | PT_WRITETHRU | \ 1277 PT_NOCACHE | PT_PAT_4K | PT_PAT_LARGE | PT_IGNORE | PT_REF | PT_MOD) 1278 1279 #define REMAPASSERT(EX) if (!(EX)) panic("hati_pte_map: " #EX) 1280 /* 1281 * Do the low-level work to get a mapping entered into a HAT's pagetables 1282 * and in the mapping list of the associated page_t. 1283 */ 1284 static int 1285 hati_pte_map( 1286 htable_t *ht, 1287 uint_t entry, 1288 page_t *pp, 1289 x86pte_t pte, 1290 int flags, 1291 void *pte_ptr) 1292 { 1293 hat_t *hat = ht->ht_hat; 1294 x86pte_t old_pte; 1295 level_t l = ht->ht_level; 1296 hment_t *hm; 1297 uint_t is_consist; 1298 uint_t is_locked; 1299 int rv = 0; 1300 1301 /* 1302 * Is this a consistant (ie. need mapping list lock) mapping? 1303 */ 1304 is_consist = (pp != NULL && (flags & HAT_LOAD_NOCONSIST) == 0); 1305 1306 /* 1307 * Track locked mapping count in the htable. Do this first, 1308 * as we track locking even if there already is a mapping present. 1309 */ 1310 is_locked = (flags & HAT_LOAD_LOCK) != 0 && hat != kas.a_hat; 1311 if (is_locked) 1312 HTABLE_LOCK_INC(ht); 1313 1314 /* 1315 * Acquire the page's mapping list lock and get an hment to use. 1316 * Note that hment_prepare() might return NULL. 1317 */ 1318 if (is_consist) { 1319 x86_hm_enter(pp); 1320 hm = hment_prepare(ht, entry, pp); 1321 } 1322 1323 /* 1324 * Set the new pte, retrieving the old one at the same time. 1325 */ 1326 old_pte = x86pte_set(ht, entry, pte, pte_ptr); 1327 1328 /* 1329 * Did we get a large page / page table collision? 1330 */ 1331 if (old_pte == LPAGE_ERROR) { 1332 if (is_locked) 1333 HTABLE_LOCK_DEC(ht); 1334 rv = -1; 1335 goto done; 1336 } 1337 1338 /* 1339 * If the mapping didn't change there is nothing more to do. 1340 */ 1341 if (PTE_EQUIV(pte, old_pte)) 1342 goto done; 1343 1344 /* 1345 * Install a new mapping in the page's mapping list 1346 */ 1347 if (!PTE_ISVALID(old_pte)) { 1348 if (is_consist) { 1349 hment_assign(ht, entry, pp, hm); 1350 x86_hm_exit(pp); 1351 } else { 1352 ASSERT(flags & HAT_LOAD_NOCONSIST); 1353 } 1354 #if defined(__amd64) 1355 if (ht->ht_flags & HTABLE_VLP) { 1356 cpu_t *cpu = CPU; 1357 x86pte_t *vlpptep = cpu->cpu_hat_info->hci_vlp_l2ptes; 1358 VLP_COPY(hat->hat_vlp_ptes, vlpptep); 1359 } 1360 #endif 1361 HTABLE_INC(ht->ht_valid_cnt); 1362 PGCNT_INC(hat, l); 1363 return (rv); 1364 } 1365 1366 /* 1367 * Remap's are more complicated: 1368 * - HAT_LOAD_REMAP must be specified if changing the pfn. 1369 * We also require that NOCONSIST be specified. 1370 * - Otherwise only permission or caching bits may change. 1371 */ 1372 if (!PTE_ISPAGE(old_pte, l)) 1373 panic("non-null/page mapping pte=" FMT_PTE, old_pte); 1374 1375 if (PTE2PFN(old_pte, l) != PTE2PFN(pte, l)) { 1376 REMAPASSERT(flags & HAT_LOAD_REMAP); 1377 REMAPASSERT(flags & HAT_LOAD_NOCONSIST); 1378 REMAPASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST); 1379 REMAPASSERT(pf_is_memory(PTE2PFN(old_pte, l)) == 1380 pf_is_memory(PTE2PFN(pte, l))); 1381 REMAPASSERT(!is_consist); 1382 } 1383 1384 /* 1385 * We only let remaps change the certain bits in the PTE. 1386 */ 1387 if (PTE_GET(old_pte, ~PT_REMAP_BITS) != PTE_GET(pte, ~PT_REMAP_BITS)) 1388 panic("remap bits changed: old_pte="FMT_PTE", pte="FMT_PTE"\n", 1389 old_pte, pte); 1390 1391 /* 1392 * We don't create any mapping list entries on a remap, so release 1393 * any allocated hment after we drop the mapping list lock. 1394 */ 1395 done: 1396 if (is_consist) { 1397 x86_hm_exit(pp); 1398 if (hm != NULL) 1399 hment_free(hm); 1400 } 1401 return (rv); 1402 } 1403 1404 /* 1405 * Internal routine to load a single page table entry. This only fails if 1406 * we attempt to overwrite a page table link with a large page. 1407 */ 1408 static int 1409 hati_load_common( 1410 hat_t *hat, 1411 uintptr_t va, 1412 page_t *pp, 1413 uint_t attr, 1414 uint_t flags, 1415 level_t level, 1416 pfn_t pfn) 1417 { 1418 htable_t *ht; 1419 uint_t entry; 1420 x86pte_t pte; 1421 int rv = 0; 1422 1423 /* 1424 * The number 16 is arbitrary and here to catch a recursion problem 1425 * early before we blow out the kernel stack. 1426 */ 1427 ++curthread->t_hatdepth; 1428 ASSERT(curthread->t_hatdepth < 16); 1429 1430 ASSERT(hat == kas.a_hat || 1431 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1432 1433 if (flags & HAT_LOAD_SHARE) 1434 hat->hat_flags |= HAT_SHARED; 1435 1436 /* 1437 * Find the page table that maps this page if it already exists. 1438 */ 1439 ht = htable_lookup(hat, va, level); 1440 1441 /* 1442 * We must have HAT_LOAD_NOCONSIST if page_t is NULL. 1443 */ 1444 if (pp == NULL) 1445 flags |= HAT_LOAD_NOCONSIST; 1446 1447 if (ht == NULL) { 1448 ht = htable_create(hat, va, level, NULL); 1449 ASSERT(ht != NULL); 1450 } 1451 entry = htable_va2entry(va, ht); 1452 1453 /* 1454 * a bunch of paranoid error checking 1455 */ 1456 ASSERT(ht->ht_busy > 0); 1457 if (ht->ht_vaddr > va || va > HTABLE_LAST_PAGE(ht)) 1458 panic("hati_load_common: bad htable %p, va %p", 1459 (void *)ht, (void *)va); 1460 ASSERT(ht->ht_level == level); 1461 1462 /* 1463 * construct the new PTE 1464 */ 1465 if (hat == kas.a_hat) 1466 attr &= ~PROT_USER; 1467 pte = hati_mkpte(pfn, attr, level, flags); 1468 if (hat == kas.a_hat && va >= kernelbase) 1469 PTE_SET(pte, mmu.pt_global); 1470 1471 /* 1472 * establish the mapping 1473 */ 1474 rv = hati_pte_map(ht, entry, pp, pte, flags, NULL); 1475 1476 /* 1477 * release the htable and any reserves 1478 */ 1479 htable_release(ht); 1480 --curthread->t_hatdepth; 1481 return (rv); 1482 } 1483 1484 /* 1485 * special case of hat_memload to deal with some kernel addrs for performance 1486 */ 1487 static void 1488 hat_kmap_load( 1489 caddr_t addr, 1490 page_t *pp, 1491 uint_t attr, 1492 uint_t flags) 1493 { 1494 uintptr_t va = (uintptr_t)addr; 1495 x86pte_t pte; 1496 pfn_t pfn = page_pptonum(pp); 1497 pgcnt_t pg_off = mmu_btop(va - mmu.kmap_addr); 1498 htable_t *ht; 1499 uint_t entry; 1500 void *pte_ptr; 1501 1502 /* 1503 * construct the requested PTE 1504 */ 1505 attr &= ~PROT_USER; 1506 attr |= HAT_STORECACHING_OK; 1507 pte = hati_mkpte(pfn, attr, 0, flags); 1508 PTE_SET(pte, mmu.pt_global); 1509 1510 /* 1511 * Figure out the pte_ptr and htable and use common code to finish up 1512 */ 1513 if (mmu.pae_hat) 1514 pte_ptr = mmu.kmap_ptes + pg_off; 1515 else 1516 pte_ptr = (x86pte32_t *)mmu.kmap_ptes + pg_off; 1517 ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) >> 1518 LEVEL_SHIFT(1)]; 1519 entry = htable_va2entry(va, ht); 1520 ++curthread->t_hatdepth; 1521 ASSERT(curthread->t_hatdepth < 16); 1522 (void) hati_pte_map(ht, entry, pp, pte, flags, pte_ptr); 1523 --curthread->t_hatdepth; 1524 } 1525 1526 /* 1527 * hat_memload() - load a translation to the given page struct 1528 * 1529 * Flags for hat_memload/hat_devload/hat_*attr. 1530 * 1531 * HAT_LOAD Default flags to load a translation to the page. 1532 * 1533 * HAT_LOAD_LOCK Lock down mapping resources; hat_map(), hat_memload(), 1534 * and hat_devload(). 1535 * 1536 * HAT_LOAD_NOCONSIST Do not add mapping to page_t mapping list. 1537 * sets PT_NOCONSIST 1538 * 1539 * HAT_LOAD_SHARE A flag to hat_memload() to indicate h/w page tables 1540 * that map some user pages (not kas) is shared by more 1541 * than one process (eg. ISM). 1542 * 1543 * HAT_LOAD_REMAP Reload a valid pte with a different page frame. 1544 * 1545 * HAT_NO_KALLOC Do not kmem_alloc while creating the mapping; at this 1546 * point, it's setting up mapping to allocate internal 1547 * hat layer data structures. This flag forces hat layer 1548 * to tap its reserves in order to prevent infinite 1549 * recursion. 1550 * 1551 * The following is a protection attribute (like PROT_READ, etc.) 1552 * 1553 * HAT_NOSYNC set PT_NOSYNC - this mapping's ref/mod bits 1554 * are never cleared. 1555 * 1556 * Installing new valid PTE's and creation of the mapping list 1557 * entry are controlled under the same lock. It's derived from the 1558 * page_t being mapped. 1559 */ 1560 static uint_t supported_memload_flags = 1561 HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST | 1562 HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT; 1563 1564 void 1565 hat_memload( 1566 hat_t *hat, 1567 caddr_t addr, 1568 page_t *pp, 1569 uint_t attr, 1570 uint_t flags) 1571 { 1572 uintptr_t va = (uintptr_t)addr; 1573 level_t level = 0; 1574 pfn_t pfn = page_pptonum(pp); 1575 1576 XPV_DISALLOW_MIGRATE(); 1577 ASSERT(IS_PAGEALIGNED(va)); 1578 ASSERT(hat == kas.a_hat || va < _userlimit); 1579 ASSERT(hat == kas.a_hat || 1580 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1581 ASSERT((flags & supported_memload_flags) == flags); 1582 1583 ASSERT(!IN_VA_HOLE(va)); 1584 ASSERT(!PP_ISFREE(pp)); 1585 1586 /* 1587 * kernel address special case for performance. 1588 */ 1589 if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) { 1590 ASSERT(hat == kas.a_hat); 1591 hat_kmap_load(addr, pp, attr, flags); 1592 XPV_ALLOW_MIGRATE(); 1593 return; 1594 } 1595 1596 /* 1597 * This is used for memory with normal caching enabled, so 1598 * always set HAT_STORECACHING_OK. 1599 */ 1600 attr |= HAT_STORECACHING_OK; 1601 if (hati_load_common(hat, va, pp, attr, flags, level, pfn) != 0) 1602 panic("unexpected hati_load_common() failure"); 1603 XPV_ALLOW_MIGRATE(); 1604 } 1605 1606 /* ARGSUSED */ 1607 void 1608 hat_memload_region(struct hat *hat, caddr_t addr, struct page *pp, 1609 uint_t attr, uint_t flags, hat_region_cookie_t rcookie) 1610 { 1611 hat_memload(hat, addr, pp, attr, flags); 1612 } 1613 1614 /* 1615 * Load the given array of page structs using large pages when possible 1616 */ 1617 void 1618 hat_memload_array( 1619 hat_t *hat, 1620 caddr_t addr, 1621 size_t len, 1622 page_t **pages, 1623 uint_t attr, 1624 uint_t flags) 1625 { 1626 uintptr_t va = (uintptr_t)addr; 1627 uintptr_t eaddr = va + len; 1628 level_t level; 1629 size_t pgsize; 1630 pgcnt_t pgindx = 0; 1631 pfn_t pfn; 1632 pgcnt_t i; 1633 1634 XPV_DISALLOW_MIGRATE(); 1635 ASSERT(IS_PAGEALIGNED(va)); 1636 ASSERT(hat == kas.a_hat || va + len <= _userlimit); 1637 ASSERT(hat == kas.a_hat || 1638 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1639 ASSERT((flags & supported_memload_flags) == flags); 1640 1641 /* 1642 * memload is used for memory with full caching enabled, so 1643 * set HAT_STORECACHING_OK. 1644 */ 1645 attr |= HAT_STORECACHING_OK; 1646 1647 /* 1648 * handle all pages using largest possible pagesize 1649 */ 1650 while (va < eaddr) { 1651 /* 1652 * decide what level mapping to use (ie. pagesize) 1653 */ 1654 pfn = page_pptonum(pages[pgindx]); 1655 for (level = mmu.max_page_level; ; --level) { 1656 pgsize = LEVEL_SIZE(level); 1657 if (level == 0) 1658 break; 1659 1660 if (!IS_P2ALIGNED(va, pgsize) || 1661 (eaddr - va) < pgsize || 1662 !IS_P2ALIGNED(pfn_to_pa(pfn), pgsize)) 1663 continue; 1664 1665 /* 1666 * To use a large mapping of this size, all the 1667 * pages we are passed must be sequential subpages 1668 * of the large page. 1669 * hat_page_demote() can't change p_szc because 1670 * all pages are locked. 1671 */ 1672 if (pages[pgindx]->p_szc >= level) { 1673 for (i = 0; i < mmu_btop(pgsize); ++i) { 1674 if (pfn + i != 1675 page_pptonum(pages[pgindx + i])) 1676 break; 1677 ASSERT(pages[pgindx + i]->p_szc >= 1678 level); 1679 ASSERT(pages[pgindx] + i == 1680 pages[pgindx + i]); 1681 } 1682 if (i == mmu_btop(pgsize)) { 1683 #ifdef DEBUG 1684 if (level == 2) 1685 map1gcnt++; 1686 #endif 1687 break; 1688 } 1689 } 1690 } 1691 1692 /* 1693 * Load this page mapping. If the load fails, try a smaller 1694 * pagesize. 1695 */ 1696 ASSERT(!IN_VA_HOLE(va)); 1697 while (hati_load_common(hat, va, pages[pgindx], attr, 1698 flags, level, pfn) != 0) { 1699 if (level == 0) 1700 panic("unexpected hati_load_common() failure"); 1701 --level; 1702 pgsize = LEVEL_SIZE(level); 1703 } 1704 1705 /* 1706 * move to next page 1707 */ 1708 va += pgsize; 1709 pgindx += mmu_btop(pgsize); 1710 } 1711 XPV_ALLOW_MIGRATE(); 1712 } 1713 1714 /* ARGSUSED */ 1715 void 1716 hat_memload_array_region(struct hat *hat, caddr_t addr, size_t len, 1717 struct page **pps, uint_t attr, uint_t flags, 1718 hat_region_cookie_t rcookie) 1719 { 1720 hat_memload_array(hat, addr, len, pps, attr, flags); 1721 } 1722 1723 /* 1724 * void hat_devload(hat, addr, len, pf, attr, flags) 1725 * load/lock the given page frame number 1726 * 1727 * Advisory ordering attributes. Apply only to device mappings. 1728 * 1729 * HAT_STRICTORDER: the CPU must issue the references in order, as the 1730 * programmer specified. This is the default. 1731 * HAT_UNORDERED_OK: the CPU may reorder the references (this is all kinds 1732 * of reordering; store or load with store or load). 1733 * HAT_MERGING_OK: merging and batching: the CPU may merge individual stores 1734 * to consecutive locations (for example, turn two consecutive byte 1735 * stores into one halfword store), and it may batch individual loads 1736 * (for example, turn two consecutive byte loads into one halfword load). 1737 * This also implies re-ordering. 1738 * HAT_LOADCACHING_OK: the CPU may cache the data it fetches and reuse it 1739 * until another store occurs. The default is to fetch new data 1740 * on every load. This also implies merging. 1741 * HAT_STORECACHING_OK: the CPU may keep the data in the cache and push it to 1742 * the device (perhaps with other data) at a later time. The default is 1743 * to push the data right away. This also implies load caching. 1744 * 1745 * Equivalent of hat_memload(), but can be used for device memory where 1746 * there are no page_t's and we support additional flags (write merging, etc). 1747 * Note that we can have large page mappings with this interface. 1748 */ 1749 int supported_devload_flags = HAT_LOAD | HAT_LOAD_LOCK | 1750 HAT_LOAD_NOCONSIST | HAT_STRICTORDER | HAT_UNORDERED_OK | 1751 HAT_MERGING_OK | HAT_LOADCACHING_OK | HAT_STORECACHING_OK; 1752 1753 void 1754 hat_devload( 1755 hat_t *hat, 1756 caddr_t addr, 1757 size_t len, 1758 pfn_t pfn, 1759 uint_t attr, 1760 int flags) 1761 { 1762 uintptr_t va = ALIGN2PAGE(addr); 1763 uintptr_t eva = va + len; 1764 level_t level; 1765 size_t pgsize; 1766 page_t *pp; 1767 int f; /* per PTE copy of flags - maybe modified */ 1768 uint_t a; /* per PTE copy of attr */ 1769 1770 XPV_DISALLOW_MIGRATE(); 1771 ASSERT(IS_PAGEALIGNED(va)); 1772 ASSERT(hat == kas.a_hat || eva <= _userlimit); 1773 ASSERT(hat == kas.a_hat || 1774 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1775 ASSERT((flags & supported_devload_flags) == flags); 1776 1777 /* 1778 * handle all pages 1779 */ 1780 while (va < eva) { 1781 1782 /* 1783 * decide what level mapping to use (ie. pagesize) 1784 */ 1785 for (level = mmu.max_page_level; ; --level) { 1786 pgsize = LEVEL_SIZE(level); 1787 if (level == 0) 1788 break; 1789 if (IS_P2ALIGNED(va, pgsize) && 1790 (eva - va) >= pgsize && 1791 IS_P2ALIGNED(pfn, mmu_btop(pgsize))) { 1792 #ifdef DEBUG 1793 if (level == 2) 1794 map1gcnt++; 1795 #endif 1796 break; 1797 } 1798 } 1799 1800 /* 1801 * If this is just memory then allow caching (this happens 1802 * for the nucleus pages) - though HAT_PLAT_NOCACHE can be used 1803 * to override that. If we don't have a page_t then make sure 1804 * NOCONSIST is set. 1805 */ 1806 a = attr; 1807 f = flags; 1808 if (!pf_is_memory(pfn)) 1809 f |= HAT_LOAD_NOCONSIST; 1810 else if (!(a & HAT_PLAT_NOCACHE)) 1811 a |= HAT_STORECACHING_OK; 1812 1813 if (f & HAT_LOAD_NOCONSIST) 1814 pp = NULL; 1815 else 1816 pp = page_numtopp_nolock(pfn); 1817 1818 /* 1819 * Check to make sure we are really trying to map a valid 1820 * memory page. The caller wishing to intentionally map 1821 * free memory pages will have passed the HAT_LOAD_NOCONSIST 1822 * flag, then pp will be NULL. 1823 */ 1824 if (pp != NULL) { 1825 if (PP_ISFREE(pp)) { 1826 panic("hat_devload: loading " 1827 "a mapping to free page %p", (void *)pp); 1828 } 1829 1830 if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) { 1831 panic("hat_devload: loading a mapping " 1832 "to an unlocked page %p", 1833 (void *)pp); 1834 } 1835 } 1836 1837 /* 1838 * load this page mapping 1839 */ 1840 ASSERT(!IN_VA_HOLE(va)); 1841 while (hati_load_common(hat, va, pp, a, f, level, pfn) != 0) { 1842 if (level == 0) 1843 panic("unexpected hati_load_common() failure"); 1844 --level; 1845 pgsize = LEVEL_SIZE(level); 1846 } 1847 1848 /* 1849 * move to next page 1850 */ 1851 va += pgsize; 1852 pfn += mmu_btop(pgsize); 1853 } 1854 XPV_ALLOW_MIGRATE(); 1855 } 1856 1857 /* 1858 * void hat_unlock(hat, addr, len) 1859 * unlock the mappings to a given range of addresses 1860 * 1861 * Locks are tracked by ht_lock_cnt in the htable. 1862 */ 1863 void 1864 hat_unlock(hat_t *hat, caddr_t addr, size_t len) 1865 { 1866 uintptr_t vaddr = (uintptr_t)addr; 1867 uintptr_t eaddr = vaddr + len; 1868 htable_t *ht = NULL; 1869 1870 /* 1871 * kernel entries are always locked, we don't track lock counts 1872 */ 1873 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); 1874 ASSERT(IS_PAGEALIGNED(vaddr)); 1875 ASSERT(IS_PAGEALIGNED(eaddr)); 1876 if (hat == kas.a_hat) 1877 return; 1878 if (eaddr > _userlimit) 1879 panic("hat_unlock() address out of range - above _userlimit"); 1880 1881 XPV_DISALLOW_MIGRATE(); 1882 ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 1883 while (vaddr < eaddr) { 1884 (void) htable_walk(hat, &ht, &vaddr, eaddr); 1885 if (ht == NULL) 1886 break; 1887 1888 ASSERT(!IN_VA_HOLE(vaddr)); 1889 1890 if (ht->ht_lock_cnt < 1) 1891 panic("hat_unlock(): lock_cnt < 1, " 1892 "htable=%p, vaddr=%p\n", (void *)ht, (void *)vaddr); 1893 HTABLE_LOCK_DEC(ht); 1894 1895 vaddr += LEVEL_SIZE(ht->ht_level); 1896 } 1897 if (ht) 1898 htable_release(ht); 1899 XPV_ALLOW_MIGRATE(); 1900 } 1901 1902 /* ARGSUSED */ 1903 void 1904 hat_unlock_region(struct hat *hat, caddr_t addr, size_t len, 1905 hat_region_cookie_t rcookie) 1906 { 1907 panic("No shared region support on x86"); 1908 } 1909 1910 #if !defined(__xpv) 1911 /* 1912 * Cross call service routine to demap a virtual page on 1913 * the current CPU or flush all mappings in TLB. 1914 */ 1915 /*ARGSUSED*/ 1916 static int 1917 hati_demap_func(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3) 1918 { 1919 hat_t *hat = (hat_t *)a1; 1920 caddr_t addr = (caddr_t)a2; 1921 1922 /* 1923 * If the target hat isn't the kernel and this CPU isn't operating 1924 * in the target hat, we can ignore the cross call. 1925 */ 1926 if (hat != kas.a_hat && hat != CPU->cpu_current_hat) 1927 return (0); 1928 1929 /* 1930 * For a normal address, we just flush one page mapping 1931 */ 1932 if ((uintptr_t)addr != DEMAP_ALL_ADDR) { 1933 mmu_tlbflush_entry(addr); 1934 return (0); 1935 } 1936 1937 /* 1938 * Otherwise we reload cr3 to effect a complete TLB flush. 1939 * 1940 * A reload of cr3 on a VLP process also means we must also recopy in 1941 * the pte values from the struct hat 1942 */ 1943 if (hat->hat_flags & HAT_VLP) { 1944 #if defined(__amd64) 1945 x86pte_t *vlpptep = CPU->cpu_hat_info->hci_vlp_l2ptes; 1946 1947 VLP_COPY(hat->hat_vlp_ptes, vlpptep); 1948 #elif defined(__i386) 1949 reload_pae32(hat, CPU); 1950 #endif 1951 } 1952 reload_cr3(); 1953 return (0); 1954 } 1955 1956 /* 1957 * Flush all TLB entries, including global (ie. kernel) ones. 1958 */ 1959 static void 1960 flush_all_tlb_entries(void) 1961 { 1962 ulong_t cr4 = getcr4(); 1963 1964 if (cr4 & CR4_PGE) { 1965 setcr4(cr4 & ~(ulong_t)CR4_PGE); 1966 setcr4(cr4); 1967 1968 /* 1969 * 32 bit PAE also needs to always reload_cr3() 1970 */ 1971 if (mmu.max_level == 2) 1972 reload_cr3(); 1973 } else { 1974 reload_cr3(); 1975 } 1976 } 1977 1978 #define TLB_CPU_HALTED (01ul) 1979 #define TLB_INVAL_ALL (02ul) 1980 #define CAS_TLB_INFO(cpu, old, new) \ 1981 caslong((ulong_t *)&(cpu)->cpu_m.mcpu_tlb_info, (old), (new)) 1982 1983 /* 1984 * Record that a CPU is going idle 1985 */ 1986 void 1987 tlb_going_idle(void) 1988 { 1989 atomic_or_long((ulong_t *)&CPU->cpu_m.mcpu_tlb_info, TLB_CPU_HALTED); 1990 } 1991 1992 /* 1993 * Service a delayed TLB flush if coming out of being idle. 1994 */ 1995 void 1996 tlb_service(void) 1997 { 1998 ulong_t flags = getflags(); 1999 ulong_t tlb_info; 2000 ulong_t found; 2001 2002 /* 2003 * Be sure interrupts are off while doing this so that 2004 * higher level interrupts correctly wait for flushes to finish. 2005 */ 2006 if (flags & PS_IE) 2007 flags = intr_clear(); 2008 2009 /* 2010 * We only have to do something if coming out of being idle. 2011 */ 2012 tlb_info = CPU->cpu_m.mcpu_tlb_info; 2013 if (tlb_info & TLB_CPU_HALTED) { 2014 ASSERT(CPU->cpu_current_hat == kas.a_hat); 2015 2016 /* 2017 * Atomic clear and fetch of old state. 2018 */ 2019 while ((found = CAS_TLB_INFO(CPU, tlb_info, 0)) != tlb_info) { 2020 ASSERT(found & TLB_CPU_HALTED); 2021 tlb_info = found; 2022 SMT_PAUSE(); 2023 } 2024 if (tlb_info & TLB_INVAL_ALL) 2025 flush_all_tlb_entries(); 2026 } 2027 2028 /* 2029 * Restore interrupt enable control bit. 2030 */ 2031 if (flags & PS_IE) 2032 sti(); 2033 } 2034 #endif /* !__xpv */ 2035 2036 /* 2037 * Internal routine to do cross calls to invalidate a range of pages on 2038 * all CPUs using a given hat. 2039 */ 2040 void 2041 hat_tlb_inval(hat_t *hat, uintptr_t va) 2042 { 2043 extern int flushes_require_xcalls; /* from mp_startup.c */ 2044 cpuset_t justme; 2045 cpuset_t cpus_to_shootdown; 2046 #ifndef __xpv 2047 cpuset_t check_cpus; 2048 cpu_t *cpup; 2049 int c; 2050 #endif 2051 2052 /* 2053 * If the hat is being destroyed, there are no more users, so 2054 * demap need not do anything. 2055 */ 2056 if (hat->hat_flags & HAT_FREEING) 2057 return; 2058 2059 /* 2060 * If demapping from a shared pagetable, we best demap the 2061 * entire set of user TLBs, since we don't know what addresses 2062 * these were shared at. 2063 */ 2064 if (hat->hat_flags & HAT_SHARED) { 2065 hat = kas.a_hat; 2066 va = DEMAP_ALL_ADDR; 2067 } 2068 2069 /* 2070 * if not running with multiple CPUs, don't use cross calls 2071 */ 2072 if (panicstr || !flushes_require_xcalls) { 2073 #ifdef __xpv 2074 if (va == DEMAP_ALL_ADDR) 2075 xen_flush_tlb(); 2076 else 2077 xen_flush_va((caddr_t)va); 2078 #else 2079 (void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL); 2080 #endif 2081 return; 2082 } 2083 2084 2085 /* 2086 * Determine CPUs to shootdown. Kernel changes always do all CPUs. 2087 * Otherwise it's just CPUs currently executing in this hat. 2088 */ 2089 kpreempt_disable(); 2090 CPUSET_ONLY(justme, CPU->cpu_id); 2091 if (hat == kas.a_hat) 2092 cpus_to_shootdown = khat_cpuset; 2093 else 2094 cpus_to_shootdown = hat->hat_cpus; 2095 2096 #ifndef __xpv 2097 /* 2098 * If any CPUs in the set are idle, just request a delayed flush 2099 * and avoid waking them up. 2100 */ 2101 check_cpus = cpus_to_shootdown; 2102 for (c = 0; c < NCPU && !CPUSET_ISNULL(check_cpus); ++c) { 2103 ulong_t tlb_info; 2104 2105 if (!CPU_IN_SET(check_cpus, c)) 2106 continue; 2107 CPUSET_DEL(check_cpus, c); 2108 cpup = cpu[c]; 2109 if (cpup == NULL) 2110 continue; 2111 2112 tlb_info = cpup->cpu_m.mcpu_tlb_info; 2113 while (tlb_info == TLB_CPU_HALTED) { 2114 (void) CAS_TLB_INFO(cpup, TLB_CPU_HALTED, 2115 TLB_CPU_HALTED | TLB_INVAL_ALL); 2116 SMT_PAUSE(); 2117 tlb_info = cpup->cpu_m.mcpu_tlb_info; 2118 } 2119 if (tlb_info == (TLB_CPU_HALTED | TLB_INVAL_ALL)) { 2120 HATSTAT_INC(hs_tlb_inval_delayed); 2121 CPUSET_DEL(cpus_to_shootdown, c); 2122 } 2123 } 2124 #endif 2125 2126 if (CPUSET_ISNULL(cpus_to_shootdown) || 2127 CPUSET_ISEQUAL(cpus_to_shootdown, justme)) { 2128 2129 #ifdef __xpv 2130 if (va == DEMAP_ALL_ADDR) 2131 xen_flush_tlb(); 2132 else 2133 xen_flush_va((caddr_t)va); 2134 #else 2135 (void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL); 2136 #endif 2137 2138 } else { 2139 2140 CPUSET_ADD(cpus_to_shootdown, CPU->cpu_id); 2141 #ifdef __xpv 2142 if (va == DEMAP_ALL_ADDR) 2143 xen_gflush_tlb(cpus_to_shootdown); 2144 else 2145 xen_gflush_va((caddr_t)va, cpus_to_shootdown); 2146 #else 2147 xc_call((xc_arg_t)hat, (xc_arg_t)va, NULL, X_CALL_HIPRI, 2148 cpus_to_shootdown, hati_demap_func); 2149 #endif 2150 2151 } 2152 kpreempt_enable(); 2153 } 2154 2155 /* 2156 * Interior routine for HAT_UNLOADs from hat_unload_callback(), 2157 * hat_kmap_unload() OR from hat_steal() code. This routine doesn't 2158 * handle releasing of the htables. 2159 */ 2160 void 2161 hat_pte_unmap( 2162 htable_t *ht, 2163 uint_t entry, 2164 uint_t flags, 2165 x86pte_t old_pte, 2166 void *pte_ptr) 2167 { 2168 hat_t *hat = ht->ht_hat; 2169 hment_t *hm = NULL; 2170 page_t *pp = NULL; 2171 level_t l = ht->ht_level; 2172 pfn_t pfn; 2173 2174 /* 2175 * We always track the locking counts, even if nothing is unmapped 2176 */ 2177 if ((flags & HAT_UNLOAD_UNLOCK) != 0 && hat != kas.a_hat) { 2178 ASSERT(ht->ht_lock_cnt > 0); 2179 HTABLE_LOCK_DEC(ht); 2180 } 2181 2182 /* 2183 * Figure out which page's mapping list lock to acquire using the PFN 2184 * passed in "old" PTE. We then attempt to invalidate the PTE. 2185 * If another thread, probably a hat_pageunload, has asynchronously 2186 * unmapped/remapped this address we'll loop here. 2187 */ 2188 ASSERT(ht->ht_busy > 0); 2189 while (PTE_ISVALID(old_pte)) { 2190 pfn = PTE2PFN(old_pte, l); 2191 if (PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST) { 2192 pp = NULL; 2193 } else { 2194 #ifdef __xpv 2195 if (pfn == PFN_INVALID) 2196 panic("Invalid PFN, but not PT_NOCONSIST"); 2197 #endif 2198 pp = page_numtopp_nolock(pfn); 2199 if (pp == NULL) { 2200 panic("no page_t, not NOCONSIST: old_pte=" 2201 FMT_PTE " ht=%lx entry=0x%x pte_ptr=%lx", 2202 old_pte, (uintptr_t)ht, entry, 2203 (uintptr_t)pte_ptr); 2204 } 2205 x86_hm_enter(pp); 2206 } 2207 2208 /* 2209 * If freeing the address space, check that the PTE 2210 * hasn't changed, as the mappings are no longer in use by 2211 * any thread, invalidation is unnecessary. 2212 * If not freeing, do a full invalidate. 2213 * 2214 * On the hypervisor we must always remove mappings, as a 2215 * writable mapping left behind could cause a page table 2216 * allocation to fail. 2217 */ 2218 #if !defined(__xpv) 2219 if (hat->hat_flags & HAT_FREEING) 2220 old_pte = x86pte_get(ht, entry); 2221 else 2222 #endif 2223 old_pte = x86pte_inval(ht, entry, old_pte, pte_ptr); 2224 2225 /* 2226 * If the page hadn't changed we've unmapped it and can proceed 2227 */ 2228 if (PTE_ISVALID(old_pte) && PTE2PFN(old_pte, l) == pfn) 2229 break; 2230 2231 /* 2232 * Otherwise, we'll have to retry with the current old_pte. 2233 * Drop the hment lock, since the pfn may have changed. 2234 */ 2235 if (pp != NULL) { 2236 x86_hm_exit(pp); 2237 pp = NULL; 2238 } else { 2239 ASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST); 2240 } 2241 } 2242 2243 /* 2244 * If the old mapping wasn't valid, there's nothing more to do 2245 */ 2246 if (!PTE_ISVALID(old_pte)) { 2247 if (pp != NULL) 2248 x86_hm_exit(pp); 2249 return; 2250 } 2251 2252 /* 2253 * Take care of syncing any MOD/REF bits and removing the hment. 2254 */ 2255 if (pp != NULL) { 2256 if (!(flags & HAT_UNLOAD_NOSYNC)) 2257 hati_sync_pte_to_page(pp, old_pte, l); 2258 hm = hment_remove(pp, ht, entry); 2259 x86_hm_exit(pp); 2260 if (hm != NULL) 2261 hment_free(hm); 2262 } 2263 2264 /* 2265 * Handle book keeping in the htable and hat 2266 */ 2267 ASSERT(ht->ht_valid_cnt > 0); 2268 HTABLE_DEC(ht->ht_valid_cnt); 2269 PGCNT_DEC(hat, l); 2270 } 2271 2272 /* 2273 * very cheap unload implementation to special case some kernel addresses 2274 */ 2275 static void 2276 hat_kmap_unload(caddr_t addr, size_t len, uint_t flags) 2277 { 2278 uintptr_t va = (uintptr_t)addr; 2279 uintptr_t eva = va + len; 2280 pgcnt_t pg_index; 2281 htable_t *ht; 2282 uint_t entry; 2283 x86pte_t *pte_ptr; 2284 x86pte_t old_pte; 2285 2286 for (; va < eva; va += MMU_PAGESIZE) { 2287 /* 2288 * Get the PTE 2289 */ 2290 pg_index = mmu_btop(va - mmu.kmap_addr); 2291 pte_ptr = PT_INDEX_PTR(mmu.kmap_ptes, pg_index); 2292 old_pte = GET_PTE(pte_ptr); 2293 2294 /* 2295 * get the htable / entry 2296 */ 2297 ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) 2298 >> LEVEL_SHIFT(1)]; 2299 entry = htable_va2entry(va, ht); 2300 2301 /* 2302 * use mostly common code to unmap it. 2303 */ 2304 hat_pte_unmap(ht, entry, flags, old_pte, pte_ptr); 2305 } 2306 } 2307 2308 2309 /* 2310 * unload a range of virtual address space (no callback) 2311 */ 2312 void 2313 hat_unload(hat_t *hat, caddr_t addr, size_t len, uint_t flags) 2314 { 2315 uintptr_t va = (uintptr_t)addr; 2316 2317 XPV_DISALLOW_MIGRATE(); 2318 ASSERT(hat == kas.a_hat || va + len <= _userlimit); 2319 2320 /* 2321 * special case for performance. 2322 */ 2323 if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) { 2324 ASSERT(hat == kas.a_hat); 2325 hat_kmap_unload(addr, len, flags); 2326 } else { 2327 hat_unload_callback(hat, addr, len, flags, NULL); 2328 } 2329 XPV_ALLOW_MIGRATE(); 2330 } 2331 2332 /* 2333 * Do the callbacks for ranges being unloaded. 2334 */ 2335 typedef struct range_info { 2336 uintptr_t rng_va; 2337 ulong_t rng_cnt; 2338 level_t rng_level; 2339 } range_info_t; 2340 2341 static void 2342 handle_ranges(hat_callback_t *cb, uint_t cnt, range_info_t *range) 2343 { 2344 /* 2345 * do callbacks to upper level VM system 2346 */ 2347 while (cb != NULL && cnt > 0) { 2348 --cnt; 2349 cb->hcb_start_addr = (caddr_t)range[cnt].rng_va; 2350 cb->hcb_end_addr = cb->hcb_start_addr; 2351 cb->hcb_end_addr += 2352 range[cnt].rng_cnt << LEVEL_SIZE(range[cnt].rng_level); 2353 cb->hcb_function(cb); 2354 } 2355 } 2356 2357 /* 2358 * Unload a given range of addresses (has optional callback) 2359 * 2360 * Flags: 2361 * define HAT_UNLOAD 0x00 2362 * define HAT_UNLOAD_NOSYNC 0x02 2363 * define HAT_UNLOAD_UNLOCK 0x04 2364 * define HAT_UNLOAD_OTHER 0x08 - not used 2365 * define HAT_UNLOAD_UNMAP 0x10 - same as HAT_UNLOAD 2366 */ 2367 #define MAX_UNLOAD_CNT (8) 2368 void 2369 hat_unload_callback( 2370 hat_t *hat, 2371 caddr_t addr, 2372 size_t len, 2373 uint_t flags, 2374 hat_callback_t *cb) 2375 { 2376 uintptr_t vaddr = (uintptr_t)addr; 2377 uintptr_t eaddr = vaddr + len; 2378 htable_t *ht = NULL; 2379 uint_t entry; 2380 uintptr_t contig_va = (uintptr_t)-1L; 2381 range_info_t r[MAX_UNLOAD_CNT]; 2382 uint_t r_cnt = 0; 2383 x86pte_t old_pte; 2384 2385 XPV_DISALLOW_MIGRATE(); 2386 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); 2387 ASSERT(IS_PAGEALIGNED(vaddr)); 2388 ASSERT(IS_PAGEALIGNED(eaddr)); 2389 2390 /* 2391 * Special case a single page being unloaded for speed. This happens 2392 * quite frequently, COW faults after a fork() for example. 2393 */ 2394 if (cb == NULL && len == MMU_PAGESIZE) { 2395 ht = htable_getpte(hat, vaddr, &entry, &old_pte, 0); 2396 if (ht != NULL) { 2397 if (PTE_ISVALID(old_pte)) 2398 hat_pte_unmap(ht, entry, flags, old_pte, NULL); 2399 htable_release(ht); 2400 } 2401 XPV_ALLOW_MIGRATE(); 2402 return; 2403 } 2404 2405 while (vaddr < eaddr) { 2406 old_pte = htable_walk(hat, &ht, &vaddr, eaddr); 2407 if (ht == NULL) 2408 break; 2409 2410 ASSERT(!IN_VA_HOLE(vaddr)); 2411 2412 if (vaddr < (uintptr_t)addr) 2413 panic("hat_unload_callback(): unmap inside large page"); 2414 2415 /* 2416 * We'll do the call backs for contiguous ranges 2417 */ 2418 if (vaddr != contig_va || 2419 (r_cnt > 0 && r[r_cnt - 1].rng_level != ht->ht_level)) { 2420 if (r_cnt == MAX_UNLOAD_CNT) { 2421 handle_ranges(cb, r_cnt, r); 2422 r_cnt = 0; 2423 } 2424 r[r_cnt].rng_va = vaddr; 2425 r[r_cnt].rng_cnt = 0; 2426 r[r_cnt].rng_level = ht->ht_level; 2427 ++r_cnt; 2428 } 2429 2430 /* 2431 * Unload one mapping from the page tables. 2432 */ 2433 entry = htable_va2entry(vaddr, ht); 2434 hat_pte_unmap(ht, entry, flags, old_pte, NULL); 2435 ASSERT(ht->ht_level <= mmu.max_page_level); 2436 vaddr += LEVEL_SIZE(ht->ht_level); 2437 contig_va = vaddr; 2438 ++r[r_cnt - 1].rng_cnt; 2439 } 2440 if (ht) 2441 htable_release(ht); 2442 2443 /* 2444 * handle last range for callbacks 2445 */ 2446 if (r_cnt > 0) 2447 handle_ranges(cb, r_cnt, r); 2448 XPV_ALLOW_MIGRATE(); 2449 } 2450 2451 /* 2452 * synchronize mapping with software data structures 2453 * 2454 * This interface is currently only used by the working set monitor 2455 * driver. 2456 */ 2457 /*ARGSUSED*/ 2458 void 2459 hat_sync(hat_t *hat, caddr_t addr, size_t len, uint_t flags) 2460 { 2461 uintptr_t vaddr = (uintptr_t)addr; 2462 uintptr_t eaddr = vaddr + len; 2463 htable_t *ht = NULL; 2464 uint_t entry; 2465 x86pte_t pte; 2466 x86pte_t save_pte; 2467 x86pte_t new; 2468 page_t *pp; 2469 2470 ASSERT(!IN_VA_HOLE(vaddr)); 2471 ASSERT(IS_PAGEALIGNED(vaddr)); 2472 ASSERT(IS_PAGEALIGNED(eaddr)); 2473 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); 2474 2475 XPV_DISALLOW_MIGRATE(); 2476 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) { 2477 try_again: 2478 pte = htable_walk(hat, &ht, &vaddr, eaddr); 2479 if (ht == NULL) 2480 break; 2481 entry = htable_va2entry(vaddr, ht); 2482 2483 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC || 2484 PTE_GET(pte, PT_REF | PT_MOD) == 0) 2485 continue; 2486 2487 /* 2488 * We need to acquire the mapping list lock to protect 2489 * against hat_pageunload(), hat_unload(), etc. 2490 */ 2491 pp = page_numtopp_nolock(PTE2PFN(pte, ht->ht_level)); 2492 if (pp == NULL) 2493 break; 2494 x86_hm_enter(pp); 2495 save_pte = pte; 2496 pte = x86pte_get(ht, entry); 2497 if (pte != save_pte) { 2498 x86_hm_exit(pp); 2499 goto try_again; 2500 } 2501 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC || 2502 PTE_GET(pte, PT_REF | PT_MOD) == 0) { 2503 x86_hm_exit(pp); 2504 continue; 2505 } 2506 2507 /* 2508 * Need to clear ref or mod bits. We may compete with 2509 * hardware updating the R/M bits and have to try again. 2510 */ 2511 if (flags == HAT_SYNC_ZERORM) { 2512 new = pte; 2513 PTE_CLR(new, PT_REF | PT_MOD); 2514 pte = hati_update_pte(ht, entry, pte, new); 2515 if (pte != 0) { 2516 x86_hm_exit(pp); 2517 goto try_again; 2518 } 2519 } else { 2520 /* 2521 * sync the PTE to the page_t 2522 */ 2523 hati_sync_pte_to_page(pp, save_pte, ht->ht_level); 2524 } 2525 x86_hm_exit(pp); 2526 } 2527 if (ht) 2528 htable_release(ht); 2529 XPV_ALLOW_MIGRATE(); 2530 } 2531 2532 /* 2533 * void hat_map(hat, addr, len, flags) 2534 */ 2535 /*ARGSUSED*/ 2536 void 2537 hat_map(hat_t *hat, caddr_t addr, size_t len, uint_t flags) 2538 { 2539 /* does nothing */ 2540 } 2541 2542 /* 2543 * uint_t hat_getattr(hat, addr, *attr) 2544 * returns attr for <hat,addr> in *attr. returns 0 if there was a 2545 * mapping and *attr is valid, nonzero if there was no mapping and 2546 * *attr is not valid. 2547 */ 2548 uint_t 2549 hat_getattr(hat_t *hat, caddr_t addr, uint_t *attr) 2550 { 2551 uintptr_t vaddr = ALIGN2PAGE(addr); 2552 htable_t *ht = NULL; 2553 x86pte_t pte; 2554 2555 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2556 2557 if (IN_VA_HOLE(vaddr)) 2558 return ((uint_t)-1); 2559 2560 ht = htable_getpte(hat, vaddr, NULL, &pte, mmu.max_page_level); 2561 if (ht == NULL) 2562 return ((uint_t)-1); 2563 2564 if (!PTE_ISVALID(pte) || !PTE_ISPAGE(pte, ht->ht_level)) { 2565 htable_release(ht); 2566 return ((uint_t)-1); 2567 } 2568 2569 *attr = PROT_READ; 2570 if (PTE_GET(pte, PT_WRITABLE)) 2571 *attr |= PROT_WRITE; 2572 if (PTE_GET(pte, PT_USER)) 2573 *attr |= PROT_USER; 2574 if (!PTE_GET(pte, mmu.pt_nx)) 2575 *attr |= PROT_EXEC; 2576 if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC) 2577 *attr |= HAT_NOSYNC; 2578 htable_release(ht); 2579 return (0); 2580 } 2581 2582 /* 2583 * hat_updateattr() applies the given attribute change to an existing mapping 2584 */ 2585 #define HAT_LOAD_ATTR 1 2586 #define HAT_SET_ATTR 2 2587 #define HAT_CLR_ATTR 3 2588 2589 static void 2590 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what) 2591 { 2592 uintptr_t vaddr = (uintptr_t)addr; 2593 uintptr_t eaddr = (uintptr_t)addr + len; 2594 htable_t *ht = NULL; 2595 uint_t entry; 2596 x86pte_t oldpte, newpte; 2597 page_t *pp; 2598 2599 XPV_DISALLOW_MIGRATE(); 2600 ASSERT(IS_PAGEALIGNED(vaddr)); 2601 ASSERT(IS_PAGEALIGNED(eaddr)); 2602 ASSERT(hat == kas.a_hat || 2603 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 2604 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) { 2605 try_again: 2606 oldpte = htable_walk(hat, &ht, &vaddr, eaddr); 2607 if (ht == NULL) 2608 break; 2609 if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST) 2610 continue; 2611 2612 pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level)); 2613 if (pp == NULL) 2614 continue; 2615 x86_hm_enter(pp); 2616 2617 newpte = oldpte; 2618 /* 2619 * We found a page table entry in the desired range, 2620 * figure out the new attributes. 2621 */ 2622 if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) { 2623 if ((attr & PROT_WRITE) && 2624 !PTE_GET(oldpte, PT_WRITABLE)) 2625 newpte |= PT_WRITABLE; 2626 2627 if ((attr & HAT_NOSYNC) && 2628 PTE_GET(oldpte, PT_SOFTWARE) < PT_NOSYNC) 2629 newpte |= PT_NOSYNC; 2630 2631 if ((attr & PROT_EXEC) && PTE_GET(oldpte, mmu.pt_nx)) 2632 newpte &= ~mmu.pt_nx; 2633 } 2634 2635 if (what == HAT_LOAD_ATTR) { 2636 if (!(attr & PROT_WRITE) && 2637 PTE_GET(oldpte, PT_WRITABLE)) 2638 newpte &= ~PT_WRITABLE; 2639 2640 if (!(attr & HAT_NOSYNC) && 2641 PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC) 2642 newpte &= ~PT_SOFTWARE; 2643 2644 if (!(attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx)) 2645 newpte |= mmu.pt_nx; 2646 } 2647 2648 if (what == HAT_CLR_ATTR) { 2649 if ((attr & PROT_WRITE) && PTE_GET(oldpte, PT_WRITABLE)) 2650 newpte &= ~PT_WRITABLE; 2651 2652 if ((attr & HAT_NOSYNC) && 2653 PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC) 2654 newpte &= ~PT_SOFTWARE; 2655 2656 if ((attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx)) 2657 newpte |= mmu.pt_nx; 2658 } 2659 2660 /* 2661 * Ensure NOSYNC/NOCONSIST mappings have REF and MOD set. 2662 * x86pte_set() depends on this. 2663 */ 2664 if (PTE_GET(newpte, PT_SOFTWARE) >= PT_NOSYNC) 2665 newpte |= PT_REF | PT_MOD; 2666 2667 /* 2668 * what about PROT_READ or others? this code only handles: 2669 * EXEC, WRITE, NOSYNC 2670 */ 2671 2672 /* 2673 * If new PTE really changed, update the table. 2674 */ 2675 if (newpte != oldpte) { 2676 entry = htable_va2entry(vaddr, ht); 2677 oldpte = hati_update_pte(ht, entry, oldpte, newpte); 2678 if (oldpte != 0) { 2679 x86_hm_exit(pp); 2680 goto try_again; 2681 } 2682 } 2683 x86_hm_exit(pp); 2684 } 2685 if (ht) 2686 htable_release(ht); 2687 XPV_ALLOW_MIGRATE(); 2688 } 2689 2690 /* 2691 * Various wrappers for hat_updateattr() 2692 */ 2693 void 2694 hat_setattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) 2695 { 2696 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2697 hat_updateattr(hat, addr, len, attr, HAT_SET_ATTR); 2698 } 2699 2700 void 2701 hat_clrattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) 2702 { 2703 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2704 hat_updateattr(hat, addr, len, attr, HAT_CLR_ATTR); 2705 } 2706 2707 void 2708 hat_chgattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) 2709 { 2710 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2711 hat_updateattr(hat, addr, len, attr, HAT_LOAD_ATTR); 2712 } 2713 2714 void 2715 hat_chgprot(hat_t *hat, caddr_t addr, size_t len, uint_t vprot) 2716 { 2717 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); 2718 hat_updateattr(hat, addr, len, vprot & HAT_PROT_MASK, HAT_LOAD_ATTR); 2719 } 2720 2721 /* 2722 * size_t hat_getpagesize(hat, addr) 2723 * returns pagesize in bytes for <hat, addr>. returns -1 of there is 2724 * no mapping. This is an advisory call. 2725 */ 2726 ssize_t 2727 hat_getpagesize(hat_t *hat, caddr_t addr) 2728 { 2729 uintptr_t vaddr = ALIGN2PAGE(addr); 2730 htable_t *ht; 2731 size_t pagesize; 2732 2733 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2734 if (IN_VA_HOLE(vaddr)) 2735 return (-1); 2736 ht = htable_getpage(hat, vaddr, NULL); 2737 if (ht == NULL) 2738 return (-1); 2739 pagesize = LEVEL_SIZE(ht->ht_level); 2740 htable_release(ht); 2741 return (pagesize); 2742 } 2743 2744 2745 2746 /* 2747 * pfn_t hat_getpfnum(hat, addr) 2748 * returns pfn for <hat, addr> or PFN_INVALID if mapping is invalid. 2749 */ 2750 pfn_t 2751 hat_getpfnum(hat_t *hat, caddr_t addr) 2752 { 2753 uintptr_t vaddr = ALIGN2PAGE(addr); 2754 htable_t *ht; 2755 uint_t entry; 2756 pfn_t pfn = PFN_INVALID; 2757 2758 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2759 if (khat_running == 0) 2760 return (PFN_INVALID); 2761 2762 if (IN_VA_HOLE(vaddr)) 2763 return (PFN_INVALID); 2764 2765 XPV_DISALLOW_MIGRATE(); 2766 /* 2767 * A very common use of hat_getpfnum() is from the DDI for kernel pages. 2768 * Use the kmap_ptes (which also covers the 32 bit heap) to speed 2769 * this up. 2770 */ 2771 if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) { 2772 x86pte_t pte; 2773 pgcnt_t pg_index; 2774 2775 pg_index = mmu_btop(vaddr - mmu.kmap_addr); 2776 pte = GET_PTE(PT_INDEX_PTR(mmu.kmap_ptes, pg_index)); 2777 if (PTE_ISVALID(pte)) 2778 /*LINTED [use of constant 0 causes a lint warning] */ 2779 pfn = PTE2PFN(pte, 0); 2780 XPV_ALLOW_MIGRATE(); 2781 return (pfn); 2782 } 2783 2784 ht = htable_getpage(hat, vaddr, &entry); 2785 if (ht == NULL) { 2786 XPV_ALLOW_MIGRATE(); 2787 return (PFN_INVALID); 2788 } 2789 ASSERT(vaddr >= ht->ht_vaddr); 2790 ASSERT(vaddr <= HTABLE_LAST_PAGE(ht)); 2791 pfn = PTE2PFN(x86pte_get(ht, entry), ht->ht_level); 2792 if (ht->ht_level > 0) 2793 pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level)); 2794 htable_release(ht); 2795 XPV_ALLOW_MIGRATE(); 2796 return (pfn); 2797 } 2798 2799 /* 2800 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 2801 * Use hat_getpfnum(kas.a_hat, ...) instead. 2802 * 2803 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 2804 * but can't right now due to the fact that some software has grown to use 2805 * this interface incorrectly. So for now when the interface is misused, 2806 * return a warning to the user that in the future it won't work in the 2807 * way they're abusing it, and carry on. 2808 * 2809 * Note that hat_getkpfnum() is never supported on amd64. 2810 */ 2811 #if !defined(__amd64) 2812 pfn_t 2813 hat_getkpfnum(caddr_t addr) 2814 { 2815 pfn_t pfn; 2816 int badcaller = 0; 2817 2818 if (khat_running == 0) 2819 panic("hat_getkpfnum(): called too early\n"); 2820 if ((uintptr_t)addr < kernelbase) 2821 return (PFN_INVALID); 2822 2823 XPV_DISALLOW_MIGRATE(); 2824 if (segkpm && IS_KPM_ADDR(addr)) { 2825 badcaller = 1; 2826 pfn = hat_kpm_va2pfn(addr); 2827 } else { 2828 pfn = hat_getpfnum(kas.a_hat, addr); 2829 badcaller = pf_is_memory(pfn); 2830 } 2831 2832 if (badcaller) 2833 hat_getkpfnum_badcall(caller()); 2834 XPV_ALLOW_MIGRATE(); 2835 return (pfn); 2836 } 2837 #endif /* __amd64 */ 2838 2839 /* 2840 * int hat_probe(hat, addr) 2841 * return 0 if no valid mapping is present. Faster version 2842 * of hat_getattr in certain architectures. 2843 */ 2844 int 2845 hat_probe(hat_t *hat, caddr_t addr) 2846 { 2847 uintptr_t vaddr = ALIGN2PAGE(addr); 2848 uint_t entry; 2849 htable_t *ht; 2850 pgcnt_t pg_off; 2851 2852 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); 2853 ASSERT(hat == kas.a_hat || 2854 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock)); 2855 if (IN_VA_HOLE(vaddr)) 2856 return (0); 2857 2858 /* 2859 * Most common use of hat_probe is from segmap. We special case it 2860 * for performance. 2861 */ 2862 if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) { 2863 pg_off = mmu_btop(vaddr - mmu.kmap_addr); 2864 if (mmu.pae_hat) 2865 return (PTE_ISVALID(mmu.kmap_ptes[pg_off])); 2866 else 2867 return (PTE_ISVALID( 2868 ((x86pte32_t *)mmu.kmap_ptes)[pg_off])); 2869 } 2870 2871 ht = htable_getpage(hat, vaddr, &entry); 2872 htable_release(ht); 2873 return (ht != NULL); 2874 } 2875 2876 /* 2877 * Find out if the segment for hat_share()/hat_unshare() is DISM or locked ISM. 2878 */ 2879 static int 2880 is_it_dism(hat_t *hat, caddr_t va) 2881 { 2882 struct seg *seg; 2883 struct shm_data *shmd; 2884 struct spt_data *sptd; 2885 2886 seg = as_findseg(hat->hat_as, va, 0); 2887 ASSERT(seg != NULL); 2888 ASSERT(seg->s_base <= va); 2889 shmd = (struct shm_data *)seg->s_data; 2890 ASSERT(shmd != NULL); 2891 sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2892 ASSERT(sptd != NULL); 2893 if (sptd->spt_flags & SHM_PAGEABLE) 2894 return (1); 2895 return (0); 2896 } 2897 2898 /* 2899 * Simple implementation of ISM. hat_share() is similar to hat_memload_array(), 2900 * except that we use the ism_hat's existing mappings to determine the pages 2901 * and protections to use for this hat. If we find a full properly aligned 2902 * and sized pagetable, we will attempt to share the pagetable itself. 2903 */ 2904 /*ARGSUSED*/ 2905 int 2906 hat_share( 2907 hat_t *hat, 2908 caddr_t addr, 2909 hat_t *ism_hat, 2910 caddr_t src_addr, 2911 size_t len, /* almost useless value, see below.. */ 2912 uint_t ismszc) 2913 { 2914 uintptr_t vaddr_start = (uintptr_t)addr; 2915 uintptr_t vaddr; 2916 uintptr_t eaddr = vaddr_start + len; 2917 uintptr_t ism_addr_start = (uintptr_t)src_addr; 2918 uintptr_t ism_addr = ism_addr_start; 2919 uintptr_t e_ism_addr = ism_addr + len; 2920 htable_t *ism_ht = NULL; 2921 htable_t *ht; 2922 x86pte_t pte; 2923 page_t *pp; 2924 pfn_t pfn; 2925 level_t l; 2926 pgcnt_t pgcnt; 2927 uint_t prot; 2928 int is_dism; 2929 int flags; 2930 2931 /* 2932 * We might be asked to share an empty DISM hat by as_dup() 2933 */ 2934 ASSERT(hat != kas.a_hat); 2935 ASSERT(eaddr <= _userlimit); 2936 if (!(ism_hat->hat_flags & HAT_SHARED)) { 2937 ASSERT(hat_get_mapped_size(ism_hat) == 0); 2938 return (0); 2939 } 2940 XPV_DISALLOW_MIGRATE(); 2941 2942 /* 2943 * The SPT segment driver often passes us a size larger than there are 2944 * valid mappings. That's because it rounds the segment size up to a 2945 * large pagesize, even if the actual memory mapped by ism_hat is less. 2946 */ 2947 ASSERT(IS_PAGEALIGNED(vaddr_start)); 2948 ASSERT(IS_PAGEALIGNED(ism_addr_start)); 2949 ASSERT(ism_hat->hat_flags & HAT_SHARED); 2950 is_dism = is_it_dism(hat, addr); 2951 while (ism_addr < e_ism_addr) { 2952 /* 2953 * use htable_walk to get the next valid ISM mapping 2954 */ 2955 pte = htable_walk(ism_hat, &ism_ht, &ism_addr, e_ism_addr); 2956 if (ism_ht == NULL) 2957 break; 2958 2959 /* 2960 * First check to see if we already share the page table. 2961 */ 2962 l = ism_ht->ht_level; 2963 vaddr = vaddr_start + (ism_addr - ism_addr_start); 2964 ht = htable_lookup(hat, vaddr, l); 2965 if (ht != NULL) { 2966 if (ht->ht_flags & HTABLE_SHARED_PFN) 2967 goto shared; 2968 htable_release(ht); 2969 goto not_shared; 2970 } 2971 2972 /* 2973 * Can't ever share top table. 2974 */ 2975 if (l == mmu.max_level) 2976 goto not_shared; 2977 2978 /* 2979 * Avoid level mismatches later due to DISM faults. 2980 */ 2981 if (is_dism && l > 0) 2982 goto not_shared; 2983 2984 /* 2985 * addresses and lengths must align 2986 * table must be fully populated 2987 * no lower level page tables 2988 */ 2989 if (ism_addr != ism_ht->ht_vaddr || 2990 (vaddr & LEVEL_OFFSET(l + 1)) != 0) 2991 goto not_shared; 2992 2993 /* 2994 * The range of address space must cover a full table. 2995 */ 2996 if (e_ism_addr - ism_addr < LEVEL_SIZE(l + 1)) 2997 goto not_shared; 2998 2999 /* 3000 * All entries in the ISM page table must be leaf PTEs. 3001 */ 3002 if (l > 0) { 3003 int e; 3004 3005 /* 3006 * We know the 0th is from htable_walk() above. 3007 */ 3008 for (e = 1; e < HTABLE_NUM_PTES(ism_ht); ++e) { 3009 x86pte_t pte; 3010 pte = x86pte_get(ism_ht, e); 3011 if (!PTE_ISPAGE(pte, l)) 3012 goto not_shared; 3013 } 3014 } 3015 3016 /* 3017 * share the page table 3018 */ 3019 ht = htable_create(hat, vaddr, l, ism_ht); 3020 shared: 3021 ASSERT(ht->ht_flags & HTABLE_SHARED_PFN); 3022 ASSERT(ht->ht_shares == ism_ht); 3023 hat->hat_ism_pgcnt += 3024 (ism_ht->ht_valid_cnt - ht->ht_valid_cnt) << 3025 (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT); 3026 ht->ht_valid_cnt = ism_ht->ht_valid_cnt; 3027 htable_release(ht); 3028 ism_addr = ism_ht->ht_vaddr + LEVEL_SIZE(l + 1); 3029 htable_release(ism_ht); 3030 ism_ht = NULL; 3031 continue; 3032 3033 not_shared: 3034 /* 3035 * Unable to share the page table. Instead we will 3036 * create new mappings from the values in the ISM mappings. 3037 * Figure out what level size mappings to use; 3038 */ 3039 for (l = ism_ht->ht_level; l > 0; --l) { 3040 if (LEVEL_SIZE(l) <= eaddr - vaddr && 3041 (vaddr & LEVEL_OFFSET(l)) == 0) 3042 break; 3043 } 3044 3045 /* 3046 * The ISM mapping might be larger than the share area, 3047 * be careful to truncate it if needed. 3048 */ 3049 if (eaddr - vaddr >= LEVEL_SIZE(ism_ht->ht_level)) { 3050 pgcnt = mmu_btop(LEVEL_SIZE(ism_ht->ht_level)); 3051 } else { 3052 pgcnt = mmu_btop(eaddr - vaddr); 3053 l = 0; 3054 } 3055 3056 pfn = PTE2PFN(pte, ism_ht->ht_level); 3057 ASSERT(pfn != PFN_INVALID); 3058 while (pgcnt > 0) { 3059 /* 3060 * Make a new pte for the PFN for this level. 3061 * Copy protections for the pte from the ISM pte. 3062 */ 3063 pp = page_numtopp_nolock(pfn); 3064 ASSERT(pp != NULL); 3065 3066 prot = PROT_USER | PROT_READ | HAT_UNORDERED_OK; 3067 if (PTE_GET(pte, PT_WRITABLE)) 3068 prot |= PROT_WRITE; 3069 if (!PTE_GET(pte, PT_NX)) 3070 prot |= PROT_EXEC; 3071 3072 flags = HAT_LOAD; 3073 if (!is_dism) 3074 flags |= HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST; 3075 while (hati_load_common(hat, vaddr, pp, prot, flags, 3076 l, pfn) != 0) { 3077 if (l == 0) 3078 panic("hati_load_common() failure"); 3079 --l; 3080 } 3081 3082 vaddr += LEVEL_SIZE(l); 3083 ism_addr += LEVEL_SIZE(l); 3084 pfn += mmu_btop(LEVEL_SIZE(l)); 3085 pgcnt -= mmu_btop(LEVEL_SIZE(l)); 3086 } 3087 } 3088 if (ism_ht != NULL) 3089 htable_release(ism_ht); 3090 XPV_ALLOW_MIGRATE(); 3091 return (0); 3092 } 3093 3094 3095 /* 3096 * hat_unshare() is similar to hat_unload_callback(), but 3097 * we have to look for empty shared pagetables. Note that 3098 * hat_unshare() is always invoked against an entire segment. 3099 */ 3100 /*ARGSUSED*/ 3101 void 3102 hat_unshare(hat_t *hat, caddr_t addr, size_t len, uint_t ismszc) 3103 { 3104 uint64_t vaddr = (uintptr_t)addr; 3105 uintptr_t eaddr = vaddr + len; 3106 htable_t *ht = NULL; 3107 uint_t need_demaps = 0; 3108 int flags = HAT_UNLOAD_UNMAP; 3109 level_t l; 3110 3111 ASSERT(hat != kas.a_hat); 3112 ASSERT(eaddr <= _userlimit); 3113 ASSERT(IS_PAGEALIGNED(vaddr)); 3114 ASSERT(IS_PAGEALIGNED(eaddr)); 3115 XPV_DISALLOW_MIGRATE(); 3116 3117 /* 3118 * First go through and remove any shared pagetables. 3119 * 3120 * Note that it's ok to delay the TLB shootdown till the entire range is 3121 * finished, because if hat_pageunload() were to unload a shared 3122 * pagetable page, its hat_tlb_inval() will do a global TLB invalidate. 3123 */ 3124 l = mmu.max_page_level; 3125 if (l == mmu.max_level) 3126 --l; 3127 for (; l >= 0; --l) { 3128 for (vaddr = (uintptr_t)addr; vaddr < eaddr; 3129 vaddr = (vaddr & LEVEL_MASK(l + 1)) + LEVEL_SIZE(l + 1)) { 3130 ASSERT(!IN_VA_HOLE(vaddr)); 3131 /* 3132 * find a pagetable that maps the current address 3133 */ 3134 ht = htable_lookup(hat, vaddr, l); 3135 if (ht == NULL) 3136 continue; 3137 if (ht->ht_flags & HTABLE_SHARED_PFN) { 3138 /* 3139 * clear page count, set valid_cnt to 0, 3140 * let htable_release() finish the job 3141 */ 3142 hat->hat_ism_pgcnt -= ht->ht_valid_cnt << 3143 (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT); 3144 ht->ht_valid_cnt = 0; 3145 need_demaps = 1; 3146 } 3147 htable_release(ht); 3148 } 3149 } 3150 3151 /* 3152 * flush the TLBs - since we're probably dealing with MANY mappings 3153 * we do just one CR3 reload. 3154 */ 3155 if (!(hat->hat_flags & HAT_FREEING) && need_demaps) 3156 hat_tlb_inval(hat, DEMAP_ALL_ADDR); 3157 3158 /* 3159 * Now go back and clean up any unaligned mappings that 3160 * couldn't share pagetables. 3161 */ 3162 if (!is_it_dism(hat, addr)) 3163 flags |= HAT_UNLOAD_UNLOCK; 3164 hat_unload(hat, addr, len, flags); 3165 XPV_ALLOW_MIGRATE(); 3166 } 3167 3168 3169 /* 3170 * hat_reserve() does nothing 3171 */ 3172 /*ARGSUSED*/ 3173 void 3174 hat_reserve(struct as *as, caddr_t addr, size_t len) 3175 { 3176 } 3177 3178 3179 /* 3180 * Called when all mappings to a page should have write permission removed. 3181 * Mostly stolem from hat_pagesync() 3182 */ 3183 static void 3184 hati_page_clrwrt(struct page *pp) 3185 { 3186 hment_t *hm = NULL; 3187 htable_t *ht; 3188 uint_t entry; 3189 x86pte_t old; 3190 x86pte_t new; 3191 uint_t pszc = 0; 3192 3193 XPV_DISALLOW_MIGRATE(); 3194 next_size: 3195 /* 3196 * walk thru the mapping list clearing write permission 3197 */ 3198 x86_hm_enter(pp); 3199 while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) { 3200 if (ht->ht_level < pszc) 3201 continue; 3202 old = x86pte_get(ht, entry); 3203 3204 for (;;) { 3205 /* 3206 * Is this mapping of interest? 3207 */ 3208 if (PTE2PFN(old, ht->ht_level) != pp->p_pagenum || 3209 PTE_GET(old, PT_WRITABLE) == 0) 3210 break; 3211 3212 /* 3213 * Clear ref/mod writable bits. This requires cross 3214 * calls to ensure any executing TLBs see cleared bits. 3215 */ 3216 new = old; 3217 PTE_CLR(new, PT_REF | PT_MOD | PT_WRITABLE); 3218 old = hati_update_pte(ht, entry, old, new); 3219 if (old != 0) 3220 continue; 3221 3222 break; 3223 } 3224 } 3225 x86_hm_exit(pp); 3226 while (pszc < pp->p_szc) { 3227 page_t *tpp; 3228 pszc++; 3229 tpp = PP_GROUPLEADER(pp, pszc); 3230 if (pp != tpp) { 3231 pp = tpp; 3232 goto next_size; 3233 } 3234 } 3235 XPV_ALLOW_MIGRATE(); 3236 } 3237 3238 /* 3239 * void hat_page_setattr(pp, flag) 3240 * void hat_page_clrattr(pp, flag) 3241 * used to set/clr ref/mod bits. 3242 */ 3243 void 3244 hat_page_setattr(struct page *pp, uint_t flag) 3245 { 3246 vnode_t *vp = pp->p_vnode; 3247 kmutex_t *vphm = NULL; 3248 page_t **listp; 3249 int noshuffle; 3250 3251 noshuffle = flag & P_NSH; 3252 flag &= ~P_NSH; 3253 3254 if (PP_GETRM(pp, flag) == flag) 3255 return; 3256 3257 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp) && 3258 !noshuffle) { 3259 vphm = page_vnode_mutex(vp); 3260 mutex_enter(vphm); 3261 } 3262 3263 PP_SETRM(pp, flag); 3264 3265 if (vphm != NULL) { 3266 3267 /* 3268 * Some File Systems examine v_pages for NULL w/o 3269 * grabbing the vphm mutex. Must not let it become NULL when 3270 * pp is the only page on the list. 3271 */ 3272 if (pp->p_vpnext != pp) { 3273 page_vpsub(&vp->v_pages, pp); 3274 if (vp->v_pages != NULL) 3275 listp = &vp->v_pages->p_vpprev->p_vpnext; 3276 else 3277 listp = &vp->v_pages; 3278 page_vpadd(listp, pp); 3279 } 3280 mutex_exit(vphm); 3281 } 3282 } 3283 3284 void 3285 hat_page_clrattr(struct page *pp, uint_t flag) 3286 { 3287 vnode_t *vp = pp->p_vnode; 3288 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 3289 3290 /* 3291 * Caller is expected to hold page's io lock for VMODSORT to work 3292 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod 3293 * bit is cleared. 3294 * We don't have assert to avoid tripping some existing third party 3295 * code. The dirty page is moved back to top of the v_page list 3296 * after IO is done in pvn_write_done(). 3297 */ 3298 PP_CLRRM(pp, flag); 3299 3300 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 3301 3302 /* 3303 * VMODSORT works by removing write permissions and getting 3304 * a fault when a page is made dirty. At this point 3305 * we need to remove write permission from all mappings 3306 * to this page. 3307 */ 3308 hati_page_clrwrt(pp); 3309 } 3310 } 3311 3312 /* 3313 * If flag is specified, returns 0 if attribute is disabled 3314 * and non zero if enabled. If flag specifes multiple attributs 3315 * then returns 0 if ALL atriibutes are disabled. This is an advisory 3316 * call. 3317 */ 3318 uint_t 3319 hat_page_getattr(struct page *pp, uint_t flag) 3320 { 3321 return (PP_GETRM(pp, flag)); 3322 } 3323 3324 3325 /* 3326 * common code used by hat_pageunload() and hment_steal() 3327 */ 3328 hment_t * 3329 hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry) 3330 { 3331 x86pte_t old_pte; 3332 pfn_t pfn = pp->p_pagenum; 3333 hment_t *hm; 3334 3335 /* 3336 * We need to acquire a hold on the htable in order to 3337 * do the invalidate. We know the htable must exist, since 3338 * unmap's don't release the htable until after removing any 3339 * hment. Having x86_hm_enter() keeps that from proceeding. 3340 */ 3341 htable_acquire(ht); 3342 3343 /* 3344 * Invalidate the PTE and remove the hment. 3345 */ 3346 old_pte = x86pte_inval(ht, entry, 0, NULL); 3347 if (PTE2PFN(old_pte, ht->ht_level) != pfn) { 3348 panic("x86pte_inval() failure found PTE = " FMT_PTE 3349 " pfn being unmapped is %lx ht=0x%lx entry=0x%x", 3350 old_pte, pfn, (uintptr_t)ht, entry); 3351 } 3352 3353 /* 3354 * Clean up all the htable information for this mapping 3355 */ 3356 ASSERT(ht->ht_valid_cnt > 0); 3357 HTABLE_DEC(ht->ht_valid_cnt); 3358 PGCNT_DEC(ht->ht_hat, ht->ht_level); 3359 3360 /* 3361 * sync ref/mod bits to the page_t 3362 */ 3363 if (PTE_GET(old_pte, PT_SOFTWARE) < PT_NOSYNC) 3364 hati_sync_pte_to_page(pp, old_pte, ht->ht_level); 3365 3366 /* 3367 * Remove the mapping list entry for this page. 3368 */ 3369 hm = hment_remove(pp, ht, entry); 3370 3371 /* 3372 * drop the mapping list lock so that we might free the 3373 * hment and htable. 3374 */ 3375 x86_hm_exit(pp); 3376 htable_release(ht); 3377 return (hm); 3378 } 3379 3380 extern int vpm_enable; 3381 /* 3382 * Unload all translations to a page. If the page is a subpage of a large 3383 * page, the large page mappings are also removed. 3384 * 3385 * The forceflags are unused. 3386 */ 3387 3388 /*ARGSUSED*/ 3389 static int 3390 hati_pageunload(struct page *pp, uint_t pg_szcd, uint_t forceflag) 3391 { 3392 page_t *cur_pp = pp; 3393 hment_t *hm; 3394 hment_t *prev; 3395 htable_t *ht; 3396 uint_t entry; 3397 level_t level; 3398 3399 XPV_DISALLOW_MIGRATE(); 3400 #if defined(__amd64) 3401 /* 3402 * clear the vpm ref. 3403 */ 3404 if (vpm_enable) { 3405 pp->p_vpmref = 0; 3406 } 3407 #endif 3408 /* 3409 * The loop with next_size handles pages with multiple pagesize mappings 3410 */ 3411 next_size: 3412 for (;;) { 3413 3414 /* 3415 * Get a mapping list entry 3416 */ 3417 x86_hm_enter(cur_pp); 3418 for (prev = NULL; ; prev = hm) { 3419 hm = hment_walk(cur_pp, &ht, &entry, prev); 3420 if (hm == NULL) { 3421 x86_hm_exit(cur_pp); 3422 3423 /* 3424 * If not part of a larger page, we're done. 3425 */ 3426 if (cur_pp->p_szc <= pg_szcd) { 3427 XPV_ALLOW_MIGRATE(); 3428 return (0); 3429 } 3430 3431 /* 3432 * Else check the next larger page size. 3433 * hat_page_demote() may decrease p_szc 3434 * but that's ok we'll just take an extra 3435 * trip discover there're no larger mappings 3436 * and return. 3437 */ 3438 ++pg_szcd; 3439 cur_pp = PP_GROUPLEADER(cur_pp, pg_szcd); 3440 goto next_size; 3441 } 3442 3443 /* 3444 * If this mapping size matches, remove it. 3445 */ 3446 level = ht->ht_level; 3447 if (level == pg_szcd) 3448 break; 3449 } 3450 3451 /* 3452 * Remove the mapping list entry for this page. 3453 * Note this does the x86_hm_exit() for us. 3454 */ 3455 hm = hati_page_unmap(cur_pp, ht, entry); 3456 if (hm != NULL) 3457 hment_free(hm); 3458 } 3459 } 3460 3461 int 3462 hat_pageunload(struct page *pp, uint_t forceflag) 3463 { 3464 ASSERT(PAGE_EXCL(pp)); 3465 return (hati_pageunload(pp, 0, forceflag)); 3466 } 3467 3468 /* 3469 * Unload all large mappings to pp and reduce by 1 p_szc field of every large 3470 * page level that included pp. 3471 * 3472 * pp must be locked EXCL. Even though no other constituent pages are locked 3473 * it's legal to unload large mappings to pp because all constituent pages of 3474 * large locked mappings have to be locked SHARED. therefore if we have EXCL 3475 * lock on one of constituent pages none of the large mappings to pp are 3476 * locked. 3477 * 3478 * Change (always decrease) p_szc field starting from the last constituent 3479 * page and ending with root constituent page so that root's pszc always shows 3480 * the area where hat_page_demote() may be active. 3481 * 3482 * This mechanism is only used for file system pages where it's not always 3483 * possible to get EXCL locks on all constituent pages to demote the size code 3484 * (as is done for anonymous or kernel large pages). 3485 */ 3486 void 3487 hat_page_demote(page_t *pp) 3488 { 3489 uint_t pszc; 3490 uint_t rszc; 3491 uint_t szc; 3492 page_t *rootpp; 3493 page_t *firstpp; 3494 page_t *lastpp; 3495 pgcnt_t pgcnt; 3496 3497 ASSERT(PAGE_EXCL(pp)); 3498 ASSERT(!PP_ISFREE(pp)); 3499 ASSERT(page_szc_lock_assert(pp)); 3500 3501 if (pp->p_szc == 0) 3502 return; 3503 3504 rootpp = PP_GROUPLEADER(pp, 1); 3505 (void) hati_pageunload(rootpp, 1, HAT_FORCE_PGUNLOAD); 3506 3507 /* 3508 * all large mappings to pp are gone 3509 * and no new can be setup since pp is locked exclusively. 3510 * 3511 * Lock the root to make sure there's only one hat_page_demote() 3512 * outstanding within the area of this root's pszc. 3513 * 3514 * Second potential hat_page_demote() is already eliminated by upper 3515 * VM layer via page_szc_lock() but we don't rely on it and use our 3516 * own locking (so that upper layer locking can be changed without 3517 * assumptions that hat depends on upper layer VM to prevent multiple 3518 * hat_page_demote() to be issued simultaneously to the same large 3519 * page). 3520 */ 3521 again: 3522 pszc = pp->p_szc; 3523 if (pszc == 0) 3524 return; 3525 rootpp = PP_GROUPLEADER(pp, pszc); 3526 x86_hm_enter(rootpp); 3527 /* 3528 * If root's p_szc is different from pszc we raced with another 3529 * hat_page_demote(). Drop the lock and try to find the root again. 3530 * If root's p_szc is greater than pszc previous hat_page_demote() is 3531 * not done yet. Take and release mlist lock of root's root to wait 3532 * for previous hat_page_demote() to complete. 3533 */ 3534 if ((rszc = rootpp->p_szc) != pszc) { 3535 x86_hm_exit(rootpp); 3536 if (rszc > pszc) { 3537 /* p_szc of a locked non free page can't increase */ 3538 ASSERT(pp != rootpp); 3539 3540 rootpp = PP_GROUPLEADER(rootpp, rszc); 3541 x86_hm_enter(rootpp); 3542 x86_hm_exit(rootpp); 3543 } 3544 goto again; 3545 } 3546 ASSERT(pp->p_szc == pszc); 3547 3548 /* 3549 * Decrement by 1 p_szc of every constituent page of a region that 3550 * covered pp. For example if original szc is 3 it gets changed to 2 3551 * everywhere except in region 2 that covered pp. Region 2 that 3552 * covered pp gets demoted to 1 everywhere except in region 1 that 3553 * covered pp. The region 1 that covered pp is demoted to region 3554 * 0. It's done this way because from region 3 we removed level 3 3555 * mappings, from region 2 that covered pp we removed level 2 mappings 3556 * and from region 1 that covered pp we removed level 1 mappings. All 3557 * changes are done from from high pfn's to low pfn's so that roots 3558 * are changed last allowing one to know the largest region where 3559 * hat_page_demote() is stil active by only looking at the root page. 3560 * 3561 * This algorithm is implemented in 2 while loops. First loop changes 3562 * p_szc of pages to the right of pp's level 1 region and second 3563 * loop changes p_szc of pages of level 1 region that covers pp 3564 * and all pages to the left of level 1 region that covers pp. 3565 * In the first loop p_szc keeps dropping with every iteration 3566 * and in the second loop it keeps increasing with every iteration. 3567 * 3568 * First loop description: Demote pages to the right of pp outside of 3569 * level 1 region that covers pp. In every iteration of the while 3570 * loop below find the last page of szc region and the first page of 3571 * (szc - 1) region that is immediately to the right of (szc - 1) 3572 * region that covers pp. From last such page to first such page 3573 * change every page's szc to szc - 1. Decrement szc and continue 3574 * looping until szc is 1. If pp belongs to the last (szc - 1) region 3575 * of szc region skip to the next iteration. 3576 */ 3577 szc = pszc; 3578 while (szc > 1) { 3579 lastpp = PP_GROUPLEADER(pp, szc); 3580 pgcnt = page_get_pagecnt(szc); 3581 lastpp += pgcnt - 1; 3582 firstpp = PP_GROUPLEADER(pp, (szc - 1)); 3583 pgcnt = page_get_pagecnt(szc - 1); 3584 if (lastpp - firstpp < pgcnt) { 3585 szc--; 3586 continue; 3587 } 3588 firstpp += pgcnt; 3589 while (lastpp != firstpp) { 3590 ASSERT(lastpp->p_szc == pszc); 3591 lastpp->p_szc = szc - 1; 3592 lastpp--; 3593 } 3594 firstpp->p_szc = szc - 1; 3595 szc--; 3596 } 3597 3598 /* 3599 * Second loop description: 3600 * First iteration changes p_szc to 0 of every 3601 * page of level 1 region that covers pp. 3602 * Subsequent iterations find last page of szc region 3603 * immediately to the left of szc region that covered pp 3604 * and first page of (szc + 1) region that covers pp. 3605 * From last to first page change p_szc of every page to szc. 3606 * Increment szc and continue looping until szc is pszc. 3607 * If pp belongs to the fist szc region of (szc + 1) region 3608 * skip to the next iteration. 3609 * 3610 */ 3611 szc = 0; 3612 while (szc < pszc) { 3613 firstpp = PP_GROUPLEADER(pp, (szc + 1)); 3614 if (szc == 0) { 3615 pgcnt = page_get_pagecnt(1); 3616 lastpp = firstpp + (pgcnt - 1); 3617 } else { 3618 lastpp = PP_GROUPLEADER(pp, szc); 3619 if (firstpp == lastpp) { 3620 szc++; 3621 continue; 3622 } 3623 lastpp--; 3624 pgcnt = page_get_pagecnt(szc); 3625 } 3626 while (lastpp != firstpp) { 3627 ASSERT(lastpp->p_szc == pszc); 3628 lastpp->p_szc = szc; 3629 lastpp--; 3630 } 3631 firstpp->p_szc = szc; 3632 if (firstpp == rootpp) 3633 break; 3634 szc++; 3635 } 3636 x86_hm_exit(rootpp); 3637 } 3638 3639 /* 3640 * get hw stats from hardware into page struct and reset hw stats 3641 * returns attributes of page 3642 * Flags for hat_pagesync, hat_getstat, hat_sync 3643 * 3644 * define HAT_SYNC_ZERORM 0x01 3645 * 3646 * Additional flags for hat_pagesync 3647 * 3648 * define HAT_SYNC_STOPON_REF 0x02 3649 * define HAT_SYNC_STOPON_MOD 0x04 3650 * define HAT_SYNC_STOPON_RM 0x06 3651 * define HAT_SYNC_STOPON_SHARED 0x08 3652 */ 3653 uint_t 3654 hat_pagesync(struct page *pp, uint_t flags) 3655 { 3656 hment_t *hm = NULL; 3657 htable_t *ht; 3658 uint_t entry; 3659 x86pte_t old, save_old; 3660 x86pte_t new; 3661 uchar_t nrmbits = P_REF|P_MOD|P_RO; 3662 extern ulong_t po_share; 3663 page_t *save_pp = pp; 3664 uint_t pszc = 0; 3665 3666 ASSERT(PAGE_LOCKED(pp) || panicstr); 3667 3668 if (PP_ISRO(pp) && (flags & HAT_SYNC_STOPON_MOD)) 3669 return (pp->p_nrm & nrmbits); 3670 3671 if ((flags & HAT_SYNC_ZERORM) == 0) { 3672 3673 if ((flags & HAT_SYNC_STOPON_REF) != 0 && PP_ISREF(pp)) 3674 return (pp->p_nrm & nrmbits); 3675 3676 if ((flags & HAT_SYNC_STOPON_MOD) != 0 && PP_ISMOD(pp)) 3677 return (pp->p_nrm & nrmbits); 3678 3679 if ((flags & HAT_SYNC_STOPON_SHARED) != 0 && 3680 hat_page_getshare(pp) > po_share) { 3681 if (PP_ISRO(pp)) 3682 PP_SETREF(pp); 3683 return (pp->p_nrm & nrmbits); 3684 } 3685 } 3686 3687 XPV_DISALLOW_MIGRATE(); 3688 next_size: 3689 /* 3690 * walk thru the mapping list syncing (and clearing) ref/mod bits. 3691 */ 3692 x86_hm_enter(pp); 3693 while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) { 3694 if (ht->ht_level < pszc) 3695 continue; 3696 old = x86pte_get(ht, entry); 3697 try_again: 3698 3699 ASSERT(PTE2PFN(old, ht->ht_level) == pp->p_pagenum); 3700 3701 if (PTE_GET(old, PT_REF | PT_MOD) == 0) 3702 continue; 3703 3704 save_old = old; 3705 if ((flags & HAT_SYNC_ZERORM) != 0) { 3706 3707 /* 3708 * Need to clear ref or mod bits. Need to demap 3709 * to make sure any executing TLBs see cleared bits. 3710 */ 3711 new = old; 3712 PTE_CLR(new, PT_REF | PT_MOD); 3713 old = hati_update_pte(ht, entry, old, new); 3714 if (old != 0) 3715 goto try_again; 3716 3717 old = save_old; 3718 } 3719 3720 /* 3721 * Sync the PTE 3722 */ 3723 if (!(flags & HAT_SYNC_ZERORM) && 3724 PTE_GET(old, PT_SOFTWARE) <= PT_NOSYNC) 3725 hati_sync_pte_to_page(pp, old, ht->ht_level); 3726 3727 /* 3728 * can stop short if we found a ref'd or mod'd page 3729 */ 3730 if ((flags & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp) || 3731 (flags & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp)) { 3732 x86_hm_exit(pp); 3733 goto done; 3734 } 3735 } 3736 x86_hm_exit(pp); 3737 while (pszc < pp->p_szc) { 3738 page_t *tpp; 3739 pszc++; 3740 tpp = PP_GROUPLEADER(pp, pszc); 3741 if (pp != tpp) { 3742 pp = tpp; 3743 goto next_size; 3744 } 3745 } 3746 done: 3747 XPV_ALLOW_MIGRATE(); 3748 return (save_pp->p_nrm & nrmbits); 3749 } 3750 3751 /* 3752 * returns approx number of mappings to this pp. A return of 0 implies 3753 * there are no mappings to the page. 3754 */ 3755 ulong_t 3756 hat_page_getshare(page_t *pp) 3757 { 3758 uint_t cnt; 3759 cnt = hment_mapcnt(pp); 3760 #if defined(__amd64) 3761 if (vpm_enable && pp->p_vpmref) { 3762 cnt += 1; 3763 } 3764 #endif 3765 return (cnt); 3766 } 3767 3768 /* 3769 * Return 1 the number of mappings exceeds sh_thresh. Return 0 3770 * otherwise. 3771 */ 3772 int 3773 hat_page_checkshare(page_t *pp, ulong_t sh_thresh) 3774 { 3775 return (hat_page_getshare(pp) > sh_thresh); 3776 } 3777 3778 /* 3779 * hat_softlock isn't supported anymore 3780 */ 3781 /*ARGSUSED*/ 3782 faultcode_t 3783 hat_softlock( 3784 hat_t *hat, 3785 caddr_t addr, 3786 size_t *len, 3787 struct page **page_array, 3788 uint_t flags) 3789 { 3790 return (FC_NOSUPPORT); 3791 } 3792 3793 3794 3795 /* 3796 * Routine to expose supported HAT features to platform independent code. 3797 */ 3798 /*ARGSUSED*/ 3799 int 3800 hat_supported(enum hat_features feature, void *arg) 3801 { 3802 switch (feature) { 3803 3804 case HAT_SHARED_PT: /* this is really ISM */ 3805 return (1); 3806 3807 case HAT_DYNAMIC_ISM_UNMAP: 3808 return (0); 3809 3810 case HAT_VMODSORT: 3811 return (1); 3812 3813 case HAT_SHARED_REGIONS: 3814 return (0); 3815 3816 default: 3817 panic("hat_supported() - unknown feature"); 3818 } 3819 return (0); 3820 } 3821 3822 /* 3823 * Called when a thread is exiting and has been switched to the kernel AS 3824 */ 3825 void 3826 hat_thread_exit(kthread_t *thd) 3827 { 3828 ASSERT(thd->t_procp->p_as == &kas); 3829 XPV_DISALLOW_MIGRATE(); 3830 hat_switch(thd->t_procp->p_as->a_hat); 3831 XPV_ALLOW_MIGRATE(); 3832 } 3833 3834 /* 3835 * Setup the given brand new hat structure as the new HAT on this cpu's mmu. 3836 */ 3837 /*ARGSUSED*/ 3838 void 3839 hat_setup(hat_t *hat, int flags) 3840 { 3841 XPV_DISALLOW_MIGRATE(); 3842 kpreempt_disable(); 3843 3844 hat_switch(hat); 3845 3846 kpreempt_enable(); 3847 XPV_ALLOW_MIGRATE(); 3848 } 3849 3850 /* 3851 * Prepare for a CPU private mapping for the given address. 3852 * 3853 * The address can only be used from a single CPU and can be remapped 3854 * using hat_mempte_remap(). Return the address of the PTE. 3855 * 3856 * We do the htable_create() if necessary and increment the valid count so 3857 * the htable can't disappear. We also hat_devload() the page table into 3858 * kernel so that the PTE is quickly accessed. 3859 */ 3860 hat_mempte_t 3861 hat_mempte_setup(caddr_t addr) 3862 { 3863 uintptr_t va = (uintptr_t)addr; 3864 htable_t *ht; 3865 uint_t entry; 3866 x86pte_t oldpte; 3867 hat_mempte_t p; 3868 3869 ASSERT(IS_PAGEALIGNED(va)); 3870 ASSERT(!IN_VA_HOLE(va)); 3871 ++curthread->t_hatdepth; 3872 XPV_DISALLOW_MIGRATE(); 3873 ht = htable_getpte(kas.a_hat, va, &entry, &oldpte, 0); 3874 if (ht == NULL) { 3875 ht = htable_create(kas.a_hat, va, 0, NULL); 3876 entry = htable_va2entry(va, ht); 3877 ASSERT(ht->ht_level == 0); 3878 oldpte = x86pte_get(ht, entry); 3879 } 3880 if (PTE_ISVALID(oldpte)) 3881 panic("hat_mempte_setup(): address already mapped" 3882 "ht=%p, entry=%d, pte=" FMT_PTE, (void *)ht, entry, oldpte); 3883 3884 /* 3885 * increment ht_valid_cnt so that the pagetable can't disappear 3886 */ 3887 HTABLE_INC(ht->ht_valid_cnt); 3888 3889 /* 3890 * return the PTE physical address to the caller. 3891 */ 3892 htable_release(ht); 3893 XPV_ALLOW_MIGRATE(); 3894 p = PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry); 3895 --curthread->t_hatdepth; 3896 return (p); 3897 } 3898 3899 /* 3900 * Release a CPU private mapping for the given address. 3901 * We decrement the htable valid count so it might be destroyed. 3902 */ 3903 /*ARGSUSED1*/ 3904 void 3905 hat_mempte_release(caddr_t addr, hat_mempte_t pte_pa) 3906 { 3907 htable_t *ht; 3908 3909 XPV_DISALLOW_MIGRATE(); 3910 /* 3911 * invalidate any left over mapping and decrement the htable valid count 3912 */ 3913 #ifdef __xpv 3914 if (HYPERVISOR_update_va_mapping((uintptr_t)addr, 0, 3915 UVMF_INVLPG | UVMF_LOCAL)) 3916 panic("HYPERVISOR_update_va_mapping() failed"); 3917 #else 3918 { 3919 x86pte_t *pteptr; 3920 3921 pteptr = x86pte_mapin(mmu_btop(pte_pa), 3922 (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL); 3923 if (mmu.pae_hat) 3924 *pteptr = 0; 3925 else 3926 *(x86pte32_t *)pteptr = 0; 3927 mmu_tlbflush_entry(addr); 3928 x86pte_mapout(); 3929 } 3930 #endif 3931 3932 ht = htable_getpte(kas.a_hat, ALIGN2PAGE(addr), NULL, NULL, 0); 3933 if (ht == NULL) 3934 panic("hat_mempte_release(): invalid address"); 3935 ASSERT(ht->ht_level == 0); 3936 HTABLE_DEC(ht->ht_valid_cnt); 3937 htable_release(ht); 3938 XPV_ALLOW_MIGRATE(); 3939 } 3940 3941 /* 3942 * Apply a temporary CPU private mapping to a page. We flush the TLB only 3943 * on this CPU, so this ought to have been called with preemption disabled. 3944 */ 3945 void 3946 hat_mempte_remap( 3947 pfn_t pfn, 3948 caddr_t addr, 3949 hat_mempte_t pte_pa, 3950 uint_t attr, 3951 uint_t flags) 3952 { 3953 uintptr_t va = (uintptr_t)addr; 3954 x86pte_t pte; 3955 3956 /* 3957 * Remap the given PTE to the new page's PFN. Invalidate only 3958 * on this CPU. 3959 */ 3960 #ifdef DEBUG 3961 htable_t *ht; 3962 uint_t entry; 3963 3964 ASSERT(IS_PAGEALIGNED(va)); 3965 ASSERT(!IN_VA_HOLE(va)); 3966 ht = htable_getpte(kas.a_hat, va, &entry, NULL, 0); 3967 ASSERT(ht != NULL); 3968 ASSERT(ht->ht_level == 0); 3969 ASSERT(ht->ht_valid_cnt > 0); 3970 ASSERT(ht->ht_pfn == mmu_btop(pte_pa)); 3971 htable_release(ht); 3972 #endif 3973 XPV_DISALLOW_MIGRATE(); 3974 pte = hati_mkpte(pfn, attr, 0, flags); 3975 #ifdef __xpv 3976 if (HYPERVISOR_update_va_mapping(va, pte, UVMF_INVLPG | UVMF_LOCAL)) 3977 panic("HYPERVISOR_update_va_mapping() failed"); 3978 #else 3979 { 3980 x86pte_t *pteptr; 3981 3982 pteptr = x86pte_mapin(mmu_btop(pte_pa), 3983 (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL); 3984 if (mmu.pae_hat) 3985 *(x86pte_t *)pteptr = pte; 3986 else 3987 *(x86pte32_t *)pteptr = (x86pte32_t)pte; 3988 mmu_tlbflush_entry(addr); 3989 x86pte_mapout(); 3990 } 3991 #endif 3992 XPV_ALLOW_MIGRATE(); 3993 } 3994 3995 3996 3997 /* 3998 * Hat locking functions 3999 * XXX - these two functions are currently being used by hatstats 4000 * they can be removed by using a per-as mutex for hatstats. 4001 */ 4002 void 4003 hat_enter(hat_t *hat) 4004 { 4005 mutex_enter(&hat->hat_mutex); 4006 } 4007 4008 void 4009 hat_exit(hat_t *hat) 4010 { 4011 mutex_exit(&hat->hat_mutex); 4012 } 4013 4014 /* 4015 * HAT part of cpu initialization. 4016 */ 4017 void 4018 hat_cpu_online(struct cpu *cpup) 4019 { 4020 if (cpup != CPU) { 4021 x86pte_cpu_init(cpup); 4022 hat_vlp_setup(cpup); 4023 } 4024 CPUSET_ATOMIC_ADD(khat_cpuset, cpup->cpu_id); 4025 } 4026 4027 /* 4028 * HAT part of cpu deletion. 4029 * (currently, we only call this after the cpu is safely passivated.) 4030 */ 4031 void 4032 hat_cpu_offline(struct cpu *cpup) 4033 { 4034 ASSERT(cpup != CPU); 4035 4036 CPUSET_ATOMIC_DEL(khat_cpuset, cpup->cpu_id); 4037 x86pte_cpu_fini(cpup); 4038 hat_vlp_teardown(cpup); 4039 } 4040 4041 /* 4042 * Function called after all CPUs are brought online. 4043 * Used to remove low address boot mappings. 4044 */ 4045 void 4046 clear_boot_mappings(uintptr_t low, uintptr_t high) 4047 { 4048 uintptr_t vaddr = low; 4049 htable_t *ht = NULL; 4050 level_t level; 4051 uint_t entry; 4052 x86pte_t pte; 4053 4054 /* 4055 * On 1st CPU we can unload the prom mappings, basically we blow away 4056 * all virtual mappings under _userlimit. 4057 */ 4058 while (vaddr < high) { 4059 pte = htable_walk(kas.a_hat, &ht, &vaddr, high); 4060 if (ht == NULL) 4061 break; 4062 4063 level = ht->ht_level; 4064 entry = htable_va2entry(vaddr, ht); 4065 ASSERT(level <= mmu.max_page_level); 4066 ASSERT(PTE_ISPAGE(pte, level)); 4067 4068 /* 4069 * Unload the mapping from the page tables. 4070 */ 4071 (void) x86pte_inval(ht, entry, 0, NULL); 4072 ASSERT(ht->ht_valid_cnt > 0); 4073 HTABLE_DEC(ht->ht_valid_cnt); 4074 PGCNT_DEC(ht->ht_hat, ht->ht_level); 4075 4076 vaddr += LEVEL_SIZE(ht->ht_level); 4077 } 4078 if (ht) 4079 htable_release(ht); 4080 } 4081 4082 /* 4083 * Atomically update a new translation for a single page. If the 4084 * currently installed PTE doesn't match the value we expect to find, 4085 * it's not updated and we return the PTE we found. 4086 * 4087 * If activating nosync or NOWRITE and the page was modified we need to sync 4088 * with the page_t. Also sync with page_t if clearing ref/mod bits. 4089 */ 4090 static x86pte_t 4091 hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, x86pte_t new) 4092 { 4093 page_t *pp; 4094 uint_t rm = 0; 4095 x86pte_t replaced; 4096 4097 if (PTE_GET(expected, PT_SOFTWARE) < PT_NOSYNC && 4098 PTE_GET(expected, PT_MOD | PT_REF) && 4099 (PTE_GET(new, PT_NOSYNC) || !PTE_GET(new, PT_WRITABLE) || 4100 !PTE_GET(new, PT_MOD | PT_REF))) { 4101 4102 ASSERT(!pfn_is_foreign(PTE2PFN(expected, ht->ht_level))); 4103 pp = page_numtopp_nolock(PTE2PFN(expected, ht->ht_level)); 4104 ASSERT(pp != NULL); 4105 if (PTE_GET(expected, PT_MOD)) 4106 rm |= P_MOD; 4107 if (PTE_GET(expected, PT_REF)) 4108 rm |= P_REF; 4109 PTE_CLR(new, PT_MOD | PT_REF); 4110 } 4111 4112 replaced = x86pte_update(ht, entry, expected, new); 4113 if (replaced != expected) 4114 return (replaced); 4115 4116 if (rm) { 4117 /* 4118 * sync to all constituent pages of a large page 4119 */ 4120 pgcnt_t pgcnt = page_get_pagecnt(ht->ht_level); 4121 ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt)); 4122 while (pgcnt-- > 0) { 4123 /* 4124 * hat_page_demote() can't decrease 4125 * pszc below this mapping size 4126 * since large mapping existed after we 4127 * took mlist lock. 4128 */ 4129 ASSERT(pp->p_szc >= ht->ht_level); 4130 hat_page_setattr(pp, rm); 4131 ++pp; 4132 } 4133 } 4134 4135 return (0); 4136 } 4137 4138 /* ARGSUSED */ 4139 void 4140 hat_join_srd(struct hat *hat, vnode_t *evp) 4141 { 4142 } 4143 4144 /* ARGSUSED */ 4145 hat_region_cookie_t 4146 hat_join_region(struct hat *hat, 4147 caddr_t r_saddr, 4148 size_t r_size, 4149 void *r_obj, 4150 u_offset_t r_objoff, 4151 uchar_t r_perm, 4152 uchar_t r_pgszc, 4153 hat_rgn_cb_func_t r_cb_function, 4154 uint_t flags) 4155 { 4156 panic("No shared region support on x86"); 4157 return (HAT_INVALID_REGION_COOKIE); 4158 } 4159 4160 /* ARGSUSED */ 4161 void 4162 hat_leave_region(struct hat *hat, hat_region_cookie_t rcookie, uint_t flags) 4163 { 4164 panic("No shared region support on x86"); 4165 } 4166 4167 /* ARGSUSED */ 4168 void 4169 hat_dup_region(struct hat *hat, hat_region_cookie_t rcookie) 4170 { 4171 panic("No shared region support on x86"); 4172 } 4173 4174 4175 /* 4176 * Kernel Physical Mapping (kpm) facility 4177 * 4178 * Most of the routines needed to support segkpm are almost no-ops on the 4179 * x86 platform. We map in the entire segment when it is created and leave 4180 * it mapped in, so there is no additional work required to set up and tear 4181 * down individual mappings. All of these routines were created to support 4182 * SPARC platforms that have to avoid aliasing in their virtually indexed 4183 * caches. 4184 * 4185 * Most of the routines have sanity checks in them (e.g. verifying that the 4186 * passed-in page is locked). We don't actually care about most of these 4187 * checks on x86, but we leave them in place to identify problems in the 4188 * upper levels. 4189 */ 4190 4191 /* 4192 * Map in a locked page and return the vaddr. 4193 */ 4194 /*ARGSUSED*/ 4195 caddr_t 4196 hat_kpm_mapin(struct page *pp, struct kpme *kpme) 4197 { 4198 caddr_t vaddr; 4199 4200 #ifdef DEBUG 4201 if (kpm_enable == 0) { 4202 cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set\n"); 4203 return ((caddr_t)NULL); 4204 } 4205 4206 if (pp == NULL || PAGE_LOCKED(pp) == 0) { 4207 cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked\n"); 4208 return ((caddr_t)NULL); 4209 } 4210 #endif 4211 4212 vaddr = hat_kpm_page2va(pp, 1); 4213 4214 return (vaddr); 4215 } 4216 4217 /* 4218 * Mapout a locked page. 4219 */ 4220 /*ARGSUSED*/ 4221 void 4222 hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr) 4223 { 4224 #ifdef DEBUG 4225 if (kpm_enable == 0) { 4226 cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set\n"); 4227 return; 4228 } 4229 4230 if (IS_KPM_ADDR(vaddr) == 0) { 4231 cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address\n"); 4232 return; 4233 } 4234 4235 if (pp == NULL || PAGE_LOCKED(pp) == 0) { 4236 cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked\n"); 4237 return; 4238 } 4239 #endif 4240 } 4241 4242 /* 4243 * Return the kpm virtual address for a specific pfn 4244 */ 4245 caddr_t 4246 hat_kpm_pfn2va(pfn_t pfn) 4247 { 4248 uintptr_t vaddr = (uintptr_t)kpm_vbase + mmu_ptob(pfn); 4249 4250 ASSERT(!pfn_is_foreign(pfn)); 4251 return ((caddr_t)vaddr); 4252 } 4253 4254 /* 4255 * Return the kpm virtual address for the page at pp. 4256 */ 4257 /*ARGSUSED*/ 4258 caddr_t 4259 hat_kpm_page2va(struct page *pp, int checkswap) 4260 { 4261 return (hat_kpm_pfn2va(pp->p_pagenum)); 4262 } 4263 4264 /* 4265 * Return the page frame number for the kpm virtual address vaddr. 4266 */ 4267 pfn_t 4268 hat_kpm_va2pfn(caddr_t vaddr) 4269 { 4270 pfn_t pfn; 4271 4272 ASSERT(IS_KPM_ADDR(vaddr)); 4273 4274 pfn = (pfn_t)btop(vaddr - kpm_vbase); 4275 4276 return (pfn); 4277 } 4278 4279 4280 /* 4281 * Return the page for the kpm virtual address vaddr. 4282 */ 4283 page_t * 4284 hat_kpm_vaddr2page(caddr_t vaddr) 4285 { 4286 pfn_t pfn; 4287 4288 ASSERT(IS_KPM_ADDR(vaddr)); 4289 4290 pfn = hat_kpm_va2pfn(vaddr); 4291 4292 return (page_numtopp_nolock(pfn)); 4293 } 4294 4295 /* 4296 * hat_kpm_fault is called from segkpm_fault when we take a page fault on a 4297 * KPM page. This should never happen on x86 4298 */ 4299 int 4300 hat_kpm_fault(hat_t *hat, caddr_t vaddr) 4301 { 4302 panic("pagefault in seg_kpm. hat: 0x%p vaddr: 0x%p", 4303 (void *)hat, (void *)vaddr); 4304 4305 return (0); 4306 } 4307 4308 /*ARGSUSED*/ 4309 void 4310 hat_kpm_mseghash_clear(int nentries) 4311 {} 4312 4313 /*ARGSUSED*/ 4314 void 4315 hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp) 4316 {} 4317 4318 #ifdef __xpv 4319 /* 4320 * There are specific Hypervisor calls to establish and remove mappings 4321 * to grant table references and the privcmd driver. We have to ensure 4322 * that a page table actually exists. 4323 */ 4324 void 4325 hat_prepare_mapping(hat_t *hat, caddr_t addr, uint64_t *pte_ma) 4326 { 4327 maddr_t base_ma; 4328 htable_t *ht; 4329 uint_t entry; 4330 4331 ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE)); 4332 XPV_DISALLOW_MIGRATE(); 4333 ht = htable_create(hat, (uintptr_t)addr, 0, NULL); 4334 4335 /* 4336 * if an address for pte_ma is passed in, return the MA of the pte 4337 * for this specific address. This address is only valid as long 4338 * as the htable stays locked. 4339 */ 4340 if (pte_ma != NULL) { 4341 entry = htable_va2entry((uintptr_t)addr, ht); 4342 base_ma = pa_to_ma(ptob(ht->ht_pfn)); 4343 *pte_ma = base_ma + (entry << mmu.pte_size_shift); 4344 } 4345 XPV_ALLOW_MIGRATE(); 4346 } 4347 4348 void 4349 hat_release_mapping(hat_t *hat, caddr_t addr) 4350 { 4351 htable_t *ht; 4352 4353 ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE)); 4354 XPV_DISALLOW_MIGRATE(); 4355 ht = htable_lookup(hat, (uintptr_t)addr, 0); 4356 ASSERT(ht != NULL); 4357 ASSERT(ht->ht_busy >= 2); 4358 htable_release(ht); 4359 htable_release(ht); 4360 XPV_ALLOW_MIGRATE(); 4361 } 4362 #endif 4363