xref: /titanic_51/usr/src/uts/i86pc/vm/hat_i86.c (revision cf5b5989488984444a152faba2a8183a71dcf485)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * VM - Hardware Address Translation management for i386 and amd64
30  *
31  * Implementation of the interfaces described in <common/vm/hat.h>
32  *
33  * Nearly all the details of how the hardware is managed should not be
34  * visible outside this layer except for misc. machine specific functions
35  * that work in conjunction with this code.
36  *
37  * Routines used only inside of i86pc/vm start with hati_ for HAT Internal.
38  */
39 
40 #include <sys/machparam.h>
41 #include <sys/machsystm.h>
42 #include <sys/mman.h>
43 #include <sys/types.h>
44 #include <sys/systm.h>
45 #include <sys/cpuvar.h>
46 #include <sys/thread.h>
47 #include <sys/proc.h>
48 #include <sys/cpu.h>
49 #include <sys/kmem.h>
50 #include <sys/disp.h>
51 #include <sys/shm.h>
52 #include <sys/sysmacros.h>
53 #include <sys/machparam.h>
54 #include <sys/vmem.h>
55 #include <sys/vmsystm.h>
56 #include <sys/promif.h>
57 #include <sys/var.h>
58 #include <sys/x86_archext.h>
59 #include <sys/atomic.h>
60 #include <sys/bitmap.h>
61 #include <sys/controlregs.h>
62 #include <sys/bootconf.h>
63 #include <sys/bootsvcs.h>
64 #include <sys/bootinfo.h>
65 #include <sys/archsystm.h>
66 
67 #include <vm/seg_kmem.h>
68 #include <vm/hat_i86.h>
69 #include <vm/as.h>
70 #include <vm/seg.h>
71 #include <vm/page.h>
72 #include <vm/seg_kp.h>
73 #include <vm/seg_kpm.h>
74 #include <vm/vm_dep.h>
75 #ifdef __xpv
76 #include <sys/hypervisor.h>
77 #endif
78 #include <vm/kboot_mmu.h>
79 #include <vm/seg_spt.h>
80 
81 #include <sys/cmn_err.h>
82 
83 /*
84  * Basic parameters for hat operation.
85  */
86 struct hat_mmu_info mmu;
87 
88 /*
89  * The page that is the kernel's top level pagetable.
90  *
91  * For 32 bit PAE support on i86pc, the kernel hat will use the 1st 4 entries
92  * on this 4K page for its top level page table. The remaining groups of
93  * 4 entries are used for per processor copies of user VLP pagetables for
94  * running threads.  See hat_switch() and reload_pae32() for details.
95  *
96  * vlp_page[0..3] - level==2 PTEs for kernel HAT
97  * vlp_page[4..7] - level==2 PTEs for user thread on cpu 0
98  * vlp_page[8..11]  - level==2 PTE for user thread on cpu 1
99  * etc...
100  */
101 static x86pte_t *vlp_page;
102 
103 /*
104  * forward declaration of internal utility routines
105  */
106 static x86pte_t hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected,
107 	x86pte_t new);
108 
109 /*
110  * The kernel address space exists in all HATs. To implement this the
111  * kernel reserves a fixed number of entries in the topmost level(s) of page
112  * tables. The values are setup during startup and then copied to every user
113  * hat created by hat_alloc(). This means that kernelbase must be:
114  *
115  *	  4Meg aligned for 32 bit kernels
116  *	512Gig aligned for x86_64 64 bit kernel
117  *
118  * The hat_kernel_range_ts describe what needs to be copied from kernel hat
119  * to each user hat.
120  */
121 typedef struct hat_kernel_range {
122 	level_t		hkr_level;
123 	uintptr_t	hkr_start_va;
124 	uintptr_t	hkr_end_va;	/* zero means to end of memory */
125 } hat_kernel_range_t;
126 #define	NUM_KERNEL_RANGE 2
127 static hat_kernel_range_t kernel_ranges[NUM_KERNEL_RANGE];
128 static int num_kernel_ranges;
129 
130 uint_t use_boot_reserve = 1;	/* cleared after early boot process */
131 uint_t can_steal_post_boot = 0;	/* set late in boot to enable stealing */
132 
133 /*
134  * A cpuset for all cpus. This is used for kernel address cross calls, since
135  * the kernel addresses apply to all cpus.
136  */
137 cpuset_t khat_cpuset;
138 
139 /*
140  * management stuff for hat structures
141  */
142 kmutex_t	hat_list_lock;
143 kcondvar_t	hat_list_cv;
144 kmem_cache_t	*hat_cache;
145 kmem_cache_t	*hat_hash_cache;
146 kmem_cache_t	*vlp_hash_cache;
147 
148 /*
149  * Simple statistics
150  */
151 struct hatstats hatstat;
152 
153 /*
154  * useful stuff for atomic access/clearing/setting REF/MOD/RO bits in page_t's.
155  */
156 extern void atomic_orb(uchar_t *addr, uchar_t val);
157 extern void atomic_andb(uchar_t *addr, uchar_t val);
158 
159 #define	PP_GETRM(pp, rmmask)    (pp->p_nrm & rmmask)
160 #define	PP_ISMOD(pp)		PP_GETRM(pp, P_MOD)
161 #define	PP_ISREF(pp)		PP_GETRM(pp, P_REF)
162 #define	PP_ISRO(pp)		PP_GETRM(pp, P_RO)
163 
164 #define	PP_SETRM(pp, rm)	atomic_orb(&(pp->p_nrm), rm)
165 #define	PP_SETMOD(pp)		PP_SETRM(pp, P_MOD)
166 #define	PP_SETREF(pp)		PP_SETRM(pp, P_REF)
167 #define	PP_SETRO(pp)		PP_SETRM(pp, P_RO)
168 
169 #define	PP_CLRRM(pp, rm)	atomic_andb(&(pp->p_nrm), ~(rm))
170 #define	PP_CLRMOD(pp)   	PP_CLRRM(pp, P_MOD)
171 #define	PP_CLRREF(pp)   	PP_CLRRM(pp, P_REF)
172 #define	PP_CLRRO(pp)    	PP_CLRRM(pp, P_RO)
173 #define	PP_CLRALL(pp)		PP_CLRRM(pp, P_MOD | P_REF | P_RO)
174 
175 /*
176  * kmem cache constructor for struct hat
177  */
178 /*ARGSUSED*/
179 static int
180 hati_constructor(void *buf, void *handle, int kmflags)
181 {
182 	hat_t	*hat = buf;
183 
184 	mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
185 	bzero(hat->hat_pages_mapped,
186 	    sizeof (pgcnt_t) * (mmu.max_page_level + 1));
187 	hat->hat_ism_pgcnt = 0;
188 	hat->hat_stats = 0;
189 	hat->hat_flags = 0;
190 	CPUSET_ZERO(hat->hat_cpus);
191 	hat->hat_htable = NULL;
192 	hat->hat_ht_hash = NULL;
193 	return (0);
194 }
195 
196 /*
197  * Allocate a hat structure for as. We also create the top level
198  * htable and initialize it to contain the kernel hat entries.
199  */
200 hat_t *
201 hat_alloc(struct as *as)
202 {
203 	hat_t			*hat;
204 	htable_t		*ht;	/* top level htable */
205 	uint_t			use_vlp;
206 	uint_t			r;
207 	hat_kernel_range_t	*rp;
208 	uintptr_t		va;
209 	uintptr_t		eva;
210 	uint_t			start;
211 	uint_t			cnt;
212 	htable_t		*src;
213 
214 	/*
215 	 * Once we start creating user process HATs we can enable
216 	 * the htable_steal() code.
217 	 */
218 	if (can_steal_post_boot == 0)
219 		can_steal_post_boot = 1;
220 
221 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
222 	hat = kmem_cache_alloc(hat_cache, KM_SLEEP);
223 	hat->hat_as = as;
224 	mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
225 	ASSERT(hat->hat_flags == 0);
226 
227 #if defined(__xpv)
228 	/*
229 	 * No VLP stuff on the hypervisor due to the 64-bit split top level
230 	 * page tables.  On 32-bit it's not needed as the hypervisor takes
231 	 * care of copying the top level PTEs to a below 4Gig page.
232 	 */
233 	use_vlp = 0;
234 #else	/* __xpv */
235 	/* 32 bit processes uses a VLP style hat when running with PAE */
236 #if defined(__amd64)
237 	use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32);
238 #elif defined(__i386)
239 	use_vlp = mmu.pae_hat;
240 #endif
241 #endif	/* __xpv */
242 	if (use_vlp) {
243 		hat->hat_flags = HAT_VLP;
244 		bzero(hat->hat_vlp_ptes, VLP_SIZE);
245 	}
246 
247 	/*
248 	 * Allocate the htable hash
249 	 */
250 	if ((hat->hat_flags & HAT_VLP)) {
251 		hat->hat_num_hash = mmu.vlp_hash_cnt;
252 		hat->hat_ht_hash = kmem_cache_alloc(vlp_hash_cache, KM_SLEEP);
253 	} else {
254 		hat->hat_num_hash = mmu.hash_cnt;
255 		hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_SLEEP);
256 	}
257 	bzero(hat->hat_ht_hash, hat->hat_num_hash * sizeof (htable_t *));
258 
259 	/*
260 	 * Initialize Kernel HAT entries at the top of the top level page
261 	 * tables for the new hat.
262 	 */
263 	hat->hat_htable = NULL;
264 	hat->hat_ht_cached = NULL;
265 	XPV_DISALLOW_MIGRATE();
266 	ht = htable_create(hat, (uintptr_t)0, TOP_LEVEL(hat), NULL);
267 	hat->hat_htable = ht;
268 
269 #if defined(__amd64)
270 	if (hat->hat_flags & HAT_VLP)
271 		goto init_done;
272 #endif
273 
274 	for (r = 0; r < num_kernel_ranges; ++r) {
275 		rp = &kernel_ranges[r];
276 		for (va = rp->hkr_start_va; va != rp->hkr_end_va;
277 		    va += cnt * LEVEL_SIZE(rp->hkr_level)) {
278 
279 			if (rp->hkr_level == TOP_LEVEL(hat))
280 				ht = hat->hat_htable;
281 			else
282 				ht = htable_create(hat, va, rp->hkr_level,
283 				    NULL);
284 
285 			start = htable_va2entry(va, ht);
286 			cnt = HTABLE_NUM_PTES(ht) - start;
287 			eva = va +
288 			    ((uintptr_t)cnt << LEVEL_SHIFT(rp->hkr_level));
289 			if (rp->hkr_end_va != 0 &&
290 			    (eva > rp->hkr_end_va || eva == 0))
291 				cnt = htable_va2entry(rp->hkr_end_va, ht) -
292 				    start;
293 
294 #if defined(__i386) && !defined(__xpv)
295 			if (ht->ht_flags & HTABLE_VLP) {
296 				bcopy(&vlp_page[start],
297 				    &hat->hat_vlp_ptes[start],
298 				    cnt * sizeof (x86pte_t));
299 				continue;
300 			}
301 #endif
302 			src = htable_lookup(kas.a_hat, va, rp->hkr_level);
303 			ASSERT(src != NULL);
304 			x86pte_copy(src, ht, start, cnt);
305 			htable_release(src);
306 		}
307 	}
308 
309 init_done:
310 	XPV_ALLOW_MIGRATE();
311 
312 #if defined(__xpv)
313 	/*
314 	 * Pin top level page tables after initializing them
315 	 */
316 	xen_pin(hat->hat_htable->ht_pfn, mmu.max_level);
317 #if defined(__amd64)
318 	xen_pin(hat->hat_user_ptable, mmu.max_level);
319 #endif
320 #endif
321 
322 	/*
323 	 * Put it at the start of the global list of all hats (used by stealing)
324 	 *
325 	 * kas.a_hat is not in the list but is instead used to find the
326 	 * first and last items in the list.
327 	 *
328 	 * - kas.a_hat->hat_next points to the start of the user hats.
329 	 *   The list ends where hat->hat_next == NULL
330 	 *
331 	 * - kas.a_hat->hat_prev points to the last of the user hats.
332 	 *   The list begins where hat->hat_prev == NULL
333 	 */
334 	mutex_enter(&hat_list_lock);
335 	hat->hat_prev = NULL;
336 	hat->hat_next = kas.a_hat->hat_next;
337 	if (hat->hat_next)
338 		hat->hat_next->hat_prev = hat;
339 	else
340 		kas.a_hat->hat_prev = hat;
341 	kas.a_hat->hat_next = hat;
342 	mutex_exit(&hat_list_lock);
343 
344 	return (hat);
345 }
346 
347 /*
348  * process has finished executing but as has not been cleaned up yet.
349  */
350 /*ARGSUSED*/
351 void
352 hat_free_start(hat_t *hat)
353 {
354 	ASSERT(AS_WRITE_HELD(hat->hat_as, &hat->hat_as->a_lock));
355 
356 	/*
357 	 * If the hat is currently a stealing victim, wait for the stealing
358 	 * to finish.  Once we mark it as HAT_FREEING, htable_steal()
359 	 * won't look at its pagetables anymore.
360 	 */
361 	mutex_enter(&hat_list_lock);
362 	while (hat->hat_flags & HAT_VICTIM)
363 		cv_wait(&hat_list_cv, &hat_list_lock);
364 	hat->hat_flags |= HAT_FREEING;
365 	mutex_exit(&hat_list_lock);
366 }
367 
368 /*
369  * An address space is being destroyed, so we destroy the associated hat.
370  */
371 void
372 hat_free_end(hat_t *hat)
373 {
374 	kmem_cache_t *cache;
375 
376 	ASSERT(hat->hat_flags & HAT_FREEING);
377 
378 	/*
379 	 * must not be running on the given hat
380 	 */
381 	ASSERT(CPU->cpu_current_hat != hat);
382 
383 	/*
384 	 * Remove it from the list of HATs
385 	 */
386 	mutex_enter(&hat_list_lock);
387 	if (hat->hat_prev)
388 		hat->hat_prev->hat_next = hat->hat_next;
389 	else
390 		kas.a_hat->hat_next = hat->hat_next;
391 	if (hat->hat_next)
392 		hat->hat_next->hat_prev = hat->hat_prev;
393 	else
394 		kas.a_hat->hat_prev = hat->hat_prev;
395 	mutex_exit(&hat_list_lock);
396 	hat->hat_next = hat->hat_prev = NULL;
397 
398 #if defined(__xpv)
399 	/*
400 	 * On the hypervisor, unpin top level page table(s)
401 	 */
402 	xen_unpin(hat->hat_htable->ht_pfn);
403 #if defined(__amd64)
404 	xen_unpin(hat->hat_user_ptable);
405 #endif
406 #endif
407 
408 	/*
409 	 * Make a pass through the htables freeing them all up.
410 	 */
411 	htable_purge_hat(hat);
412 
413 	/*
414 	 * Decide which kmem cache the hash table came from, then free it.
415 	 */
416 	if (hat->hat_flags & HAT_VLP)
417 		cache = vlp_hash_cache;
418 	else
419 		cache = hat_hash_cache;
420 	kmem_cache_free(cache, hat->hat_ht_hash);
421 	hat->hat_ht_hash = NULL;
422 
423 	hat->hat_flags = 0;
424 	kmem_cache_free(hat_cache, hat);
425 }
426 
427 /*
428  * round kernelbase down to a supported value to use for _userlimit
429  *
430  * userlimit must be aligned down to an entry in the top level htable.
431  * The one exception is for 32 bit HAT's running PAE.
432  */
433 uintptr_t
434 hat_kernelbase(uintptr_t va)
435 {
436 #if defined(__i386)
437 	va &= LEVEL_MASK(1);
438 #endif
439 	if (IN_VA_HOLE(va))
440 		panic("_userlimit %p will fall in VA hole\n", (void *)va);
441 	return (va);
442 }
443 
444 /*
445  * Initialize hat data structures based on processor MMU information.
446  */
447 void
448 mmu_init(void)
449 {
450 	uint_t max_htables;
451 	uint_t pa_bits;
452 	uint_t va_bits;
453 	int i;
454 
455 	/*
456 	 * If CPU enabled the page table global bit, use it for the kernel
457 	 * This is bit 7 in CR4 (PGE - Page Global Enable).
458 	 */
459 	if ((x86_feature & X86_PGE) != 0 && (getcr4() & CR4_PGE) != 0)
460 		mmu.pt_global = PT_GLOBAL;
461 
462 	/*
463 	 * Detect NX and PAE usage.
464 	 */
465 	mmu.pae_hat = kbm_pae_support;
466 	if (kbm_nx_support)
467 		mmu.pt_nx = PT_NX;
468 	else
469 		mmu.pt_nx = 0;
470 
471 	/*
472 	 * Use CPU info to set various MMU parameters
473 	 */
474 	cpuid_get_addrsize(CPU, &pa_bits, &va_bits);
475 
476 	if (va_bits < sizeof (void *) * NBBY) {
477 		mmu.hole_start = (1ul << (va_bits - 1));
478 		mmu.hole_end = 0ul - mmu.hole_start - 1;
479 	} else {
480 		mmu.hole_end = 0;
481 		mmu.hole_start = mmu.hole_end - 1;
482 	}
483 #if defined(OPTERON_ERRATUM_121)
484 	/*
485 	 * If erratum 121 has already been detected at this time, hole_start
486 	 * contains the value to be subtracted from mmu.hole_start.
487 	 */
488 	ASSERT(hole_start == 0 || opteron_erratum_121 != 0);
489 	hole_start = mmu.hole_start - hole_start;
490 #else
491 	hole_start = mmu.hole_start;
492 #endif
493 	hole_end = mmu.hole_end;
494 
495 	mmu.highest_pfn = mmu_btop((1ull << pa_bits) - 1);
496 	if (mmu.pae_hat == 0 && pa_bits > 32)
497 		mmu.highest_pfn = PFN_4G - 1;
498 
499 	if (mmu.pae_hat) {
500 		mmu.pte_size = 8;	/* 8 byte PTEs */
501 		mmu.pte_size_shift = 3;
502 	} else {
503 		mmu.pte_size = 4;	/* 4 byte PTEs */
504 		mmu.pte_size_shift = 2;
505 	}
506 
507 	if (mmu.pae_hat && (x86_feature & X86_PAE) == 0)
508 		panic("Processor does not support PAE");
509 
510 	if ((x86_feature & X86_CX8) == 0)
511 		panic("Processor does not support cmpxchg8b instruction");
512 
513 	/*
514 	 * Initialize parameters based on the 64 or 32 bit kernels and
515 	 * for the 32 bit kernel decide if we should use PAE.
516 	 */
517 	if (kbm_largepage_support)
518 		mmu.max_page_level = 1;
519 	else
520 		mmu.max_page_level = 0;
521 	mmu_page_sizes = mmu.max_page_level + 1;
522 	mmu_exported_page_sizes = mmu_page_sizes;
523 
524 #if defined(__amd64)
525 
526 	mmu.num_level = 4;
527 	mmu.max_level = 3;
528 	mmu.ptes_per_table = 512;
529 	mmu.top_level_count = 512;
530 
531 	mmu.level_shift[0] = 12;
532 	mmu.level_shift[1] = 21;
533 	mmu.level_shift[2] = 30;
534 	mmu.level_shift[3] = 39;
535 
536 #elif defined(__i386)
537 
538 	if (mmu.pae_hat) {
539 		mmu.num_level = 3;
540 		mmu.max_level = 2;
541 		mmu.ptes_per_table = 512;
542 		mmu.top_level_count = 4;
543 
544 		mmu.level_shift[0] = 12;
545 		mmu.level_shift[1] = 21;
546 		mmu.level_shift[2] = 30;
547 
548 	} else {
549 		mmu.num_level = 2;
550 		mmu.max_level = 1;
551 		mmu.ptes_per_table = 1024;
552 		mmu.top_level_count = 1024;
553 
554 		mmu.level_shift[0] = 12;
555 		mmu.level_shift[1] = 22;
556 	}
557 
558 #endif	/* __i386 */
559 
560 	for (i = 0; i < mmu.num_level; ++i) {
561 		mmu.level_size[i] = 1UL << mmu.level_shift[i];
562 		mmu.level_offset[i] = mmu.level_size[i] - 1;
563 		mmu.level_mask[i] = ~mmu.level_offset[i];
564 	}
565 
566 	for (i = 0; i <= mmu.max_page_level; ++i) {
567 		mmu.pte_bits[i] = PT_VALID;
568 #if defined(__xpv) && defined(__amd64)
569 		mmu.pte_bits[i] |= PT_USER;
570 #endif
571 		if (i > 0)
572 			mmu.pte_bits[i] |= PT_PAGESIZE;
573 	}
574 
575 	/*
576 	 * NOTE Legacy 32 bit PAE mode only has the P_VALID bit at top level.
577 	 */
578 	for (i = 1; i < mmu.num_level; ++i)
579 		mmu.ptp_bits[i] = PT_PTPBITS;
580 
581 #if defined(__i386)
582 	mmu.ptp_bits[2] = PT_VALID;
583 #endif
584 
585 	/*
586 	 * Compute how many hash table entries to have per process for htables.
587 	 * We start with 1 page's worth of entries.
588 	 *
589 	 * If physical memory is small, reduce the amount need to cover it.
590 	 */
591 	max_htables = physmax / mmu.ptes_per_table;
592 	mmu.hash_cnt = MMU_PAGESIZE / sizeof (htable_t *);
593 	while (mmu.hash_cnt > 16 && mmu.hash_cnt >= max_htables)
594 		mmu.hash_cnt >>= 1;
595 	mmu.vlp_hash_cnt = mmu.hash_cnt;
596 
597 #if defined(__amd64)
598 	/*
599 	 * If running in 64 bits and physical memory is large,
600 	 * increase the size of the cache to cover all of memory for
601 	 * a 64 bit process.
602 	 */
603 #define	HASH_MAX_LENGTH 4
604 	while (mmu.hash_cnt * HASH_MAX_LENGTH < max_htables)
605 		mmu.hash_cnt <<= 1;
606 #endif
607 }
608 
609 
610 /*
611  * initialize hat data structures
612  */
613 void
614 hat_init()
615 {
616 #if defined(__i386)
617 	/*
618 	 * _userlimit must be aligned correctly
619 	 */
620 	if ((_userlimit & LEVEL_MASK(1)) != _userlimit) {
621 		prom_printf("hat_init(): _userlimit=%p, not aligned at %p\n",
622 		    (void *)_userlimit, (void *)LEVEL_SIZE(1));
623 		halt("hat_init(): Unable to continue");
624 	}
625 #endif
626 
627 	cv_init(&hat_list_cv, NULL, CV_DEFAULT, NULL);
628 
629 	/*
630 	 * initialize kmem caches
631 	 */
632 	htable_init();
633 	hment_init();
634 
635 	hat_cache = kmem_cache_create("hat_t",
636 	    sizeof (hat_t), 0, hati_constructor, NULL, NULL,
637 	    NULL, 0, 0);
638 
639 	hat_hash_cache = kmem_cache_create("HatHash",
640 	    mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
641 	    NULL, 0, 0);
642 
643 	/*
644 	 * VLP hats can use a smaller hash table size on large memroy machines
645 	 */
646 	if (mmu.hash_cnt == mmu.vlp_hash_cnt) {
647 		vlp_hash_cache = hat_hash_cache;
648 	} else {
649 		vlp_hash_cache = kmem_cache_create("HatVlpHash",
650 		    mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
651 		    NULL, 0, 0);
652 	}
653 
654 	/*
655 	 * Set up the kernel's hat
656 	 */
657 	AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER);
658 	kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP);
659 	mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
660 	kas.a_hat->hat_as = &kas;
661 	kas.a_hat->hat_flags = 0;
662 	AS_LOCK_EXIT(&kas, &kas.a_lock);
663 
664 	CPUSET_ZERO(khat_cpuset);
665 	CPUSET_ADD(khat_cpuset, CPU->cpu_id);
666 
667 	/*
668 	 * The kernel hat's next pointer serves as the head of the hat list .
669 	 * The kernel hat's prev pointer tracks the last hat on the list for
670 	 * htable_steal() to use.
671 	 */
672 	kas.a_hat->hat_next = NULL;
673 	kas.a_hat->hat_prev = NULL;
674 
675 	/*
676 	 * Allocate an htable hash bucket for the kernel
677 	 * XX64 - tune for 64 bit procs
678 	 */
679 	kas.a_hat->hat_num_hash = mmu.hash_cnt;
680 	kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP);
681 	bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *));
682 
683 	/*
684 	 * zero out the top level and cached htable pointers
685 	 */
686 	kas.a_hat->hat_ht_cached = NULL;
687 	kas.a_hat->hat_htable = NULL;
688 
689 	/*
690 	 * Pre-allocate hrm_hashtab before enabling the collection of
691 	 * refmod statistics.  Allocating on the fly would mean us
692 	 * running the risk of suffering recursive mutex enters or
693 	 * deadlocks.
694 	 */
695 	hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *),
696 	    KM_SLEEP);
697 }
698 
699 /*
700  * Prepare CPU specific pagetables for VLP processes on 64 bit kernels.
701  *
702  * Each CPU has a set of 2 pagetables that are reused for any 32 bit
703  * process it runs. They are the top level pagetable, hci_vlp_l3ptes, and
704  * the next to top level table for the bottom 512 Gig, hci_vlp_l2ptes.
705  */
706 /*ARGSUSED*/
707 static void
708 hat_vlp_setup(struct cpu *cpu)
709 {
710 #if defined(__amd64) && !defined(__xpv)
711 	struct hat_cpu_info *hci = cpu->cpu_hat_info;
712 	pfn_t pfn;
713 
714 	/*
715 	 * allocate the level==2 page table for the bottom most
716 	 * 512Gig of address space (this is where 32 bit apps live)
717 	 */
718 	ASSERT(hci != NULL);
719 	hci->hci_vlp_l2ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
720 
721 	/*
722 	 * Allocate a top level pagetable and copy the kernel's
723 	 * entries into it. Then link in hci_vlp_l2ptes in the 1st entry.
724 	 */
725 	hci->hci_vlp_l3ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
726 	hci->hci_vlp_pfn =
727 	    hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l3ptes);
728 	ASSERT(hci->hci_vlp_pfn != PFN_INVALID);
729 	bcopy(vlp_page, hci->hci_vlp_l3ptes, MMU_PAGESIZE);
730 
731 	pfn = hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l2ptes);
732 	ASSERT(pfn != PFN_INVALID);
733 	hci->hci_vlp_l3ptes[0] = MAKEPTP(pfn, 2);
734 #endif /* __amd64 && !__xpv */
735 }
736 
737 /*ARGSUSED*/
738 static void
739 hat_vlp_teardown(cpu_t *cpu)
740 {
741 #if defined(__amd64) && !defined(__xpv)
742 	struct hat_cpu_info *hci;
743 
744 	if ((hci = cpu->cpu_hat_info) == NULL)
745 		return;
746 	if (hci->hci_vlp_l2ptes)
747 		kmem_free(hci->hci_vlp_l2ptes, MMU_PAGESIZE);
748 	if (hci->hci_vlp_l3ptes)
749 		kmem_free(hci->hci_vlp_l3ptes, MMU_PAGESIZE);
750 #endif
751 }
752 
753 #define	NEXT_HKR(r, l, s, e) {			\
754 	kernel_ranges[r].hkr_level = l;		\
755 	kernel_ranges[r].hkr_start_va = s;	\
756 	kernel_ranges[r].hkr_end_va = e;	\
757 	++r;					\
758 }
759 
760 /*
761  * Finish filling in the kernel hat.
762  * Pre fill in all top level kernel page table entries for the kernel's
763  * part of the address range.  From this point on we can't use any new
764  * kernel large pages if they need PTE's at max_level
765  *
766  * create the kmap mappings.
767  */
768 void
769 hat_init_finish(void)
770 {
771 	size_t		size;
772 	uint_t		r = 0;
773 	uintptr_t	va;
774 	hat_kernel_range_t *rp;
775 
776 
777 	/*
778 	 * We are now effectively running on the kernel hat.
779 	 * Clearing use_boot_reserve shuts off using the pre-allocated boot
780 	 * reserve for all HAT allocations.  From here on, the reserves are
781 	 * only used when avoiding recursion in kmem_alloc().
782 	 */
783 	use_boot_reserve = 0;
784 	htable_adjust_reserve();
785 
786 	/*
787 	 * User HATs are initialized with copies of all kernel mappings in
788 	 * higher level page tables. Ensure that those entries exist.
789 	 */
790 #if defined(__amd64)
791 
792 	NEXT_HKR(r, 3, kernelbase, 0);
793 #if defined(__xpv)
794 	NEXT_HKR(r, 3, HYPERVISOR_VIRT_START, HYPERVISOR_VIRT_END);
795 #endif
796 
797 #elif defined(__i386)
798 
799 #if !defined(__xpv)
800 	if (mmu.pae_hat) {
801 		va = kernelbase;
802 		if ((va & LEVEL_MASK(2)) != va) {
803 			va = P2ROUNDUP(va, LEVEL_SIZE(2));
804 			NEXT_HKR(r, 1, kernelbase, va);
805 		}
806 		if (va != 0)
807 			NEXT_HKR(r, 2, va, 0);
808 	} else
809 #endif /* __xpv */
810 		NEXT_HKR(r, 1, kernelbase, 0);
811 
812 #endif /* __i386 */
813 
814 	num_kernel_ranges = r;
815 
816 	/*
817 	 * Create all the kernel pagetables that will have entries
818 	 * shared to user HATs.
819 	 */
820 	for (r = 0; r < num_kernel_ranges; ++r) {
821 		rp = &kernel_ranges[r];
822 		for (va = rp->hkr_start_va; va != rp->hkr_end_va;
823 		    va += LEVEL_SIZE(rp->hkr_level)) {
824 			htable_t *ht;
825 
826 			if (IN_HYPERVISOR_VA(va))
827 				continue;
828 
829 			/* can/must skip if a page mapping already exists */
830 			if (rp->hkr_level <= mmu.max_page_level &&
831 			    (ht = htable_getpage(kas.a_hat, va, NULL)) !=
832 			    NULL) {
833 				htable_release(ht);
834 				continue;
835 			}
836 
837 			(void) htable_create(kas.a_hat, va, rp->hkr_level - 1,
838 			    NULL);
839 		}
840 	}
841 
842 	/*
843 	 * 32 bit PAE metal kernels use only 4 of the 512 entries in the
844 	 * page holding the top level pagetable. We use the remainder for
845 	 * the "per CPU" page tables for VLP processes.
846 	 * Map the top level kernel pagetable into the kernel to make
847 	 * it easy to use bcopy access these tables.
848 	 */
849 	if (mmu.pae_hat) {
850 		vlp_page = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP);
851 		hat_devload(kas.a_hat, (caddr_t)vlp_page, MMU_PAGESIZE,
852 		    kas.a_hat->hat_htable->ht_pfn,
853 #if !defined(__xpv)
854 		    PROT_WRITE |
855 #endif
856 		    PROT_READ | HAT_NOSYNC | HAT_UNORDERED_OK,
857 		    HAT_LOAD | HAT_LOAD_NOCONSIST);
858 	}
859 	hat_vlp_setup(CPU);
860 
861 	/*
862 	 * Create kmap (cached mappings of kernel PTEs)
863 	 * for 32 bit we map from segmap_start .. ekernelheap
864 	 * for 64 bit we map from segmap_start .. segmap_start + segmapsize;
865 	 */
866 #if defined(__i386)
867 	size = (uintptr_t)ekernelheap - segmap_start;
868 #elif defined(__amd64)
869 	size = segmapsize;
870 #endif
871 	hat_kmap_init((uintptr_t)segmap_start, size);
872 }
873 
874 /*
875  * On 32 bit PAE mode, PTE's are 64 bits, but ordinary atomic memory references
876  * are 32 bit, so for safety we must use cas64() to install these.
877  */
878 #ifdef __i386
879 static void
880 reload_pae32(hat_t *hat, cpu_t *cpu)
881 {
882 	x86pte_t *src;
883 	x86pte_t *dest;
884 	x86pte_t pte;
885 	int i;
886 
887 	/*
888 	 * Load the 4 entries of the level 2 page table into this
889 	 * cpu's range of the vlp_page and point cr3 at them.
890 	 */
891 	ASSERT(mmu.pae_hat);
892 	src = hat->hat_vlp_ptes;
893 	dest = vlp_page + (cpu->cpu_id + 1) * VLP_NUM_PTES;
894 	for (i = 0; i < VLP_NUM_PTES; ++i) {
895 		for (;;) {
896 			pte = dest[i];
897 			if (pte == src[i])
898 				break;
899 			if (cas64(dest + i, pte, src[i]) != src[i])
900 				break;
901 		}
902 	}
903 }
904 #endif
905 
906 /*
907  * Switch to a new active hat, maintaining bit masks to track active CPUs.
908  *
909  * On the 32-bit PAE hypervisor, %cr3 is a 64-bit value, on metal it
910  * remains a 32-bit value.
911  */
912 void
913 hat_switch(hat_t *hat)
914 {
915 	uint64_t	newcr3;
916 	cpu_t		*cpu = CPU;
917 	hat_t		*old = cpu->cpu_current_hat;
918 
919 	/*
920 	 * set up this information first, so we don't miss any cross calls
921 	 */
922 	if (old != NULL) {
923 		if (old == hat)
924 			return;
925 		if (old != kas.a_hat)
926 			CPUSET_ATOMIC_DEL(old->hat_cpus, cpu->cpu_id);
927 	}
928 
929 	/*
930 	 * Add this CPU to the active set for this HAT.
931 	 */
932 	if (hat != kas.a_hat) {
933 		CPUSET_ATOMIC_ADD(hat->hat_cpus, cpu->cpu_id);
934 	}
935 	cpu->cpu_current_hat = hat;
936 
937 	/*
938 	 * now go ahead and load cr3
939 	 */
940 	if (hat->hat_flags & HAT_VLP) {
941 #if defined(__amd64)
942 		x86pte_t *vlpptep = cpu->cpu_hat_info->hci_vlp_l2ptes;
943 
944 		VLP_COPY(hat->hat_vlp_ptes, vlpptep);
945 		newcr3 = MAKECR3(cpu->cpu_hat_info->hci_vlp_pfn);
946 #elif defined(__i386)
947 		reload_pae32(hat, cpu);
948 		newcr3 = MAKECR3(kas.a_hat->hat_htable->ht_pfn) +
949 		    (cpu->cpu_id + 1) * VLP_SIZE;
950 #endif
951 	} else {
952 		newcr3 = MAKECR3((uint64_t)hat->hat_htable->ht_pfn);
953 	}
954 #ifdef __xpv
955 	{
956 		struct mmuext_op t[2];
957 		uint_t retcnt;
958 		uint_t opcnt = 1;
959 
960 		t[0].cmd = MMUEXT_NEW_BASEPTR;
961 		t[0].arg1.mfn = mmu_btop(pa_to_ma(newcr3));
962 #if defined(__amd64)
963 		/*
964 		 * There's an interesting problem here, as to what to
965 		 * actually specify when switching to the kernel hat.
966 		 * For now we'll reuse the kernel hat again.
967 		 */
968 		t[1].cmd = MMUEXT_NEW_USER_BASEPTR;
969 		if (hat == kas.a_hat)
970 			t[1].arg1.mfn = mmu_btop(pa_to_ma(newcr3));
971 		else
972 			t[1].arg1.mfn = pfn_to_mfn(hat->hat_user_ptable);
973 		++opcnt;
974 #endif	/* __amd64 */
975 		if (HYPERVISOR_mmuext_op(t, opcnt, &retcnt, DOMID_SELF) < 0)
976 			panic("HYPERVISOR_mmu_update() failed");
977 		ASSERT(retcnt == opcnt);
978 
979 	}
980 #else
981 	setcr3(newcr3);
982 #endif
983 	ASSERT(cpu == CPU);
984 }
985 
986 /*
987  * Utility to return a valid x86pte_t from protections, pfn, and level number
988  */
989 static x86pte_t
990 hati_mkpte(pfn_t pfn, uint_t attr, level_t level, uint_t flags)
991 {
992 	x86pte_t	pte;
993 	uint_t		cache_attr = attr & HAT_ORDER_MASK;
994 
995 	pte = MAKEPTE(pfn, level);
996 
997 	if (attr & PROT_WRITE)
998 		PTE_SET(pte, PT_WRITABLE);
999 
1000 	if (attr & PROT_USER)
1001 		PTE_SET(pte, PT_USER);
1002 
1003 	if (!(attr & PROT_EXEC))
1004 		PTE_SET(pte, mmu.pt_nx);
1005 
1006 	/*
1007 	 * Set the software bits used track ref/mod sync's and hments.
1008 	 * If not using REF/MOD, set them to avoid h/w rewriting PTEs.
1009 	 */
1010 	if (flags & HAT_LOAD_NOCONSIST)
1011 		PTE_SET(pte, PT_NOCONSIST | PT_REF | PT_MOD);
1012 	else if (attr & HAT_NOSYNC)
1013 		PTE_SET(pte, PT_NOSYNC | PT_REF | PT_MOD);
1014 
1015 	/*
1016 	 * Set the caching attributes in the PTE. The combination
1017 	 * of attributes are poorly defined, so we pay attention
1018 	 * to them in the given order.
1019 	 *
1020 	 * The test for HAT_STRICTORDER is different because it's defined
1021 	 * as "0" - which was a stupid thing to do, but is too late to change!
1022 	 */
1023 	if (cache_attr == HAT_STRICTORDER) {
1024 		PTE_SET(pte, PT_NOCACHE);
1025 	/*LINTED [Lint hates empty ifs, but it's the obvious way to do this] */
1026 	} else if (cache_attr & (HAT_UNORDERED_OK | HAT_STORECACHING_OK)) {
1027 		/* nothing to set */;
1028 	} else if (cache_attr & (HAT_MERGING_OK | HAT_LOADCACHING_OK)) {
1029 		PTE_SET(pte, PT_NOCACHE);
1030 		if (x86_feature & X86_PAT)
1031 			PTE_SET(pte, (level == 0) ? PT_PAT_4K : PT_PAT_LARGE);
1032 		else
1033 			PTE_SET(pte, PT_WRITETHRU);
1034 	} else {
1035 		panic("hati_mkpte(): bad caching attributes: %x\n", cache_attr);
1036 	}
1037 
1038 	return (pte);
1039 }
1040 
1041 /*
1042  * Duplicate address translations of the parent to the child.
1043  * This function really isn't used anymore.
1044  */
1045 /*ARGSUSED*/
1046 int
1047 hat_dup(hat_t *old, hat_t *new, caddr_t addr, size_t len, uint_t flag)
1048 {
1049 	ASSERT((uintptr_t)addr < kernelbase);
1050 	ASSERT(new != kas.a_hat);
1051 	ASSERT(old != kas.a_hat);
1052 	return (0);
1053 }
1054 
1055 /*
1056  * Allocate any hat resources required for a process being swapped in.
1057  */
1058 /*ARGSUSED*/
1059 void
1060 hat_swapin(hat_t *hat)
1061 {
1062 	/* do nothing - we let everything fault back in */
1063 }
1064 
1065 /*
1066  * Unload all translations associated with an address space of a process
1067  * that is being swapped out.
1068  */
1069 void
1070 hat_swapout(hat_t *hat)
1071 {
1072 	uintptr_t	vaddr = (uintptr_t)0;
1073 	uintptr_t	eaddr = _userlimit;
1074 	htable_t	*ht = NULL;
1075 	level_t		l;
1076 
1077 	XPV_DISALLOW_MIGRATE();
1078 	/*
1079 	 * We can't just call hat_unload(hat, 0, _userlimit...)  here, because
1080 	 * seg_spt and shared pagetables can't be swapped out.
1081 	 * Take a look at segspt_shmswapout() - it's a big no-op.
1082 	 *
1083 	 * Instead we'll walk through all the address space and unload
1084 	 * any mappings which we are sure are not shared, not locked.
1085 	 */
1086 	ASSERT(IS_PAGEALIGNED(vaddr));
1087 	ASSERT(IS_PAGEALIGNED(eaddr));
1088 	ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1089 	if ((uintptr_t)hat->hat_as->a_userlimit < eaddr)
1090 		eaddr = (uintptr_t)hat->hat_as->a_userlimit;
1091 
1092 	while (vaddr < eaddr) {
1093 		(void) htable_walk(hat, &ht, &vaddr, eaddr);
1094 		if (ht == NULL)
1095 			break;
1096 
1097 		ASSERT(!IN_VA_HOLE(vaddr));
1098 
1099 		/*
1100 		 * If the page table is shared skip its entire range.
1101 		 * This code knows that only level 0 page tables are shared
1102 		 */
1103 		l = ht->ht_level;
1104 		if (ht->ht_flags & HTABLE_SHARED_PFN) {
1105 			ASSERT(l == 0);
1106 			vaddr = ht->ht_vaddr + LEVEL_SIZE(1);
1107 			htable_release(ht);
1108 			ht = NULL;
1109 			continue;
1110 		}
1111 
1112 		/*
1113 		 * If the page table has no locked entries, unload this one.
1114 		 */
1115 		if (ht->ht_lock_cnt == 0)
1116 			hat_unload(hat, (caddr_t)vaddr, LEVEL_SIZE(l),
1117 			    HAT_UNLOAD_UNMAP);
1118 
1119 		/*
1120 		 * If we have a level 0 page table with locked entries,
1121 		 * skip the entire page table, otherwise skip just one entry.
1122 		 */
1123 		if (ht->ht_lock_cnt > 0 && l == 0)
1124 			vaddr = ht->ht_vaddr + LEVEL_SIZE(1);
1125 		else
1126 			vaddr += LEVEL_SIZE(l);
1127 	}
1128 	if (ht)
1129 		htable_release(ht);
1130 
1131 	/*
1132 	 * We're in swapout because the system is low on memory, so
1133 	 * go back and flush all the htables off the cached list.
1134 	 */
1135 	htable_purge_hat(hat);
1136 	XPV_ALLOW_MIGRATE();
1137 }
1138 
1139 /*
1140  * returns number of bytes that have valid mappings in hat.
1141  */
1142 size_t
1143 hat_get_mapped_size(hat_t *hat)
1144 {
1145 	size_t total = 0;
1146 	int l;
1147 
1148 	for (l = 0; l <= mmu.max_page_level; l++)
1149 		total += (hat->hat_pages_mapped[l] << LEVEL_SHIFT(l));
1150 	total += hat->hat_ism_pgcnt;
1151 
1152 	return (total);
1153 }
1154 
1155 /*
1156  * enable/disable collection of stats for hat.
1157  */
1158 int
1159 hat_stats_enable(hat_t *hat)
1160 {
1161 	atomic_add_32(&hat->hat_stats, 1);
1162 	return (1);
1163 }
1164 
1165 void
1166 hat_stats_disable(hat_t *hat)
1167 {
1168 	atomic_add_32(&hat->hat_stats, -1);
1169 }
1170 
1171 /*
1172  * Utility to sync the ref/mod bits from a page table entry to the page_t
1173  * We must be holding the mapping list lock when this is called.
1174  */
1175 static void
1176 hati_sync_pte_to_page(page_t *pp, x86pte_t pte, level_t level)
1177 {
1178 	uint_t	rm = 0;
1179 	pgcnt_t	pgcnt;
1180 
1181 	if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC)
1182 		return;
1183 
1184 	if (PTE_GET(pte, PT_REF))
1185 		rm |= P_REF;
1186 
1187 	if (PTE_GET(pte, PT_MOD))
1188 		rm |= P_MOD;
1189 
1190 	if (rm == 0)
1191 		return;
1192 
1193 	/*
1194 	 * sync to all constituent pages of a large page
1195 	 */
1196 	ASSERT(x86_hm_held(pp));
1197 	pgcnt = page_get_pagecnt(level);
1198 	ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt));
1199 	for (; pgcnt > 0; --pgcnt) {
1200 		/*
1201 		 * hat_page_demote() can't decrease
1202 		 * pszc below this mapping size
1203 		 * since this large mapping existed after we
1204 		 * took mlist lock.
1205 		 */
1206 		ASSERT(pp->p_szc >= level);
1207 		hat_page_setattr(pp, rm);
1208 		++pp;
1209 	}
1210 }
1211 
1212 /*
1213  * This the set of PTE bits for PFN, permissions and caching
1214  * that are allowed to change on a HAT_LOAD_REMAP
1215  */
1216 #define	PT_REMAP_BITS							\
1217 	(PT_PADDR | PT_NX | PT_WRITABLE | PT_WRITETHRU |		\
1218 	PT_NOCACHE | PT_PAT_4K | PT_PAT_LARGE | PT_IGNORE | PT_REF | PT_MOD)
1219 
1220 #define	REMAPASSERT(EX)	if (!(EX)) panic("hati_pte_map: " #EX)
1221 /*
1222  * Do the low-level work to get a mapping entered into a HAT's pagetables
1223  * and in the mapping list of the associated page_t.
1224  */
1225 static int
1226 hati_pte_map(
1227 	htable_t	*ht,
1228 	uint_t		entry,
1229 	page_t		*pp,
1230 	x86pte_t	pte,
1231 	int		flags,
1232 	void		*pte_ptr)
1233 {
1234 	hat_t		*hat = ht->ht_hat;
1235 	x86pte_t	old_pte;
1236 	level_t		l = ht->ht_level;
1237 	hment_t		*hm;
1238 	uint_t		is_consist;
1239 	int		rv = 0;
1240 
1241 	/*
1242 	 * Is this a consistant (ie. need mapping list lock) mapping?
1243 	 */
1244 	is_consist = (pp != NULL && (flags & HAT_LOAD_NOCONSIST) == 0);
1245 
1246 	/*
1247 	 * Track locked mapping count in the htable.  Do this first,
1248 	 * as we track locking even if there already is a mapping present.
1249 	 */
1250 	if ((flags & HAT_LOAD_LOCK) != 0 && hat != kas.a_hat)
1251 		HTABLE_LOCK_INC(ht);
1252 
1253 	/*
1254 	 * Acquire the page's mapping list lock and get an hment to use.
1255 	 * Note that hment_prepare() might return NULL.
1256 	 */
1257 	if (is_consist) {
1258 		x86_hm_enter(pp);
1259 		hm = hment_prepare(ht, entry, pp);
1260 	}
1261 
1262 	/*
1263 	 * Set the new pte, retrieving the old one at the same time.
1264 	 */
1265 	old_pte = x86pte_set(ht, entry, pte, pte_ptr);
1266 
1267 	/*
1268 	 * did we get a large page / page table collision?
1269 	 */
1270 	if (old_pte == LPAGE_ERROR) {
1271 		rv = -1;
1272 		goto done;
1273 	}
1274 
1275 	/*
1276 	 * If the mapping didn't change there is nothing more to do.
1277 	 */
1278 	if (PTE_EQUIV(pte, old_pte))
1279 		goto done;
1280 
1281 	/*
1282 	 * Install a new mapping in the page's mapping list
1283 	 */
1284 	if (!PTE_ISVALID(old_pte)) {
1285 		if (is_consist) {
1286 			hment_assign(ht, entry, pp, hm);
1287 			x86_hm_exit(pp);
1288 		} else {
1289 			ASSERT(flags & HAT_LOAD_NOCONSIST);
1290 		}
1291 		HTABLE_INC(ht->ht_valid_cnt);
1292 		PGCNT_INC(hat, l);
1293 		return (rv);
1294 	}
1295 
1296 	/*
1297 	 * Remap's are more complicated:
1298 	 *  - HAT_LOAD_REMAP must be specified if changing the pfn.
1299 	 *    We also require that NOCONSIST be specified.
1300 	 *  - Otherwise only permission or caching bits may change.
1301 	 */
1302 	if (!PTE_ISPAGE(old_pte, l))
1303 		panic("non-null/page mapping pte=" FMT_PTE, old_pte);
1304 
1305 	if (PTE2PFN(old_pte, l) != PTE2PFN(pte, l)) {
1306 		REMAPASSERT(flags & HAT_LOAD_REMAP);
1307 		REMAPASSERT(flags & HAT_LOAD_NOCONSIST);
1308 		REMAPASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST);
1309 		REMAPASSERT(pf_is_memory(PTE2PFN(old_pte, l)) ==
1310 		    pf_is_memory(PTE2PFN(pte, l)));
1311 		REMAPASSERT(!is_consist);
1312 	}
1313 
1314 	/*
1315 	 * We only let remaps change the certain bits in the PTE.
1316 	 */
1317 	if (PTE_GET(old_pte, ~PT_REMAP_BITS) != PTE_GET(pte, ~PT_REMAP_BITS))
1318 		panic("remap bits changed: old_pte="FMT_PTE", pte="FMT_PTE"\n",
1319 		    old_pte, pte);
1320 
1321 	/*
1322 	 * We don't create any mapping list entries on a remap, so release
1323 	 * any allocated hment after we drop the mapping list lock.
1324 	 */
1325 done:
1326 	if (is_consist) {
1327 		x86_hm_exit(pp);
1328 		if (hm != NULL)
1329 			hment_free(hm);
1330 	}
1331 	return (rv);
1332 }
1333 
1334 /*
1335  * Internal routine to load a single page table entry. This only fails if
1336  * we attempt to overwrite a page table link with a large page.
1337  */
1338 static int
1339 hati_load_common(
1340 	hat_t		*hat,
1341 	uintptr_t	va,
1342 	page_t		*pp,
1343 	uint_t		attr,
1344 	uint_t		flags,
1345 	level_t		level,
1346 	pfn_t		pfn)
1347 {
1348 	htable_t	*ht;
1349 	uint_t		entry;
1350 	x86pte_t	pte;
1351 	int		rv = 0;
1352 
1353 	/*
1354 	 * The number 16 is arbitrary and here to catch a recursion problem
1355 	 * early before we blow out the kernel stack.
1356 	 */
1357 	++curthread->t_hatdepth;
1358 	ASSERT(curthread->t_hatdepth < 16);
1359 
1360 	ASSERT(hat == kas.a_hat ||
1361 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1362 
1363 	if (flags & HAT_LOAD_SHARE)
1364 		hat->hat_flags |= HAT_SHARED;
1365 
1366 	/*
1367 	 * Find the page table that maps this page if it already exists.
1368 	 */
1369 	ht = htable_lookup(hat, va, level);
1370 
1371 	/*
1372 	 * We must have HAT_LOAD_NOCONSIST if page_t is NULL.
1373 	 */
1374 	if (pp == NULL)
1375 		flags |= HAT_LOAD_NOCONSIST;
1376 
1377 	if (ht == NULL) {
1378 		ht = htable_create(hat, va, level, NULL);
1379 		ASSERT(ht != NULL);
1380 	}
1381 	entry = htable_va2entry(va, ht);
1382 
1383 	/*
1384 	 * a bunch of paranoid error checking
1385 	 */
1386 	ASSERT(ht->ht_busy > 0);
1387 	if (ht->ht_vaddr > va || va > HTABLE_LAST_PAGE(ht))
1388 		panic("hati_load_common: bad htable %p, va %p", ht, (void *)va);
1389 	ASSERT(ht->ht_level == level);
1390 
1391 	/*
1392 	 * construct the new PTE
1393 	 */
1394 	if (hat == kas.a_hat)
1395 		attr &= ~PROT_USER;
1396 	pte = hati_mkpte(pfn, attr, level, flags);
1397 	if (hat == kas.a_hat && va >= kernelbase)
1398 		PTE_SET(pte, mmu.pt_global);
1399 
1400 	/*
1401 	 * establish the mapping
1402 	 */
1403 	rv = hati_pte_map(ht, entry, pp, pte, flags, NULL);
1404 
1405 	/*
1406 	 * release the htable and any reserves
1407 	 */
1408 	htable_release(ht);
1409 	--curthread->t_hatdepth;
1410 	return (rv);
1411 }
1412 
1413 /*
1414  * special case of hat_memload to deal with some kernel addrs for performance
1415  */
1416 static void
1417 hat_kmap_load(
1418 	caddr_t		addr,
1419 	page_t		*pp,
1420 	uint_t		attr,
1421 	uint_t		flags)
1422 {
1423 	uintptr_t	va = (uintptr_t)addr;
1424 	x86pte_t	pte;
1425 	pfn_t		pfn = page_pptonum(pp);
1426 	pgcnt_t		pg_off = mmu_btop(va - mmu.kmap_addr);
1427 	htable_t	*ht;
1428 	uint_t		entry;
1429 	void		*pte_ptr;
1430 
1431 	/*
1432 	 * construct the requested PTE
1433 	 */
1434 	attr &= ~PROT_USER;
1435 	attr |= HAT_STORECACHING_OK;
1436 	pte = hati_mkpte(pfn, attr, 0, flags);
1437 	PTE_SET(pte, mmu.pt_global);
1438 
1439 	/*
1440 	 * Figure out the pte_ptr and htable and use common code to finish up
1441 	 */
1442 	if (mmu.pae_hat)
1443 		pte_ptr = mmu.kmap_ptes + pg_off;
1444 	else
1445 		pte_ptr = (x86pte32_t *)mmu.kmap_ptes + pg_off;
1446 	ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) >>
1447 	    LEVEL_SHIFT(1)];
1448 	entry = htable_va2entry(va, ht);
1449 	++curthread->t_hatdepth;
1450 	ASSERT(curthread->t_hatdepth < 16);
1451 	(void) hati_pte_map(ht, entry, pp, pte, flags, pte_ptr);
1452 	--curthread->t_hatdepth;
1453 }
1454 
1455 /*
1456  * hat_memload() - load a translation to the given page struct
1457  *
1458  * Flags for hat_memload/hat_devload/hat_*attr.
1459  *
1460  * 	HAT_LOAD	Default flags to load a translation to the page.
1461  *
1462  * 	HAT_LOAD_LOCK	Lock down mapping resources; hat_map(), hat_memload(),
1463  *			and hat_devload().
1464  *
1465  *	HAT_LOAD_NOCONSIST Do not add mapping to page_t mapping list.
1466  *			sets PT_NOCONSIST
1467  *
1468  *	HAT_LOAD_SHARE	A flag to hat_memload() to indicate h/w page tables
1469  *			that map some user pages (not kas) is shared by more
1470  *			than one process (eg. ISM).
1471  *
1472  *	HAT_LOAD_REMAP	Reload a valid pte with a different page frame.
1473  *
1474  *	HAT_NO_KALLOC	Do not kmem_alloc while creating the mapping; at this
1475  *			point, it's setting up mapping to allocate internal
1476  *			hat layer data structures.  This flag forces hat layer
1477  *			to tap its reserves in order to prevent infinite
1478  *			recursion.
1479  *
1480  * The following is a protection attribute (like PROT_READ, etc.)
1481  *
1482  *	HAT_NOSYNC	set PT_NOSYNC - this mapping's ref/mod bits
1483  *			are never cleared.
1484  *
1485  * Installing new valid PTE's and creation of the mapping list
1486  * entry are controlled under the same lock. It's derived from the
1487  * page_t being mapped.
1488  */
1489 static uint_t supported_memload_flags =
1490 	HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST |
1491 	HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT;
1492 
1493 void
1494 hat_memload(
1495 	hat_t		*hat,
1496 	caddr_t		addr,
1497 	page_t		*pp,
1498 	uint_t		attr,
1499 	uint_t		flags)
1500 {
1501 	uintptr_t	va = (uintptr_t)addr;
1502 	level_t		level = 0;
1503 	pfn_t		pfn = page_pptonum(pp);
1504 
1505 	XPV_DISALLOW_MIGRATE();
1506 	ASSERT(IS_PAGEALIGNED(va));
1507 	ASSERT(hat == kas.a_hat || va < _userlimit);
1508 	ASSERT(hat == kas.a_hat ||
1509 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1510 	ASSERT((flags & supported_memload_flags) == flags);
1511 
1512 	ASSERT(!IN_VA_HOLE(va));
1513 	ASSERT(!PP_ISFREE(pp));
1514 
1515 	/*
1516 	 * kernel address special case for performance.
1517 	 */
1518 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
1519 		ASSERT(hat == kas.a_hat);
1520 		hat_kmap_load(addr, pp, attr, flags);
1521 		XPV_ALLOW_MIGRATE();
1522 		return;
1523 	}
1524 
1525 	/*
1526 	 * This is used for memory with normal caching enabled, so
1527 	 * always set HAT_STORECACHING_OK.
1528 	 */
1529 	attr |= HAT_STORECACHING_OK;
1530 	if (hati_load_common(hat, va, pp, attr, flags, level, pfn) != 0)
1531 		panic("unexpected hati_load_common() failure");
1532 	XPV_ALLOW_MIGRATE();
1533 }
1534 
1535 /* ARGSUSED */
1536 void
1537 hat_memload_region(struct hat *hat, caddr_t addr, struct page *pp,
1538     uint_t attr, uint_t flags, hat_region_cookie_t rcookie)
1539 {
1540 	hat_memload(hat, addr, pp, attr, flags);
1541 }
1542 
1543 /*
1544  * Load the given array of page structs using large pages when possible
1545  */
1546 void
1547 hat_memload_array(
1548 	hat_t		*hat,
1549 	caddr_t		addr,
1550 	size_t		len,
1551 	page_t		**pages,
1552 	uint_t		attr,
1553 	uint_t		flags)
1554 {
1555 	uintptr_t	va = (uintptr_t)addr;
1556 	uintptr_t	eaddr = va + len;
1557 	level_t		level;
1558 	size_t		pgsize;
1559 	pgcnt_t		pgindx = 0;
1560 	pfn_t		pfn;
1561 	pgcnt_t		i;
1562 
1563 	XPV_DISALLOW_MIGRATE();
1564 	ASSERT(IS_PAGEALIGNED(va));
1565 	ASSERT(hat == kas.a_hat || va + len <= _userlimit);
1566 	ASSERT(hat == kas.a_hat ||
1567 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1568 	ASSERT((flags & supported_memload_flags) == flags);
1569 
1570 	/*
1571 	 * memload is used for memory with full caching enabled, so
1572 	 * set HAT_STORECACHING_OK.
1573 	 */
1574 	attr |= HAT_STORECACHING_OK;
1575 
1576 	/*
1577 	 * handle all pages using largest possible pagesize
1578 	 */
1579 	while (va < eaddr) {
1580 		/*
1581 		 * decide what level mapping to use (ie. pagesize)
1582 		 */
1583 		pfn = page_pptonum(pages[pgindx]);
1584 		for (level = mmu.max_page_level; ; --level) {
1585 			pgsize = LEVEL_SIZE(level);
1586 			if (level == 0)
1587 				break;
1588 
1589 			if (!IS_P2ALIGNED(va, pgsize) ||
1590 			    (eaddr - va) < pgsize ||
1591 			    !IS_P2ALIGNED(pfn_to_pa(pfn), pgsize))
1592 				continue;
1593 
1594 			/*
1595 			 * To use a large mapping of this size, all the
1596 			 * pages we are passed must be sequential subpages
1597 			 * of the large page.
1598 			 * hat_page_demote() can't change p_szc because
1599 			 * all pages are locked.
1600 			 */
1601 			if (pages[pgindx]->p_szc >= level) {
1602 				for (i = 0; i < mmu_btop(pgsize); ++i) {
1603 					if (pfn + i !=
1604 					    page_pptonum(pages[pgindx + i]))
1605 						break;
1606 					ASSERT(pages[pgindx + i]->p_szc >=
1607 					    level);
1608 					ASSERT(pages[pgindx] + i ==
1609 					    pages[pgindx + i]);
1610 				}
1611 				if (i == mmu_btop(pgsize))
1612 					break;
1613 			}
1614 		}
1615 
1616 		/*
1617 		 * Load this page mapping. If the load fails, try a smaller
1618 		 * pagesize.
1619 		 */
1620 		ASSERT(!IN_VA_HOLE(va));
1621 		while (hati_load_common(hat, va, pages[pgindx], attr,
1622 		    flags, level, pfn) != 0) {
1623 			if (level == 0)
1624 				panic("unexpected hati_load_common() failure");
1625 			--level;
1626 			pgsize = LEVEL_SIZE(level);
1627 		}
1628 
1629 		/*
1630 		 * move to next page
1631 		 */
1632 		va += pgsize;
1633 		pgindx += mmu_btop(pgsize);
1634 	}
1635 	XPV_ALLOW_MIGRATE();
1636 }
1637 
1638 /* ARGSUSED */
1639 void
1640 hat_memload_array_region(struct hat *hat, caddr_t addr, size_t len,
1641     struct page **pps, uint_t attr, uint_t flags,
1642     hat_region_cookie_t rcookie)
1643 {
1644 	hat_memload_array(hat, addr, len, pps, attr, flags);
1645 }
1646 
1647 /*
1648  * void hat_devload(hat, addr, len, pf, attr, flags)
1649  *	load/lock the given page frame number
1650  *
1651  * Advisory ordering attributes. Apply only to device mappings.
1652  *
1653  * HAT_STRICTORDER: the CPU must issue the references in order, as the
1654  *	programmer specified.  This is the default.
1655  * HAT_UNORDERED_OK: the CPU may reorder the references (this is all kinds
1656  *	of reordering; store or load with store or load).
1657  * HAT_MERGING_OK: merging and batching: the CPU may merge individual stores
1658  *	to consecutive locations (for example, turn two consecutive byte
1659  *	stores into one halfword store), and it may batch individual loads
1660  *	(for example, turn two consecutive byte loads into one halfword load).
1661  *	This also implies re-ordering.
1662  * HAT_LOADCACHING_OK: the CPU may cache the data it fetches and reuse it
1663  *	until another store occurs.  The default is to fetch new data
1664  *	on every load.  This also implies merging.
1665  * HAT_STORECACHING_OK: the CPU may keep the data in the cache and push it to
1666  *	the device (perhaps with other data) at a later time.  The default is
1667  *	to push the data right away.  This also implies load caching.
1668  *
1669  * Equivalent of hat_memload(), but can be used for device memory where
1670  * there are no page_t's and we support additional flags (write merging, etc).
1671  * Note that we can have large page mappings with this interface.
1672  */
1673 int supported_devload_flags = HAT_LOAD | HAT_LOAD_LOCK |
1674 	HAT_LOAD_NOCONSIST | HAT_STRICTORDER | HAT_UNORDERED_OK |
1675 	HAT_MERGING_OK | HAT_LOADCACHING_OK | HAT_STORECACHING_OK;
1676 
1677 void
1678 hat_devload(
1679 	hat_t		*hat,
1680 	caddr_t		addr,
1681 	size_t		len,
1682 	pfn_t		pfn,
1683 	uint_t		attr,
1684 	int		flags)
1685 {
1686 	uintptr_t	va = ALIGN2PAGE(addr);
1687 	uintptr_t	eva = va + len;
1688 	level_t		level;
1689 	size_t		pgsize;
1690 	page_t		*pp;
1691 	int		f;	/* per PTE copy of flags  - maybe modified */
1692 	uint_t		a;	/* per PTE copy of attr */
1693 
1694 	XPV_DISALLOW_MIGRATE();
1695 	ASSERT(IS_PAGEALIGNED(va));
1696 	ASSERT(hat == kas.a_hat || eva <= _userlimit);
1697 	ASSERT(hat == kas.a_hat ||
1698 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1699 	ASSERT((flags & supported_devload_flags) == flags);
1700 
1701 	/*
1702 	 * handle all pages
1703 	 */
1704 	while (va < eva) {
1705 
1706 		/*
1707 		 * decide what level mapping to use (ie. pagesize)
1708 		 */
1709 		for (level = mmu.max_page_level; ; --level) {
1710 			pgsize = LEVEL_SIZE(level);
1711 			if (level == 0)
1712 				break;
1713 			if (IS_P2ALIGNED(va, pgsize) &&
1714 			    (eva - va) >= pgsize &&
1715 			    IS_P2ALIGNED(pfn, mmu_btop(pgsize)))
1716 				break;
1717 		}
1718 
1719 		/*
1720 		 * If this is just memory then allow caching (this happens
1721 		 * for the nucleus pages) - though HAT_PLAT_NOCACHE can be used
1722 		 * to override that. If we don't have a page_t then make sure
1723 		 * NOCONSIST is set.
1724 		 */
1725 		a = attr;
1726 		f = flags;
1727 		if (!pf_is_memory(pfn))
1728 			f |= HAT_LOAD_NOCONSIST;
1729 		else if (!(a & HAT_PLAT_NOCACHE))
1730 			a |= HAT_STORECACHING_OK;
1731 
1732 		if (f & HAT_LOAD_NOCONSIST)
1733 			pp = NULL;
1734 		else
1735 			pp = page_numtopp_nolock(pfn);
1736 
1737 		/*
1738 		 * load this page mapping
1739 		 */
1740 		ASSERT(!IN_VA_HOLE(va));
1741 		while (hati_load_common(hat, va, pp, a, f, level, pfn) != 0) {
1742 			if (level == 0)
1743 				panic("unexpected hati_load_common() failure");
1744 			--level;
1745 			pgsize = LEVEL_SIZE(level);
1746 		}
1747 
1748 		/*
1749 		 * move to next page
1750 		 */
1751 		va += pgsize;
1752 		pfn += mmu_btop(pgsize);
1753 	}
1754 	XPV_ALLOW_MIGRATE();
1755 }
1756 
1757 /*
1758  * void hat_unlock(hat, addr, len)
1759  *	unlock the mappings to a given range of addresses
1760  *
1761  * Locks are tracked by ht_lock_cnt in the htable.
1762  */
1763 void
1764 hat_unlock(hat_t *hat, caddr_t addr, size_t len)
1765 {
1766 	uintptr_t	vaddr = (uintptr_t)addr;
1767 	uintptr_t	eaddr = vaddr + len;
1768 	htable_t	*ht = NULL;
1769 
1770 	/*
1771 	 * kernel entries are always locked, we don't track lock counts
1772 	 */
1773 	ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
1774 	ASSERT(IS_PAGEALIGNED(vaddr));
1775 	ASSERT(IS_PAGEALIGNED(eaddr));
1776 	if (hat == kas.a_hat)
1777 		return;
1778 	if (eaddr > _userlimit)
1779 		panic("hat_unlock() address out of range - above _userlimit");
1780 
1781 	XPV_DISALLOW_MIGRATE();
1782 	ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1783 	while (vaddr < eaddr) {
1784 		(void) htable_walk(hat, &ht, &vaddr, eaddr);
1785 		if (ht == NULL)
1786 			break;
1787 
1788 		ASSERT(!IN_VA_HOLE(vaddr));
1789 
1790 		if (ht->ht_lock_cnt < 1)
1791 			panic("hat_unlock(): lock_cnt < 1, "
1792 			    "htable=%p, vaddr=%p\n", ht, (caddr_t)vaddr);
1793 		HTABLE_LOCK_DEC(ht);
1794 
1795 		vaddr += LEVEL_SIZE(ht->ht_level);
1796 	}
1797 	if (ht)
1798 		htable_release(ht);
1799 	XPV_ALLOW_MIGRATE();
1800 }
1801 
1802 /* ARGSUSED */
1803 void
1804 hat_unlock_region(struct hat *hat, caddr_t addr, size_t len,
1805     hat_region_cookie_t rcookie)
1806 {
1807 	panic("No shared region support on x86");
1808 }
1809 
1810 #if !defined(__xpv)
1811 /*
1812  * Cross call service routine to demap a virtual page on
1813  * the current CPU or flush all mappings in TLB.
1814  */
1815 /*ARGSUSED*/
1816 static int
1817 hati_demap_func(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3)
1818 {
1819 	hat_t	*hat = (hat_t *)a1;
1820 	caddr_t	addr = (caddr_t)a2;
1821 
1822 	/*
1823 	 * If the target hat isn't the kernel and this CPU isn't operating
1824 	 * in the target hat, we can ignore the cross call.
1825 	 */
1826 	if (hat != kas.a_hat && hat != CPU->cpu_current_hat)
1827 		return (0);
1828 
1829 	/*
1830 	 * For a normal address, we just flush one page mapping
1831 	 */
1832 	if ((uintptr_t)addr != DEMAP_ALL_ADDR) {
1833 		mmu_tlbflush_entry(addr);
1834 		return (0);
1835 	}
1836 
1837 	/*
1838 	 * Otherwise we reload cr3 to effect a complete TLB flush.
1839 	 *
1840 	 * A reload of cr3 on a VLP process also means we must also recopy in
1841 	 * the pte values from the struct hat
1842 	 */
1843 	if (hat->hat_flags & HAT_VLP) {
1844 #if defined(__amd64)
1845 		x86pte_t *vlpptep = CPU->cpu_hat_info->hci_vlp_l2ptes;
1846 
1847 		VLP_COPY(hat->hat_vlp_ptes, vlpptep);
1848 #elif defined(__i386)
1849 		reload_pae32(hat, CPU);
1850 #endif
1851 	}
1852 	reload_cr3();
1853 	return (0);
1854 }
1855 
1856 /*
1857  * Flush all TLB entries, including global (ie. kernel) ones.
1858  */
1859 static void
1860 flush_all_tlb_entries(void)
1861 {
1862 	ulong_t cr4 = getcr4();
1863 
1864 	if (cr4 & CR4_PGE) {
1865 		setcr4(cr4 & ~(ulong_t)CR4_PGE);
1866 		setcr4(cr4);
1867 
1868 		/*
1869 		 * 32 bit PAE also needs to always reload_cr3()
1870 		 */
1871 		if (mmu.max_level == 2)
1872 			reload_cr3();
1873 	} else {
1874 		reload_cr3();
1875 	}
1876 }
1877 
1878 #define	TLB_CPU_HALTED	(01ul)
1879 #define	TLB_INVAL_ALL	(02ul)
1880 #define	CAS_TLB_INFO(cpu, old, new)	\
1881 	caslong((ulong_t *)&(cpu)->cpu_m.mcpu_tlb_info, (old), (new))
1882 
1883 /*
1884  * Record that a CPU is going idle
1885  */
1886 void
1887 tlb_going_idle(void)
1888 {
1889 	atomic_or_long((ulong_t *)&CPU->cpu_m.mcpu_tlb_info, TLB_CPU_HALTED);
1890 }
1891 
1892 /*
1893  * Service a delayed TLB flush if coming out of being idle.
1894  */
1895 void
1896 tlb_service(void)
1897 {
1898 	ulong_t flags = getflags();
1899 	ulong_t tlb_info;
1900 	ulong_t found;
1901 
1902 	/*
1903 	 * Be sure interrupts are off while doing this so that
1904 	 * higher level interrupts correctly wait for flushes to finish.
1905 	 */
1906 	if (flags & PS_IE)
1907 		flags = intr_clear();
1908 
1909 	/*
1910 	 * We only have to do something if coming out of being idle.
1911 	 */
1912 	tlb_info = CPU->cpu_m.mcpu_tlb_info;
1913 	if (tlb_info & TLB_CPU_HALTED) {
1914 		ASSERT(CPU->cpu_current_hat == kas.a_hat);
1915 
1916 		/*
1917 		 * Atomic clear and fetch of old state.
1918 		 */
1919 		while ((found = CAS_TLB_INFO(CPU, tlb_info, 0)) != tlb_info) {
1920 			ASSERT(found & TLB_CPU_HALTED);
1921 			tlb_info = found;
1922 			SMT_PAUSE();
1923 		}
1924 		if (tlb_info & TLB_INVAL_ALL)
1925 			flush_all_tlb_entries();
1926 	}
1927 
1928 	/*
1929 	 * Restore interrupt enable control bit.
1930 	 */
1931 	if (flags & PS_IE)
1932 		sti();
1933 }
1934 #endif /* !__xpv */
1935 
1936 /*
1937  * Internal routine to do cross calls to invalidate a range of pages on
1938  * all CPUs using a given hat.
1939  */
1940 void
1941 hat_tlb_inval(hat_t *hat, uintptr_t va)
1942 {
1943 	extern int	flushes_require_xcalls;	/* from mp_startup.c */
1944 	cpuset_t	justme;
1945 	cpuset_t	cpus_to_shootdown;
1946 #ifndef __xpv
1947 	cpuset_t	check_cpus;
1948 	cpu_t		*cpup;
1949 	int		c;
1950 #endif
1951 
1952 	/*
1953 	 * If the hat is being destroyed, there are no more users, so
1954 	 * demap need not do anything.
1955 	 */
1956 	if (hat->hat_flags & HAT_FREEING)
1957 		return;
1958 
1959 	/*
1960 	 * If demapping from a shared pagetable, we best demap the
1961 	 * entire set of user TLBs, since we don't know what addresses
1962 	 * these were shared at.
1963 	 */
1964 	if (hat->hat_flags & HAT_SHARED) {
1965 		hat = kas.a_hat;
1966 		va = DEMAP_ALL_ADDR;
1967 	}
1968 
1969 	/*
1970 	 * if not running with multiple CPUs, don't use cross calls
1971 	 */
1972 	if (panicstr || !flushes_require_xcalls) {
1973 #ifdef __xpv
1974 		if (va == DEMAP_ALL_ADDR)
1975 			xen_flush_tlb();
1976 		else
1977 			xen_flush_va((caddr_t)va);
1978 #else
1979 		(void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL);
1980 #endif
1981 		return;
1982 	}
1983 
1984 
1985 	/*
1986 	 * Determine CPUs to shootdown. Kernel changes always do all CPUs.
1987 	 * Otherwise it's just CPUs currently executing in this hat.
1988 	 */
1989 	kpreempt_disable();
1990 	CPUSET_ONLY(justme, CPU->cpu_id);
1991 	if (hat == kas.a_hat)
1992 		cpus_to_shootdown = khat_cpuset;
1993 	else
1994 		cpus_to_shootdown = hat->hat_cpus;
1995 
1996 #ifndef __xpv
1997 	/*
1998 	 * If any CPUs in the set are idle, just request a delayed flush
1999 	 * and avoid waking them up.
2000 	 */
2001 	check_cpus = cpus_to_shootdown;
2002 	for (c = 0; c < NCPU && !CPUSET_ISNULL(check_cpus); ++c) {
2003 		ulong_t tlb_info;
2004 
2005 		if (!CPU_IN_SET(check_cpus, c))
2006 			continue;
2007 		CPUSET_DEL(check_cpus, c);
2008 		cpup = cpu[c];
2009 		if (cpup == NULL)
2010 			continue;
2011 
2012 		tlb_info = cpup->cpu_m.mcpu_tlb_info;
2013 		while (tlb_info == TLB_CPU_HALTED) {
2014 			(void) CAS_TLB_INFO(cpup, TLB_CPU_HALTED,
2015 			    TLB_CPU_HALTED | TLB_INVAL_ALL);
2016 			SMT_PAUSE();
2017 			tlb_info = cpup->cpu_m.mcpu_tlb_info;
2018 		}
2019 		if (tlb_info == (TLB_CPU_HALTED | TLB_INVAL_ALL)) {
2020 			HATSTAT_INC(hs_tlb_inval_delayed);
2021 			CPUSET_DEL(cpus_to_shootdown, c);
2022 		}
2023 	}
2024 #endif
2025 
2026 	if (CPUSET_ISNULL(cpus_to_shootdown) ||
2027 	    CPUSET_ISEQUAL(cpus_to_shootdown, justme)) {
2028 
2029 #ifdef __xpv
2030 		if (va == DEMAP_ALL_ADDR)
2031 			xen_flush_tlb();
2032 		else
2033 			xen_flush_va((caddr_t)va);
2034 #else
2035 		(void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL);
2036 #endif
2037 
2038 	} else {
2039 
2040 		CPUSET_ADD(cpus_to_shootdown, CPU->cpu_id);
2041 #ifdef __xpv
2042 		if (va == DEMAP_ALL_ADDR)
2043 			xen_gflush_tlb(cpus_to_shootdown);
2044 		else
2045 			xen_gflush_va((caddr_t)va, cpus_to_shootdown);
2046 #else
2047 		xc_call((xc_arg_t)hat, (xc_arg_t)va, NULL, X_CALL_HIPRI,
2048 		    cpus_to_shootdown, hati_demap_func);
2049 #endif
2050 
2051 	}
2052 	kpreempt_enable();
2053 }
2054 
2055 /*
2056  * Interior routine for HAT_UNLOADs from hat_unload_callback(),
2057  * hat_kmap_unload() OR from hat_steal() code.  This routine doesn't
2058  * handle releasing of the htables.
2059  */
2060 void
2061 hat_pte_unmap(
2062 	htable_t	*ht,
2063 	uint_t		entry,
2064 	uint_t		flags,
2065 	x86pte_t	old_pte,
2066 	void		*pte_ptr)
2067 {
2068 	hat_t		*hat = ht->ht_hat;
2069 	hment_t		*hm = NULL;
2070 	page_t		*pp = NULL;
2071 	level_t		l = ht->ht_level;
2072 	pfn_t		pfn;
2073 
2074 	/*
2075 	 * We always track the locking counts, even if nothing is unmapped
2076 	 */
2077 	if ((flags & HAT_UNLOAD_UNLOCK) != 0 && hat != kas.a_hat) {
2078 		ASSERT(ht->ht_lock_cnt > 0);
2079 		HTABLE_LOCK_DEC(ht);
2080 	}
2081 
2082 	/*
2083 	 * Figure out which page's mapping list lock to acquire using the PFN
2084 	 * passed in "old" PTE. We then attempt to invalidate the PTE.
2085 	 * If another thread, probably a hat_pageunload, has asynchronously
2086 	 * unmapped/remapped this address we'll loop here.
2087 	 */
2088 	ASSERT(ht->ht_busy > 0);
2089 	while (PTE_ISVALID(old_pte)) {
2090 		pfn = PTE2PFN(old_pte, l);
2091 		if (PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST) {
2092 			pp = NULL;
2093 		} else {
2094 #ifdef __xpv
2095 			if (pfn == PFN_INVALID)
2096 				panic("Invalid PFN, but not PT_NOCONSIST");
2097 #endif
2098 			pp = page_numtopp_nolock(pfn);
2099 			if (pp == NULL) {
2100 				panic("no page_t, not NOCONSIST: old_pte="
2101 				    FMT_PTE " ht=%lx entry=0x%x pte_ptr=%lx",
2102 				    old_pte, (uintptr_t)ht, entry,
2103 				    (uintptr_t)pte_ptr);
2104 			}
2105 			x86_hm_enter(pp);
2106 		}
2107 
2108 		/*
2109 		 * If freeing the address space, check that the PTE
2110 		 * hasn't changed, as the mappings are no longer in use by
2111 		 * any thread, invalidation is unnecessary.
2112 		 * If not freeing, do a full invalidate.
2113 		 *
2114 		 * On the hypervisor we must always remove mappings, as a
2115 		 * writable mapping left behind could cause a page table
2116 		 * allocation to fail.
2117 		 */
2118 #if !defined(__xpv)
2119 		if (hat->hat_flags & HAT_FREEING)
2120 			old_pte = x86pte_get(ht, entry);
2121 		else
2122 #endif
2123 			old_pte = x86pte_inval(ht, entry, old_pte, pte_ptr);
2124 
2125 		/*
2126 		 * If the page hadn't changed we've unmapped it and can proceed
2127 		 */
2128 		if (PTE_ISVALID(old_pte) && PTE2PFN(old_pte, l) == pfn)
2129 			break;
2130 
2131 		/*
2132 		 * Otherwise, we'll have to retry with the current old_pte.
2133 		 * Drop the hment lock, since the pfn may have changed.
2134 		 */
2135 		if (pp != NULL) {
2136 			x86_hm_exit(pp);
2137 			pp = NULL;
2138 		} else {
2139 			ASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST);
2140 		}
2141 	}
2142 
2143 	/*
2144 	 * If the old mapping wasn't valid, there's nothing more to do
2145 	 */
2146 	if (!PTE_ISVALID(old_pte)) {
2147 		if (pp != NULL)
2148 			x86_hm_exit(pp);
2149 		return;
2150 	}
2151 
2152 	/*
2153 	 * Take care of syncing any MOD/REF bits and removing the hment.
2154 	 */
2155 	if (pp != NULL) {
2156 		if (!(flags & HAT_UNLOAD_NOSYNC))
2157 			hati_sync_pte_to_page(pp, old_pte, l);
2158 		hm = hment_remove(pp, ht, entry);
2159 		x86_hm_exit(pp);
2160 		if (hm != NULL)
2161 			hment_free(hm);
2162 	}
2163 
2164 	/*
2165 	 * Handle book keeping in the htable and hat
2166 	 */
2167 	ASSERT(ht->ht_valid_cnt > 0);
2168 	HTABLE_DEC(ht->ht_valid_cnt);
2169 	PGCNT_DEC(hat, l);
2170 }
2171 
2172 /*
2173  * very cheap unload implementation to special case some kernel addresses
2174  */
2175 static void
2176 hat_kmap_unload(caddr_t addr, size_t len, uint_t flags)
2177 {
2178 	uintptr_t	va = (uintptr_t)addr;
2179 	uintptr_t	eva = va + len;
2180 	pgcnt_t		pg_index;
2181 	htable_t	*ht;
2182 	uint_t		entry;
2183 	x86pte_t	*pte_ptr;
2184 	x86pte_t	old_pte;
2185 
2186 	for (; va < eva; va += MMU_PAGESIZE) {
2187 		/*
2188 		 * Get the PTE
2189 		 */
2190 		pg_index = mmu_btop(va - mmu.kmap_addr);
2191 		pte_ptr = PT_INDEX_PTR(mmu.kmap_ptes, pg_index);
2192 		old_pte = GET_PTE(pte_ptr);
2193 
2194 		/*
2195 		 * get the htable / entry
2196 		 */
2197 		ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr)
2198 		    >> LEVEL_SHIFT(1)];
2199 		entry = htable_va2entry(va, ht);
2200 
2201 		/*
2202 		 * use mostly common code to unmap it.
2203 		 */
2204 		hat_pte_unmap(ht, entry, flags, old_pte, pte_ptr);
2205 	}
2206 }
2207 
2208 
2209 /*
2210  * unload a range of virtual address space (no callback)
2211  */
2212 void
2213 hat_unload(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2214 {
2215 	uintptr_t va = (uintptr_t)addr;
2216 
2217 	XPV_DISALLOW_MIGRATE();
2218 	ASSERT(hat == kas.a_hat || va + len <= _userlimit);
2219 
2220 	/*
2221 	 * special case for performance.
2222 	 */
2223 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
2224 		ASSERT(hat == kas.a_hat);
2225 		hat_kmap_unload(addr, len, flags);
2226 	} else {
2227 		hat_unload_callback(hat, addr, len, flags, NULL);
2228 	}
2229 	XPV_ALLOW_MIGRATE();
2230 }
2231 
2232 /*
2233  * Do the callbacks for ranges being unloaded.
2234  */
2235 typedef struct range_info {
2236 	uintptr_t	rng_va;
2237 	ulong_t		rng_cnt;
2238 	level_t		rng_level;
2239 } range_info_t;
2240 
2241 static void
2242 handle_ranges(hat_callback_t *cb, uint_t cnt, range_info_t *range)
2243 {
2244 	/*
2245 	 * do callbacks to upper level VM system
2246 	 */
2247 	while (cb != NULL && cnt > 0) {
2248 		--cnt;
2249 		cb->hcb_start_addr = (caddr_t)range[cnt].rng_va;
2250 		cb->hcb_end_addr = cb->hcb_start_addr;
2251 		cb->hcb_end_addr +=
2252 		    range[cnt].rng_cnt << LEVEL_SIZE(range[cnt].rng_level);
2253 		cb->hcb_function(cb);
2254 	}
2255 }
2256 
2257 /*
2258  * Unload a given range of addresses (has optional callback)
2259  *
2260  * Flags:
2261  * define	HAT_UNLOAD		0x00
2262  * define	HAT_UNLOAD_NOSYNC	0x02
2263  * define	HAT_UNLOAD_UNLOCK	0x04
2264  * define	HAT_UNLOAD_OTHER	0x08 - not used
2265  * define	HAT_UNLOAD_UNMAP	0x10 - same as HAT_UNLOAD
2266  */
2267 #define	MAX_UNLOAD_CNT (8)
2268 void
2269 hat_unload_callback(
2270 	hat_t		*hat,
2271 	caddr_t		addr,
2272 	size_t		len,
2273 	uint_t		flags,
2274 	hat_callback_t	*cb)
2275 {
2276 	uintptr_t	vaddr = (uintptr_t)addr;
2277 	uintptr_t	eaddr = vaddr + len;
2278 	htable_t	*ht = NULL;
2279 	uint_t		entry;
2280 	uintptr_t	contig_va = (uintptr_t)-1L;
2281 	range_info_t	r[MAX_UNLOAD_CNT];
2282 	uint_t		r_cnt = 0;
2283 	x86pte_t	old_pte;
2284 
2285 	XPV_DISALLOW_MIGRATE();
2286 	ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
2287 	ASSERT(IS_PAGEALIGNED(vaddr));
2288 	ASSERT(IS_PAGEALIGNED(eaddr));
2289 
2290 	/*
2291 	 * Special case a single page being unloaded for speed. This happens
2292 	 * quite frequently, COW faults after a fork() for example.
2293 	 */
2294 	if (cb == NULL && len == MMU_PAGESIZE) {
2295 		ht = htable_getpte(hat, vaddr, &entry, &old_pte, 0);
2296 		if (ht != NULL) {
2297 			if (PTE_ISVALID(old_pte))
2298 				hat_pte_unmap(ht, entry, flags, old_pte, NULL);
2299 			htable_release(ht);
2300 		}
2301 		XPV_ALLOW_MIGRATE();
2302 		return;
2303 	}
2304 
2305 	while (vaddr < eaddr) {
2306 		old_pte = htable_walk(hat, &ht, &vaddr, eaddr);
2307 		if (ht == NULL)
2308 			break;
2309 
2310 		ASSERT(!IN_VA_HOLE(vaddr));
2311 
2312 		if (vaddr < (uintptr_t)addr)
2313 			panic("hat_unload_callback(): unmap inside large page");
2314 
2315 		/*
2316 		 * We'll do the call backs for contiguous ranges
2317 		 */
2318 		if (vaddr != contig_va ||
2319 		    (r_cnt > 0 && r[r_cnt - 1].rng_level != ht->ht_level)) {
2320 			if (r_cnt == MAX_UNLOAD_CNT) {
2321 				handle_ranges(cb, r_cnt, r);
2322 				r_cnt = 0;
2323 			}
2324 			r[r_cnt].rng_va = vaddr;
2325 			r[r_cnt].rng_cnt = 0;
2326 			r[r_cnt].rng_level = ht->ht_level;
2327 			++r_cnt;
2328 		}
2329 
2330 		/*
2331 		 * Unload one mapping from the page tables.
2332 		 */
2333 		entry = htable_va2entry(vaddr, ht);
2334 		hat_pte_unmap(ht, entry, flags, old_pte, NULL);
2335 		ASSERT(ht->ht_level <= mmu.max_page_level);
2336 		vaddr += LEVEL_SIZE(ht->ht_level);
2337 		contig_va = vaddr;
2338 		++r[r_cnt - 1].rng_cnt;
2339 	}
2340 	if (ht)
2341 		htable_release(ht);
2342 
2343 	/*
2344 	 * handle last range for callbacks
2345 	 */
2346 	if (r_cnt > 0)
2347 		handle_ranges(cb, r_cnt, r);
2348 	XPV_ALLOW_MIGRATE();
2349 }
2350 
2351 /*
2352  * synchronize mapping with software data structures
2353  *
2354  * This interface is currently only used by the working set monitor
2355  * driver.
2356  */
2357 /*ARGSUSED*/
2358 void
2359 hat_sync(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2360 {
2361 	uintptr_t	vaddr = (uintptr_t)addr;
2362 	uintptr_t	eaddr = vaddr + len;
2363 	htable_t	*ht = NULL;
2364 	uint_t		entry;
2365 	x86pte_t	pte;
2366 	x86pte_t	save_pte;
2367 	x86pte_t	new;
2368 	page_t		*pp;
2369 
2370 	ASSERT(!IN_VA_HOLE(vaddr));
2371 	ASSERT(IS_PAGEALIGNED(vaddr));
2372 	ASSERT(IS_PAGEALIGNED(eaddr));
2373 	ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
2374 
2375 	XPV_DISALLOW_MIGRATE();
2376 	for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2377 try_again:
2378 		pte = htable_walk(hat, &ht, &vaddr, eaddr);
2379 		if (ht == NULL)
2380 			break;
2381 		entry = htable_va2entry(vaddr, ht);
2382 
2383 		if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC ||
2384 		    PTE_GET(pte, PT_REF | PT_MOD) == 0)
2385 			continue;
2386 
2387 		/*
2388 		 * We need to acquire the mapping list lock to protect
2389 		 * against hat_pageunload(), hat_unload(), etc.
2390 		 */
2391 		pp = page_numtopp_nolock(PTE2PFN(pte, ht->ht_level));
2392 		if (pp == NULL)
2393 			break;
2394 		x86_hm_enter(pp);
2395 		save_pte = pte;
2396 		pte = x86pte_get(ht, entry);
2397 		if (pte != save_pte) {
2398 			x86_hm_exit(pp);
2399 			goto try_again;
2400 		}
2401 		if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC ||
2402 		    PTE_GET(pte, PT_REF | PT_MOD) == 0) {
2403 			x86_hm_exit(pp);
2404 			continue;
2405 		}
2406 
2407 		/*
2408 		 * Need to clear ref or mod bits. We may compete with
2409 		 * hardware updating the R/M bits and have to try again.
2410 		 */
2411 		if (flags == HAT_SYNC_ZERORM) {
2412 			new = pte;
2413 			PTE_CLR(new, PT_REF | PT_MOD);
2414 			pte = hati_update_pte(ht, entry, pte, new);
2415 			if (pte != 0) {
2416 				x86_hm_exit(pp);
2417 				goto try_again;
2418 			}
2419 		} else {
2420 			/*
2421 			 * sync the PTE to the page_t
2422 			 */
2423 			hati_sync_pte_to_page(pp, save_pte, ht->ht_level);
2424 		}
2425 		x86_hm_exit(pp);
2426 	}
2427 	if (ht)
2428 		htable_release(ht);
2429 	XPV_ALLOW_MIGRATE();
2430 }
2431 
2432 /*
2433  * void	hat_map(hat, addr, len, flags)
2434  */
2435 /*ARGSUSED*/
2436 void
2437 hat_map(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2438 {
2439 	/* does nothing */
2440 }
2441 
2442 /*
2443  * uint_t hat_getattr(hat, addr, *attr)
2444  *	returns attr for <hat,addr> in *attr.  returns 0 if there was a
2445  *	mapping and *attr is valid, nonzero if there was no mapping and
2446  *	*attr is not valid.
2447  */
2448 uint_t
2449 hat_getattr(hat_t *hat, caddr_t addr, uint_t *attr)
2450 {
2451 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2452 	htable_t	*ht = NULL;
2453 	x86pte_t	pte;
2454 
2455 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2456 
2457 	if (IN_VA_HOLE(vaddr))
2458 		return ((uint_t)-1);
2459 
2460 	ht = htable_getpte(hat, vaddr, NULL, &pte, mmu.max_page_level);
2461 	if (ht == NULL)
2462 		return ((uint_t)-1);
2463 
2464 	if (!PTE_ISVALID(pte) || !PTE_ISPAGE(pte, ht->ht_level)) {
2465 		htable_release(ht);
2466 		return ((uint_t)-1);
2467 	}
2468 
2469 	*attr = PROT_READ;
2470 	if (PTE_GET(pte, PT_WRITABLE))
2471 		*attr |= PROT_WRITE;
2472 	if (PTE_GET(pte, PT_USER))
2473 		*attr |= PROT_USER;
2474 	if (!PTE_GET(pte, mmu.pt_nx))
2475 		*attr |= PROT_EXEC;
2476 	if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC)
2477 		*attr |= HAT_NOSYNC;
2478 	htable_release(ht);
2479 	return (0);
2480 }
2481 
2482 /*
2483  * hat_updateattr() applies the given attribute change to an existing mapping
2484  */
2485 #define	HAT_LOAD_ATTR		1
2486 #define	HAT_SET_ATTR		2
2487 #define	HAT_CLR_ATTR		3
2488 
2489 static void
2490 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what)
2491 {
2492 	uintptr_t	vaddr = (uintptr_t)addr;
2493 	uintptr_t	eaddr = (uintptr_t)addr + len;
2494 	htable_t	*ht = NULL;
2495 	uint_t		entry;
2496 	x86pte_t	oldpte, newpte;
2497 	page_t		*pp;
2498 
2499 	XPV_DISALLOW_MIGRATE();
2500 	ASSERT(IS_PAGEALIGNED(vaddr));
2501 	ASSERT(IS_PAGEALIGNED(eaddr));
2502 	ASSERT(hat == kas.a_hat ||
2503 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2504 	for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2505 try_again:
2506 		oldpte = htable_walk(hat, &ht, &vaddr, eaddr);
2507 		if (ht == NULL)
2508 			break;
2509 		if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST)
2510 			continue;
2511 
2512 		pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level));
2513 		if (pp == NULL)
2514 			continue;
2515 		x86_hm_enter(pp);
2516 
2517 		newpte = oldpte;
2518 		/*
2519 		 * We found a page table entry in the desired range,
2520 		 * figure out the new attributes.
2521 		 */
2522 		if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) {
2523 			if ((attr & PROT_WRITE) &&
2524 			    !PTE_GET(oldpte, PT_WRITABLE))
2525 				newpte |= PT_WRITABLE;
2526 
2527 			if ((attr & HAT_NOSYNC) &&
2528 			    PTE_GET(oldpte, PT_SOFTWARE) < PT_NOSYNC)
2529 				newpte |= PT_NOSYNC;
2530 
2531 			if ((attr & PROT_EXEC) && PTE_GET(oldpte, mmu.pt_nx))
2532 				newpte &= ~mmu.pt_nx;
2533 		}
2534 
2535 		if (what == HAT_LOAD_ATTR) {
2536 			if (!(attr & PROT_WRITE) &&
2537 			    PTE_GET(oldpte, PT_WRITABLE))
2538 				newpte &= ~PT_WRITABLE;
2539 
2540 			if (!(attr & HAT_NOSYNC) &&
2541 			    PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC)
2542 				newpte &= ~PT_SOFTWARE;
2543 
2544 			if (!(attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx))
2545 				newpte |= mmu.pt_nx;
2546 		}
2547 
2548 		if (what == HAT_CLR_ATTR) {
2549 			if ((attr & PROT_WRITE) && PTE_GET(oldpte, PT_WRITABLE))
2550 				newpte &= ~PT_WRITABLE;
2551 
2552 			if ((attr & HAT_NOSYNC) &&
2553 			    PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC)
2554 				newpte &= ~PT_SOFTWARE;
2555 
2556 			if ((attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx))
2557 				newpte |= mmu.pt_nx;
2558 		}
2559 
2560 		/*
2561 		 * Ensure NOSYNC/NOCONSIST mappings have REF and MOD set.
2562 		 * x86pte_set() depends on this.
2563 		 */
2564 		if (PTE_GET(newpte, PT_SOFTWARE) >= PT_NOSYNC)
2565 			newpte |= PT_REF | PT_MOD;
2566 
2567 		/*
2568 		 * what about PROT_READ or others? this code only handles:
2569 		 * EXEC, WRITE, NOSYNC
2570 		 */
2571 
2572 		/*
2573 		 * If new PTE really changed, update the table.
2574 		 */
2575 		if (newpte != oldpte) {
2576 			entry = htable_va2entry(vaddr, ht);
2577 			oldpte = hati_update_pte(ht, entry, oldpte, newpte);
2578 			if (oldpte != 0) {
2579 				x86_hm_exit(pp);
2580 				goto try_again;
2581 			}
2582 		}
2583 		x86_hm_exit(pp);
2584 	}
2585 	if (ht)
2586 		htable_release(ht);
2587 	XPV_ALLOW_MIGRATE();
2588 }
2589 
2590 /*
2591  * Various wrappers for hat_updateattr()
2592  */
2593 void
2594 hat_setattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2595 {
2596 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2597 	hat_updateattr(hat, addr, len, attr, HAT_SET_ATTR);
2598 }
2599 
2600 void
2601 hat_clrattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2602 {
2603 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2604 	hat_updateattr(hat, addr, len, attr, HAT_CLR_ATTR);
2605 }
2606 
2607 void
2608 hat_chgattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2609 {
2610 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2611 	hat_updateattr(hat, addr, len, attr, HAT_LOAD_ATTR);
2612 }
2613 
2614 void
2615 hat_chgprot(hat_t *hat, caddr_t addr, size_t len, uint_t vprot)
2616 {
2617 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2618 	hat_updateattr(hat, addr, len, vprot & HAT_PROT_MASK, HAT_LOAD_ATTR);
2619 }
2620 
2621 /*
2622  * size_t hat_getpagesize(hat, addr)
2623  *	returns pagesize in bytes for <hat, addr>. returns -1 of there is
2624  *	no mapping. This is an advisory call.
2625  */
2626 ssize_t
2627 hat_getpagesize(hat_t *hat, caddr_t addr)
2628 {
2629 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2630 	htable_t	*ht;
2631 	size_t		pagesize;
2632 
2633 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2634 	if (IN_VA_HOLE(vaddr))
2635 		return (-1);
2636 	ht = htable_getpage(hat, vaddr, NULL);
2637 	if (ht == NULL)
2638 		return (-1);
2639 	pagesize = LEVEL_SIZE(ht->ht_level);
2640 	htable_release(ht);
2641 	return (pagesize);
2642 }
2643 
2644 
2645 
2646 /*
2647  * pfn_t hat_getpfnum(hat, addr)
2648  *	returns pfn for <hat, addr> or PFN_INVALID if mapping is invalid.
2649  */
2650 pfn_t
2651 hat_getpfnum(hat_t *hat, caddr_t addr)
2652 {
2653 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2654 	htable_t	*ht;
2655 	uint_t		entry;
2656 	pfn_t		pfn = PFN_INVALID;
2657 
2658 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2659 	if (khat_running == 0)
2660 		return (PFN_INVALID);
2661 
2662 	if (IN_VA_HOLE(vaddr))
2663 		return (PFN_INVALID);
2664 
2665 	XPV_DISALLOW_MIGRATE();
2666 	/*
2667 	 * A very common use of hat_getpfnum() is from the DDI for kernel pages.
2668 	 * Use the kmap_ptes (which also covers the 32 bit heap) to speed
2669 	 * this up.
2670 	 */
2671 	if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2672 		x86pte_t pte;
2673 		pgcnt_t pg_index;
2674 
2675 		pg_index = mmu_btop(vaddr - mmu.kmap_addr);
2676 		pte = GET_PTE(PT_INDEX_PTR(mmu.kmap_ptes, pg_index));
2677 		if (PTE_ISVALID(pte))
2678 			/*LINTED [use of constant 0 causes a lint warning] */
2679 			pfn = PTE2PFN(pte, 0);
2680 		XPV_ALLOW_MIGRATE();
2681 		return (pfn);
2682 	}
2683 
2684 	ht = htable_getpage(hat, vaddr, &entry);
2685 	if (ht == NULL) {
2686 		XPV_ALLOW_MIGRATE();
2687 		return (PFN_INVALID);
2688 	}
2689 	ASSERT(vaddr >= ht->ht_vaddr);
2690 	ASSERT(vaddr <= HTABLE_LAST_PAGE(ht));
2691 	pfn = PTE2PFN(x86pte_get(ht, entry), ht->ht_level);
2692 	if (ht->ht_level > 0)
2693 		pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level));
2694 	htable_release(ht);
2695 	XPV_ALLOW_MIGRATE();
2696 	return (pfn);
2697 }
2698 
2699 /*
2700  * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged.
2701  * Use hat_getpfnum(kas.a_hat, ...) instead.
2702  *
2703  * We'd like to return PFN_INVALID if the mappings have underlying page_t's
2704  * but can't right now due to the fact that some software has grown to use
2705  * this interface incorrectly. So for now when the interface is misused,
2706  * return a warning to the user that in the future it won't work in the
2707  * way they're abusing it, and carry on.
2708  *
2709  * Note that hat_getkpfnum() is never supported on amd64.
2710  */
2711 #if !defined(__amd64)
2712 pfn_t
2713 hat_getkpfnum(caddr_t addr)
2714 {
2715 	pfn_t	pfn;
2716 	int badcaller = 0;
2717 
2718 	if (khat_running == 0)
2719 		panic("hat_getkpfnum(): called too early\n");
2720 	if ((uintptr_t)addr < kernelbase)
2721 		return (PFN_INVALID);
2722 
2723 	XPV_DISALLOW_MIGRATE();
2724 	if (segkpm && IS_KPM_ADDR(addr)) {
2725 		badcaller = 1;
2726 		pfn = hat_kpm_va2pfn(addr);
2727 	} else {
2728 		pfn = hat_getpfnum(kas.a_hat, addr);
2729 		badcaller = pf_is_memory(pfn);
2730 	}
2731 
2732 	if (badcaller)
2733 		hat_getkpfnum_badcall(caller());
2734 	XPV_ALLOW_MIGRATE();
2735 	return (pfn);
2736 }
2737 #endif /* __amd64 */
2738 
2739 /*
2740  * int hat_probe(hat, addr)
2741  *	return 0 if no valid mapping is present.  Faster version
2742  *	of hat_getattr in certain architectures.
2743  */
2744 int
2745 hat_probe(hat_t *hat, caddr_t addr)
2746 {
2747 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2748 	uint_t		entry;
2749 	htable_t	*ht;
2750 	pgcnt_t		pg_off;
2751 
2752 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2753 	ASSERT(hat == kas.a_hat ||
2754 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2755 	if (IN_VA_HOLE(vaddr))
2756 		return (0);
2757 
2758 	/*
2759 	 * Most common use of hat_probe is from segmap. We special case it
2760 	 * for performance.
2761 	 */
2762 	if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2763 		pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2764 		if (mmu.pae_hat)
2765 			return (PTE_ISVALID(mmu.kmap_ptes[pg_off]));
2766 		else
2767 			return (PTE_ISVALID(
2768 			    ((x86pte32_t *)mmu.kmap_ptes)[pg_off]));
2769 	}
2770 
2771 	ht = htable_getpage(hat, vaddr, &entry);
2772 	htable_release(ht);
2773 	return (ht != NULL);
2774 }
2775 
2776 /*
2777  * Find out if the segment for hat_share()/hat_unshare() is DISM or locked ISM.
2778  */
2779 static int
2780 is_it_dism(hat_t *hat, caddr_t va)
2781 {
2782 	struct seg *seg;
2783 	struct shm_data *shmd;
2784 	struct spt_data *sptd;
2785 
2786 	seg = as_findseg(hat->hat_as, va, 0);
2787 	ASSERT(seg != NULL);
2788 	ASSERT(seg->s_base <= va);
2789 	shmd = (struct shm_data *)seg->s_data;
2790 	ASSERT(shmd != NULL);
2791 	sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2792 	ASSERT(sptd != NULL);
2793 	if (sptd->spt_flags & SHM_PAGEABLE)
2794 		return (1);
2795 	return (0);
2796 }
2797 
2798 /*
2799  * Simple implementation of ISM. hat_share() is similar to hat_memload_array(),
2800  * except that we use the ism_hat's existing mappings to determine the pages
2801  * and protections to use for this hat. If we find a full properly aligned
2802  * and sized pagetable, we will attempt to share the pagetable itself.
2803  */
2804 /*ARGSUSED*/
2805 int
2806 hat_share(
2807 	hat_t		*hat,
2808 	caddr_t		addr,
2809 	hat_t		*ism_hat,
2810 	caddr_t		src_addr,
2811 	size_t		len,	/* almost useless value, see below.. */
2812 	uint_t		ismszc)
2813 {
2814 	uintptr_t	vaddr_start = (uintptr_t)addr;
2815 	uintptr_t	vaddr;
2816 	uintptr_t	eaddr = vaddr_start + len;
2817 	uintptr_t	ism_addr_start = (uintptr_t)src_addr;
2818 	uintptr_t	ism_addr = ism_addr_start;
2819 	uintptr_t	e_ism_addr = ism_addr + len;
2820 	htable_t	*ism_ht = NULL;
2821 	htable_t	*ht;
2822 	x86pte_t	pte;
2823 	page_t		*pp;
2824 	pfn_t		pfn;
2825 	level_t		l;
2826 	pgcnt_t		pgcnt;
2827 	uint_t		prot;
2828 	int		is_dism;
2829 	int		flags;
2830 
2831 	/*
2832 	 * We might be asked to share an empty DISM hat by as_dup()
2833 	 */
2834 	ASSERT(hat != kas.a_hat);
2835 	ASSERT(eaddr <= _userlimit);
2836 	if (!(ism_hat->hat_flags & HAT_SHARED)) {
2837 		ASSERT(hat_get_mapped_size(ism_hat) == 0);
2838 		return (0);
2839 	}
2840 	XPV_DISALLOW_MIGRATE();
2841 
2842 	/*
2843 	 * The SPT segment driver often passes us a size larger than there are
2844 	 * valid mappings. That's because it rounds the segment size up to a
2845 	 * large pagesize, even if the actual memory mapped by ism_hat is less.
2846 	 */
2847 	ASSERT(IS_PAGEALIGNED(vaddr_start));
2848 	ASSERT(IS_PAGEALIGNED(ism_addr_start));
2849 	ASSERT(ism_hat->hat_flags & HAT_SHARED);
2850 	is_dism = is_it_dism(hat, addr);
2851 	while (ism_addr < e_ism_addr) {
2852 		/*
2853 		 * use htable_walk to get the next valid ISM mapping
2854 		 */
2855 		pte = htable_walk(ism_hat, &ism_ht, &ism_addr, e_ism_addr);
2856 		if (ism_ht == NULL)
2857 			break;
2858 
2859 		/*
2860 		 * First check to see if we already share the page table.
2861 		 */
2862 		l = ism_ht->ht_level;
2863 		vaddr = vaddr_start + (ism_addr - ism_addr_start);
2864 		ht = htable_lookup(hat, vaddr, l);
2865 		if (ht != NULL) {
2866 			if (ht->ht_flags & HTABLE_SHARED_PFN)
2867 				goto shared;
2868 			htable_release(ht);
2869 			goto not_shared;
2870 		}
2871 
2872 		/*
2873 		 * Can't ever share top table.
2874 		 */
2875 		if (l == mmu.max_level)
2876 			goto not_shared;
2877 
2878 		/*
2879 		 * Avoid level mismatches later due to DISM faults.
2880 		 */
2881 		if (is_dism && l > 0)
2882 			goto not_shared;
2883 
2884 		/*
2885 		 * addresses and lengths must align
2886 		 * table must be fully populated
2887 		 * no lower level page tables
2888 		 */
2889 		if (ism_addr != ism_ht->ht_vaddr ||
2890 		    (vaddr & LEVEL_OFFSET(l + 1)) != 0)
2891 			goto not_shared;
2892 
2893 		/*
2894 		 * The range of address space must cover a full table.
2895 		 */
2896 		if (e_ism_addr - ism_addr < LEVEL_SIZE(1 + 1))
2897 			goto not_shared;
2898 
2899 		/*
2900 		 * All entries in the ISM page table must be leaf PTEs.
2901 		 */
2902 		if (l > 0) {
2903 			int e;
2904 
2905 			/*
2906 			 * We know the 0th is from htable_walk() above.
2907 			 */
2908 			for (e = 1; e < HTABLE_NUM_PTES(ism_ht); ++e) {
2909 				x86pte_t pte;
2910 				pte = x86pte_get(ism_ht, e);
2911 				if (!PTE_ISPAGE(pte, l))
2912 					goto not_shared;
2913 			}
2914 		}
2915 
2916 		/*
2917 		 * share the page table
2918 		 */
2919 		ht = htable_create(hat, vaddr, l, ism_ht);
2920 shared:
2921 		ASSERT(ht->ht_flags & HTABLE_SHARED_PFN);
2922 		ASSERT(ht->ht_shares == ism_ht);
2923 		hat->hat_ism_pgcnt +=
2924 		    (ism_ht->ht_valid_cnt - ht->ht_valid_cnt) <<
2925 		    (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT);
2926 		ht->ht_valid_cnt = ism_ht->ht_valid_cnt;
2927 		htable_release(ht);
2928 		ism_addr = ism_ht->ht_vaddr + LEVEL_SIZE(l + 1);
2929 		htable_release(ism_ht);
2930 		ism_ht = NULL;
2931 		continue;
2932 
2933 not_shared:
2934 		/*
2935 		 * Unable to share the page table. Instead we will
2936 		 * create new mappings from the values in the ISM mappings.
2937 		 * Figure out what level size mappings to use;
2938 		 */
2939 		for (l = ism_ht->ht_level; l > 0; --l) {
2940 			if (LEVEL_SIZE(l) <= eaddr - vaddr &&
2941 			    (vaddr & LEVEL_OFFSET(l)) == 0)
2942 				break;
2943 		}
2944 
2945 		/*
2946 		 * The ISM mapping might be larger than the share area,
2947 		 * be careful to truncate it if needed.
2948 		 */
2949 		if (eaddr - vaddr >= LEVEL_SIZE(ism_ht->ht_level)) {
2950 			pgcnt = mmu_btop(LEVEL_SIZE(ism_ht->ht_level));
2951 		} else {
2952 			pgcnt = mmu_btop(eaddr - vaddr);
2953 			l = 0;
2954 		}
2955 
2956 		pfn = PTE2PFN(pte, ism_ht->ht_level);
2957 		ASSERT(pfn != PFN_INVALID);
2958 		while (pgcnt > 0) {
2959 			/*
2960 			 * Make a new pte for the PFN for this level.
2961 			 * Copy protections for the pte from the ISM pte.
2962 			 */
2963 			pp = page_numtopp_nolock(pfn);
2964 			ASSERT(pp != NULL);
2965 
2966 			prot = PROT_USER | PROT_READ | HAT_UNORDERED_OK;
2967 			if (PTE_GET(pte, PT_WRITABLE))
2968 				prot |= PROT_WRITE;
2969 			if (!PTE_GET(pte, PT_NX))
2970 				prot |= PROT_EXEC;
2971 
2972 			flags = HAT_LOAD;
2973 			if (!is_dism)
2974 				flags |= HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST;
2975 			while (hati_load_common(hat, vaddr, pp, prot, flags,
2976 			    l, pfn) != 0) {
2977 				if (l == 0)
2978 					panic("hati_load_common() failure");
2979 				--l;
2980 			}
2981 
2982 			vaddr += LEVEL_SIZE(l);
2983 			ism_addr += LEVEL_SIZE(l);
2984 			pfn += mmu_btop(LEVEL_SIZE(l));
2985 			pgcnt -= mmu_btop(LEVEL_SIZE(l));
2986 		}
2987 	}
2988 	if (ism_ht != NULL)
2989 		htable_release(ism_ht);
2990 	XPV_ALLOW_MIGRATE();
2991 	return (0);
2992 }
2993 
2994 
2995 /*
2996  * hat_unshare() is similar to hat_unload_callback(), but
2997  * we have to look for empty shared pagetables. Note that
2998  * hat_unshare() is always invoked against an entire segment.
2999  */
3000 /*ARGSUSED*/
3001 void
3002 hat_unshare(hat_t *hat, caddr_t addr, size_t len, uint_t ismszc)
3003 {
3004 	uint64_t	vaddr = (uintptr_t)addr;
3005 	uintptr_t	eaddr = vaddr + len;
3006 	htable_t	*ht = NULL;
3007 	uint_t		need_demaps = 0;
3008 	int		flags = HAT_UNLOAD_UNMAP;
3009 	level_t		l;
3010 
3011 	ASSERT(hat != kas.a_hat);
3012 	ASSERT(eaddr <= _userlimit);
3013 	ASSERT(IS_PAGEALIGNED(vaddr));
3014 	ASSERT(IS_PAGEALIGNED(eaddr));
3015 	XPV_DISALLOW_MIGRATE();
3016 
3017 	/*
3018 	 * First go through and remove any shared pagetables.
3019 	 *
3020 	 * Note that it's ok to delay the TLB shootdown till the entire range is
3021 	 * finished, because if hat_pageunload() were to unload a shared
3022 	 * pagetable page, its hat_tlb_inval() will do a global TLB invalidate.
3023 	 */
3024 	l = mmu.max_page_level;
3025 	if (l == mmu.max_level)
3026 		--l;
3027 	for (; l >= 0; --l) {
3028 		for (vaddr = (uintptr_t)addr; vaddr < eaddr;
3029 		    vaddr = (vaddr & LEVEL_MASK(l + 1)) + LEVEL_SIZE(l + 1)) {
3030 			ASSERT(!IN_VA_HOLE(vaddr));
3031 			/*
3032 			 * find a pagetable that maps the current address
3033 			 */
3034 			ht = htable_lookup(hat, vaddr, l);
3035 			if (ht == NULL)
3036 				continue;
3037 			if (ht->ht_flags & HTABLE_SHARED_PFN) {
3038 				/*
3039 				 * clear page count, set valid_cnt to 0,
3040 				 * let htable_release() finish the job
3041 				 */
3042 				hat->hat_ism_pgcnt -= ht->ht_valid_cnt <<
3043 				    (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT);
3044 				ht->ht_valid_cnt = 0;
3045 				need_demaps = 1;
3046 			}
3047 			htable_release(ht);
3048 		}
3049 	}
3050 
3051 	/*
3052 	 * flush the TLBs - since we're probably dealing with MANY mappings
3053 	 * we do just one CR3 reload.
3054 	 */
3055 	if (!(hat->hat_flags & HAT_FREEING) && need_demaps)
3056 		hat_tlb_inval(hat, DEMAP_ALL_ADDR);
3057 
3058 	/*
3059 	 * Now go back and clean up any unaligned mappings that
3060 	 * couldn't share pagetables.
3061 	 */
3062 	if (!is_it_dism(hat, addr))
3063 		flags |= HAT_UNLOAD_UNLOCK;
3064 	hat_unload(hat, addr, len, flags);
3065 	XPV_ALLOW_MIGRATE();
3066 }
3067 
3068 
3069 /*
3070  * hat_reserve() does nothing
3071  */
3072 /*ARGSUSED*/
3073 void
3074 hat_reserve(struct as *as, caddr_t addr, size_t len)
3075 {
3076 }
3077 
3078 
3079 /*
3080  * Called when all mappings to a page should have write permission removed.
3081  * Mostly stolem from hat_pagesync()
3082  */
3083 static void
3084 hati_page_clrwrt(struct page *pp)
3085 {
3086 	hment_t		*hm = NULL;
3087 	htable_t	*ht;
3088 	uint_t		entry;
3089 	x86pte_t	old;
3090 	x86pte_t	new;
3091 	uint_t		pszc = 0;
3092 
3093 	XPV_DISALLOW_MIGRATE();
3094 next_size:
3095 	/*
3096 	 * walk thru the mapping list clearing write permission
3097 	 */
3098 	x86_hm_enter(pp);
3099 	while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) {
3100 		if (ht->ht_level < pszc)
3101 			continue;
3102 		old = x86pte_get(ht, entry);
3103 
3104 		for (;;) {
3105 			/*
3106 			 * Is this mapping of interest?
3107 			 */
3108 			if (PTE2PFN(old, ht->ht_level) != pp->p_pagenum ||
3109 			    PTE_GET(old, PT_WRITABLE) == 0)
3110 				break;
3111 
3112 			/*
3113 			 * Clear ref/mod writable bits. This requires cross
3114 			 * calls to ensure any executing TLBs see cleared bits.
3115 			 */
3116 			new = old;
3117 			PTE_CLR(new, PT_REF | PT_MOD | PT_WRITABLE);
3118 			old = hati_update_pte(ht, entry, old, new);
3119 			if (old != 0)
3120 				continue;
3121 
3122 			break;
3123 		}
3124 	}
3125 	x86_hm_exit(pp);
3126 	while (pszc < pp->p_szc) {
3127 		page_t *tpp;
3128 		pszc++;
3129 		tpp = PP_GROUPLEADER(pp, pszc);
3130 		if (pp != tpp) {
3131 			pp = tpp;
3132 			goto next_size;
3133 		}
3134 	}
3135 	XPV_ALLOW_MIGRATE();
3136 }
3137 
3138 /*
3139  * void hat_page_setattr(pp, flag)
3140  * void hat_page_clrattr(pp, flag)
3141  *	used to set/clr ref/mod bits.
3142  */
3143 void
3144 hat_page_setattr(struct page *pp, uint_t flag)
3145 {
3146 	vnode_t		*vp = pp->p_vnode;
3147 	kmutex_t	*vphm = NULL;
3148 	page_t		**listp;
3149 	int		noshuffle;
3150 
3151 	noshuffle = flag & P_NSH;
3152 	flag &= ~P_NSH;
3153 
3154 	if (PP_GETRM(pp, flag) == flag)
3155 		return;
3156 
3157 	if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp) &&
3158 	    !noshuffle) {
3159 		vphm = page_vnode_mutex(vp);
3160 		mutex_enter(vphm);
3161 	}
3162 
3163 	PP_SETRM(pp, flag);
3164 
3165 	if (vphm != NULL) {
3166 
3167 		/*
3168 		 * Some File Systems examine v_pages for NULL w/o
3169 		 * grabbing the vphm mutex. Must not let it become NULL when
3170 		 * pp is the only page on the list.
3171 		 */
3172 		if (pp->p_vpnext != pp) {
3173 			page_vpsub(&vp->v_pages, pp);
3174 			if (vp->v_pages != NULL)
3175 				listp = &vp->v_pages->p_vpprev->p_vpnext;
3176 			else
3177 				listp = &vp->v_pages;
3178 			page_vpadd(listp, pp);
3179 		}
3180 		mutex_exit(vphm);
3181 	}
3182 }
3183 
3184 void
3185 hat_page_clrattr(struct page *pp, uint_t flag)
3186 {
3187 	vnode_t		*vp = pp->p_vnode;
3188 	ASSERT(!(flag & ~(P_MOD | P_REF | P_RO)));
3189 
3190 	/*
3191 	 * Caller is expected to hold page's io lock for VMODSORT to work
3192 	 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod
3193 	 * bit is cleared.
3194 	 * We don't have assert to avoid tripping some existing third party
3195 	 * code. The dirty page is moved back to top of the v_page list
3196 	 * after IO is done in pvn_write_done().
3197 	 */
3198 	PP_CLRRM(pp, flag);
3199 
3200 	if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) {
3201 
3202 		/*
3203 		 * VMODSORT works by removing write permissions and getting
3204 		 * a fault when a page is made dirty. At this point
3205 		 * we need to remove write permission from all mappings
3206 		 * to this page.
3207 		 */
3208 		hati_page_clrwrt(pp);
3209 	}
3210 }
3211 
3212 /*
3213  *	If flag is specified, returns 0 if attribute is disabled
3214  *	and non zero if enabled.  If flag specifes multiple attributs
3215  *	then returns 0 if ALL atriibutes are disabled.  This is an advisory
3216  *	call.
3217  */
3218 uint_t
3219 hat_page_getattr(struct page *pp, uint_t flag)
3220 {
3221 	return (PP_GETRM(pp, flag));
3222 }
3223 
3224 
3225 /*
3226  * common code used by hat_pageunload() and hment_steal()
3227  */
3228 hment_t *
3229 hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry)
3230 {
3231 	x86pte_t old_pte;
3232 	pfn_t pfn = pp->p_pagenum;
3233 	hment_t *hm;
3234 
3235 	/*
3236 	 * We need to acquire a hold on the htable in order to
3237 	 * do the invalidate. We know the htable must exist, since
3238 	 * unmap's don't release the htable until after removing any
3239 	 * hment. Having x86_hm_enter() keeps that from proceeding.
3240 	 */
3241 	htable_acquire(ht);
3242 
3243 	/*
3244 	 * Invalidate the PTE and remove the hment.
3245 	 */
3246 	old_pte = x86pte_inval(ht, entry, 0, NULL);
3247 	if (PTE2PFN(old_pte, ht->ht_level) != pfn) {
3248 		panic("x86pte_inval() failure found PTE = " FMT_PTE
3249 		    " pfn being unmapped is %lx ht=0x%lx entry=0x%x",
3250 		    old_pte, pfn, (uintptr_t)ht, entry);
3251 	}
3252 
3253 	/*
3254 	 * Clean up all the htable information for this mapping
3255 	 */
3256 	ASSERT(ht->ht_valid_cnt > 0);
3257 	HTABLE_DEC(ht->ht_valid_cnt);
3258 	PGCNT_DEC(ht->ht_hat, ht->ht_level);
3259 
3260 	/*
3261 	 * sync ref/mod bits to the page_t
3262 	 */
3263 	if (PTE_GET(old_pte, PT_SOFTWARE) < PT_NOSYNC)
3264 		hati_sync_pte_to_page(pp, old_pte, ht->ht_level);
3265 
3266 	/*
3267 	 * Remove the mapping list entry for this page.
3268 	 */
3269 	hm = hment_remove(pp, ht, entry);
3270 
3271 	/*
3272 	 * drop the mapping list lock so that we might free the
3273 	 * hment and htable.
3274 	 */
3275 	x86_hm_exit(pp);
3276 	htable_release(ht);
3277 	return (hm);
3278 }
3279 
3280 extern int	vpm_enable;
3281 /*
3282  * Unload all translations to a page. If the page is a subpage of a large
3283  * page, the large page mappings are also removed.
3284  *
3285  * The forceflags are unused.
3286  */
3287 
3288 /*ARGSUSED*/
3289 static int
3290 hati_pageunload(struct page *pp, uint_t pg_szcd, uint_t forceflag)
3291 {
3292 	page_t		*cur_pp = pp;
3293 	hment_t		*hm;
3294 	hment_t		*prev;
3295 	htable_t	*ht;
3296 	uint_t		entry;
3297 	level_t		level;
3298 
3299 	XPV_DISALLOW_MIGRATE();
3300 #if defined(__amd64)
3301 	/*
3302 	 * clear the vpm ref.
3303 	 */
3304 	if (vpm_enable) {
3305 		pp->p_vpmref = 0;
3306 	}
3307 #endif
3308 	/*
3309 	 * The loop with next_size handles pages with multiple pagesize mappings
3310 	 */
3311 next_size:
3312 	for (;;) {
3313 
3314 		/*
3315 		 * Get a mapping list entry
3316 		 */
3317 		x86_hm_enter(cur_pp);
3318 		for (prev = NULL; ; prev = hm) {
3319 			hm = hment_walk(cur_pp, &ht, &entry, prev);
3320 			if (hm == NULL) {
3321 				x86_hm_exit(cur_pp);
3322 
3323 				/*
3324 				 * If not part of a larger page, we're done.
3325 				 */
3326 				if (cur_pp->p_szc <= pg_szcd) {
3327 					XPV_ALLOW_MIGRATE();
3328 					return (0);
3329 				}
3330 
3331 				/*
3332 				 * Else check the next larger page size.
3333 				 * hat_page_demote() may decrease p_szc
3334 				 * but that's ok we'll just take an extra
3335 				 * trip discover there're no larger mappings
3336 				 * and return.
3337 				 */
3338 				++pg_szcd;
3339 				cur_pp = PP_GROUPLEADER(cur_pp, pg_szcd);
3340 				goto next_size;
3341 			}
3342 
3343 			/*
3344 			 * If this mapping size matches, remove it.
3345 			 */
3346 			level = ht->ht_level;
3347 			if (level == pg_szcd)
3348 				break;
3349 		}
3350 
3351 		/*
3352 		 * Remove the mapping list entry for this page.
3353 		 * Note this does the x86_hm_exit() for us.
3354 		 */
3355 		hm = hati_page_unmap(cur_pp, ht, entry);
3356 		if (hm != NULL)
3357 			hment_free(hm);
3358 	}
3359 }
3360 
3361 int
3362 hat_pageunload(struct page *pp, uint_t forceflag)
3363 {
3364 	ASSERT(PAGE_EXCL(pp));
3365 	return (hati_pageunload(pp, 0, forceflag));
3366 }
3367 
3368 /*
3369  * Unload all large mappings to pp and reduce by 1 p_szc field of every large
3370  * page level that included pp.
3371  *
3372  * pp must be locked EXCL. Even though no other constituent pages are locked
3373  * it's legal to unload large mappings to pp because all constituent pages of
3374  * large locked mappings have to be locked SHARED.  therefore if we have EXCL
3375  * lock on one of constituent pages none of the large mappings to pp are
3376  * locked.
3377  *
3378  * Change (always decrease) p_szc field starting from the last constituent
3379  * page and ending with root constituent page so that root's pszc always shows
3380  * the area where hat_page_demote() may be active.
3381  *
3382  * This mechanism is only used for file system pages where it's not always
3383  * possible to get EXCL locks on all constituent pages to demote the size code
3384  * (as is done for anonymous or kernel large pages).
3385  */
3386 void
3387 hat_page_demote(page_t *pp)
3388 {
3389 	uint_t		pszc;
3390 	uint_t		rszc;
3391 	uint_t		szc;
3392 	page_t		*rootpp;
3393 	page_t		*firstpp;
3394 	page_t		*lastpp;
3395 	pgcnt_t		pgcnt;
3396 
3397 	ASSERT(PAGE_EXCL(pp));
3398 	ASSERT(!PP_ISFREE(pp));
3399 	ASSERT(page_szc_lock_assert(pp));
3400 
3401 	if (pp->p_szc == 0)
3402 		return;
3403 
3404 	rootpp = PP_GROUPLEADER(pp, 1);
3405 	(void) hati_pageunload(rootpp, 1, HAT_FORCE_PGUNLOAD);
3406 
3407 	/*
3408 	 * all large mappings to pp are gone
3409 	 * and no new can be setup since pp is locked exclusively.
3410 	 *
3411 	 * Lock the root to make sure there's only one hat_page_demote()
3412 	 * outstanding within the area of this root's pszc.
3413 	 *
3414 	 * Second potential hat_page_demote() is already eliminated by upper
3415 	 * VM layer via page_szc_lock() but we don't rely on it and use our
3416 	 * own locking (so that upper layer locking can be changed without
3417 	 * assumptions that hat depends on upper layer VM to prevent multiple
3418 	 * hat_page_demote() to be issued simultaneously to the same large
3419 	 * page).
3420 	 */
3421 again:
3422 	pszc = pp->p_szc;
3423 	if (pszc == 0)
3424 		return;
3425 	rootpp = PP_GROUPLEADER(pp, pszc);
3426 	x86_hm_enter(rootpp);
3427 	/*
3428 	 * If root's p_szc is different from pszc we raced with another
3429 	 * hat_page_demote().  Drop the lock and try to find the root again.
3430 	 * If root's p_szc is greater than pszc previous hat_page_demote() is
3431 	 * not done yet.  Take and release mlist lock of root's root to wait
3432 	 * for previous hat_page_demote() to complete.
3433 	 */
3434 	if ((rszc = rootpp->p_szc) != pszc) {
3435 		x86_hm_exit(rootpp);
3436 		if (rszc > pszc) {
3437 			/* p_szc of a locked non free page can't increase */
3438 			ASSERT(pp != rootpp);
3439 
3440 			rootpp = PP_GROUPLEADER(rootpp, rszc);
3441 			x86_hm_enter(rootpp);
3442 			x86_hm_exit(rootpp);
3443 		}
3444 		goto again;
3445 	}
3446 	ASSERT(pp->p_szc == pszc);
3447 
3448 	/*
3449 	 * Decrement by 1 p_szc of every constituent page of a region that
3450 	 * covered pp. For example if original szc is 3 it gets changed to 2
3451 	 * everywhere except in region 2 that covered pp. Region 2 that
3452 	 * covered pp gets demoted to 1 everywhere except in region 1 that
3453 	 * covered pp. The region 1 that covered pp is demoted to region
3454 	 * 0. It's done this way because from region 3 we removed level 3
3455 	 * mappings, from region 2 that covered pp we removed level 2 mappings
3456 	 * and from region 1 that covered pp we removed level 1 mappings.  All
3457 	 * changes are done from from high pfn's to low pfn's so that roots
3458 	 * are changed last allowing one to know the largest region where
3459 	 * hat_page_demote() is stil active by only looking at the root page.
3460 	 *
3461 	 * This algorithm is implemented in 2 while loops. First loop changes
3462 	 * p_szc of pages to the right of pp's level 1 region and second
3463 	 * loop changes p_szc of pages of level 1 region that covers pp
3464 	 * and all pages to the left of level 1 region that covers pp.
3465 	 * In the first loop p_szc keeps dropping with every iteration
3466 	 * and in the second loop it keeps increasing with every iteration.
3467 	 *
3468 	 * First loop description: Demote pages to the right of pp outside of
3469 	 * level 1 region that covers pp.  In every iteration of the while
3470 	 * loop below find the last page of szc region and the first page of
3471 	 * (szc - 1) region that is immediately to the right of (szc - 1)
3472 	 * region that covers pp.  From last such page to first such page
3473 	 * change every page's szc to szc - 1. Decrement szc and continue
3474 	 * looping until szc is 1. If pp belongs to the last (szc - 1) region
3475 	 * of szc region skip to the next iteration.
3476 	 */
3477 	szc = pszc;
3478 	while (szc > 1) {
3479 		lastpp = PP_GROUPLEADER(pp, szc);
3480 		pgcnt = page_get_pagecnt(szc);
3481 		lastpp += pgcnt - 1;
3482 		firstpp = PP_GROUPLEADER(pp, (szc - 1));
3483 		pgcnt = page_get_pagecnt(szc - 1);
3484 		if (lastpp - firstpp < pgcnt) {
3485 			szc--;
3486 			continue;
3487 		}
3488 		firstpp += pgcnt;
3489 		while (lastpp != firstpp) {
3490 			ASSERT(lastpp->p_szc == pszc);
3491 			lastpp->p_szc = szc - 1;
3492 			lastpp--;
3493 		}
3494 		firstpp->p_szc = szc - 1;
3495 		szc--;
3496 	}
3497 
3498 	/*
3499 	 * Second loop description:
3500 	 * First iteration changes p_szc to 0 of every
3501 	 * page of level 1 region that covers pp.
3502 	 * Subsequent iterations find last page of szc region
3503 	 * immediately to the left of szc region that covered pp
3504 	 * and first page of (szc + 1) region that covers pp.
3505 	 * From last to first page change p_szc of every page to szc.
3506 	 * Increment szc and continue looping until szc is pszc.
3507 	 * If pp belongs to the fist szc region of (szc + 1) region
3508 	 * skip to the next iteration.
3509 	 *
3510 	 */
3511 	szc = 0;
3512 	while (szc < pszc) {
3513 		firstpp = PP_GROUPLEADER(pp, (szc + 1));
3514 		if (szc == 0) {
3515 			pgcnt = page_get_pagecnt(1);
3516 			lastpp = firstpp + (pgcnt - 1);
3517 		} else {
3518 			lastpp = PP_GROUPLEADER(pp, szc);
3519 			if (firstpp == lastpp) {
3520 				szc++;
3521 				continue;
3522 			}
3523 			lastpp--;
3524 			pgcnt = page_get_pagecnt(szc);
3525 		}
3526 		while (lastpp != firstpp) {
3527 			ASSERT(lastpp->p_szc == pszc);
3528 			lastpp->p_szc = szc;
3529 			lastpp--;
3530 		}
3531 		firstpp->p_szc = szc;
3532 		if (firstpp == rootpp)
3533 			break;
3534 		szc++;
3535 	}
3536 	x86_hm_exit(rootpp);
3537 }
3538 
3539 /*
3540  * get hw stats from hardware into page struct and reset hw stats
3541  * returns attributes of page
3542  * Flags for hat_pagesync, hat_getstat, hat_sync
3543  *
3544  * define	HAT_SYNC_ZERORM		0x01
3545  *
3546  * Additional flags for hat_pagesync
3547  *
3548  * define	HAT_SYNC_STOPON_REF	0x02
3549  * define	HAT_SYNC_STOPON_MOD	0x04
3550  * define	HAT_SYNC_STOPON_RM	0x06
3551  * define	HAT_SYNC_STOPON_SHARED	0x08
3552  */
3553 uint_t
3554 hat_pagesync(struct page *pp, uint_t flags)
3555 {
3556 	hment_t		*hm = NULL;
3557 	htable_t	*ht;
3558 	uint_t		entry;
3559 	x86pte_t	old, save_old;
3560 	x86pte_t	new;
3561 	uchar_t		nrmbits = P_REF|P_MOD|P_RO;
3562 	extern ulong_t	po_share;
3563 	page_t		*save_pp = pp;
3564 	uint_t		pszc = 0;
3565 
3566 	ASSERT(PAGE_LOCKED(pp) || panicstr);
3567 
3568 	if (PP_ISRO(pp) && (flags & HAT_SYNC_STOPON_MOD))
3569 		return (pp->p_nrm & nrmbits);
3570 
3571 	if ((flags & HAT_SYNC_ZERORM) == 0) {
3572 
3573 		if ((flags & HAT_SYNC_STOPON_REF) != 0 && PP_ISREF(pp))
3574 			return (pp->p_nrm & nrmbits);
3575 
3576 		if ((flags & HAT_SYNC_STOPON_MOD) != 0 && PP_ISMOD(pp))
3577 			return (pp->p_nrm & nrmbits);
3578 
3579 		if ((flags & HAT_SYNC_STOPON_SHARED) != 0 &&
3580 		    hat_page_getshare(pp) > po_share) {
3581 			if (PP_ISRO(pp))
3582 				PP_SETREF(pp);
3583 			return (pp->p_nrm & nrmbits);
3584 		}
3585 	}
3586 
3587 	XPV_DISALLOW_MIGRATE();
3588 next_size:
3589 	/*
3590 	 * walk thru the mapping list syncing (and clearing) ref/mod bits.
3591 	 */
3592 	x86_hm_enter(pp);
3593 	while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) {
3594 		if (ht->ht_level < pszc)
3595 			continue;
3596 		old = x86pte_get(ht, entry);
3597 try_again:
3598 
3599 		ASSERT(PTE2PFN(old, ht->ht_level) == pp->p_pagenum);
3600 
3601 		if (PTE_GET(old, PT_REF | PT_MOD) == 0)
3602 			continue;
3603 
3604 		save_old = old;
3605 		if ((flags & HAT_SYNC_ZERORM) != 0) {
3606 
3607 			/*
3608 			 * Need to clear ref or mod bits. Need to demap
3609 			 * to make sure any executing TLBs see cleared bits.
3610 			 */
3611 			new = old;
3612 			PTE_CLR(new, PT_REF | PT_MOD);
3613 			old = hati_update_pte(ht, entry, old, new);
3614 			if (old != 0)
3615 				goto try_again;
3616 
3617 			old = save_old;
3618 		}
3619 
3620 		/*
3621 		 * Sync the PTE
3622 		 */
3623 		if (!(flags & HAT_SYNC_ZERORM) &&
3624 		    PTE_GET(old, PT_SOFTWARE) <= PT_NOSYNC)
3625 			hati_sync_pte_to_page(pp, old, ht->ht_level);
3626 
3627 		/*
3628 		 * can stop short if we found a ref'd or mod'd page
3629 		 */
3630 		if ((flags & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp) ||
3631 		    (flags & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp)) {
3632 			x86_hm_exit(pp);
3633 			goto done;
3634 		}
3635 	}
3636 	x86_hm_exit(pp);
3637 	while (pszc < pp->p_szc) {
3638 		page_t *tpp;
3639 		pszc++;
3640 		tpp = PP_GROUPLEADER(pp, pszc);
3641 		if (pp != tpp) {
3642 			pp = tpp;
3643 			goto next_size;
3644 		}
3645 	}
3646 done:
3647 	XPV_ALLOW_MIGRATE();
3648 	return (save_pp->p_nrm & nrmbits);
3649 }
3650 
3651 /*
3652  * returns approx number of mappings to this pp.  A return of 0 implies
3653  * there are no mappings to the page.
3654  */
3655 ulong_t
3656 hat_page_getshare(page_t *pp)
3657 {
3658 	uint_t cnt;
3659 	cnt = hment_mapcnt(pp);
3660 #if defined(__amd64)
3661 	if (vpm_enable && pp->p_vpmref) {
3662 		cnt += 1;
3663 	}
3664 #endif
3665 	return (cnt);
3666 }
3667 
3668 /*
3669  * Return 1 the number of mappings exceeds sh_thresh. Return 0
3670  * otherwise.
3671  */
3672 int
3673 hat_page_checkshare(page_t *pp, ulong_t sh_thresh)
3674 {
3675 	return (hat_page_getshare(pp) > sh_thresh);
3676 }
3677 
3678 /*
3679  * hat_softlock isn't supported anymore
3680  */
3681 /*ARGSUSED*/
3682 faultcode_t
3683 hat_softlock(
3684 	hat_t *hat,
3685 	caddr_t addr,
3686 	size_t *len,
3687 	struct page **page_array,
3688 	uint_t flags)
3689 {
3690 	return (FC_NOSUPPORT);
3691 }
3692 
3693 
3694 
3695 /*
3696  * Routine to expose supported HAT features to platform independent code.
3697  */
3698 /*ARGSUSED*/
3699 int
3700 hat_supported(enum hat_features feature, void *arg)
3701 {
3702 	switch (feature) {
3703 
3704 	case HAT_SHARED_PT:	/* this is really ISM */
3705 		return (1);
3706 
3707 	case HAT_DYNAMIC_ISM_UNMAP:
3708 		return (0);
3709 
3710 	case HAT_VMODSORT:
3711 		return (1);
3712 
3713 	case HAT_SHARED_REGIONS:
3714 		return (0);
3715 
3716 	default:
3717 		panic("hat_supported() - unknown feature");
3718 	}
3719 	return (0);
3720 }
3721 
3722 /*
3723  * Called when a thread is exiting and has been switched to the kernel AS
3724  */
3725 void
3726 hat_thread_exit(kthread_t *thd)
3727 {
3728 	ASSERT(thd->t_procp->p_as == &kas);
3729 	XPV_DISALLOW_MIGRATE();
3730 	hat_switch(thd->t_procp->p_as->a_hat);
3731 	XPV_ALLOW_MIGRATE();
3732 }
3733 
3734 /*
3735  * Setup the given brand new hat structure as the new HAT on this cpu's mmu.
3736  */
3737 /*ARGSUSED*/
3738 void
3739 hat_setup(hat_t *hat, int flags)
3740 {
3741 	XPV_DISALLOW_MIGRATE();
3742 	kpreempt_disable();
3743 
3744 	hat_switch(hat);
3745 
3746 	kpreempt_enable();
3747 	XPV_ALLOW_MIGRATE();
3748 }
3749 
3750 /*
3751  * Prepare for a CPU private mapping for the given address.
3752  *
3753  * The address can only be used from a single CPU and can be remapped
3754  * using hat_mempte_remap().  Return the address of the PTE.
3755  *
3756  * We do the htable_create() if necessary and increment the valid count so
3757  * the htable can't disappear.  We also hat_devload() the page table into
3758  * kernel so that the PTE is quickly accessed.
3759  */
3760 hat_mempte_t
3761 hat_mempte_setup(caddr_t addr)
3762 {
3763 	uintptr_t	va = (uintptr_t)addr;
3764 	htable_t	*ht;
3765 	uint_t		entry;
3766 	x86pte_t	oldpte;
3767 	hat_mempte_t	p;
3768 
3769 	ASSERT(IS_PAGEALIGNED(va));
3770 	ASSERT(!IN_VA_HOLE(va));
3771 	++curthread->t_hatdepth;
3772 	ht = htable_getpte(kas.a_hat, va, &entry, &oldpte, 0);
3773 	if (ht == NULL) {
3774 		ht = htable_create(kas.a_hat, va, 0, NULL);
3775 		entry = htable_va2entry(va, ht);
3776 		ASSERT(ht->ht_level == 0);
3777 		oldpte = x86pte_get(ht, entry);
3778 	}
3779 	if (PTE_ISVALID(oldpte))
3780 		panic("hat_mempte_setup(): address already mapped"
3781 		    "ht=%p, entry=%d, pte=" FMT_PTE, ht, entry, oldpte);
3782 
3783 	/*
3784 	 * increment ht_valid_cnt so that the pagetable can't disappear
3785 	 */
3786 	HTABLE_INC(ht->ht_valid_cnt);
3787 
3788 	/*
3789 	 * return the PTE physical address to the caller.
3790 	 */
3791 	htable_release(ht);
3792 	p = PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry);
3793 	--curthread->t_hatdepth;
3794 	return (p);
3795 }
3796 
3797 /*
3798  * Release a CPU private mapping for the given address.
3799  * We decrement the htable valid count so it might be destroyed.
3800  */
3801 /*ARGSUSED1*/
3802 void
3803 hat_mempte_release(caddr_t addr, hat_mempte_t pte_pa)
3804 {
3805 	htable_t	*ht;
3806 
3807 	/*
3808 	 * invalidate any left over mapping and decrement the htable valid count
3809 	 */
3810 #ifdef __xpv
3811 	if (HYPERVISOR_update_va_mapping((uintptr_t)addr, 0,
3812 	    UVMF_INVLPG | UVMF_LOCAL))
3813 		panic("HYPERVISOR_update_va_mapping() failed");
3814 #else
3815 	{
3816 		x86pte_t *pteptr;
3817 
3818 		pteptr = x86pte_mapin(mmu_btop(pte_pa),
3819 		    (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL);
3820 		if (mmu.pae_hat)
3821 			*pteptr = 0;
3822 		else
3823 			*(x86pte32_t *)pteptr = 0;
3824 		mmu_tlbflush_entry(addr);
3825 		x86pte_mapout();
3826 	}
3827 #endif
3828 
3829 	ht = htable_getpte(kas.a_hat, ALIGN2PAGE(addr), NULL, NULL, 0);
3830 	if (ht == NULL)
3831 		panic("hat_mempte_release(): invalid address");
3832 	ASSERT(ht->ht_level == 0);
3833 	HTABLE_DEC(ht->ht_valid_cnt);
3834 	htable_release(ht);
3835 }
3836 
3837 /*
3838  * Apply a temporary CPU private mapping to a page. We flush the TLB only
3839  * on this CPU, so this ought to have been called with preemption disabled.
3840  */
3841 void
3842 hat_mempte_remap(
3843 	pfn_t		pfn,
3844 	caddr_t		addr,
3845 	hat_mempte_t	pte_pa,
3846 	uint_t		attr,
3847 	uint_t		flags)
3848 {
3849 	uintptr_t	va = (uintptr_t)addr;
3850 	x86pte_t	pte;
3851 
3852 	/*
3853 	 * Remap the given PTE to the new page's PFN. Invalidate only
3854 	 * on this CPU.
3855 	 */
3856 #ifdef DEBUG
3857 	htable_t	*ht;
3858 	uint_t		entry;
3859 
3860 	ASSERT(IS_PAGEALIGNED(va));
3861 	ASSERT(!IN_VA_HOLE(va));
3862 	ht = htable_getpte(kas.a_hat, va, &entry, NULL, 0);
3863 	ASSERT(ht != NULL);
3864 	ASSERT(ht->ht_level == 0);
3865 	ASSERT(ht->ht_valid_cnt > 0);
3866 	ASSERT(ht->ht_pfn == mmu_btop(pte_pa));
3867 	htable_release(ht);
3868 #endif
3869 	XPV_DISALLOW_MIGRATE();
3870 	pte = hati_mkpte(pfn, attr, 0, flags);
3871 #ifdef __xpv
3872 	if (HYPERVISOR_update_va_mapping(va, pte, UVMF_INVLPG | UVMF_LOCAL))
3873 		panic("HYPERVISOR_update_va_mapping() failed");
3874 #else
3875 	{
3876 		x86pte_t *pteptr;
3877 
3878 		pteptr = x86pte_mapin(mmu_btop(pte_pa),
3879 		    (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL);
3880 		if (mmu.pae_hat)
3881 			*(x86pte_t *)pteptr = pte;
3882 		else
3883 			*(x86pte32_t *)pteptr = (x86pte32_t)pte;
3884 		mmu_tlbflush_entry(addr);
3885 		x86pte_mapout();
3886 	}
3887 #endif
3888 	XPV_ALLOW_MIGRATE();
3889 }
3890 
3891 
3892 
3893 /*
3894  * Hat locking functions
3895  * XXX - these two functions are currently being used by hatstats
3896  * 	they can be removed by using a per-as mutex for hatstats.
3897  */
3898 void
3899 hat_enter(hat_t *hat)
3900 {
3901 	mutex_enter(&hat->hat_mutex);
3902 }
3903 
3904 void
3905 hat_exit(hat_t *hat)
3906 {
3907 	mutex_exit(&hat->hat_mutex);
3908 }
3909 
3910 /*
3911  * HAT part of cpu initialization.
3912  */
3913 void
3914 hat_cpu_online(struct cpu *cpup)
3915 {
3916 	if (cpup != CPU) {
3917 		x86pte_cpu_init(cpup);
3918 		hat_vlp_setup(cpup);
3919 	}
3920 	CPUSET_ATOMIC_ADD(khat_cpuset, cpup->cpu_id);
3921 }
3922 
3923 /*
3924  * HAT part of cpu deletion.
3925  * (currently, we only call this after the cpu is safely passivated.)
3926  */
3927 void
3928 hat_cpu_offline(struct cpu *cpup)
3929 {
3930 	ASSERT(cpup != CPU);
3931 
3932 	CPUSET_ATOMIC_DEL(khat_cpuset, cpup->cpu_id);
3933 	x86pte_cpu_fini(cpup);
3934 	hat_vlp_teardown(cpup);
3935 }
3936 
3937 /*
3938  * Function called after all CPUs are brought online.
3939  * Used to remove low address boot mappings.
3940  */
3941 void
3942 clear_boot_mappings(uintptr_t low, uintptr_t high)
3943 {
3944 	uintptr_t vaddr = low;
3945 	htable_t *ht = NULL;
3946 	level_t level;
3947 	uint_t entry;
3948 	x86pte_t pte;
3949 
3950 	/*
3951 	 * On 1st CPU we can unload the prom mappings, basically we blow away
3952 	 * all virtual mappings under _userlimit.
3953 	 */
3954 	while (vaddr < high) {
3955 		pte = htable_walk(kas.a_hat, &ht, &vaddr, high);
3956 		if (ht == NULL)
3957 			break;
3958 
3959 		level = ht->ht_level;
3960 		entry = htable_va2entry(vaddr, ht);
3961 		ASSERT(level <= mmu.max_page_level);
3962 		ASSERT(PTE_ISPAGE(pte, level));
3963 
3964 		/*
3965 		 * Unload the mapping from the page tables.
3966 		 */
3967 		(void) x86pte_inval(ht, entry, 0, NULL);
3968 		ASSERT(ht->ht_valid_cnt > 0);
3969 		HTABLE_DEC(ht->ht_valid_cnt);
3970 		PGCNT_DEC(ht->ht_hat, ht->ht_level);
3971 
3972 		vaddr += LEVEL_SIZE(ht->ht_level);
3973 	}
3974 	if (ht)
3975 		htable_release(ht);
3976 }
3977 
3978 /*
3979  * Atomically update a new translation for a single page.  If the
3980  * currently installed PTE doesn't match the value we expect to find,
3981  * it's not updated and we return the PTE we found.
3982  *
3983  * If activating nosync or NOWRITE and the page was modified we need to sync
3984  * with the page_t. Also sync with page_t if clearing ref/mod bits.
3985  */
3986 static x86pte_t
3987 hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, x86pte_t new)
3988 {
3989 	page_t		*pp;
3990 	uint_t		rm = 0;
3991 	x86pte_t	replaced;
3992 
3993 	if (PTE_GET(expected, PT_SOFTWARE) < PT_NOSYNC &&
3994 	    PTE_GET(expected, PT_MOD | PT_REF) &&
3995 	    (PTE_GET(new, PT_NOSYNC) || !PTE_GET(new, PT_WRITABLE) ||
3996 	    !PTE_GET(new, PT_MOD | PT_REF))) {
3997 
3998 		ASSERT(!pfn_is_foreign(PTE2PFN(expected, ht->ht_level)));
3999 		pp = page_numtopp_nolock(PTE2PFN(expected, ht->ht_level));
4000 		ASSERT(pp != NULL);
4001 		if (PTE_GET(expected, PT_MOD))
4002 			rm |= P_MOD;
4003 		if (PTE_GET(expected, PT_REF))
4004 			rm |= P_REF;
4005 		PTE_CLR(new, PT_MOD | PT_REF);
4006 	}
4007 
4008 	replaced = x86pte_update(ht, entry, expected, new);
4009 	if (replaced != expected)
4010 		return (replaced);
4011 
4012 	if (rm) {
4013 		/*
4014 		 * sync to all constituent pages of a large page
4015 		 */
4016 		pgcnt_t pgcnt = page_get_pagecnt(ht->ht_level);
4017 		ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt));
4018 		while (pgcnt-- > 0) {
4019 			/*
4020 			 * hat_page_demote() can't decrease
4021 			 * pszc below this mapping size
4022 			 * since large mapping existed after we
4023 			 * took mlist lock.
4024 			 */
4025 			ASSERT(pp->p_szc >= ht->ht_level);
4026 			hat_page_setattr(pp, rm);
4027 			++pp;
4028 		}
4029 	}
4030 
4031 	return (0);
4032 }
4033 
4034 /* ARGSUSED */
4035 void
4036 hat_join_srd(struct hat *hat, vnode_t *evp)
4037 {
4038 }
4039 
4040 /* ARGSUSED */
4041 hat_region_cookie_t
4042 hat_join_region(struct hat *hat,
4043     caddr_t r_saddr,
4044     size_t r_size,
4045     void *r_obj,
4046     u_offset_t r_objoff,
4047     uchar_t r_perm,
4048     uchar_t r_pgszc,
4049     hat_rgn_cb_func_t r_cb_function,
4050     uint_t flags)
4051 {
4052 	panic("No shared region support on x86");
4053 	return (HAT_INVALID_REGION_COOKIE);
4054 }
4055 
4056 /* ARGSUSED */
4057 void
4058 hat_leave_region(struct hat *hat, hat_region_cookie_t rcookie, uint_t flags)
4059 {
4060 	panic("No shared region support on x86");
4061 }
4062 
4063 /* ARGSUSED */
4064 void
4065 hat_dup_region(struct hat *hat, hat_region_cookie_t rcookie)
4066 {
4067 	panic("No shared region support on x86");
4068 }
4069 
4070 
4071 /*
4072  * Kernel Physical Mapping (kpm) facility
4073  *
4074  * Most of the routines needed to support segkpm are almost no-ops on the
4075  * x86 platform.  We map in the entire segment when it is created and leave
4076  * it mapped in, so there is no additional work required to set up and tear
4077  * down individual mappings.  All of these routines were created to support
4078  * SPARC platforms that have to avoid aliasing in their virtually indexed
4079  * caches.
4080  *
4081  * Most of the routines have sanity checks in them (e.g. verifying that the
4082  * passed-in page is locked).  We don't actually care about most of these
4083  * checks on x86, but we leave them in place to identify problems in the
4084  * upper levels.
4085  */
4086 
4087 /*
4088  * Map in a locked page and return the vaddr.
4089  */
4090 /*ARGSUSED*/
4091 caddr_t
4092 hat_kpm_mapin(struct page *pp, struct kpme *kpme)
4093 {
4094 	caddr_t		vaddr;
4095 
4096 #ifdef DEBUG
4097 	if (kpm_enable == 0) {
4098 		cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set\n");
4099 		return ((caddr_t)NULL);
4100 	}
4101 
4102 	if (pp == NULL || PAGE_LOCKED(pp) == 0) {
4103 		cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked\n");
4104 		return ((caddr_t)NULL);
4105 	}
4106 #endif
4107 
4108 	vaddr = hat_kpm_page2va(pp, 1);
4109 
4110 	return (vaddr);
4111 }
4112 
4113 /*
4114  * Mapout a locked page.
4115  */
4116 /*ARGSUSED*/
4117 void
4118 hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr)
4119 {
4120 #ifdef DEBUG
4121 	if (kpm_enable == 0) {
4122 		cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set\n");
4123 		return;
4124 	}
4125 
4126 	if (IS_KPM_ADDR(vaddr) == 0) {
4127 		cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address\n");
4128 		return;
4129 	}
4130 
4131 	if (pp == NULL || PAGE_LOCKED(pp) == 0) {
4132 		cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked\n");
4133 		return;
4134 	}
4135 #endif
4136 }
4137 
4138 /*
4139  * Return the kpm virtual address for a specific pfn
4140  */
4141 caddr_t
4142 hat_kpm_pfn2va(pfn_t pfn)
4143 {
4144 	uintptr_t vaddr = (uintptr_t)kpm_vbase + mmu_ptob(pfn);
4145 
4146 	return ((caddr_t)vaddr);
4147 }
4148 
4149 /*
4150  * Return the kpm virtual address for the page at pp.
4151  */
4152 /*ARGSUSED*/
4153 caddr_t
4154 hat_kpm_page2va(struct page *pp, int checkswap)
4155 {
4156 	return (hat_kpm_pfn2va(pp->p_pagenum));
4157 }
4158 
4159 /*
4160  * Return the page frame number for the kpm virtual address vaddr.
4161  */
4162 pfn_t
4163 hat_kpm_va2pfn(caddr_t vaddr)
4164 {
4165 	pfn_t		pfn;
4166 
4167 	ASSERT(IS_KPM_ADDR(vaddr));
4168 
4169 	pfn = (pfn_t)btop(vaddr - kpm_vbase);
4170 
4171 	return (pfn);
4172 }
4173 
4174 
4175 /*
4176  * Return the page for the kpm virtual address vaddr.
4177  */
4178 page_t *
4179 hat_kpm_vaddr2page(caddr_t vaddr)
4180 {
4181 	pfn_t		pfn;
4182 
4183 	ASSERT(IS_KPM_ADDR(vaddr));
4184 
4185 	pfn = hat_kpm_va2pfn(vaddr);
4186 
4187 	return (page_numtopp_nolock(pfn));
4188 }
4189 
4190 /*
4191  * hat_kpm_fault is called from segkpm_fault when we take a page fault on a
4192  * KPM page.  This should never happen on x86
4193  */
4194 int
4195 hat_kpm_fault(hat_t *hat, caddr_t vaddr)
4196 {
4197 	panic("pagefault in seg_kpm.  hat: 0x%p  vaddr: 0x%p", hat, vaddr);
4198 
4199 	return (0);
4200 }
4201 
4202 /*ARGSUSED*/
4203 void
4204 hat_kpm_mseghash_clear(int nentries)
4205 {}
4206 
4207 /*ARGSUSED*/
4208 void
4209 hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp)
4210 {}
4211 
4212 #ifdef __xpv
4213 /*
4214  * There are specific Hypervisor calls to establish and remove mappings
4215  * to grant table references and the privcmd driver. We have to ensure
4216  * that a page table actually exists.
4217  */
4218 void
4219 hat_prepare_mapping(hat_t *hat, caddr_t addr)
4220 {
4221 	ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE));
4222 	(void) htable_create(hat, (uintptr_t)addr, 0, NULL);
4223 }
4224 
4225 void
4226 hat_release_mapping(hat_t *hat, caddr_t addr)
4227 {
4228 	htable_t *ht;
4229 
4230 	ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE));
4231 	ht = htable_lookup(hat, (uintptr_t)addr, 0);
4232 	ASSERT(ht != NULL);
4233 	ASSERT(ht->ht_busy >= 2);
4234 	htable_release(ht);
4235 	htable_release(ht);
4236 }
4237 #endif
4238