xref: /titanic_50/usr/src/uts/i86pc/vm/hat_i86.c (revision ab4a9beb2e4d596be0b3288c7d92919e27781b57)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * VM - Hardware Address Translation management for i386 and amd64
30  *
31  * Implementation of the interfaces described in <common/vm/hat.h>
32  *
33  * Nearly all the details of how the hardware is managed should not be
34  * visible outside this layer except for misc. machine specific functions
35  * that work in conjunction with this code.
36  *
37  * Routines used only inside of i86pc/vm start with hati_ for HAT Internal.
38  */
39 
40 #include <sys/machparam.h>
41 #include <sys/machsystm.h>
42 #include <sys/mman.h>
43 #include <sys/types.h>
44 #include <sys/systm.h>
45 #include <sys/cpuvar.h>
46 #include <sys/thread.h>
47 #include <sys/proc.h>
48 #include <sys/cpu.h>
49 #include <sys/kmem.h>
50 #include <sys/disp.h>
51 #include <sys/shm.h>
52 #include <sys/sysmacros.h>
53 #include <sys/machparam.h>
54 #include <sys/vmem.h>
55 #include <sys/vmsystm.h>
56 #include <sys/promif.h>
57 #include <sys/var.h>
58 #include <sys/x86_archext.h>
59 #include <sys/atomic.h>
60 #include <sys/bitmap.h>
61 #include <sys/controlregs.h>
62 #include <sys/bootconf.h>
63 #include <sys/bootsvcs.h>
64 #include <sys/bootinfo.h>
65 #include <sys/archsystm.h>
66 
67 #include <vm/seg_kmem.h>
68 #include <vm/hat_i86.h>
69 #include <vm/as.h>
70 #include <vm/seg.h>
71 #include <vm/page.h>
72 #include <vm/seg_kp.h>
73 #include <vm/seg_kpm.h>
74 #include <vm/vm_dep.h>
75 #ifdef __xpv
76 #include <sys/hypervisor.h>
77 #endif
78 #include <vm/kboot_mmu.h>
79 #include <vm/seg_spt.h>
80 
81 #include <sys/cmn_err.h>
82 
83 /*
84  * Basic parameters for hat operation.
85  */
86 struct hat_mmu_info mmu;
87 
88 /*
89  * The page that is the kernel's top level pagetable.
90  *
91  * For 32 bit PAE support on i86pc, the kernel hat will use the 1st 4 entries
92  * on this 4K page for its top level page table. The remaining groups of
93  * 4 entries are used for per processor copies of user VLP pagetables for
94  * running threads.  See hat_switch() and reload_pae32() for details.
95  *
96  * vlp_page[0..3] - level==2 PTEs for kernel HAT
97  * vlp_page[4..7] - level==2 PTEs for user thread on cpu 0
98  * vlp_page[8..11]  - level==2 PTE for user thread on cpu 1
99  * etc...
100  */
101 static x86pte_t *vlp_page;
102 
103 /*
104  * forward declaration of internal utility routines
105  */
106 static x86pte_t hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected,
107 	x86pte_t new);
108 
109 /*
110  * The kernel address space exists in all HATs. To implement this the
111  * kernel reserves a fixed number of entries in the topmost level(s) of page
112  * tables. The values are setup during startup and then copied to every user
113  * hat created by hat_alloc(). This means that kernelbase must be:
114  *
115  *	  4Meg aligned for 32 bit kernels
116  *	512Gig aligned for x86_64 64 bit kernel
117  *
118  * The hat_kernel_range_ts describe what needs to be copied from kernel hat
119  * to each user hat.
120  */
121 typedef struct hat_kernel_range {
122 	level_t		hkr_level;
123 	uintptr_t	hkr_start_va;
124 	uintptr_t	hkr_end_va;	/* zero means to end of memory */
125 } hat_kernel_range_t;
126 #define	NUM_KERNEL_RANGE 2
127 static hat_kernel_range_t kernel_ranges[NUM_KERNEL_RANGE];
128 static int num_kernel_ranges;
129 
130 uint_t use_boot_reserve = 1;	/* cleared after early boot process */
131 uint_t can_steal_post_boot = 0;	/* set late in boot to enable stealing */
132 
133 /*
134  * A cpuset for all cpus. This is used for kernel address cross calls, since
135  * the kernel addresses apply to all cpus.
136  */
137 cpuset_t khat_cpuset;
138 
139 /*
140  * management stuff for hat structures
141  */
142 kmutex_t	hat_list_lock;
143 kcondvar_t	hat_list_cv;
144 kmem_cache_t	*hat_cache;
145 kmem_cache_t	*hat_hash_cache;
146 kmem_cache_t	*vlp_hash_cache;
147 
148 /*
149  * Simple statistics
150  */
151 struct hatstats hatstat;
152 
153 /*
154  * Some earlier hypervisor versions do not emulate cmpxchg of PTEs
155  * correctly.  For such hypervisors we must set PT_USER for kernel
156  * entries ourselves (normally the emulation would set PT_USER for
157  * kernel entries and PT_USER|PT_GLOBAL for user entries).  pt_kern is
158  * thus set appropriately.  Note that dboot/kbm is OK, as only the full
159  * HAT uses cmpxchg() and the other paths (hypercall etc.) were never
160  * incorrect.
161  */
162 int pt_kern;
163 
164 /*
165  * useful stuff for atomic access/clearing/setting REF/MOD/RO bits in page_t's.
166  */
167 extern void atomic_orb(uchar_t *addr, uchar_t val);
168 extern void atomic_andb(uchar_t *addr, uchar_t val);
169 
170 #define	PP_GETRM(pp, rmmask)    (pp->p_nrm & rmmask)
171 #define	PP_ISMOD(pp)		PP_GETRM(pp, P_MOD)
172 #define	PP_ISREF(pp)		PP_GETRM(pp, P_REF)
173 #define	PP_ISRO(pp)		PP_GETRM(pp, P_RO)
174 
175 #define	PP_SETRM(pp, rm)	atomic_orb(&(pp->p_nrm), rm)
176 #define	PP_SETMOD(pp)		PP_SETRM(pp, P_MOD)
177 #define	PP_SETREF(pp)		PP_SETRM(pp, P_REF)
178 #define	PP_SETRO(pp)		PP_SETRM(pp, P_RO)
179 
180 #define	PP_CLRRM(pp, rm)	atomic_andb(&(pp->p_nrm), ~(rm))
181 #define	PP_CLRMOD(pp)   	PP_CLRRM(pp, P_MOD)
182 #define	PP_CLRREF(pp)   	PP_CLRRM(pp, P_REF)
183 #define	PP_CLRRO(pp)    	PP_CLRRM(pp, P_RO)
184 #define	PP_CLRALL(pp)		PP_CLRRM(pp, P_MOD | P_REF | P_RO)
185 
186 /*
187  * kmem cache constructor for struct hat
188  */
189 /*ARGSUSED*/
190 static int
191 hati_constructor(void *buf, void *handle, int kmflags)
192 {
193 	hat_t	*hat = buf;
194 
195 	mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
196 	bzero(hat->hat_pages_mapped,
197 	    sizeof (pgcnt_t) * (mmu.max_page_level + 1));
198 	hat->hat_ism_pgcnt = 0;
199 	hat->hat_stats = 0;
200 	hat->hat_flags = 0;
201 	CPUSET_ZERO(hat->hat_cpus);
202 	hat->hat_htable = NULL;
203 	hat->hat_ht_hash = NULL;
204 	return (0);
205 }
206 
207 /*
208  * Allocate a hat structure for as. We also create the top level
209  * htable and initialize it to contain the kernel hat entries.
210  */
211 hat_t *
212 hat_alloc(struct as *as)
213 {
214 	hat_t			*hat;
215 	htable_t		*ht;	/* top level htable */
216 	uint_t			use_vlp;
217 	uint_t			r;
218 	hat_kernel_range_t	*rp;
219 	uintptr_t		va;
220 	uintptr_t		eva;
221 	uint_t			start;
222 	uint_t			cnt;
223 	htable_t		*src;
224 
225 	/*
226 	 * Once we start creating user process HATs we can enable
227 	 * the htable_steal() code.
228 	 */
229 	if (can_steal_post_boot == 0)
230 		can_steal_post_boot = 1;
231 
232 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
233 	hat = kmem_cache_alloc(hat_cache, KM_SLEEP);
234 	hat->hat_as = as;
235 	mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
236 	ASSERT(hat->hat_flags == 0);
237 
238 #if defined(__xpv)
239 	/*
240 	 * No VLP stuff on the hypervisor due to the 64-bit split top level
241 	 * page tables.  On 32-bit it's not needed as the hypervisor takes
242 	 * care of copying the top level PTEs to a below 4Gig page.
243 	 */
244 	use_vlp = 0;
245 #else	/* __xpv */
246 	/* 32 bit processes uses a VLP style hat when running with PAE */
247 #if defined(__amd64)
248 	use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32);
249 #elif defined(__i386)
250 	use_vlp = mmu.pae_hat;
251 #endif
252 #endif	/* __xpv */
253 	if (use_vlp) {
254 		hat->hat_flags = HAT_VLP;
255 		bzero(hat->hat_vlp_ptes, VLP_SIZE);
256 	}
257 
258 	/*
259 	 * Allocate the htable hash
260 	 */
261 	if ((hat->hat_flags & HAT_VLP)) {
262 		hat->hat_num_hash = mmu.vlp_hash_cnt;
263 		hat->hat_ht_hash = kmem_cache_alloc(vlp_hash_cache, KM_SLEEP);
264 	} else {
265 		hat->hat_num_hash = mmu.hash_cnt;
266 		hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_SLEEP);
267 	}
268 	bzero(hat->hat_ht_hash, hat->hat_num_hash * sizeof (htable_t *));
269 
270 	/*
271 	 * Initialize Kernel HAT entries at the top of the top level page
272 	 * tables for the new hat.
273 	 */
274 	hat->hat_htable = NULL;
275 	hat->hat_ht_cached = NULL;
276 	XPV_DISALLOW_MIGRATE();
277 	ht = htable_create(hat, (uintptr_t)0, TOP_LEVEL(hat), NULL);
278 	hat->hat_htable = ht;
279 
280 #if defined(__amd64)
281 	if (hat->hat_flags & HAT_VLP)
282 		goto init_done;
283 #endif
284 
285 	for (r = 0; r < num_kernel_ranges; ++r) {
286 		rp = &kernel_ranges[r];
287 		for (va = rp->hkr_start_va; va != rp->hkr_end_va;
288 		    va += cnt * LEVEL_SIZE(rp->hkr_level)) {
289 
290 			if (rp->hkr_level == TOP_LEVEL(hat))
291 				ht = hat->hat_htable;
292 			else
293 				ht = htable_create(hat, va, rp->hkr_level,
294 				    NULL);
295 
296 			start = htable_va2entry(va, ht);
297 			cnt = HTABLE_NUM_PTES(ht) - start;
298 			eva = va +
299 			    ((uintptr_t)cnt << LEVEL_SHIFT(rp->hkr_level));
300 			if (rp->hkr_end_va != 0 &&
301 			    (eva > rp->hkr_end_va || eva == 0))
302 				cnt = htable_va2entry(rp->hkr_end_va, ht) -
303 				    start;
304 
305 #if defined(__i386) && !defined(__xpv)
306 			if (ht->ht_flags & HTABLE_VLP) {
307 				bcopy(&vlp_page[start],
308 				    &hat->hat_vlp_ptes[start],
309 				    cnt * sizeof (x86pte_t));
310 				continue;
311 			}
312 #endif
313 			src = htable_lookup(kas.a_hat, va, rp->hkr_level);
314 			ASSERT(src != NULL);
315 			x86pte_copy(src, ht, start, cnt);
316 			htable_release(src);
317 		}
318 	}
319 
320 init_done:
321 	XPV_ALLOW_MIGRATE();
322 
323 #if defined(__xpv)
324 	/*
325 	 * Pin top level page tables after initializing them
326 	 */
327 	xen_pin(hat->hat_htable->ht_pfn, mmu.max_level);
328 #if defined(__amd64)
329 	xen_pin(hat->hat_user_ptable, mmu.max_level);
330 #endif
331 #endif
332 
333 	/*
334 	 * Put it at the start of the global list of all hats (used by stealing)
335 	 *
336 	 * kas.a_hat is not in the list but is instead used to find the
337 	 * first and last items in the list.
338 	 *
339 	 * - kas.a_hat->hat_next points to the start of the user hats.
340 	 *   The list ends where hat->hat_next == NULL
341 	 *
342 	 * - kas.a_hat->hat_prev points to the last of the user hats.
343 	 *   The list begins where hat->hat_prev == NULL
344 	 */
345 	mutex_enter(&hat_list_lock);
346 	hat->hat_prev = NULL;
347 	hat->hat_next = kas.a_hat->hat_next;
348 	if (hat->hat_next)
349 		hat->hat_next->hat_prev = hat;
350 	else
351 		kas.a_hat->hat_prev = hat;
352 	kas.a_hat->hat_next = hat;
353 	mutex_exit(&hat_list_lock);
354 
355 	return (hat);
356 }
357 
358 /*
359  * process has finished executing but as has not been cleaned up yet.
360  */
361 /*ARGSUSED*/
362 void
363 hat_free_start(hat_t *hat)
364 {
365 	ASSERT(AS_WRITE_HELD(hat->hat_as, &hat->hat_as->a_lock));
366 
367 	/*
368 	 * If the hat is currently a stealing victim, wait for the stealing
369 	 * to finish.  Once we mark it as HAT_FREEING, htable_steal()
370 	 * won't look at its pagetables anymore.
371 	 */
372 	mutex_enter(&hat_list_lock);
373 	while (hat->hat_flags & HAT_VICTIM)
374 		cv_wait(&hat_list_cv, &hat_list_lock);
375 	hat->hat_flags |= HAT_FREEING;
376 	mutex_exit(&hat_list_lock);
377 }
378 
379 /*
380  * An address space is being destroyed, so we destroy the associated hat.
381  */
382 void
383 hat_free_end(hat_t *hat)
384 {
385 	kmem_cache_t *cache;
386 
387 	ASSERT(hat->hat_flags & HAT_FREEING);
388 
389 	/*
390 	 * must not be running on the given hat
391 	 */
392 	ASSERT(CPU->cpu_current_hat != hat);
393 
394 	/*
395 	 * Remove it from the list of HATs
396 	 */
397 	mutex_enter(&hat_list_lock);
398 	if (hat->hat_prev)
399 		hat->hat_prev->hat_next = hat->hat_next;
400 	else
401 		kas.a_hat->hat_next = hat->hat_next;
402 	if (hat->hat_next)
403 		hat->hat_next->hat_prev = hat->hat_prev;
404 	else
405 		kas.a_hat->hat_prev = hat->hat_prev;
406 	mutex_exit(&hat_list_lock);
407 	hat->hat_next = hat->hat_prev = NULL;
408 
409 #if defined(__xpv)
410 	/*
411 	 * On the hypervisor, unpin top level page table(s)
412 	 */
413 	xen_unpin(hat->hat_htable->ht_pfn);
414 #if defined(__amd64)
415 	xen_unpin(hat->hat_user_ptable);
416 #endif
417 #endif
418 
419 	/*
420 	 * Make a pass through the htables freeing them all up.
421 	 */
422 	htable_purge_hat(hat);
423 
424 	/*
425 	 * Decide which kmem cache the hash table came from, then free it.
426 	 */
427 	if (hat->hat_flags & HAT_VLP)
428 		cache = vlp_hash_cache;
429 	else
430 		cache = hat_hash_cache;
431 	kmem_cache_free(cache, hat->hat_ht_hash);
432 	hat->hat_ht_hash = NULL;
433 
434 	hat->hat_flags = 0;
435 	kmem_cache_free(hat_cache, hat);
436 }
437 
438 /*
439  * round kernelbase down to a supported value to use for _userlimit
440  *
441  * userlimit must be aligned down to an entry in the top level htable.
442  * The one exception is for 32 bit HAT's running PAE.
443  */
444 uintptr_t
445 hat_kernelbase(uintptr_t va)
446 {
447 #if defined(__i386)
448 	va &= LEVEL_MASK(1);
449 #endif
450 	if (IN_VA_HOLE(va))
451 		panic("_userlimit %p will fall in VA hole\n", (void *)va);
452 	return (va);
453 }
454 
455 /*
456  * Initialize hat data structures based on processor MMU information.
457  */
458 void
459 mmu_init(void)
460 {
461 	uint_t max_htables;
462 	uint_t pa_bits;
463 	uint_t va_bits;
464 	int i;
465 
466 	/*
467 	 * If CPU enabled the page table global bit, use it for the kernel
468 	 * This is bit 7 in CR4 (PGE - Page Global Enable).
469 	 */
470 	if ((x86_feature & X86_PGE) != 0 && (getcr4() & CR4_PGE) != 0)
471 		mmu.pt_global = PT_GLOBAL;
472 
473 	/*
474 	 * Detect NX and PAE usage.
475 	 */
476 	mmu.pae_hat = kbm_pae_support;
477 	if (kbm_nx_support)
478 		mmu.pt_nx = PT_NX;
479 	else
480 		mmu.pt_nx = 0;
481 
482 	/*
483 	 * Use CPU info to set various MMU parameters
484 	 */
485 	cpuid_get_addrsize(CPU, &pa_bits, &va_bits);
486 
487 	if (va_bits < sizeof (void *) * NBBY) {
488 		mmu.hole_start = (1ul << (va_bits - 1));
489 		mmu.hole_end = 0ul - mmu.hole_start - 1;
490 	} else {
491 		mmu.hole_end = 0;
492 		mmu.hole_start = mmu.hole_end - 1;
493 	}
494 #if defined(OPTERON_ERRATUM_121)
495 	/*
496 	 * If erratum 121 has already been detected at this time, hole_start
497 	 * contains the value to be subtracted from mmu.hole_start.
498 	 */
499 	ASSERT(hole_start == 0 || opteron_erratum_121 != 0);
500 	hole_start = mmu.hole_start - hole_start;
501 #else
502 	hole_start = mmu.hole_start;
503 #endif
504 	hole_end = mmu.hole_end;
505 
506 	mmu.highest_pfn = mmu_btop((1ull << pa_bits) - 1);
507 	if (mmu.pae_hat == 0 && pa_bits > 32)
508 		mmu.highest_pfn = PFN_4G - 1;
509 
510 	if (mmu.pae_hat) {
511 		mmu.pte_size = 8;	/* 8 byte PTEs */
512 		mmu.pte_size_shift = 3;
513 	} else {
514 		mmu.pte_size = 4;	/* 4 byte PTEs */
515 		mmu.pte_size_shift = 2;
516 	}
517 
518 	if (mmu.pae_hat && (x86_feature & X86_PAE) == 0)
519 		panic("Processor does not support PAE");
520 
521 	if ((x86_feature & X86_CX8) == 0)
522 		panic("Processor does not support cmpxchg8b instruction");
523 
524 	/*
525 	 * Initialize parameters based on the 64 or 32 bit kernels and
526 	 * for the 32 bit kernel decide if we should use PAE.
527 	 */
528 	if (kbm_largepage_support)
529 		mmu.max_page_level = 1;
530 	else
531 		mmu.max_page_level = 0;
532 	mmu_page_sizes = mmu.max_page_level + 1;
533 	mmu_exported_page_sizes = mmu_page_sizes;
534 
535 #if defined(__amd64)
536 
537 	mmu.num_level = 4;
538 	mmu.max_level = 3;
539 	mmu.ptes_per_table = 512;
540 	mmu.top_level_count = 512;
541 
542 	mmu.level_shift[0] = 12;
543 	mmu.level_shift[1] = 21;
544 	mmu.level_shift[2] = 30;
545 	mmu.level_shift[3] = 39;
546 
547 #elif defined(__i386)
548 
549 	if (mmu.pae_hat) {
550 		mmu.num_level = 3;
551 		mmu.max_level = 2;
552 		mmu.ptes_per_table = 512;
553 		mmu.top_level_count = 4;
554 
555 		mmu.level_shift[0] = 12;
556 		mmu.level_shift[1] = 21;
557 		mmu.level_shift[2] = 30;
558 
559 	} else {
560 		mmu.num_level = 2;
561 		mmu.max_level = 1;
562 		mmu.ptes_per_table = 1024;
563 		mmu.top_level_count = 1024;
564 
565 		mmu.level_shift[0] = 12;
566 		mmu.level_shift[1] = 22;
567 	}
568 
569 #endif	/* __i386 */
570 
571 	for (i = 0; i < mmu.num_level; ++i) {
572 		mmu.level_size[i] = 1UL << mmu.level_shift[i];
573 		mmu.level_offset[i] = mmu.level_size[i] - 1;
574 		mmu.level_mask[i] = ~mmu.level_offset[i];
575 	}
576 
577 	for (i = 0; i <= mmu.max_page_level; ++i) {
578 		mmu.pte_bits[i] = PT_VALID | pt_kern;
579 		if (i > 0)
580 			mmu.pte_bits[i] |= PT_PAGESIZE;
581 	}
582 
583 	/*
584 	 * NOTE Legacy 32 bit PAE mode only has the P_VALID bit at top level.
585 	 */
586 	for (i = 1; i < mmu.num_level; ++i)
587 		mmu.ptp_bits[i] = PT_PTPBITS;
588 
589 #if defined(__i386)
590 	mmu.ptp_bits[2] = PT_VALID;
591 #endif
592 
593 	/*
594 	 * Compute how many hash table entries to have per process for htables.
595 	 * We start with 1 page's worth of entries.
596 	 *
597 	 * If physical memory is small, reduce the amount need to cover it.
598 	 */
599 	max_htables = physmax / mmu.ptes_per_table;
600 	mmu.hash_cnt = MMU_PAGESIZE / sizeof (htable_t *);
601 	while (mmu.hash_cnt > 16 && mmu.hash_cnt >= max_htables)
602 		mmu.hash_cnt >>= 1;
603 	mmu.vlp_hash_cnt = mmu.hash_cnt;
604 
605 #if defined(__amd64)
606 	/*
607 	 * If running in 64 bits and physical memory is large,
608 	 * increase the size of the cache to cover all of memory for
609 	 * a 64 bit process.
610 	 */
611 #define	HASH_MAX_LENGTH 4
612 	while (mmu.hash_cnt * HASH_MAX_LENGTH < max_htables)
613 		mmu.hash_cnt <<= 1;
614 #endif
615 }
616 
617 
618 /*
619  * initialize hat data structures
620  */
621 void
622 hat_init()
623 {
624 #if defined(__i386)
625 	/*
626 	 * _userlimit must be aligned correctly
627 	 */
628 	if ((_userlimit & LEVEL_MASK(1)) != _userlimit) {
629 		prom_printf("hat_init(): _userlimit=%p, not aligned at %p\n",
630 		    (void *)_userlimit, (void *)LEVEL_SIZE(1));
631 		halt("hat_init(): Unable to continue");
632 	}
633 #endif
634 
635 	cv_init(&hat_list_cv, NULL, CV_DEFAULT, NULL);
636 
637 	/*
638 	 * initialize kmem caches
639 	 */
640 	htable_init();
641 	hment_init();
642 
643 	hat_cache = kmem_cache_create("hat_t",
644 	    sizeof (hat_t), 0, hati_constructor, NULL, NULL,
645 	    NULL, 0, 0);
646 
647 	hat_hash_cache = kmem_cache_create("HatHash",
648 	    mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
649 	    NULL, 0, 0);
650 
651 	/*
652 	 * VLP hats can use a smaller hash table size on large memroy machines
653 	 */
654 	if (mmu.hash_cnt == mmu.vlp_hash_cnt) {
655 		vlp_hash_cache = hat_hash_cache;
656 	} else {
657 		vlp_hash_cache = kmem_cache_create("HatVlpHash",
658 		    mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
659 		    NULL, 0, 0);
660 	}
661 
662 	/*
663 	 * Set up the kernel's hat
664 	 */
665 	AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER);
666 	kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP);
667 	mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
668 	kas.a_hat->hat_as = &kas;
669 	kas.a_hat->hat_flags = 0;
670 	AS_LOCK_EXIT(&kas, &kas.a_lock);
671 
672 	CPUSET_ZERO(khat_cpuset);
673 	CPUSET_ADD(khat_cpuset, CPU->cpu_id);
674 
675 	/*
676 	 * The kernel hat's next pointer serves as the head of the hat list .
677 	 * The kernel hat's prev pointer tracks the last hat on the list for
678 	 * htable_steal() to use.
679 	 */
680 	kas.a_hat->hat_next = NULL;
681 	kas.a_hat->hat_prev = NULL;
682 
683 	/*
684 	 * Allocate an htable hash bucket for the kernel
685 	 * XX64 - tune for 64 bit procs
686 	 */
687 	kas.a_hat->hat_num_hash = mmu.hash_cnt;
688 	kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP);
689 	bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *));
690 
691 	/*
692 	 * zero out the top level and cached htable pointers
693 	 */
694 	kas.a_hat->hat_ht_cached = NULL;
695 	kas.a_hat->hat_htable = NULL;
696 
697 	/*
698 	 * Pre-allocate hrm_hashtab before enabling the collection of
699 	 * refmod statistics.  Allocating on the fly would mean us
700 	 * running the risk of suffering recursive mutex enters or
701 	 * deadlocks.
702 	 */
703 	hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *),
704 	    KM_SLEEP);
705 }
706 
707 /*
708  * Prepare CPU specific pagetables for VLP processes on 64 bit kernels.
709  *
710  * Each CPU has a set of 2 pagetables that are reused for any 32 bit
711  * process it runs. They are the top level pagetable, hci_vlp_l3ptes, and
712  * the next to top level table for the bottom 512 Gig, hci_vlp_l2ptes.
713  */
714 /*ARGSUSED*/
715 static void
716 hat_vlp_setup(struct cpu *cpu)
717 {
718 #if defined(__amd64) && !defined(__xpv)
719 	struct hat_cpu_info *hci = cpu->cpu_hat_info;
720 	pfn_t pfn;
721 
722 	/*
723 	 * allocate the level==2 page table for the bottom most
724 	 * 512Gig of address space (this is where 32 bit apps live)
725 	 */
726 	ASSERT(hci != NULL);
727 	hci->hci_vlp_l2ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
728 
729 	/*
730 	 * Allocate a top level pagetable and copy the kernel's
731 	 * entries into it. Then link in hci_vlp_l2ptes in the 1st entry.
732 	 */
733 	hci->hci_vlp_l3ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
734 	hci->hci_vlp_pfn =
735 	    hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l3ptes);
736 	ASSERT(hci->hci_vlp_pfn != PFN_INVALID);
737 	bcopy(vlp_page, hci->hci_vlp_l3ptes, MMU_PAGESIZE);
738 
739 	pfn = hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l2ptes);
740 	ASSERT(pfn != PFN_INVALID);
741 	hci->hci_vlp_l3ptes[0] = MAKEPTP(pfn, 2);
742 #endif /* __amd64 && !__xpv */
743 }
744 
745 /*ARGSUSED*/
746 static void
747 hat_vlp_teardown(cpu_t *cpu)
748 {
749 #if defined(__amd64) && !defined(__xpv)
750 	struct hat_cpu_info *hci;
751 
752 	if ((hci = cpu->cpu_hat_info) == NULL)
753 		return;
754 	if (hci->hci_vlp_l2ptes)
755 		kmem_free(hci->hci_vlp_l2ptes, MMU_PAGESIZE);
756 	if (hci->hci_vlp_l3ptes)
757 		kmem_free(hci->hci_vlp_l3ptes, MMU_PAGESIZE);
758 #endif
759 }
760 
761 #define	NEXT_HKR(r, l, s, e) {			\
762 	kernel_ranges[r].hkr_level = l;		\
763 	kernel_ranges[r].hkr_start_va = s;	\
764 	kernel_ranges[r].hkr_end_va = e;	\
765 	++r;					\
766 }
767 
768 /*
769  * Finish filling in the kernel hat.
770  * Pre fill in all top level kernel page table entries for the kernel's
771  * part of the address range.  From this point on we can't use any new
772  * kernel large pages if they need PTE's at max_level
773  *
774  * create the kmap mappings.
775  */
776 void
777 hat_init_finish(void)
778 {
779 	size_t		size;
780 	uint_t		r = 0;
781 	uintptr_t	va;
782 	hat_kernel_range_t *rp;
783 
784 
785 	/*
786 	 * We are now effectively running on the kernel hat.
787 	 * Clearing use_boot_reserve shuts off using the pre-allocated boot
788 	 * reserve for all HAT allocations.  From here on, the reserves are
789 	 * only used when avoiding recursion in kmem_alloc().
790 	 */
791 	use_boot_reserve = 0;
792 	htable_adjust_reserve();
793 
794 	/*
795 	 * User HATs are initialized with copies of all kernel mappings in
796 	 * higher level page tables. Ensure that those entries exist.
797 	 */
798 #if defined(__amd64)
799 
800 	NEXT_HKR(r, 3, kernelbase, 0);
801 #if defined(__xpv)
802 	NEXT_HKR(r, 3, HYPERVISOR_VIRT_START, HYPERVISOR_VIRT_END);
803 #endif
804 
805 #elif defined(__i386)
806 
807 #if !defined(__xpv)
808 	if (mmu.pae_hat) {
809 		va = kernelbase;
810 		if ((va & LEVEL_MASK(2)) != va) {
811 			va = P2ROUNDUP(va, LEVEL_SIZE(2));
812 			NEXT_HKR(r, 1, kernelbase, va);
813 		}
814 		if (va != 0)
815 			NEXT_HKR(r, 2, va, 0);
816 	} else
817 #endif /* __xpv */
818 		NEXT_HKR(r, 1, kernelbase, 0);
819 
820 #endif /* __i386 */
821 
822 	num_kernel_ranges = r;
823 
824 	/*
825 	 * Create all the kernel pagetables that will have entries
826 	 * shared to user HATs.
827 	 */
828 	for (r = 0; r < num_kernel_ranges; ++r) {
829 		rp = &kernel_ranges[r];
830 		for (va = rp->hkr_start_va; va != rp->hkr_end_va;
831 		    va += LEVEL_SIZE(rp->hkr_level)) {
832 			htable_t *ht;
833 
834 			if (IN_HYPERVISOR_VA(va))
835 				continue;
836 
837 			/* can/must skip if a page mapping already exists */
838 			if (rp->hkr_level <= mmu.max_page_level &&
839 			    (ht = htable_getpage(kas.a_hat, va, NULL)) !=
840 			    NULL) {
841 				htable_release(ht);
842 				continue;
843 			}
844 
845 			(void) htable_create(kas.a_hat, va, rp->hkr_level - 1,
846 			    NULL);
847 		}
848 	}
849 
850 	/*
851 	 * 32 bit PAE metal kernels use only 4 of the 512 entries in the
852 	 * page holding the top level pagetable. We use the remainder for
853 	 * the "per CPU" page tables for VLP processes.
854 	 * Map the top level kernel pagetable into the kernel to make
855 	 * it easy to use bcopy access these tables.
856 	 */
857 	if (mmu.pae_hat) {
858 		vlp_page = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP);
859 		hat_devload(kas.a_hat, (caddr_t)vlp_page, MMU_PAGESIZE,
860 		    kas.a_hat->hat_htable->ht_pfn,
861 #if !defined(__xpv)
862 		    PROT_WRITE |
863 #endif
864 		    PROT_READ | HAT_NOSYNC | HAT_UNORDERED_OK,
865 		    HAT_LOAD | HAT_LOAD_NOCONSIST);
866 	}
867 	hat_vlp_setup(CPU);
868 
869 	/*
870 	 * Create kmap (cached mappings of kernel PTEs)
871 	 * for 32 bit we map from segmap_start .. ekernelheap
872 	 * for 64 bit we map from segmap_start .. segmap_start + segmapsize;
873 	 */
874 #if defined(__i386)
875 	size = (uintptr_t)ekernelheap - segmap_start;
876 #elif defined(__amd64)
877 	size = segmapsize;
878 #endif
879 	hat_kmap_init((uintptr_t)segmap_start, size);
880 }
881 
882 /*
883  * On 32 bit PAE mode, PTE's are 64 bits, but ordinary atomic memory references
884  * are 32 bit, so for safety we must use cas64() to install these.
885  */
886 #ifdef __i386
887 static void
888 reload_pae32(hat_t *hat, cpu_t *cpu)
889 {
890 	x86pte_t *src;
891 	x86pte_t *dest;
892 	x86pte_t pte;
893 	int i;
894 
895 	/*
896 	 * Load the 4 entries of the level 2 page table into this
897 	 * cpu's range of the vlp_page and point cr3 at them.
898 	 */
899 	ASSERT(mmu.pae_hat);
900 	src = hat->hat_vlp_ptes;
901 	dest = vlp_page + (cpu->cpu_id + 1) * VLP_NUM_PTES;
902 	for (i = 0; i < VLP_NUM_PTES; ++i) {
903 		for (;;) {
904 			pte = dest[i];
905 			if (pte == src[i])
906 				break;
907 			if (cas64(dest + i, pte, src[i]) != src[i])
908 				break;
909 		}
910 	}
911 }
912 #endif
913 
914 /*
915  * Switch to a new active hat, maintaining bit masks to track active CPUs.
916  *
917  * On the 32-bit PAE hypervisor, %cr3 is a 64-bit value, on metal it
918  * remains a 32-bit value.
919  */
920 void
921 hat_switch(hat_t *hat)
922 {
923 	uint64_t	newcr3;
924 	cpu_t		*cpu = CPU;
925 	hat_t		*old = cpu->cpu_current_hat;
926 
927 	/*
928 	 * set up this information first, so we don't miss any cross calls
929 	 */
930 	if (old != NULL) {
931 		if (old == hat)
932 			return;
933 		if (old != kas.a_hat)
934 			CPUSET_ATOMIC_DEL(old->hat_cpus, cpu->cpu_id);
935 	}
936 
937 	/*
938 	 * Add this CPU to the active set for this HAT.
939 	 */
940 	if (hat != kas.a_hat) {
941 		CPUSET_ATOMIC_ADD(hat->hat_cpus, cpu->cpu_id);
942 	}
943 	cpu->cpu_current_hat = hat;
944 
945 	/*
946 	 * now go ahead and load cr3
947 	 */
948 	if (hat->hat_flags & HAT_VLP) {
949 #if defined(__amd64)
950 		x86pte_t *vlpptep = cpu->cpu_hat_info->hci_vlp_l2ptes;
951 
952 		VLP_COPY(hat->hat_vlp_ptes, vlpptep);
953 		newcr3 = MAKECR3(cpu->cpu_hat_info->hci_vlp_pfn);
954 #elif defined(__i386)
955 		reload_pae32(hat, cpu);
956 		newcr3 = MAKECR3(kas.a_hat->hat_htable->ht_pfn) +
957 		    (cpu->cpu_id + 1) * VLP_SIZE;
958 #endif
959 	} else {
960 		newcr3 = MAKECR3((uint64_t)hat->hat_htable->ht_pfn);
961 	}
962 #ifdef __xpv
963 	{
964 		struct mmuext_op t[2];
965 		uint_t retcnt;
966 		uint_t opcnt = 1;
967 
968 		t[0].cmd = MMUEXT_NEW_BASEPTR;
969 		t[0].arg1.mfn = mmu_btop(pa_to_ma(newcr3));
970 #if defined(__amd64)
971 		/*
972 		 * There's an interesting problem here, as to what to
973 		 * actually specify when switching to the kernel hat.
974 		 * For now we'll reuse the kernel hat again.
975 		 */
976 		t[1].cmd = MMUEXT_NEW_USER_BASEPTR;
977 		if (hat == kas.a_hat)
978 			t[1].arg1.mfn = mmu_btop(pa_to_ma(newcr3));
979 		else
980 			t[1].arg1.mfn = pfn_to_mfn(hat->hat_user_ptable);
981 		++opcnt;
982 #endif	/* __amd64 */
983 		if (HYPERVISOR_mmuext_op(t, opcnt, &retcnt, DOMID_SELF) < 0)
984 			panic("HYPERVISOR_mmu_update() failed");
985 		ASSERT(retcnt == opcnt);
986 
987 	}
988 #else
989 	setcr3(newcr3);
990 #endif
991 	ASSERT(cpu == CPU);
992 }
993 
994 /*
995  * Utility to return a valid x86pte_t from protections, pfn, and level number
996  */
997 static x86pte_t
998 hati_mkpte(pfn_t pfn, uint_t attr, level_t level, uint_t flags)
999 {
1000 	x86pte_t	pte;
1001 	uint_t		cache_attr = attr & HAT_ORDER_MASK;
1002 
1003 	pte = MAKEPTE(pfn, level);
1004 
1005 	if (attr & PROT_WRITE)
1006 		PTE_SET(pte, PT_WRITABLE);
1007 
1008 	if (attr & PROT_USER)
1009 		PTE_SET(pte, PT_USER);
1010 
1011 	if (!(attr & PROT_EXEC))
1012 		PTE_SET(pte, mmu.pt_nx);
1013 
1014 	/*
1015 	 * Set the software bits used track ref/mod sync's and hments.
1016 	 * If not using REF/MOD, set them to avoid h/w rewriting PTEs.
1017 	 */
1018 	if (flags & HAT_LOAD_NOCONSIST)
1019 		PTE_SET(pte, PT_NOCONSIST | PT_REF | PT_MOD);
1020 	else if (attr & HAT_NOSYNC)
1021 		PTE_SET(pte, PT_NOSYNC | PT_REF | PT_MOD);
1022 
1023 	/*
1024 	 * Set the caching attributes in the PTE. The combination
1025 	 * of attributes are poorly defined, so we pay attention
1026 	 * to them in the given order.
1027 	 *
1028 	 * The test for HAT_STRICTORDER is different because it's defined
1029 	 * as "0" - which was a stupid thing to do, but is too late to change!
1030 	 */
1031 	if (cache_attr == HAT_STRICTORDER) {
1032 		PTE_SET(pte, PT_NOCACHE);
1033 	/*LINTED [Lint hates empty ifs, but it's the obvious way to do this] */
1034 	} else if (cache_attr & (HAT_UNORDERED_OK | HAT_STORECACHING_OK)) {
1035 		/* nothing to set */;
1036 	} else if (cache_attr & (HAT_MERGING_OK | HAT_LOADCACHING_OK)) {
1037 		PTE_SET(pte, PT_NOCACHE);
1038 		if (x86_feature & X86_PAT)
1039 			PTE_SET(pte, (level == 0) ? PT_PAT_4K : PT_PAT_LARGE);
1040 		else
1041 			PTE_SET(pte, PT_WRITETHRU);
1042 	} else {
1043 		panic("hati_mkpte(): bad caching attributes: %x\n", cache_attr);
1044 	}
1045 
1046 	return (pte);
1047 }
1048 
1049 /*
1050  * Duplicate address translations of the parent to the child.
1051  * This function really isn't used anymore.
1052  */
1053 /*ARGSUSED*/
1054 int
1055 hat_dup(hat_t *old, hat_t *new, caddr_t addr, size_t len, uint_t flag)
1056 {
1057 	ASSERT((uintptr_t)addr < kernelbase);
1058 	ASSERT(new != kas.a_hat);
1059 	ASSERT(old != kas.a_hat);
1060 	return (0);
1061 }
1062 
1063 /*
1064  * Allocate any hat resources required for a process being swapped in.
1065  */
1066 /*ARGSUSED*/
1067 void
1068 hat_swapin(hat_t *hat)
1069 {
1070 	/* do nothing - we let everything fault back in */
1071 }
1072 
1073 /*
1074  * Unload all translations associated with an address space of a process
1075  * that is being swapped out.
1076  */
1077 void
1078 hat_swapout(hat_t *hat)
1079 {
1080 	uintptr_t	vaddr = (uintptr_t)0;
1081 	uintptr_t	eaddr = _userlimit;
1082 	htable_t	*ht = NULL;
1083 	level_t		l;
1084 
1085 	XPV_DISALLOW_MIGRATE();
1086 	/*
1087 	 * We can't just call hat_unload(hat, 0, _userlimit...)  here, because
1088 	 * seg_spt and shared pagetables can't be swapped out.
1089 	 * Take a look at segspt_shmswapout() - it's a big no-op.
1090 	 *
1091 	 * Instead we'll walk through all the address space and unload
1092 	 * any mappings which we are sure are not shared, not locked.
1093 	 */
1094 	ASSERT(IS_PAGEALIGNED(vaddr));
1095 	ASSERT(IS_PAGEALIGNED(eaddr));
1096 	ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1097 	if ((uintptr_t)hat->hat_as->a_userlimit < eaddr)
1098 		eaddr = (uintptr_t)hat->hat_as->a_userlimit;
1099 
1100 	while (vaddr < eaddr) {
1101 		(void) htable_walk(hat, &ht, &vaddr, eaddr);
1102 		if (ht == NULL)
1103 			break;
1104 
1105 		ASSERT(!IN_VA_HOLE(vaddr));
1106 
1107 		/*
1108 		 * If the page table is shared skip its entire range.
1109 		 * This code knows that only level 0 page tables are shared
1110 		 */
1111 		l = ht->ht_level;
1112 		if (ht->ht_flags & HTABLE_SHARED_PFN) {
1113 			ASSERT(l == 0);
1114 			vaddr = ht->ht_vaddr + LEVEL_SIZE(1);
1115 			htable_release(ht);
1116 			ht = NULL;
1117 			continue;
1118 		}
1119 
1120 		/*
1121 		 * If the page table has no locked entries, unload this one.
1122 		 */
1123 		if (ht->ht_lock_cnt == 0)
1124 			hat_unload(hat, (caddr_t)vaddr, LEVEL_SIZE(l),
1125 			    HAT_UNLOAD_UNMAP);
1126 
1127 		/*
1128 		 * If we have a level 0 page table with locked entries,
1129 		 * skip the entire page table, otherwise skip just one entry.
1130 		 */
1131 		if (ht->ht_lock_cnt > 0 && l == 0)
1132 			vaddr = ht->ht_vaddr + LEVEL_SIZE(1);
1133 		else
1134 			vaddr += LEVEL_SIZE(l);
1135 	}
1136 	if (ht)
1137 		htable_release(ht);
1138 
1139 	/*
1140 	 * We're in swapout because the system is low on memory, so
1141 	 * go back and flush all the htables off the cached list.
1142 	 */
1143 	htable_purge_hat(hat);
1144 	XPV_ALLOW_MIGRATE();
1145 }
1146 
1147 /*
1148  * returns number of bytes that have valid mappings in hat.
1149  */
1150 size_t
1151 hat_get_mapped_size(hat_t *hat)
1152 {
1153 	size_t total = 0;
1154 	int l;
1155 
1156 	for (l = 0; l <= mmu.max_page_level; l++)
1157 		total += (hat->hat_pages_mapped[l] << LEVEL_SHIFT(l));
1158 	total += hat->hat_ism_pgcnt;
1159 
1160 	return (total);
1161 }
1162 
1163 /*
1164  * enable/disable collection of stats for hat.
1165  */
1166 int
1167 hat_stats_enable(hat_t *hat)
1168 {
1169 	atomic_add_32(&hat->hat_stats, 1);
1170 	return (1);
1171 }
1172 
1173 void
1174 hat_stats_disable(hat_t *hat)
1175 {
1176 	atomic_add_32(&hat->hat_stats, -1);
1177 }
1178 
1179 /*
1180  * Utility to sync the ref/mod bits from a page table entry to the page_t
1181  * We must be holding the mapping list lock when this is called.
1182  */
1183 static void
1184 hati_sync_pte_to_page(page_t *pp, x86pte_t pte, level_t level)
1185 {
1186 	uint_t	rm = 0;
1187 	pgcnt_t	pgcnt;
1188 
1189 	if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC)
1190 		return;
1191 
1192 	if (PTE_GET(pte, PT_REF))
1193 		rm |= P_REF;
1194 
1195 	if (PTE_GET(pte, PT_MOD))
1196 		rm |= P_MOD;
1197 
1198 	if (rm == 0)
1199 		return;
1200 
1201 	/*
1202 	 * sync to all constituent pages of a large page
1203 	 */
1204 	ASSERT(x86_hm_held(pp));
1205 	pgcnt = page_get_pagecnt(level);
1206 	ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt));
1207 	for (; pgcnt > 0; --pgcnt) {
1208 		/*
1209 		 * hat_page_demote() can't decrease
1210 		 * pszc below this mapping size
1211 		 * since this large mapping existed after we
1212 		 * took mlist lock.
1213 		 */
1214 		ASSERT(pp->p_szc >= level);
1215 		hat_page_setattr(pp, rm);
1216 		++pp;
1217 	}
1218 }
1219 
1220 /*
1221  * This the set of PTE bits for PFN, permissions and caching
1222  * that are allowed to change on a HAT_LOAD_REMAP
1223  */
1224 #define	PT_REMAP_BITS							\
1225 	(PT_PADDR | PT_NX | PT_WRITABLE | PT_WRITETHRU |		\
1226 	PT_NOCACHE | PT_PAT_4K | PT_PAT_LARGE | PT_IGNORE | PT_REF | PT_MOD)
1227 
1228 #define	REMAPASSERT(EX)	if (!(EX)) panic("hati_pte_map: " #EX)
1229 /*
1230  * Do the low-level work to get a mapping entered into a HAT's pagetables
1231  * and in the mapping list of the associated page_t.
1232  */
1233 static int
1234 hati_pte_map(
1235 	htable_t	*ht,
1236 	uint_t		entry,
1237 	page_t		*pp,
1238 	x86pte_t	pte,
1239 	int		flags,
1240 	void		*pte_ptr)
1241 {
1242 	hat_t		*hat = ht->ht_hat;
1243 	x86pte_t	old_pte;
1244 	level_t		l = ht->ht_level;
1245 	hment_t		*hm;
1246 	uint_t		is_consist;
1247 	int		rv = 0;
1248 
1249 	/*
1250 	 * Is this a consistant (ie. need mapping list lock) mapping?
1251 	 */
1252 	is_consist = (pp != NULL && (flags & HAT_LOAD_NOCONSIST) == 0);
1253 
1254 	/*
1255 	 * Track locked mapping count in the htable.  Do this first,
1256 	 * as we track locking even if there already is a mapping present.
1257 	 */
1258 	if ((flags & HAT_LOAD_LOCK) != 0 && hat != kas.a_hat)
1259 		HTABLE_LOCK_INC(ht);
1260 
1261 	/*
1262 	 * Acquire the page's mapping list lock and get an hment to use.
1263 	 * Note that hment_prepare() might return NULL.
1264 	 */
1265 	if (is_consist) {
1266 		x86_hm_enter(pp);
1267 		hm = hment_prepare(ht, entry, pp);
1268 	}
1269 
1270 	/*
1271 	 * Set the new pte, retrieving the old one at the same time.
1272 	 */
1273 	old_pte = x86pte_set(ht, entry, pte, pte_ptr);
1274 
1275 	/*
1276 	 * did we get a large page / page table collision?
1277 	 */
1278 	if (old_pte == LPAGE_ERROR) {
1279 		rv = -1;
1280 		goto done;
1281 	}
1282 
1283 	/*
1284 	 * If the mapping didn't change there is nothing more to do.
1285 	 */
1286 	if (PTE_EQUIV(pte, old_pte))
1287 		goto done;
1288 
1289 	/*
1290 	 * Install a new mapping in the page's mapping list
1291 	 */
1292 	if (!PTE_ISVALID(old_pte)) {
1293 		if (is_consist) {
1294 			hment_assign(ht, entry, pp, hm);
1295 			x86_hm_exit(pp);
1296 		} else {
1297 			ASSERT(flags & HAT_LOAD_NOCONSIST);
1298 		}
1299 		HTABLE_INC(ht->ht_valid_cnt);
1300 		PGCNT_INC(hat, l);
1301 		return (rv);
1302 	}
1303 
1304 	/*
1305 	 * Remap's are more complicated:
1306 	 *  - HAT_LOAD_REMAP must be specified if changing the pfn.
1307 	 *    We also require that NOCONSIST be specified.
1308 	 *  - Otherwise only permission or caching bits may change.
1309 	 */
1310 	if (!PTE_ISPAGE(old_pte, l))
1311 		panic("non-null/page mapping pte=" FMT_PTE, old_pte);
1312 
1313 	if (PTE2PFN(old_pte, l) != PTE2PFN(pte, l)) {
1314 		REMAPASSERT(flags & HAT_LOAD_REMAP);
1315 		REMAPASSERT(flags & HAT_LOAD_NOCONSIST);
1316 		REMAPASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST);
1317 		REMAPASSERT(pf_is_memory(PTE2PFN(old_pte, l)) ==
1318 		    pf_is_memory(PTE2PFN(pte, l)));
1319 		REMAPASSERT(!is_consist);
1320 	}
1321 
1322 	/*
1323 	 * We only let remaps change the certain bits in the PTE.
1324 	 */
1325 	if (PTE_GET(old_pte, ~PT_REMAP_BITS) != PTE_GET(pte, ~PT_REMAP_BITS))
1326 		panic("remap bits changed: old_pte="FMT_PTE", pte="FMT_PTE"\n",
1327 		    old_pte, pte);
1328 
1329 	/*
1330 	 * We don't create any mapping list entries on a remap, so release
1331 	 * any allocated hment after we drop the mapping list lock.
1332 	 */
1333 done:
1334 	if (is_consist) {
1335 		x86_hm_exit(pp);
1336 		if (hm != NULL)
1337 			hment_free(hm);
1338 	}
1339 	return (rv);
1340 }
1341 
1342 /*
1343  * Internal routine to load a single page table entry. This only fails if
1344  * we attempt to overwrite a page table link with a large page.
1345  */
1346 static int
1347 hati_load_common(
1348 	hat_t		*hat,
1349 	uintptr_t	va,
1350 	page_t		*pp,
1351 	uint_t		attr,
1352 	uint_t		flags,
1353 	level_t		level,
1354 	pfn_t		pfn)
1355 {
1356 	htable_t	*ht;
1357 	uint_t		entry;
1358 	x86pte_t	pte;
1359 	int		rv = 0;
1360 
1361 	/*
1362 	 * The number 16 is arbitrary and here to catch a recursion problem
1363 	 * early before we blow out the kernel stack.
1364 	 */
1365 	++curthread->t_hatdepth;
1366 	ASSERT(curthread->t_hatdepth < 16);
1367 
1368 	ASSERT(hat == kas.a_hat ||
1369 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1370 
1371 	if (flags & HAT_LOAD_SHARE)
1372 		hat->hat_flags |= HAT_SHARED;
1373 
1374 	/*
1375 	 * Find the page table that maps this page if it already exists.
1376 	 */
1377 	ht = htable_lookup(hat, va, level);
1378 
1379 	/*
1380 	 * We must have HAT_LOAD_NOCONSIST if page_t is NULL.
1381 	 */
1382 	if (pp == NULL)
1383 		flags |= HAT_LOAD_NOCONSIST;
1384 
1385 	if (ht == NULL) {
1386 		ht = htable_create(hat, va, level, NULL);
1387 		ASSERT(ht != NULL);
1388 	}
1389 	entry = htable_va2entry(va, ht);
1390 
1391 	/*
1392 	 * a bunch of paranoid error checking
1393 	 */
1394 	ASSERT(ht->ht_busy > 0);
1395 	if (ht->ht_vaddr > va || va > HTABLE_LAST_PAGE(ht))
1396 		panic("hati_load_common: bad htable %p, va %p", ht, (void *)va);
1397 	ASSERT(ht->ht_level == level);
1398 
1399 	/*
1400 	 * construct the new PTE
1401 	 */
1402 	if (hat == kas.a_hat)
1403 		attr &= ~PROT_USER;
1404 	pte = hati_mkpte(pfn, attr, level, flags);
1405 	if (hat == kas.a_hat && va >= kernelbase)
1406 		PTE_SET(pte, mmu.pt_global);
1407 
1408 	/*
1409 	 * establish the mapping
1410 	 */
1411 	rv = hati_pte_map(ht, entry, pp, pte, flags, NULL);
1412 
1413 	/*
1414 	 * release the htable and any reserves
1415 	 */
1416 	htable_release(ht);
1417 	--curthread->t_hatdepth;
1418 	return (rv);
1419 }
1420 
1421 /*
1422  * special case of hat_memload to deal with some kernel addrs for performance
1423  */
1424 static void
1425 hat_kmap_load(
1426 	caddr_t		addr,
1427 	page_t		*pp,
1428 	uint_t		attr,
1429 	uint_t		flags)
1430 {
1431 	uintptr_t	va = (uintptr_t)addr;
1432 	x86pte_t	pte;
1433 	pfn_t		pfn = page_pptonum(pp);
1434 	pgcnt_t		pg_off = mmu_btop(va - mmu.kmap_addr);
1435 	htable_t	*ht;
1436 	uint_t		entry;
1437 	void		*pte_ptr;
1438 
1439 	/*
1440 	 * construct the requested PTE
1441 	 */
1442 	attr &= ~PROT_USER;
1443 	attr |= HAT_STORECACHING_OK;
1444 	pte = hati_mkpte(pfn, attr, 0, flags);
1445 	PTE_SET(pte, mmu.pt_global);
1446 
1447 	/*
1448 	 * Figure out the pte_ptr and htable and use common code to finish up
1449 	 */
1450 	if (mmu.pae_hat)
1451 		pte_ptr = mmu.kmap_ptes + pg_off;
1452 	else
1453 		pte_ptr = (x86pte32_t *)mmu.kmap_ptes + pg_off;
1454 	ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) >>
1455 	    LEVEL_SHIFT(1)];
1456 	entry = htable_va2entry(va, ht);
1457 	++curthread->t_hatdepth;
1458 	ASSERT(curthread->t_hatdepth < 16);
1459 	(void) hati_pte_map(ht, entry, pp, pte, flags, pte_ptr);
1460 	--curthread->t_hatdepth;
1461 }
1462 
1463 /*
1464  * hat_memload() - load a translation to the given page struct
1465  *
1466  * Flags for hat_memload/hat_devload/hat_*attr.
1467  *
1468  * 	HAT_LOAD	Default flags to load a translation to the page.
1469  *
1470  * 	HAT_LOAD_LOCK	Lock down mapping resources; hat_map(), hat_memload(),
1471  *			and hat_devload().
1472  *
1473  *	HAT_LOAD_NOCONSIST Do not add mapping to page_t mapping list.
1474  *			sets PT_NOCONSIST
1475  *
1476  *	HAT_LOAD_SHARE	A flag to hat_memload() to indicate h/w page tables
1477  *			that map some user pages (not kas) is shared by more
1478  *			than one process (eg. ISM).
1479  *
1480  *	HAT_LOAD_REMAP	Reload a valid pte with a different page frame.
1481  *
1482  *	HAT_NO_KALLOC	Do not kmem_alloc while creating the mapping; at this
1483  *			point, it's setting up mapping to allocate internal
1484  *			hat layer data structures.  This flag forces hat layer
1485  *			to tap its reserves in order to prevent infinite
1486  *			recursion.
1487  *
1488  * The following is a protection attribute (like PROT_READ, etc.)
1489  *
1490  *	HAT_NOSYNC	set PT_NOSYNC - this mapping's ref/mod bits
1491  *			are never cleared.
1492  *
1493  * Installing new valid PTE's and creation of the mapping list
1494  * entry are controlled under the same lock. It's derived from the
1495  * page_t being mapped.
1496  */
1497 static uint_t supported_memload_flags =
1498 	HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST |
1499 	HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT;
1500 
1501 void
1502 hat_memload(
1503 	hat_t		*hat,
1504 	caddr_t		addr,
1505 	page_t		*pp,
1506 	uint_t		attr,
1507 	uint_t		flags)
1508 {
1509 	uintptr_t	va = (uintptr_t)addr;
1510 	level_t		level = 0;
1511 	pfn_t		pfn = page_pptonum(pp);
1512 
1513 	XPV_DISALLOW_MIGRATE();
1514 	ASSERT(IS_PAGEALIGNED(va));
1515 	ASSERT(hat == kas.a_hat || va < _userlimit);
1516 	ASSERT(hat == kas.a_hat ||
1517 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1518 	ASSERT((flags & supported_memload_flags) == flags);
1519 
1520 	ASSERT(!IN_VA_HOLE(va));
1521 	ASSERT(!PP_ISFREE(pp));
1522 
1523 	/*
1524 	 * kernel address special case for performance.
1525 	 */
1526 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
1527 		ASSERT(hat == kas.a_hat);
1528 		hat_kmap_load(addr, pp, attr, flags);
1529 		XPV_ALLOW_MIGRATE();
1530 		return;
1531 	}
1532 
1533 	/*
1534 	 * This is used for memory with normal caching enabled, so
1535 	 * always set HAT_STORECACHING_OK.
1536 	 */
1537 	attr |= HAT_STORECACHING_OK;
1538 	if (hati_load_common(hat, va, pp, attr, flags, level, pfn) != 0)
1539 		panic("unexpected hati_load_common() failure");
1540 	XPV_ALLOW_MIGRATE();
1541 }
1542 
1543 /* ARGSUSED */
1544 void
1545 hat_memload_region(struct hat *hat, caddr_t addr, struct page *pp,
1546     uint_t attr, uint_t flags, hat_region_cookie_t rcookie)
1547 {
1548 	hat_memload(hat, addr, pp, attr, flags);
1549 }
1550 
1551 /*
1552  * Load the given array of page structs using large pages when possible
1553  */
1554 void
1555 hat_memload_array(
1556 	hat_t		*hat,
1557 	caddr_t		addr,
1558 	size_t		len,
1559 	page_t		**pages,
1560 	uint_t		attr,
1561 	uint_t		flags)
1562 {
1563 	uintptr_t	va = (uintptr_t)addr;
1564 	uintptr_t	eaddr = va + len;
1565 	level_t		level;
1566 	size_t		pgsize;
1567 	pgcnt_t		pgindx = 0;
1568 	pfn_t		pfn;
1569 	pgcnt_t		i;
1570 
1571 	XPV_DISALLOW_MIGRATE();
1572 	ASSERT(IS_PAGEALIGNED(va));
1573 	ASSERT(hat == kas.a_hat || va + len <= _userlimit);
1574 	ASSERT(hat == kas.a_hat ||
1575 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1576 	ASSERT((flags & supported_memload_flags) == flags);
1577 
1578 	/*
1579 	 * memload is used for memory with full caching enabled, so
1580 	 * set HAT_STORECACHING_OK.
1581 	 */
1582 	attr |= HAT_STORECACHING_OK;
1583 
1584 	/*
1585 	 * handle all pages using largest possible pagesize
1586 	 */
1587 	while (va < eaddr) {
1588 		/*
1589 		 * decide what level mapping to use (ie. pagesize)
1590 		 */
1591 		pfn = page_pptonum(pages[pgindx]);
1592 		for (level = mmu.max_page_level; ; --level) {
1593 			pgsize = LEVEL_SIZE(level);
1594 			if (level == 0)
1595 				break;
1596 
1597 			if (!IS_P2ALIGNED(va, pgsize) ||
1598 			    (eaddr - va) < pgsize ||
1599 			    !IS_P2ALIGNED(pfn_to_pa(pfn), pgsize))
1600 				continue;
1601 
1602 			/*
1603 			 * To use a large mapping of this size, all the
1604 			 * pages we are passed must be sequential subpages
1605 			 * of the large page.
1606 			 * hat_page_demote() can't change p_szc because
1607 			 * all pages are locked.
1608 			 */
1609 			if (pages[pgindx]->p_szc >= level) {
1610 				for (i = 0; i < mmu_btop(pgsize); ++i) {
1611 					if (pfn + i !=
1612 					    page_pptonum(pages[pgindx + i]))
1613 						break;
1614 					ASSERT(pages[pgindx + i]->p_szc >=
1615 					    level);
1616 					ASSERT(pages[pgindx] + i ==
1617 					    pages[pgindx + i]);
1618 				}
1619 				if (i == mmu_btop(pgsize))
1620 					break;
1621 			}
1622 		}
1623 
1624 		/*
1625 		 * Load this page mapping. If the load fails, try a smaller
1626 		 * pagesize.
1627 		 */
1628 		ASSERT(!IN_VA_HOLE(va));
1629 		while (hati_load_common(hat, va, pages[pgindx], attr,
1630 		    flags, level, pfn) != 0) {
1631 			if (level == 0)
1632 				panic("unexpected hati_load_common() failure");
1633 			--level;
1634 			pgsize = LEVEL_SIZE(level);
1635 		}
1636 
1637 		/*
1638 		 * move to next page
1639 		 */
1640 		va += pgsize;
1641 		pgindx += mmu_btop(pgsize);
1642 	}
1643 	XPV_ALLOW_MIGRATE();
1644 }
1645 
1646 /* ARGSUSED */
1647 void
1648 hat_memload_array_region(struct hat *hat, caddr_t addr, size_t len,
1649     struct page **pps, uint_t attr, uint_t flags,
1650     hat_region_cookie_t rcookie)
1651 {
1652 	hat_memload_array(hat, addr, len, pps, attr, flags);
1653 }
1654 
1655 /*
1656  * void hat_devload(hat, addr, len, pf, attr, flags)
1657  *	load/lock the given page frame number
1658  *
1659  * Advisory ordering attributes. Apply only to device mappings.
1660  *
1661  * HAT_STRICTORDER: the CPU must issue the references in order, as the
1662  *	programmer specified.  This is the default.
1663  * HAT_UNORDERED_OK: the CPU may reorder the references (this is all kinds
1664  *	of reordering; store or load with store or load).
1665  * HAT_MERGING_OK: merging and batching: the CPU may merge individual stores
1666  *	to consecutive locations (for example, turn two consecutive byte
1667  *	stores into one halfword store), and it may batch individual loads
1668  *	(for example, turn two consecutive byte loads into one halfword load).
1669  *	This also implies re-ordering.
1670  * HAT_LOADCACHING_OK: the CPU may cache the data it fetches and reuse it
1671  *	until another store occurs.  The default is to fetch new data
1672  *	on every load.  This also implies merging.
1673  * HAT_STORECACHING_OK: the CPU may keep the data in the cache and push it to
1674  *	the device (perhaps with other data) at a later time.  The default is
1675  *	to push the data right away.  This also implies load caching.
1676  *
1677  * Equivalent of hat_memload(), but can be used for device memory where
1678  * there are no page_t's and we support additional flags (write merging, etc).
1679  * Note that we can have large page mappings with this interface.
1680  */
1681 int supported_devload_flags = HAT_LOAD | HAT_LOAD_LOCK |
1682 	HAT_LOAD_NOCONSIST | HAT_STRICTORDER | HAT_UNORDERED_OK |
1683 	HAT_MERGING_OK | HAT_LOADCACHING_OK | HAT_STORECACHING_OK;
1684 
1685 void
1686 hat_devload(
1687 	hat_t		*hat,
1688 	caddr_t		addr,
1689 	size_t		len,
1690 	pfn_t		pfn,
1691 	uint_t		attr,
1692 	int		flags)
1693 {
1694 	uintptr_t	va = ALIGN2PAGE(addr);
1695 	uintptr_t	eva = va + len;
1696 	level_t		level;
1697 	size_t		pgsize;
1698 	page_t		*pp;
1699 	int		f;	/* per PTE copy of flags  - maybe modified */
1700 	uint_t		a;	/* per PTE copy of attr */
1701 
1702 	XPV_DISALLOW_MIGRATE();
1703 	ASSERT(IS_PAGEALIGNED(va));
1704 	ASSERT(hat == kas.a_hat || eva <= _userlimit);
1705 	ASSERT(hat == kas.a_hat ||
1706 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1707 	ASSERT((flags & supported_devload_flags) == flags);
1708 
1709 	/*
1710 	 * handle all pages
1711 	 */
1712 	while (va < eva) {
1713 
1714 		/*
1715 		 * decide what level mapping to use (ie. pagesize)
1716 		 */
1717 		for (level = mmu.max_page_level; ; --level) {
1718 			pgsize = LEVEL_SIZE(level);
1719 			if (level == 0)
1720 				break;
1721 			if (IS_P2ALIGNED(va, pgsize) &&
1722 			    (eva - va) >= pgsize &&
1723 			    IS_P2ALIGNED(pfn, mmu_btop(pgsize)))
1724 				break;
1725 		}
1726 
1727 		/*
1728 		 * If this is just memory then allow caching (this happens
1729 		 * for the nucleus pages) - though HAT_PLAT_NOCACHE can be used
1730 		 * to override that. If we don't have a page_t then make sure
1731 		 * NOCONSIST is set.
1732 		 */
1733 		a = attr;
1734 		f = flags;
1735 		if (!pf_is_memory(pfn))
1736 			f |= HAT_LOAD_NOCONSIST;
1737 		else if (!(a & HAT_PLAT_NOCACHE))
1738 			a |= HAT_STORECACHING_OK;
1739 
1740 		if (f & HAT_LOAD_NOCONSIST)
1741 			pp = NULL;
1742 		else
1743 			pp = page_numtopp_nolock(pfn);
1744 
1745 		/*
1746 		 * load this page mapping
1747 		 */
1748 		ASSERT(!IN_VA_HOLE(va));
1749 		while (hati_load_common(hat, va, pp, a, f, level, pfn) != 0) {
1750 			if (level == 0)
1751 				panic("unexpected hati_load_common() failure");
1752 			--level;
1753 			pgsize = LEVEL_SIZE(level);
1754 		}
1755 
1756 		/*
1757 		 * move to next page
1758 		 */
1759 		va += pgsize;
1760 		pfn += mmu_btop(pgsize);
1761 	}
1762 	XPV_ALLOW_MIGRATE();
1763 }
1764 
1765 /*
1766  * void hat_unlock(hat, addr, len)
1767  *	unlock the mappings to a given range of addresses
1768  *
1769  * Locks are tracked by ht_lock_cnt in the htable.
1770  */
1771 void
1772 hat_unlock(hat_t *hat, caddr_t addr, size_t len)
1773 {
1774 	uintptr_t	vaddr = (uintptr_t)addr;
1775 	uintptr_t	eaddr = vaddr + len;
1776 	htable_t	*ht = NULL;
1777 
1778 	/*
1779 	 * kernel entries are always locked, we don't track lock counts
1780 	 */
1781 	ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
1782 	ASSERT(IS_PAGEALIGNED(vaddr));
1783 	ASSERT(IS_PAGEALIGNED(eaddr));
1784 	if (hat == kas.a_hat)
1785 		return;
1786 	if (eaddr > _userlimit)
1787 		panic("hat_unlock() address out of range - above _userlimit");
1788 
1789 	XPV_DISALLOW_MIGRATE();
1790 	ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1791 	while (vaddr < eaddr) {
1792 		(void) htable_walk(hat, &ht, &vaddr, eaddr);
1793 		if (ht == NULL)
1794 			break;
1795 
1796 		ASSERT(!IN_VA_HOLE(vaddr));
1797 
1798 		if (ht->ht_lock_cnt < 1)
1799 			panic("hat_unlock(): lock_cnt < 1, "
1800 			    "htable=%p, vaddr=%p\n", ht, (caddr_t)vaddr);
1801 		HTABLE_LOCK_DEC(ht);
1802 
1803 		vaddr += LEVEL_SIZE(ht->ht_level);
1804 	}
1805 	if (ht)
1806 		htable_release(ht);
1807 	XPV_ALLOW_MIGRATE();
1808 }
1809 
1810 /* ARGSUSED */
1811 void
1812 hat_unlock_region(struct hat *hat, caddr_t addr, size_t len,
1813     hat_region_cookie_t rcookie)
1814 {
1815 	panic("No shared region support on x86");
1816 }
1817 
1818 #if !defined(__xpv)
1819 /*
1820  * Cross call service routine to demap a virtual page on
1821  * the current CPU or flush all mappings in TLB.
1822  */
1823 /*ARGSUSED*/
1824 static int
1825 hati_demap_func(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3)
1826 {
1827 	hat_t	*hat = (hat_t *)a1;
1828 	caddr_t	addr = (caddr_t)a2;
1829 
1830 	/*
1831 	 * If the target hat isn't the kernel and this CPU isn't operating
1832 	 * in the target hat, we can ignore the cross call.
1833 	 */
1834 	if (hat != kas.a_hat && hat != CPU->cpu_current_hat)
1835 		return (0);
1836 
1837 	/*
1838 	 * For a normal address, we just flush one page mapping
1839 	 */
1840 	if ((uintptr_t)addr != DEMAP_ALL_ADDR) {
1841 		mmu_tlbflush_entry(addr);
1842 		return (0);
1843 	}
1844 
1845 	/*
1846 	 * Otherwise we reload cr3 to effect a complete TLB flush.
1847 	 *
1848 	 * A reload of cr3 on a VLP process also means we must also recopy in
1849 	 * the pte values from the struct hat
1850 	 */
1851 	if (hat->hat_flags & HAT_VLP) {
1852 #if defined(__amd64)
1853 		x86pte_t *vlpptep = CPU->cpu_hat_info->hci_vlp_l2ptes;
1854 
1855 		VLP_COPY(hat->hat_vlp_ptes, vlpptep);
1856 #elif defined(__i386)
1857 		reload_pae32(hat, CPU);
1858 #endif
1859 	}
1860 	reload_cr3();
1861 	return (0);
1862 }
1863 
1864 /*
1865  * Flush all TLB entries, including global (ie. kernel) ones.
1866  */
1867 static void
1868 flush_all_tlb_entries(void)
1869 {
1870 	ulong_t cr4 = getcr4();
1871 
1872 	if (cr4 & CR4_PGE) {
1873 		setcr4(cr4 & ~(ulong_t)CR4_PGE);
1874 		setcr4(cr4);
1875 
1876 		/*
1877 		 * 32 bit PAE also needs to always reload_cr3()
1878 		 */
1879 		if (mmu.max_level == 2)
1880 			reload_cr3();
1881 	} else {
1882 		reload_cr3();
1883 	}
1884 }
1885 
1886 #define	TLB_CPU_HALTED	(01ul)
1887 #define	TLB_INVAL_ALL	(02ul)
1888 #define	CAS_TLB_INFO(cpu, old, new)	\
1889 	caslong((ulong_t *)&(cpu)->cpu_m.mcpu_tlb_info, (old), (new))
1890 
1891 /*
1892  * Record that a CPU is going idle
1893  */
1894 void
1895 tlb_going_idle(void)
1896 {
1897 	atomic_or_long((ulong_t *)&CPU->cpu_m.mcpu_tlb_info, TLB_CPU_HALTED);
1898 }
1899 
1900 /*
1901  * Service a delayed TLB flush if coming out of being idle.
1902  */
1903 void
1904 tlb_service(void)
1905 {
1906 	ulong_t flags = getflags();
1907 	ulong_t tlb_info;
1908 	ulong_t found;
1909 
1910 	/*
1911 	 * Be sure interrupts are off while doing this so that
1912 	 * higher level interrupts correctly wait for flushes to finish.
1913 	 */
1914 	if (flags & PS_IE)
1915 		flags = intr_clear();
1916 
1917 	/*
1918 	 * We only have to do something if coming out of being idle.
1919 	 */
1920 	tlb_info = CPU->cpu_m.mcpu_tlb_info;
1921 	if (tlb_info & TLB_CPU_HALTED) {
1922 		ASSERT(CPU->cpu_current_hat == kas.a_hat);
1923 
1924 		/*
1925 		 * Atomic clear and fetch of old state.
1926 		 */
1927 		while ((found = CAS_TLB_INFO(CPU, tlb_info, 0)) != tlb_info) {
1928 			ASSERT(found & TLB_CPU_HALTED);
1929 			tlb_info = found;
1930 			SMT_PAUSE();
1931 		}
1932 		if (tlb_info & TLB_INVAL_ALL)
1933 			flush_all_tlb_entries();
1934 	}
1935 
1936 	/*
1937 	 * Restore interrupt enable control bit.
1938 	 */
1939 	if (flags & PS_IE)
1940 		sti();
1941 }
1942 #endif /* !__xpv */
1943 
1944 /*
1945  * Internal routine to do cross calls to invalidate a range of pages on
1946  * all CPUs using a given hat.
1947  */
1948 void
1949 hat_tlb_inval(hat_t *hat, uintptr_t va)
1950 {
1951 	extern int	flushes_require_xcalls;	/* from mp_startup.c */
1952 	cpuset_t	justme;
1953 	cpuset_t	cpus_to_shootdown;
1954 #ifndef __xpv
1955 	cpuset_t	check_cpus;
1956 	cpu_t		*cpup;
1957 	int		c;
1958 #endif
1959 
1960 	/*
1961 	 * If the hat is being destroyed, there are no more users, so
1962 	 * demap need not do anything.
1963 	 */
1964 	if (hat->hat_flags & HAT_FREEING)
1965 		return;
1966 
1967 	/*
1968 	 * If demapping from a shared pagetable, we best demap the
1969 	 * entire set of user TLBs, since we don't know what addresses
1970 	 * these were shared at.
1971 	 */
1972 	if (hat->hat_flags & HAT_SHARED) {
1973 		hat = kas.a_hat;
1974 		va = DEMAP_ALL_ADDR;
1975 	}
1976 
1977 	/*
1978 	 * if not running with multiple CPUs, don't use cross calls
1979 	 */
1980 	if (panicstr || !flushes_require_xcalls) {
1981 #ifdef __xpv
1982 		if (va == DEMAP_ALL_ADDR)
1983 			xen_flush_tlb();
1984 		else
1985 			xen_flush_va((caddr_t)va);
1986 #else
1987 		(void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL);
1988 #endif
1989 		return;
1990 	}
1991 
1992 
1993 	/*
1994 	 * Determine CPUs to shootdown. Kernel changes always do all CPUs.
1995 	 * Otherwise it's just CPUs currently executing in this hat.
1996 	 */
1997 	kpreempt_disable();
1998 	CPUSET_ONLY(justme, CPU->cpu_id);
1999 	if (hat == kas.a_hat)
2000 		cpus_to_shootdown = khat_cpuset;
2001 	else
2002 		cpus_to_shootdown = hat->hat_cpus;
2003 
2004 #ifndef __xpv
2005 	/*
2006 	 * If any CPUs in the set are idle, just request a delayed flush
2007 	 * and avoid waking them up.
2008 	 */
2009 	check_cpus = cpus_to_shootdown;
2010 	for (c = 0; c < NCPU && !CPUSET_ISNULL(check_cpus); ++c) {
2011 		ulong_t tlb_info;
2012 
2013 		if (!CPU_IN_SET(check_cpus, c))
2014 			continue;
2015 		CPUSET_DEL(check_cpus, c);
2016 		cpup = cpu[c];
2017 		if (cpup == NULL)
2018 			continue;
2019 
2020 		tlb_info = cpup->cpu_m.mcpu_tlb_info;
2021 		while (tlb_info == TLB_CPU_HALTED) {
2022 			(void) CAS_TLB_INFO(cpup, TLB_CPU_HALTED,
2023 			    TLB_CPU_HALTED | TLB_INVAL_ALL);
2024 			SMT_PAUSE();
2025 			tlb_info = cpup->cpu_m.mcpu_tlb_info;
2026 		}
2027 		if (tlb_info == (TLB_CPU_HALTED | TLB_INVAL_ALL)) {
2028 			HATSTAT_INC(hs_tlb_inval_delayed);
2029 			CPUSET_DEL(cpus_to_shootdown, c);
2030 		}
2031 	}
2032 #endif
2033 
2034 	if (CPUSET_ISNULL(cpus_to_shootdown) ||
2035 	    CPUSET_ISEQUAL(cpus_to_shootdown, justme)) {
2036 
2037 #ifdef __xpv
2038 		if (va == DEMAP_ALL_ADDR)
2039 			xen_flush_tlb();
2040 		else
2041 			xen_flush_va((caddr_t)va);
2042 #else
2043 		(void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL);
2044 #endif
2045 
2046 	} else {
2047 
2048 		CPUSET_ADD(cpus_to_shootdown, CPU->cpu_id);
2049 #ifdef __xpv
2050 		if (va == DEMAP_ALL_ADDR)
2051 			xen_gflush_tlb(cpus_to_shootdown);
2052 		else
2053 			xen_gflush_va((caddr_t)va, cpus_to_shootdown);
2054 #else
2055 		xc_call((xc_arg_t)hat, (xc_arg_t)va, NULL, X_CALL_HIPRI,
2056 		    cpus_to_shootdown, hati_demap_func);
2057 #endif
2058 
2059 	}
2060 	kpreempt_enable();
2061 }
2062 
2063 /*
2064  * Interior routine for HAT_UNLOADs from hat_unload_callback(),
2065  * hat_kmap_unload() OR from hat_steal() code.  This routine doesn't
2066  * handle releasing of the htables.
2067  */
2068 void
2069 hat_pte_unmap(
2070 	htable_t	*ht,
2071 	uint_t		entry,
2072 	uint_t		flags,
2073 	x86pte_t	old_pte,
2074 	void		*pte_ptr)
2075 {
2076 	hat_t		*hat = ht->ht_hat;
2077 	hment_t		*hm = NULL;
2078 	page_t		*pp = NULL;
2079 	level_t		l = ht->ht_level;
2080 	pfn_t		pfn;
2081 
2082 	/*
2083 	 * We always track the locking counts, even if nothing is unmapped
2084 	 */
2085 	if ((flags & HAT_UNLOAD_UNLOCK) != 0 && hat != kas.a_hat) {
2086 		ASSERT(ht->ht_lock_cnt > 0);
2087 		HTABLE_LOCK_DEC(ht);
2088 	}
2089 
2090 	/*
2091 	 * Figure out which page's mapping list lock to acquire using the PFN
2092 	 * passed in "old" PTE. We then attempt to invalidate the PTE.
2093 	 * If another thread, probably a hat_pageunload, has asynchronously
2094 	 * unmapped/remapped this address we'll loop here.
2095 	 */
2096 	ASSERT(ht->ht_busy > 0);
2097 	while (PTE_ISVALID(old_pte)) {
2098 		pfn = PTE2PFN(old_pte, l);
2099 		if (PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST) {
2100 			pp = NULL;
2101 		} else {
2102 #ifdef __xpv
2103 			if (pfn == PFN_INVALID)
2104 				panic("Invalid PFN, but not PT_NOCONSIST");
2105 #endif
2106 			pp = page_numtopp_nolock(pfn);
2107 			if (pp == NULL) {
2108 				panic("no page_t, not NOCONSIST: old_pte="
2109 				    FMT_PTE " ht=%lx entry=0x%x pte_ptr=%lx",
2110 				    old_pte, (uintptr_t)ht, entry,
2111 				    (uintptr_t)pte_ptr);
2112 			}
2113 			x86_hm_enter(pp);
2114 		}
2115 
2116 		/*
2117 		 * If freeing the address space, check that the PTE
2118 		 * hasn't changed, as the mappings are no longer in use by
2119 		 * any thread, invalidation is unnecessary.
2120 		 * If not freeing, do a full invalidate.
2121 		 *
2122 		 * On the hypervisor we must always remove mappings, as a
2123 		 * writable mapping left behind could cause a page table
2124 		 * allocation to fail.
2125 		 */
2126 #if !defined(__xpv)
2127 		if (hat->hat_flags & HAT_FREEING)
2128 			old_pte = x86pte_get(ht, entry);
2129 		else
2130 #endif
2131 			old_pte = x86pte_inval(ht, entry, old_pte, pte_ptr);
2132 
2133 		/*
2134 		 * If the page hadn't changed we've unmapped it and can proceed
2135 		 */
2136 		if (PTE_ISVALID(old_pte) && PTE2PFN(old_pte, l) == pfn)
2137 			break;
2138 
2139 		/*
2140 		 * Otherwise, we'll have to retry with the current old_pte.
2141 		 * Drop the hment lock, since the pfn may have changed.
2142 		 */
2143 		if (pp != NULL) {
2144 			x86_hm_exit(pp);
2145 			pp = NULL;
2146 		} else {
2147 			ASSERT(PTE_GET(old_pte, PT_SOFTWARE) >= PT_NOCONSIST);
2148 		}
2149 	}
2150 
2151 	/*
2152 	 * If the old mapping wasn't valid, there's nothing more to do
2153 	 */
2154 	if (!PTE_ISVALID(old_pte)) {
2155 		if (pp != NULL)
2156 			x86_hm_exit(pp);
2157 		return;
2158 	}
2159 
2160 	/*
2161 	 * Take care of syncing any MOD/REF bits and removing the hment.
2162 	 */
2163 	if (pp != NULL) {
2164 		if (!(flags & HAT_UNLOAD_NOSYNC))
2165 			hati_sync_pte_to_page(pp, old_pte, l);
2166 		hm = hment_remove(pp, ht, entry);
2167 		x86_hm_exit(pp);
2168 		if (hm != NULL)
2169 			hment_free(hm);
2170 	}
2171 
2172 	/*
2173 	 * Handle book keeping in the htable and hat
2174 	 */
2175 	ASSERT(ht->ht_valid_cnt > 0);
2176 	HTABLE_DEC(ht->ht_valid_cnt);
2177 	PGCNT_DEC(hat, l);
2178 }
2179 
2180 /*
2181  * very cheap unload implementation to special case some kernel addresses
2182  */
2183 static void
2184 hat_kmap_unload(caddr_t addr, size_t len, uint_t flags)
2185 {
2186 	uintptr_t	va = (uintptr_t)addr;
2187 	uintptr_t	eva = va + len;
2188 	pgcnt_t		pg_index;
2189 	htable_t	*ht;
2190 	uint_t		entry;
2191 	x86pte_t	*pte_ptr;
2192 	x86pte_t	old_pte;
2193 
2194 	for (; va < eva; va += MMU_PAGESIZE) {
2195 		/*
2196 		 * Get the PTE
2197 		 */
2198 		pg_index = mmu_btop(va - mmu.kmap_addr);
2199 		pte_ptr = PT_INDEX_PTR(mmu.kmap_ptes, pg_index);
2200 		old_pte = GET_PTE(pte_ptr);
2201 
2202 		/*
2203 		 * get the htable / entry
2204 		 */
2205 		ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr)
2206 		    >> LEVEL_SHIFT(1)];
2207 		entry = htable_va2entry(va, ht);
2208 
2209 		/*
2210 		 * use mostly common code to unmap it.
2211 		 */
2212 		hat_pte_unmap(ht, entry, flags, old_pte, pte_ptr);
2213 	}
2214 }
2215 
2216 
2217 /*
2218  * unload a range of virtual address space (no callback)
2219  */
2220 void
2221 hat_unload(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2222 {
2223 	uintptr_t va = (uintptr_t)addr;
2224 
2225 	XPV_DISALLOW_MIGRATE();
2226 	ASSERT(hat == kas.a_hat || va + len <= _userlimit);
2227 
2228 	/*
2229 	 * special case for performance.
2230 	 */
2231 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
2232 		ASSERT(hat == kas.a_hat);
2233 		hat_kmap_unload(addr, len, flags);
2234 	} else {
2235 		hat_unload_callback(hat, addr, len, flags, NULL);
2236 	}
2237 	XPV_ALLOW_MIGRATE();
2238 }
2239 
2240 /*
2241  * Do the callbacks for ranges being unloaded.
2242  */
2243 typedef struct range_info {
2244 	uintptr_t	rng_va;
2245 	ulong_t		rng_cnt;
2246 	level_t		rng_level;
2247 } range_info_t;
2248 
2249 static void
2250 handle_ranges(hat_callback_t *cb, uint_t cnt, range_info_t *range)
2251 {
2252 	/*
2253 	 * do callbacks to upper level VM system
2254 	 */
2255 	while (cb != NULL && cnt > 0) {
2256 		--cnt;
2257 		cb->hcb_start_addr = (caddr_t)range[cnt].rng_va;
2258 		cb->hcb_end_addr = cb->hcb_start_addr;
2259 		cb->hcb_end_addr +=
2260 		    range[cnt].rng_cnt << LEVEL_SIZE(range[cnt].rng_level);
2261 		cb->hcb_function(cb);
2262 	}
2263 }
2264 
2265 /*
2266  * Unload a given range of addresses (has optional callback)
2267  *
2268  * Flags:
2269  * define	HAT_UNLOAD		0x00
2270  * define	HAT_UNLOAD_NOSYNC	0x02
2271  * define	HAT_UNLOAD_UNLOCK	0x04
2272  * define	HAT_UNLOAD_OTHER	0x08 - not used
2273  * define	HAT_UNLOAD_UNMAP	0x10 - same as HAT_UNLOAD
2274  */
2275 #define	MAX_UNLOAD_CNT (8)
2276 void
2277 hat_unload_callback(
2278 	hat_t		*hat,
2279 	caddr_t		addr,
2280 	size_t		len,
2281 	uint_t		flags,
2282 	hat_callback_t	*cb)
2283 {
2284 	uintptr_t	vaddr = (uintptr_t)addr;
2285 	uintptr_t	eaddr = vaddr + len;
2286 	htable_t	*ht = NULL;
2287 	uint_t		entry;
2288 	uintptr_t	contig_va = (uintptr_t)-1L;
2289 	range_info_t	r[MAX_UNLOAD_CNT];
2290 	uint_t		r_cnt = 0;
2291 	x86pte_t	old_pte;
2292 
2293 	XPV_DISALLOW_MIGRATE();
2294 	ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
2295 	ASSERT(IS_PAGEALIGNED(vaddr));
2296 	ASSERT(IS_PAGEALIGNED(eaddr));
2297 
2298 	/*
2299 	 * Special case a single page being unloaded for speed. This happens
2300 	 * quite frequently, COW faults after a fork() for example.
2301 	 */
2302 	if (cb == NULL && len == MMU_PAGESIZE) {
2303 		ht = htable_getpte(hat, vaddr, &entry, &old_pte, 0);
2304 		if (ht != NULL) {
2305 			if (PTE_ISVALID(old_pte))
2306 				hat_pte_unmap(ht, entry, flags, old_pte, NULL);
2307 			htable_release(ht);
2308 		}
2309 		XPV_ALLOW_MIGRATE();
2310 		return;
2311 	}
2312 
2313 	while (vaddr < eaddr) {
2314 		old_pte = htable_walk(hat, &ht, &vaddr, eaddr);
2315 		if (ht == NULL)
2316 			break;
2317 
2318 		ASSERT(!IN_VA_HOLE(vaddr));
2319 
2320 		if (vaddr < (uintptr_t)addr)
2321 			panic("hat_unload_callback(): unmap inside large page");
2322 
2323 		/*
2324 		 * We'll do the call backs for contiguous ranges
2325 		 */
2326 		if (vaddr != contig_va ||
2327 		    (r_cnt > 0 && r[r_cnt - 1].rng_level != ht->ht_level)) {
2328 			if (r_cnt == MAX_UNLOAD_CNT) {
2329 				handle_ranges(cb, r_cnt, r);
2330 				r_cnt = 0;
2331 			}
2332 			r[r_cnt].rng_va = vaddr;
2333 			r[r_cnt].rng_cnt = 0;
2334 			r[r_cnt].rng_level = ht->ht_level;
2335 			++r_cnt;
2336 		}
2337 
2338 		/*
2339 		 * Unload one mapping from the page tables.
2340 		 */
2341 		entry = htable_va2entry(vaddr, ht);
2342 		hat_pte_unmap(ht, entry, flags, old_pte, NULL);
2343 		ASSERT(ht->ht_level <= mmu.max_page_level);
2344 		vaddr += LEVEL_SIZE(ht->ht_level);
2345 		contig_va = vaddr;
2346 		++r[r_cnt - 1].rng_cnt;
2347 	}
2348 	if (ht)
2349 		htable_release(ht);
2350 
2351 	/*
2352 	 * handle last range for callbacks
2353 	 */
2354 	if (r_cnt > 0)
2355 		handle_ranges(cb, r_cnt, r);
2356 	XPV_ALLOW_MIGRATE();
2357 }
2358 
2359 /*
2360  * synchronize mapping with software data structures
2361  *
2362  * This interface is currently only used by the working set monitor
2363  * driver.
2364  */
2365 /*ARGSUSED*/
2366 void
2367 hat_sync(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2368 {
2369 	uintptr_t	vaddr = (uintptr_t)addr;
2370 	uintptr_t	eaddr = vaddr + len;
2371 	htable_t	*ht = NULL;
2372 	uint_t		entry;
2373 	x86pte_t	pte;
2374 	x86pte_t	save_pte;
2375 	x86pte_t	new;
2376 	page_t		*pp;
2377 
2378 	ASSERT(!IN_VA_HOLE(vaddr));
2379 	ASSERT(IS_PAGEALIGNED(vaddr));
2380 	ASSERT(IS_PAGEALIGNED(eaddr));
2381 	ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
2382 
2383 	XPV_DISALLOW_MIGRATE();
2384 	for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2385 try_again:
2386 		pte = htable_walk(hat, &ht, &vaddr, eaddr);
2387 		if (ht == NULL)
2388 			break;
2389 		entry = htable_va2entry(vaddr, ht);
2390 
2391 		if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC ||
2392 		    PTE_GET(pte, PT_REF | PT_MOD) == 0)
2393 			continue;
2394 
2395 		/*
2396 		 * We need to acquire the mapping list lock to protect
2397 		 * against hat_pageunload(), hat_unload(), etc.
2398 		 */
2399 		pp = page_numtopp_nolock(PTE2PFN(pte, ht->ht_level));
2400 		if (pp == NULL)
2401 			break;
2402 		x86_hm_enter(pp);
2403 		save_pte = pte;
2404 		pte = x86pte_get(ht, entry);
2405 		if (pte != save_pte) {
2406 			x86_hm_exit(pp);
2407 			goto try_again;
2408 		}
2409 		if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC ||
2410 		    PTE_GET(pte, PT_REF | PT_MOD) == 0) {
2411 			x86_hm_exit(pp);
2412 			continue;
2413 		}
2414 
2415 		/*
2416 		 * Need to clear ref or mod bits. We may compete with
2417 		 * hardware updating the R/M bits and have to try again.
2418 		 */
2419 		if (flags == HAT_SYNC_ZERORM) {
2420 			new = pte;
2421 			PTE_CLR(new, PT_REF | PT_MOD);
2422 			pte = hati_update_pte(ht, entry, pte, new);
2423 			if (pte != 0) {
2424 				x86_hm_exit(pp);
2425 				goto try_again;
2426 			}
2427 		} else {
2428 			/*
2429 			 * sync the PTE to the page_t
2430 			 */
2431 			hati_sync_pte_to_page(pp, save_pte, ht->ht_level);
2432 		}
2433 		x86_hm_exit(pp);
2434 	}
2435 	if (ht)
2436 		htable_release(ht);
2437 	XPV_ALLOW_MIGRATE();
2438 }
2439 
2440 /*
2441  * void	hat_map(hat, addr, len, flags)
2442  */
2443 /*ARGSUSED*/
2444 void
2445 hat_map(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2446 {
2447 	/* does nothing */
2448 }
2449 
2450 /*
2451  * uint_t hat_getattr(hat, addr, *attr)
2452  *	returns attr for <hat,addr> in *attr.  returns 0 if there was a
2453  *	mapping and *attr is valid, nonzero if there was no mapping and
2454  *	*attr is not valid.
2455  */
2456 uint_t
2457 hat_getattr(hat_t *hat, caddr_t addr, uint_t *attr)
2458 {
2459 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2460 	htable_t	*ht = NULL;
2461 	x86pte_t	pte;
2462 
2463 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2464 
2465 	if (IN_VA_HOLE(vaddr))
2466 		return ((uint_t)-1);
2467 
2468 	ht = htable_getpte(hat, vaddr, NULL, &pte, mmu.max_page_level);
2469 	if (ht == NULL)
2470 		return ((uint_t)-1);
2471 
2472 	if (!PTE_ISVALID(pte) || !PTE_ISPAGE(pte, ht->ht_level)) {
2473 		htable_release(ht);
2474 		return ((uint_t)-1);
2475 	}
2476 
2477 	*attr = PROT_READ;
2478 	if (PTE_GET(pte, PT_WRITABLE))
2479 		*attr |= PROT_WRITE;
2480 	if (PTE_GET(pte, PT_USER))
2481 		*attr |= PROT_USER;
2482 	if (!PTE_GET(pte, mmu.pt_nx))
2483 		*attr |= PROT_EXEC;
2484 	if (PTE_GET(pte, PT_SOFTWARE) >= PT_NOSYNC)
2485 		*attr |= HAT_NOSYNC;
2486 	htable_release(ht);
2487 	return (0);
2488 }
2489 
2490 /*
2491  * hat_updateattr() applies the given attribute change to an existing mapping
2492  */
2493 #define	HAT_LOAD_ATTR		1
2494 #define	HAT_SET_ATTR		2
2495 #define	HAT_CLR_ATTR		3
2496 
2497 static void
2498 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what)
2499 {
2500 	uintptr_t	vaddr = (uintptr_t)addr;
2501 	uintptr_t	eaddr = (uintptr_t)addr + len;
2502 	htable_t	*ht = NULL;
2503 	uint_t		entry;
2504 	x86pte_t	oldpte, newpte;
2505 	page_t		*pp;
2506 
2507 	XPV_DISALLOW_MIGRATE();
2508 	ASSERT(IS_PAGEALIGNED(vaddr));
2509 	ASSERT(IS_PAGEALIGNED(eaddr));
2510 	ASSERT(hat == kas.a_hat ||
2511 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2512 	for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2513 try_again:
2514 		oldpte = htable_walk(hat, &ht, &vaddr, eaddr);
2515 		if (ht == NULL)
2516 			break;
2517 		if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST)
2518 			continue;
2519 
2520 		pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level));
2521 		if (pp == NULL)
2522 			continue;
2523 		x86_hm_enter(pp);
2524 
2525 		newpte = oldpte;
2526 		/*
2527 		 * We found a page table entry in the desired range,
2528 		 * figure out the new attributes.
2529 		 */
2530 		if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) {
2531 			if ((attr & PROT_WRITE) &&
2532 			    !PTE_GET(oldpte, PT_WRITABLE))
2533 				newpte |= PT_WRITABLE;
2534 
2535 			if ((attr & HAT_NOSYNC) &&
2536 			    PTE_GET(oldpte, PT_SOFTWARE) < PT_NOSYNC)
2537 				newpte |= PT_NOSYNC;
2538 
2539 			if ((attr & PROT_EXEC) && PTE_GET(oldpte, mmu.pt_nx))
2540 				newpte &= ~mmu.pt_nx;
2541 		}
2542 
2543 		if (what == HAT_LOAD_ATTR) {
2544 			if (!(attr & PROT_WRITE) &&
2545 			    PTE_GET(oldpte, PT_WRITABLE))
2546 				newpte &= ~PT_WRITABLE;
2547 
2548 			if (!(attr & HAT_NOSYNC) &&
2549 			    PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC)
2550 				newpte &= ~PT_SOFTWARE;
2551 
2552 			if (!(attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx))
2553 				newpte |= mmu.pt_nx;
2554 		}
2555 
2556 		if (what == HAT_CLR_ATTR) {
2557 			if ((attr & PROT_WRITE) && PTE_GET(oldpte, PT_WRITABLE))
2558 				newpte &= ~PT_WRITABLE;
2559 
2560 			if ((attr & HAT_NOSYNC) &&
2561 			    PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOSYNC)
2562 				newpte &= ~PT_SOFTWARE;
2563 
2564 			if ((attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx))
2565 				newpte |= mmu.pt_nx;
2566 		}
2567 
2568 		/*
2569 		 * Ensure NOSYNC/NOCONSIST mappings have REF and MOD set.
2570 		 * x86pte_set() depends on this.
2571 		 */
2572 		if (PTE_GET(newpte, PT_SOFTWARE) >= PT_NOSYNC)
2573 			newpte |= PT_REF | PT_MOD;
2574 
2575 		/*
2576 		 * what about PROT_READ or others? this code only handles:
2577 		 * EXEC, WRITE, NOSYNC
2578 		 */
2579 
2580 		/*
2581 		 * If new PTE really changed, update the table.
2582 		 */
2583 		if (newpte != oldpte) {
2584 			entry = htable_va2entry(vaddr, ht);
2585 			oldpte = hati_update_pte(ht, entry, oldpte, newpte);
2586 			if (oldpte != 0) {
2587 				x86_hm_exit(pp);
2588 				goto try_again;
2589 			}
2590 		}
2591 		x86_hm_exit(pp);
2592 	}
2593 	if (ht)
2594 		htable_release(ht);
2595 	XPV_ALLOW_MIGRATE();
2596 }
2597 
2598 /*
2599  * Various wrappers for hat_updateattr()
2600  */
2601 void
2602 hat_setattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2603 {
2604 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2605 	hat_updateattr(hat, addr, len, attr, HAT_SET_ATTR);
2606 }
2607 
2608 void
2609 hat_clrattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2610 {
2611 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2612 	hat_updateattr(hat, addr, len, attr, HAT_CLR_ATTR);
2613 }
2614 
2615 void
2616 hat_chgattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2617 {
2618 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2619 	hat_updateattr(hat, addr, len, attr, HAT_LOAD_ATTR);
2620 }
2621 
2622 void
2623 hat_chgprot(hat_t *hat, caddr_t addr, size_t len, uint_t vprot)
2624 {
2625 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit);
2626 	hat_updateattr(hat, addr, len, vprot & HAT_PROT_MASK, HAT_LOAD_ATTR);
2627 }
2628 
2629 /*
2630  * size_t hat_getpagesize(hat, addr)
2631  *	returns pagesize in bytes for <hat, addr>. returns -1 of there is
2632  *	no mapping. This is an advisory call.
2633  */
2634 ssize_t
2635 hat_getpagesize(hat_t *hat, caddr_t addr)
2636 {
2637 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2638 	htable_t	*ht;
2639 	size_t		pagesize;
2640 
2641 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2642 	if (IN_VA_HOLE(vaddr))
2643 		return (-1);
2644 	ht = htable_getpage(hat, vaddr, NULL);
2645 	if (ht == NULL)
2646 		return (-1);
2647 	pagesize = LEVEL_SIZE(ht->ht_level);
2648 	htable_release(ht);
2649 	return (pagesize);
2650 }
2651 
2652 
2653 
2654 /*
2655  * pfn_t hat_getpfnum(hat, addr)
2656  *	returns pfn for <hat, addr> or PFN_INVALID if mapping is invalid.
2657  */
2658 pfn_t
2659 hat_getpfnum(hat_t *hat, caddr_t addr)
2660 {
2661 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2662 	htable_t	*ht;
2663 	uint_t		entry;
2664 	pfn_t		pfn = PFN_INVALID;
2665 
2666 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2667 	if (khat_running == 0)
2668 		return (PFN_INVALID);
2669 
2670 	if (IN_VA_HOLE(vaddr))
2671 		return (PFN_INVALID);
2672 
2673 	XPV_DISALLOW_MIGRATE();
2674 	/*
2675 	 * A very common use of hat_getpfnum() is from the DDI for kernel pages.
2676 	 * Use the kmap_ptes (which also covers the 32 bit heap) to speed
2677 	 * this up.
2678 	 */
2679 	if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2680 		x86pte_t pte;
2681 		pgcnt_t pg_index;
2682 
2683 		pg_index = mmu_btop(vaddr - mmu.kmap_addr);
2684 		pte = GET_PTE(PT_INDEX_PTR(mmu.kmap_ptes, pg_index));
2685 		if (PTE_ISVALID(pte))
2686 			/*LINTED [use of constant 0 causes a lint warning] */
2687 			pfn = PTE2PFN(pte, 0);
2688 		XPV_ALLOW_MIGRATE();
2689 		return (pfn);
2690 	}
2691 
2692 	ht = htable_getpage(hat, vaddr, &entry);
2693 	if (ht == NULL) {
2694 		XPV_ALLOW_MIGRATE();
2695 		return (PFN_INVALID);
2696 	}
2697 	ASSERT(vaddr >= ht->ht_vaddr);
2698 	ASSERT(vaddr <= HTABLE_LAST_PAGE(ht));
2699 	pfn = PTE2PFN(x86pte_get(ht, entry), ht->ht_level);
2700 	if (ht->ht_level > 0)
2701 		pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level));
2702 	htable_release(ht);
2703 	XPV_ALLOW_MIGRATE();
2704 	return (pfn);
2705 }
2706 
2707 /*
2708  * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged.
2709  * Use hat_getpfnum(kas.a_hat, ...) instead.
2710  *
2711  * We'd like to return PFN_INVALID if the mappings have underlying page_t's
2712  * but can't right now due to the fact that some software has grown to use
2713  * this interface incorrectly. So for now when the interface is misused,
2714  * return a warning to the user that in the future it won't work in the
2715  * way they're abusing it, and carry on.
2716  *
2717  * Note that hat_getkpfnum() is never supported on amd64.
2718  */
2719 #if !defined(__amd64)
2720 pfn_t
2721 hat_getkpfnum(caddr_t addr)
2722 {
2723 	pfn_t	pfn;
2724 	int badcaller = 0;
2725 
2726 	if (khat_running == 0)
2727 		panic("hat_getkpfnum(): called too early\n");
2728 	if ((uintptr_t)addr < kernelbase)
2729 		return (PFN_INVALID);
2730 
2731 	XPV_DISALLOW_MIGRATE();
2732 	if (segkpm && IS_KPM_ADDR(addr)) {
2733 		badcaller = 1;
2734 		pfn = hat_kpm_va2pfn(addr);
2735 	} else {
2736 		pfn = hat_getpfnum(kas.a_hat, addr);
2737 		badcaller = pf_is_memory(pfn);
2738 	}
2739 
2740 	if (badcaller)
2741 		hat_getkpfnum_badcall(caller());
2742 	XPV_ALLOW_MIGRATE();
2743 	return (pfn);
2744 }
2745 #endif /* __amd64 */
2746 
2747 /*
2748  * int hat_probe(hat, addr)
2749  *	return 0 if no valid mapping is present.  Faster version
2750  *	of hat_getattr in certain architectures.
2751  */
2752 int
2753 hat_probe(hat_t *hat, caddr_t addr)
2754 {
2755 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2756 	uint_t		entry;
2757 	htable_t	*ht;
2758 	pgcnt_t		pg_off;
2759 
2760 	ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2761 	ASSERT(hat == kas.a_hat ||
2762 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2763 	if (IN_VA_HOLE(vaddr))
2764 		return (0);
2765 
2766 	/*
2767 	 * Most common use of hat_probe is from segmap. We special case it
2768 	 * for performance.
2769 	 */
2770 	if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2771 		pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2772 		if (mmu.pae_hat)
2773 			return (PTE_ISVALID(mmu.kmap_ptes[pg_off]));
2774 		else
2775 			return (PTE_ISVALID(
2776 			    ((x86pte32_t *)mmu.kmap_ptes)[pg_off]));
2777 	}
2778 
2779 	ht = htable_getpage(hat, vaddr, &entry);
2780 	htable_release(ht);
2781 	return (ht != NULL);
2782 }
2783 
2784 /*
2785  * Find out if the segment for hat_share()/hat_unshare() is DISM or locked ISM.
2786  */
2787 static int
2788 is_it_dism(hat_t *hat, caddr_t va)
2789 {
2790 	struct seg *seg;
2791 	struct shm_data *shmd;
2792 	struct spt_data *sptd;
2793 
2794 	seg = as_findseg(hat->hat_as, va, 0);
2795 	ASSERT(seg != NULL);
2796 	ASSERT(seg->s_base <= va);
2797 	shmd = (struct shm_data *)seg->s_data;
2798 	ASSERT(shmd != NULL);
2799 	sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2800 	ASSERT(sptd != NULL);
2801 	if (sptd->spt_flags & SHM_PAGEABLE)
2802 		return (1);
2803 	return (0);
2804 }
2805 
2806 /*
2807  * Simple implementation of ISM. hat_share() is similar to hat_memload_array(),
2808  * except that we use the ism_hat's existing mappings to determine the pages
2809  * and protections to use for this hat. If we find a full properly aligned
2810  * and sized pagetable, we will attempt to share the pagetable itself.
2811  */
2812 /*ARGSUSED*/
2813 int
2814 hat_share(
2815 	hat_t		*hat,
2816 	caddr_t		addr,
2817 	hat_t		*ism_hat,
2818 	caddr_t		src_addr,
2819 	size_t		len,	/* almost useless value, see below.. */
2820 	uint_t		ismszc)
2821 {
2822 	uintptr_t	vaddr_start = (uintptr_t)addr;
2823 	uintptr_t	vaddr;
2824 	uintptr_t	eaddr = vaddr_start + len;
2825 	uintptr_t	ism_addr_start = (uintptr_t)src_addr;
2826 	uintptr_t	ism_addr = ism_addr_start;
2827 	uintptr_t	e_ism_addr = ism_addr + len;
2828 	htable_t	*ism_ht = NULL;
2829 	htable_t	*ht;
2830 	x86pte_t	pte;
2831 	page_t		*pp;
2832 	pfn_t		pfn;
2833 	level_t		l;
2834 	pgcnt_t		pgcnt;
2835 	uint_t		prot;
2836 	int		is_dism;
2837 	int		flags;
2838 
2839 	/*
2840 	 * We might be asked to share an empty DISM hat by as_dup()
2841 	 */
2842 	ASSERT(hat != kas.a_hat);
2843 	ASSERT(eaddr <= _userlimit);
2844 	if (!(ism_hat->hat_flags & HAT_SHARED)) {
2845 		ASSERT(hat_get_mapped_size(ism_hat) == 0);
2846 		return (0);
2847 	}
2848 	XPV_DISALLOW_MIGRATE();
2849 
2850 	/*
2851 	 * The SPT segment driver often passes us a size larger than there are
2852 	 * valid mappings. That's because it rounds the segment size up to a
2853 	 * large pagesize, even if the actual memory mapped by ism_hat is less.
2854 	 */
2855 	ASSERT(IS_PAGEALIGNED(vaddr_start));
2856 	ASSERT(IS_PAGEALIGNED(ism_addr_start));
2857 	ASSERT(ism_hat->hat_flags & HAT_SHARED);
2858 	is_dism = is_it_dism(hat, addr);
2859 	while (ism_addr < e_ism_addr) {
2860 		/*
2861 		 * use htable_walk to get the next valid ISM mapping
2862 		 */
2863 		pte = htable_walk(ism_hat, &ism_ht, &ism_addr, e_ism_addr);
2864 		if (ism_ht == NULL)
2865 			break;
2866 
2867 		/*
2868 		 * First check to see if we already share the page table.
2869 		 */
2870 		l = ism_ht->ht_level;
2871 		vaddr = vaddr_start + (ism_addr - ism_addr_start);
2872 		ht = htable_lookup(hat, vaddr, l);
2873 		if (ht != NULL) {
2874 			if (ht->ht_flags & HTABLE_SHARED_PFN)
2875 				goto shared;
2876 			htable_release(ht);
2877 			goto not_shared;
2878 		}
2879 
2880 		/*
2881 		 * Can't ever share top table.
2882 		 */
2883 		if (l == mmu.max_level)
2884 			goto not_shared;
2885 
2886 		/*
2887 		 * Avoid level mismatches later due to DISM faults.
2888 		 */
2889 		if (is_dism && l > 0)
2890 			goto not_shared;
2891 
2892 		/*
2893 		 * addresses and lengths must align
2894 		 * table must be fully populated
2895 		 * no lower level page tables
2896 		 */
2897 		if (ism_addr != ism_ht->ht_vaddr ||
2898 		    (vaddr & LEVEL_OFFSET(l + 1)) != 0)
2899 			goto not_shared;
2900 
2901 		/*
2902 		 * The range of address space must cover a full table.
2903 		 */
2904 		if (e_ism_addr - ism_addr < LEVEL_SIZE(l + 1))
2905 			goto not_shared;
2906 
2907 		/*
2908 		 * All entries in the ISM page table must be leaf PTEs.
2909 		 */
2910 		if (l > 0) {
2911 			int e;
2912 
2913 			/*
2914 			 * We know the 0th is from htable_walk() above.
2915 			 */
2916 			for (e = 1; e < HTABLE_NUM_PTES(ism_ht); ++e) {
2917 				x86pte_t pte;
2918 				pte = x86pte_get(ism_ht, e);
2919 				if (!PTE_ISPAGE(pte, l))
2920 					goto not_shared;
2921 			}
2922 		}
2923 
2924 		/*
2925 		 * share the page table
2926 		 */
2927 		ht = htable_create(hat, vaddr, l, ism_ht);
2928 shared:
2929 		ASSERT(ht->ht_flags & HTABLE_SHARED_PFN);
2930 		ASSERT(ht->ht_shares == ism_ht);
2931 		hat->hat_ism_pgcnt +=
2932 		    (ism_ht->ht_valid_cnt - ht->ht_valid_cnt) <<
2933 		    (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT);
2934 		ht->ht_valid_cnt = ism_ht->ht_valid_cnt;
2935 		htable_release(ht);
2936 		ism_addr = ism_ht->ht_vaddr + LEVEL_SIZE(l + 1);
2937 		htable_release(ism_ht);
2938 		ism_ht = NULL;
2939 		continue;
2940 
2941 not_shared:
2942 		/*
2943 		 * Unable to share the page table. Instead we will
2944 		 * create new mappings from the values in the ISM mappings.
2945 		 * Figure out what level size mappings to use;
2946 		 */
2947 		for (l = ism_ht->ht_level; l > 0; --l) {
2948 			if (LEVEL_SIZE(l) <= eaddr - vaddr &&
2949 			    (vaddr & LEVEL_OFFSET(l)) == 0)
2950 				break;
2951 		}
2952 
2953 		/*
2954 		 * The ISM mapping might be larger than the share area,
2955 		 * be careful to truncate it if needed.
2956 		 */
2957 		if (eaddr - vaddr >= LEVEL_SIZE(ism_ht->ht_level)) {
2958 			pgcnt = mmu_btop(LEVEL_SIZE(ism_ht->ht_level));
2959 		} else {
2960 			pgcnt = mmu_btop(eaddr - vaddr);
2961 			l = 0;
2962 		}
2963 
2964 		pfn = PTE2PFN(pte, ism_ht->ht_level);
2965 		ASSERT(pfn != PFN_INVALID);
2966 		while (pgcnt > 0) {
2967 			/*
2968 			 * Make a new pte for the PFN for this level.
2969 			 * Copy protections for the pte from the ISM pte.
2970 			 */
2971 			pp = page_numtopp_nolock(pfn);
2972 			ASSERT(pp != NULL);
2973 
2974 			prot = PROT_USER | PROT_READ | HAT_UNORDERED_OK;
2975 			if (PTE_GET(pte, PT_WRITABLE))
2976 				prot |= PROT_WRITE;
2977 			if (!PTE_GET(pte, PT_NX))
2978 				prot |= PROT_EXEC;
2979 
2980 			flags = HAT_LOAD;
2981 			if (!is_dism)
2982 				flags |= HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST;
2983 			while (hati_load_common(hat, vaddr, pp, prot, flags,
2984 			    l, pfn) != 0) {
2985 				if (l == 0)
2986 					panic("hati_load_common() failure");
2987 				--l;
2988 			}
2989 
2990 			vaddr += LEVEL_SIZE(l);
2991 			ism_addr += LEVEL_SIZE(l);
2992 			pfn += mmu_btop(LEVEL_SIZE(l));
2993 			pgcnt -= mmu_btop(LEVEL_SIZE(l));
2994 		}
2995 	}
2996 	if (ism_ht != NULL)
2997 		htable_release(ism_ht);
2998 	XPV_ALLOW_MIGRATE();
2999 	return (0);
3000 }
3001 
3002 
3003 /*
3004  * hat_unshare() is similar to hat_unload_callback(), but
3005  * we have to look for empty shared pagetables. Note that
3006  * hat_unshare() is always invoked against an entire segment.
3007  */
3008 /*ARGSUSED*/
3009 void
3010 hat_unshare(hat_t *hat, caddr_t addr, size_t len, uint_t ismszc)
3011 {
3012 	uint64_t	vaddr = (uintptr_t)addr;
3013 	uintptr_t	eaddr = vaddr + len;
3014 	htable_t	*ht = NULL;
3015 	uint_t		need_demaps = 0;
3016 	int		flags = HAT_UNLOAD_UNMAP;
3017 	level_t		l;
3018 
3019 	ASSERT(hat != kas.a_hat);
3020 	ASSERT(eaddr <= _userlimit);
3021 	ASSERT(IS_PAGEALIGNED(vaddr));
3022 	ASSERT(IS_PAGEALIGNED(eaddr));
3023 	XPV_DISALLOW_MIGRATE();
3024 
3025 	/*
3026 	 * First go through and remove any shared pagetables.
3027 	 *
3028 	 * Note that it's ok to delay the TLB shootdown till the entire range is
3029 	 * finished, because if hat_pageunload() were to unload a shared
3030 	 * pagetable page, its hat_tlb_inval() will do a global TLB invalidate.
3031 	 */
3032 	l = mmu.max_page_level;
3033 	if (l == mmu.max_level)
3034 		--l;
3035 	for (; l >= 0; --l) {
3036 		for (vaddr = (uintptr_t)addr; vaddr < eaddr;
3037 		    vaddr = (vaddr & LEVEL_MASK(l + 1)) + LEVEL_SIZE(l + 1)) {
3038 			ASSERT(!IN_VA_HOLE(vaddr));
3039 			/*
3040 			 * find a pagetable that maps the current address
3041 			 */
3042 			ht = htable_lookup(hat, vaddr, l);
3043 			if (ht == NULL)
3044 				continue;
3045 			if (ht->ht_flags & HTABLE_SHARED_PFN) {
3046 				/*
3047 				 * clear page count, set valid_cnt to 0,
3048 				 * let htable_release() finish the job
3049 				 */
3050 				hat->hat_ism_pgcnt -= ht->ht_valid_cnt <<
3051 				    (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT);
3052 				ht->ht_valid_cnt = 0;
3053 				need_demaps = 1;
3054 			}
3055 			htable_release(ht);
3056 		}
3057 	}
3058 
3059 	/*
3060 	 * flush the TLBs - since we're probably dealing with MANY mappings
3061 	 * we do just one CR3 reload.
3062 	 */
3063 	if (!(hat->hat_flags & HAT_FREEING) && need_demaps)
3064 		hat_tlb_inval(hat, DEMAP_ALL_ADDR);
3065 
3066 	/*
3067 	 * Now go back and clean up any unaligned mappings that
3068 	 * couldn't share pagetables.
3069 	 */
3070 	if (!is_it_dism(hat, addr))
3071 		flags |= HAT_UNLOAD_UNLOCK;
3072 	hat_unload(hat, addr, len, flags);
3073 	XPV_ALLOW_MIGRATE();
3074 }
3075 
3076 
3077 /*
3078  * hat_reserve() does nothing
3079  */
3080 /*ARGSUSED*/
3081 void
3082 hat_reserve(struct as *as, caddr_t addr, size_t len)
3083 {
3084 }
3085 
3086 
3087 /*
3088  * Called when all mappings to a page should have write permission removed.
3089  * Mostly stolem from hat_pagesync()
3090  */
3091 static void
3092 hati_page_clrwrt(struct page *pp)
3093 {
3094 	hment_t		*hm = NULL;
3095 	htable_t	*ht;
3096 	uint_t		entry;
3097 	x86pte_t	old;
3098 	x86pte_t	new;
3099 	uint_t		pszc = 0;
3100 
3101 	XPV_DISALLOW_MIGRATE();
3102 next_size:
3103 	/*
3104 	 * walk thru the mapping list clearing write permission
3105 	 */
3106 	x86_hm_enter(pp);
3107 	while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) {
3108 		if (ht->ht_level < pszc)
3109 			continue;
3110 		old = x86pte_get(ht, entry);
3111 
3112 		for (;;) {
3113 			/*
3114 			 * Is this mapping of interest?
3115 			 */
3116 			if (PTE2PFN(old, ht->ht_level) != pp->p_pagenum ||
3117 			    PTE_GET(old, PT_WRITABLE) == 0)
3118 				break;
3119 
3120 			/*
3121 			 * Clear ref/mod writable bits. This requires cross
3122 			 * calls to ensure any executing TLBs see cleared bits.
3123 			 */
3124 			new = old;
3125 			PTE_CLR(new, PT_REF | PT_MOD | PT_WRITABLE);
3126 			old = hati_update_pte(ht, entry, old, new);
3127 			if (old != 0)
3128 				continue;
3129 
3130 			break;
3131 		}
3132 	}
3133 	x86_hm_exit(pp);
3134 	while (pszc < pp->p_szc) {
3135 		page_t *tpp;
3136 		pszc++;
3137 		tpp = PP_GROUPLEADER(pp, pszc);
3138 		if (pp != tpp) {
3139 			pp = tpp;
3140 			goto next_size;
3141 		}
3142 	}
3143 	XPV_ALLOW_MIGRATE();
3144 }
3145 
3146 /*
3147  * void hat_page_setattr(pp, flag)
3148  * void hat_page_clrattr(pp, flag)
3149  *	used to set/clr ref/mod bits.
3150  */
3151 void
3152 hat_page_setattr(struct page *pp, uint_t flag)
3153 {
3154 	vnode_t		*vp = pp->p_vnode;
3155 	kmutex_t	*vphm = NULL;
3156 	page_t		**listp;
3157 	int		noshuffle;
3158 
3159 	noshuffle = flag & P_NSH;
3160 	flag &= ~P_NSH;
3161 
3162 	if (PP_GETRM(pp, flag) == flag)
3163 		return;
3164 
3165 	if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp) &&
3166 	    !noshuffle) {
3167 		vphm = page_vnode_mutex(vp);
3168 		mutex_enter(vphm);
3169 	}
3170 
3171 	PP_SETRM(pp, flag);
3172 
3173 	if (vphm != NULL) {
3174 
3175 		/*
3176 		 * Some File Systems examine v_pages for NULL w/o
3177 		 * grabbing the vphm mutex. Must not let it become NULL when
3178 		 * pp is the only page on the list.
3179 		 */
3180 		if (pp->p_vpnext != pp) {
3181 			page_vpsub(&vp->v_pages, pp);
3182 			if (vp->v_pages != NULL)
3183 				listp = &vp->v_pages->p_vpprev->p_vpnext;
3184 			else
3185 				listp = &vp->v_pages;
3186 			page_vpadd(listp, pp);
3187 		}
3188 		mutex_exit(vphm);
3189 	}
3190 }
3191 
3192 void
3193 hat_page_clrattr(struct page *pp, uint_t flag)
3194 {
3195 	vnode_t		*vp = pp->p_vnode;
3196 	ASSERT(!(flag & ~(P_MOD | P_REF | P_RO)));
3197 
3198 	/*
3199 	 * Caller is expected to hold page's io lock for VMODSORT to work
3200 	 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod
3201 	 * bit is cleared.
3202 	 * We don't have assert to avoid tripping some existing third party
3203 	 * code. The dirty page is moved back to top of the v_page list
3204 	 * after IO is done in pvn_write_done().
3205 	 */
3206 	PP_CLRRM(pp, flag);
3207 
3208 	if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) {
3209 
3210 		/*
3211 		 * VMODSORT works by removing write permissions and getting
3212 		 * a fault when a page is made dirty. At this point
3213 		 * we need to remove write permission from all mappings
3214 		 * to this page.
3215 		 */
3216 		hati_page_clrwrt(pp);
3217 	}
3218 }
3219 
3220 /*
3221  *	If flag is specified, returns 0 if attribute is disabled
3222  *	and non zero if enabled.  If flag specifes multiple attributs
3223  *	then returns 0 if ALL atriibutes are disabled.  This is an advisory
3224  *	call.
3225  */
3226 uint_t
3227 hat_page_getattr(struct page *pp, uint_t flag)
3228 {
3229 	return (PP_GETRM(pp, flag));
3230 }
3231 
3232 
3233 /*
3234  * common code used by hat_pageunload() and hment_steal()
3235  */
3236 hment_t *
3237 hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry)
3238 {
3239 	x86pte_t old_pte;
3240 	pfn_t pfn = pp->p_pagenum;
3241 	hment_t *hm;
3242 
3243 	/*
3244 	 * We need to acquire a hold on the htable in order to
3245 	 * do the invalidate. We know the htable must exist, since
3246 	 * unmap's don't release the htable until after removing any
3247 	 * hment. Having x86_hm_enter() keeps that from proceeding.
3248 	 */
3249 	htable_acquire(ht);
3250 
3251 	/*
3252 	 * Invalidate the PTE and remove the hment.
3253 	 */
3254 	old_pte = x86pte_inval(ht, entry, 0, NULL);
3255 	if (PTE2PFN(old_pte, ht->ht_level) != pfn) {
3256 		panic("x86pte_inval() failure found PTE = " FMT_PTE
3257 		    " pfn being unmapped is %lx ht=0x%lx entry=0x%x",
3258 		    old_pte, pfn, (uintptr_t)ht, entry);
3259 	}
3260 
3261 	/*
3262 	 * Clean up all the htable information for this mapping
3263 	 */
3264 	ASSERT(ht->ht_valid_cnt > 0);
3265 	HTABLE_DEC(ht->ht_valid_cnt);
3266 	PGCNT_DEC(ht->ht_hat, ht->ht_level);
3267 
3268 	/*
3269 	 * sync ref/mod bits to the page_t
3270 	 */
3271 	if (PTE_GET(old_pte, PT_SOFTWARE) < PT_NOSYNC)
3272 		hati_sync_pte_to_page(pp, old_pte, ht->ht_level);
3273 
3274 	/*
3275 	 * Remove the mapping list entry for this page.
3276 	 */
3277 	hm = hment_remove(pp, ht, entry);
3278 
3279 	/*
3280 	 * drop the mapping list lock so that we might free the
3281 	 * hment and htable.
3282 	 */
3283 	x86_hm_exit(pp);
3284 	htable_release(ht);
3285 	return (hm);
3286 }
3287 
3288 extern int	vpm_enable;
3289 /*
3290  * Unload all translations to a page. If the page is a subpage of a large
3291  * page, the large page mappings are also removed.
3292  *
3293  * The forceflags are unused.
3294  */
3295 
3296 /*ARGSUSED*/
3297 static int
3298 hati_pageunload(struct page *pp, uint_t pg_szcd, uint_t forceflag)
3299 {
3300 	page_t		*cur_pp = pp;
3301 	hment_t		*hm;
3302 	hment_t		*prev;
3303 	htable_t	*ht;
3304 	uint_t		entry;
3305 	level_t		level;
3306 
3307 	XPV_DISALLOW_MIGRATE();
3308 #if defined(__amd64)
3309 	/*
3310 	 * clear the vpm ref.
3311 	 */
3312 	if (vpm_enable) {
3313 		pp->p_vpmref = 0;
3314 	}
3315 #endif
3316 	/*
3317 	 * The loop with next_size handles pages with multiple pagesize mappings
3318 	 */
3319 next_size:
3320 	for (;;) {
3321 
3322 		/*
3323 		 * Get a mapping list entry
3324 		 */
3325 		x86_hm_enter(cur_pp);
3326 		for (prev = NULL; ; prev = hm) {
3327 			hm = hment_walk(cur_pp, &ht, &entry, prev);
3328 			if (hm == NULL) {
3329 				x86_hm_exit(cur_pp);
3330 
3331 				/*
3332 				 * If not part of a larger page, we're done.
3333 				 */
3334 				if (cur_pp->p_szc <= pg_szcd) {
3335 					XPV_ALLOW_MIGRATE();
3336 					return (0);
3337 				}
3338 
3339 				/*
3340 				 * Else check the next larger page size.
3341 				 * hat_page_demote() may decrease p_szc
3342 				 * but that's ok we'll just take an extra
3343 				 * trip discover there're no larger mappings
3344 				 * and return.
3345 				 */
3346 				++pg_szcd;
3347 				cur_pp = PP_GROUPLEADER(cur_pp, pg_szcd);
3348 				goto next_size;
3349 			}
3350 
3351 			/*
3352 			 * If this mapping size matches, remove it.
3353 			 */
3354 			level = ht->ht_level;
3355 			if (level == pg_szcd)
3356 				break;
3357 		}
3358 
3359 		/*
3360 		 * Remove the mapping list entry for this page.
3361 		 * Note this does the x86_hm_exit() for us.
3362 		 */
3363 		hm = hati_page_unmap(cur_pp, ht, entry);
3364 		if (hm != NULL)
3365 			hment_free(hm);
3366 	}
3367 }
3368 
3369 int
3370 hat_pageunload(struct page *pp, uint_t forceflag)
3371 {
3372 	ASSERT(PAGE_EXCL(pp));
3373 	return (hati_pageunload(pp, 0, forceflag));
3374 }
3375 
3376 /*
3377  * Unload all large mappings to pp and reduce by 1 p_szc field of every large
3378  * page level that included pp.
3379  *
3380  * pp must be locked EXCL. Even though no other constituent pages are locked
3381  * it's legal to unload large mappings to pp because all constituent pages of
3382  * large locked mappings have to be locked SHARED.  therefore if we have EXCL
3383  * lock on one of constituent pages none of the large mappings to pp are
3384  * locked.
3385  *
3386  * Change (always decrease) p_szc field starting from the last constituent
3387  * page and ending with root constituent page so that root's pszc always shows
3388  * the area where hat_page_demote() may be active.
3389  *
3390  * This mechanism is only used for file system pages where it's not always
3391  * possible to get EXCL locks on all constituent pages to demote the size code
3392  * (as is done for anonymous or kernel large pages).
3393  */
3394 void
3395 hat_page_demote(page_t *pp)
3396 {
3397 	uint_t		pszc;
3398 	uint_t		rszc;
3399 	uint_t		szc;
3400 	page_t		*rootpp;
3401 	page_t		*firstpp;
3402 	page_t		*lastpp;
3403 	pgcnt_t		pgcnt;
3404 
3405 	ASSERT(PAGE_EXCL(pp));
3406 	ASSERT(!PP_ISFREE(pp));
3407 	ASSERT(page_szc_lock_assert(pp));
3408 
3409 	if (pp->p_szc == 0)
3410 		return;
3411 
3412 	rootpp = PP_GROUPLEADER(pp, 1);
3413 	(void) hati_pageunload(rootpp, 1, HAT_FORCE_PGUNLOAD);
3414 
3415 	/*
3416 	 * all large mappings to pp are gone
3417 	 * and no new can be setup since pp is locked exclusively.
3418 	 *
3419 	 * Lock the root to make sure there's only one hat_page_demote()
3420 	 * outstanding within the area of this root's pszc.
3421 	 *
3422 	 * Second potential hat_page_demote() is already eliminated by upper
3423 	 * VM layer via page_szc_lock() but we don't rely on it and use our
3424 	 * own locking (so that upper layer locking can be changed without
3425 	 * assumptions that hat depends on upper layer VM to prevent multiple
3426 	 * hat_page_demote() to be issued simultaneously to the same large
3427 	 * page).
3428 	 */
3429 again:
3430 	pszc = pp->p_szc;
3431 	if (pszc == 0)
3432 		return;
3433 	rootpp = PP_GROUPLEADER(pp, pszc);
3434 	x86_hm_enter(rootpp);
3435 	/*
3436 	 * If root's p_szc is different from pszc we raced with another
3437 	 * hat_page_demote().  Drop the lock and try to find the root again.
3438 	 * If root's p_szc is greater than pszc previous hat_page_demote() is
3439 	 * not done yet.  Take and release mlist lock of root's root to wait
3440 	 * for previous hat_page_demote() to complete.
3441 	 */
3442 	if ((rszc = rootpp->p_szc) != pszc) {
3443 		x86_hm_exit(rootpp);
3444 		if (rszc > pszc) {
3445 			/* p_szc of a locked non free page can't increase */
3446 			ASSERT(pp != rootpp);
3447 
3448 			rootpp = PP_GROUPLEADER(rootpp, rszc);
3449 			x86_hm_enter(rootpp);
3450 			x86_hm_exit(rootpp);
3451 		}
3452 		goto again;
3453 	}
3454 	ASSERT(pp->p_szc == pszc);
3455 
3456 	/*
3457 	 * Decrement by 1 p_szc of every constituent page of a region that
3458 	 * covered pp. For example if original szc is 3 it gets changed to 2
3459 	 * everywhere except in region 2 that covered pp. Region 2 that
3460 	 * covered pp gets demoted to 1 everywhere except in region 1 that
3461 	 * covered pp. The region 1 that covered pp is demoted to region
3462 	 * 0. It's done this way because from region 3 we removed level 3
3463 	 * mappings, from region 2 that covered pp we removed level 2 mappings
3464 	 * and from region 1 that covered pp we removed level 1 mappings.  All
3465 	 * changes are done from from high pfn's to low pfn's so that roots
3466 	 * are changed last allowing one to know the largest region where
3467 	 * hat_page_demote() is stil active by only looking at the root page.
3468 	 *
3469 	 * This algorithm is implemented in 2 while loops. First loop changes
3470 	 * p_szc of pages to the right of pp's level 1 region and second
3471 	 * loop changes p_szc of pages of level 1 region that covers pp
3472 	 * and all pages to the left of level 1 region that covers pp.
3473 	 * In the first loop p_szc keeps dropping with every iteration
3474 	 * and in the second loop it keeps increasing with every iteration.
3475 	 *
3476 	 * First loop description: Demote pages to the right of pp outside of
3477 	 * level 1 region that covers pp.  In every iteration of the while
3478 	 * loop below find the last page of szc region and the first page of
3479 	 * (szc - 1) region that is immediately to the right of (szc - 1)
3480 	 * region that covers pp.  From last such page to first such page
3481 	 * change every page's szc to szc - 1. Decrement szc and continue
3482 	 * looping until szc is 1. If pp belongs to the last (szc - 1) region
3483 	 * of szc region skip to the next iteration.
3484 	 */
3485 	szc = pszc;
3486 	while (szc > 1) {
3487 		lastpp = PP_GROUPLEADER(pp, szc);
3488 		pgcnt = page_get_pagecnt(szc);
3489 		lastpp += pgcnt - 1;
3490 		firstpp = PP_GROUPLEADER(pp, (szc - 1));
3491 		pgcnt = page_get_pagecnt(szc - 1);
3492 		if (lastpp - firstpp < pgcnt) {
3493 			szc--;
3494 			continue;
3495 		}
3496 		firstpp += pgcnt;
3497 		while (lastpp != firstpp) {
3498 			ASSERT(lastpp->p_szc == pszc);
3499 			lastpp->p_szc = szc - 1;
3500 			lastpp--;
3501 		}
3502 		firstpp->p_szc = szc - 1;
3503 		szc--;
3504 	}
3505 
3506 	/*
3507 	 * Second loop description:
3508 	 * First iteration changes p_szc to 0 of every
3509 	 * page of level 1 region that covers pp.
3510 	 * Subsequent iterations find last page of szc region
3511 	 * immediately to the left of szc region that covered pp
3512 	 * and first page of (szc + 1) region that covers pp.
3513 	 * From last to first page change p_szc of every page to szc.
3514 	 * Increment szc and continue looping until szc is pszc.
3515 	 * If pp belongs to the fist szc region of (szc + 1) region
3516 	 * skip to the next iteration.
3517 	 *
3518 	 */
3519 	szc = 0;
3520 	while (szc < pszc) {
3521 		firstpp = PP_GROUPLEADER(pp, (szc + 1));
3522 		if (szc == 0) {
3523 			pgcnt = page_get_pagecnt(1);
3524 			lastpp = firstpp + (pgcnt - 1);
3525 		} else {
3526 			lastpp = PP_GROUPLEADER(pp, szc);
3527 			if (firstpp == lastpp) {
3528 				szc++;
3529 				continue;
3530 			}
3531 			lastpp--;
3532 			pgcnt = page_get_pagecnt(szc);
3533 		}
3534 		while (lastpp != firstpp) {
3535 			ASSERT(lastpp->p_szc == pszc);
3536 			lastpp->p_szc = szc;
3537 			lastpp--;
3538 		}
3539 		firstpp->p_szc = szc;
3540 		if (firstpp == rootpp)
3541 			break;
3542 		szc++;
3543 	}
3544 	x86_hm_exit(rootpp);
3545 }
3546 
3547 /*
3548  * get hw stats from hardware into page struct and reset hw stats
3549  * returns attributes of page
3550  * Flags for hat_pagesync, hat_getstat, hat_sync
3551  *
3552  * define	HAT_SYNC_ZERORM		0x01
3553  *
3554  * Additional flags for hat_pagesync
3555  *
3556  * define	HAT_SYNC_STOPON_REF	0x02
3557  * define	HAT_SYNC_STOPON_MOD	0x04
3558  * define	HAT_SYNC_STOPON_RM	0x06
3559  * define	HAT_SYNC_STOPON_SHARED	0x08
3560  */
3561 uint_t
3562 hat_pagesync(struct page *pp, uint_t flags)
3563 {
3564 	hment_t		*hm = NULL;
3565 	htable_t	*ht;
3566 	uint_t		entry;
3567 	x86pte_t	old, save_old;
3568 	x86pte_t	new;
3569 	uchar_t		nrmbits = P_REF|P_MOD|P_RO;
3570 	extern ulong_t	po_share;
3571 	page_t		*save_pp = pp;
3572 	uint_t		pszc = 0;
3573 
3574 	ASSERT(PAGE_LOCKED(pp) || panicstr);
3575 
3576 	if (PP_ISRO(pp) && (flags & HAT_SYNC_STOPON_MOD))
3577 		return (pp->p_nrm & nrmbits);
3578 
3579 	if ((flags & HAT_SYNC_ZERORM) == 0) {
3580 
3581 		if ((flags & HAT_SYNC_STOPON_REF) != 0 && PP_ISREF(pp))
3582 			return (pp->p_nrm & nrmbits);
3583 
3584 		if ((flags & HAT_SYNC_STOPON_MOD) != 0 && PP_ISMOD(pp))
3585 			return (pp->p_nrm & nrmbits);
3586 
3587 		if ((flags & HAT_SYNC_STOPON_SHARED) != 0 &&
3588 		    hat_page_getshare(pp) > po_share) {
3589 			if (PP_ISRO(pp))
3590 				PP_SETREF(pp);
3591 			return (pp->p_nrm & nrmbits);
3592 		}
3593 	}
3594 
3595 	XPV_DISALLOW_MIGRATE();
3596 next_size:
3597 	/*
3598 	 * walk thru the mapping list syncing (and clearing) ref/mod bits.
3599 	 */
3600 	x86_hm_enter(pp);
3601 	while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) {
3602 		if (ht->ht_level < pszc)
3603 			continue;
3604 		old = x86pte_get(ht, entry);
3605 try_again:
3606 
3607 		ASSERT(PTE2PFN(old, ht->ht_level) == pp->p_pagenum);
3608 
3609 		if (PTE_GET(old, PT_REF | PT_MOD) == 0)
3610 			continue;
3611 
3612 		save_old = old;
3613 		if ((flags & HAT_SYNC_ZERORM) != 0) {
3614 
3615 			/*
3616 			 * Need to clear ref or mod bits. Need to demap
3617 			 * to make sure any executing TLBs see cleared bits.
3618 			 */
3619 			new = old;
3620 			PTE_CLR(new, PT_REF | PT_MOD);
3621 			old = hati_update_pte(ht, entry, old, new);
3622 			if (old != 0)
3623 				goto try_again;
3624 
3625 			old = save_old;
3626 		}
3627 
3628 		/*
3629 		 * Sync the PTE
3630 		 */
3631 		if (!(flags & HAT_SYNC_ZERORM) &&
3632 		    PTE_GET(old, PT_SOFTWARE) <= PT_NOSYNC)
3633 			hati_sync_pte_to_page(pp, old, ht->ht_level);
3634 
3635 		/*
3636 		 * can stop short if we found a ref'd or mod'd page
3637 		 */
3638 		if ((flags & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp) ||
3639 		    (flags & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp)) {
3640 			x86_hm_exit(pp);
3641 			goto done;
3642 		}
3643 	}
3644 	x86_hm_exit(pp);
3645 	while (pszc < pp->p_szc) {
3646 		page_t *tpp;
3647 		pszc++;
3648 		tpp = PP_GROUPLEADER(pp, pszc);
3649 		if (pp != tpp) {
3650 			pp = tpp;
3651 			goto next_size;
3652 		}
3653 	}
3654 done:
3655 	XPV_ALLOW_MIGRATE();
3656 	return (save_pp->p_nrm & nrmbits);
3657 }
3658 
3659 /*
3660  * returns approx number of mappings to this pp.  A return of 0 implies
3661  * there are no mappings to the page.
3662  */
3663 ulong_t
3664 hat_page_getshare(page_t *pp)
3665 {
3666 	uint_t cnt;
3667 	cnt = hment_mapcnt(pp);
3668 #if defined(__amd64)
3669 	if (vpm_enable && pp->p_vpmref) {
3670 		cnt += 1;
3671 	}
3672 #endif
3673 	return (cnt);
3674 }
3675 
3676 /*
3677  * Return 1 the number of mappings exceeds sh_thresh. Return 0
3678  * otherwise.
3679  */
3680 int
3681 hat_page_checkshare(page_t *pp, ulong_t sh_thresh)
3682 {
3683 	return (hat_page_getshare(pp) > sh_thresh);
3684 }
3685 
3686 /*
3687  * hat_softlock isn't supported anymore
3688  */
3689 /*ARGSUSED*/
3690 faultcode_t
3691 hat_softlock(
3692 	hat_t *hat,
3693 	caddr_t addr,
3694 	size_t *len,
3695 	struct page **page_array,
3696 	uint_t flags)
3697 {
3698 	return (FC_NOSUPPORT);
3699 }
3700 
3701 
3702 
3703 /*
3704  * Routine to expose supported HAT features to platform independent code.
3705  */
3706 /*ARGSUSED*/
3707 int
3708 hat_supported(enum hat_features feature, void *arg)
3709 {
3710 	switch (feature) {
3711 
3712 	case HAT_SHARED_PT:	/* this is really ISM */
3713 		return (1);
3714 
3715 	case HAT_DYNAMIC_ISM_UNMAP:
3716 		return (0);
3717 
3718 	case HAT_VMODSORT:
3719 		return (1);
3720 
3721 	case HAT_SHARED_REGIONS:
3722 		return (0);
3723 
3724 	default:
3725 		panic("hat_supported() - unknown feature");
3726 	}
3727 	return (0);
3728 }
3729 
3730 /*
3731  * Called when a thread is exiting and has been switched to the kernel AS
3732  */
3733 void
3734 hat_thread_exit(kthread_t *thd)
3735 {
3736 	ASSERT(thd->t_procp->p_as == &kas);
3737 	XPV_DISALLOW_MIGRATE();
3738 	hat_switch(thd->t_procp->p_as->a_hat);
3739 	XPV_ALLOW_MIGRATE();
3740 }
3741 
3742 /*
3743  * Setup the given brand new hat structure as the new HAT on this cpu's mmu.
3744  */
3745 /*ARGSUSED*/
3746 void
3747 hat_setup(hat_t *hat, int flags)
3748 {
3749 	XPV_DISALLOW_MIGRATE();
3750 	kpreempt_disable();
3751 
3752 	hat_switch(hat);
3753 
3754 	kpreempt_enable();
3755 	XPV_ALLOW_MIGRATE();
3756 }
3757 
3758 /*
3759  * Prepare for a CPU private mapping for the given address.
3760  *
3761  * The address can only be used from a single CPU and can be remapped
3762  * using hat_mempte_remap().  Return the address of the PTE.
3763  *
3764  * We do the htable_create() if necessary and increment the valid count so
3765  * the htable can't disappear.  We also hat_devload() the page table into
3766  * kernel so that the PTE is quickly accessed.
3767  */
3768 hat_mempte_t
3769 hat_mempte_setup(caddr_t addr)
3770 {
3771 	uintptr_t	va = (uintptr_t)addr;
3772 	htable_t	*ht;
3773 	uint_t		entry;
3774 	x86pte_t	oldpte;
3775 	hat_mempte_t	p;
3776 
3777 	ASSERT(IS_PAGEALIGNED(va));
3778 	ASSERT(!IN_VA_HOLE(va));
3779 	++curthread->t_hatdepth;
3780 	ht = htable_getpte(kas.a_hat, va, &entry, &oldpte, 0);
3781 	if (ht == NULL) {
3782 		ht = htable_create(kas.a_hat, va, 0, NULL);
3783 		entry = htable_va2entry(va, ht);
3784 		ASSERT(ht->ht_level == 0);
3785 		oldpte = x86pte_get(ht, entry);
3786 	}
3787 	if (PTE_ISVALID(oldpte))
3788 		panic("hat_mempte_setup(): address already mapped"
3789 		    "ht=%p, entry=%d, pte=" FMT_PTE, ht, entry, oldpte);
3790 
3791 	/*
3792 	 * increment ht_valid_cnt so that the pagetable can't disappear
3793 	 */
3794 	HTABLE_INC(ht->ht_valid_cnt);
3795 
3796 	/*
3797 	 * return the PTE physical address to the caller.
3798 	 */
3799 	htable_release(ht);
3800 	p = PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry);
3801 	--curthread->t_hatdepth;
3802 	return (p);
3803 }
3804 
3805 /*
3806  * Release a CPU private mapping for the given address.
3807  * We decrement the htable valid count so it might be destroyed.
3808  */
3809 /*ARGSUSED1*/
3810 void
3811 hat_mempte_release(caddr_t addr, hat_mempte_t pte_pa)
3812 {
3813 	htable_t	*ht;
3814 
3815 	/*
3816 	 * invalidate any left over mapping and decrement the htable valid count
3817 	 */
3818 #ifdef __xpv
3819 	if (HYPERVISOR_update_va_mapping((uintptr_t)addr, 0,
3820 	    UVMF_INVLPG | UVMF_LOCAL))
3821 		panic("HYPERVISOR_update_va_mapping() failed");
3822 #else
3823 	{
3824 		x86pte_t *pteptr;
3825 
3826 		pteptr = x86pte_mapin(mmu_btop(pte_pa),
3827 		    (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL);
3828 		if (mmu.pae_hat)
3829 			*pteptr = 0;
3830 		else
3831 			*(x86pte32_t *)pteptr = 0;
3832 		mmu_tlbflush_entry(addr);
3833 		x86pte_mapout();
3834 	}
3835 #endif
3836 
3837 	ht = htable_getpte(kas.a_hat, ALIGN2PAGE(addr), NULL, NULL, 0);
3838 	if (ht == NULL)
3839 		panic("hat_mempte_release(): invalid address");
3840 	ASSERT(ht->ht_level == 0);
3841 	HTABLE_DEC(ht->ht_valid_cnt);
3842 	htable_release(ht);
3843 }
3844 
3845 /*
3846  * Apply a temporary CPU private mapping to a page. We flush the TLB only
3847  * on this CPU, so this ought to have been called with preemption disabled.
3848  */
3849 void
3850 hat_mempte_remap(
3851 	pfn_t		pfn,
3852 	caddr_t		addr,
3853 	hat_mempte_t	pte_pa,
3854 	uint_t		attr,
3855 	uint_t		flags)
3856 {
3857 	uintptr_t	va = (uintptr_t)addr;
3858 	x86pte_t	pte;
3859 
3860 	/*
3861 	 * Remap the given PTE to the new page's PFN. Invalidate only
3862 	 * on this CPU.
3863 	 */
3864 #ifdef DEBUG
3865 	htable_t	*ht;
3866 	uint_t		entry;
3867 
3868 	ASSERT(IS_PAGEALIGNED(va));
3869 	ASSERT(!IN_VA_HOLE(va));
3870 	ht = htable_getpte(kas.a_hat, va, &entry, NULL, 0);
3871 	ASSERT(ht != NULL);
3872 	ASSERT(ht->ht_level == 0);
3873 	ASSERT(ht->ht_valid_cnt > 0);
3874 	ASSERT(ht->ht_pfn == mmu_btop(pte_pa));
3875 	htable_release(ht);
3876 #endif
3877 	XPV_DISALLOW_MIGRATE();
3878 	pte = hati_mkpte(pfn, attr, 0, flags);
3879 #ifdef __xpv
3880 	if (HYPERVISOR_update_va_mapping(va, pte, UVMF_INVLPG | UVMF_LOCAL))
3881 		panic("HYPERVISOR_update_va_mapping() failed");
3882 #else
3883 	{
3884 		x86pte_t *pteptr;
3885 
3886 		pteptr = x86pte_mapin(mmu_btop(pte_pa),
3887 		    (pte_pa & MMU_PAGEOFFSET) >> mmu.pte_size_shift, NULL);
3888 		if (mmu.pae_hat)
3889 			*(x86pte_t *)pteptr = pte;
3890 		else
3891 			*(x86pte32_t *)pteptr = (x86pte32_t)pte;
3892 		mmu_tlbflush_entry(addr);
3893 		x86pte_mapout();
3894 	}
3895 #endif
3896 	XPV_ALLOW_MIGRATE();
3897 }
3898 
3899 
3900 
3901 /*
3902  * Hat locking functions
3903  * XXX - these two functions are currently being used by hatstats
3904  * 	they can be removed by using a per-as mutex for hatstats.
3905  */
3906 void
3907 hat_enter(hat_t *hat)
3908 {
3909 	mutex_enter(&hat->hat_mutex);
3910 }
3911 
3912 void
3913 hat_exit(hat_t *hat)
3914 {
3915 	mutex_exit(&hat->hat_mutex);
3916 }
3917 
3918 /*
3919  * HAT part of cpu initialization.
3920  */
3921 void
3922 hat_cpu_online(struct cpu *cpup)
3923 {
3924 	if (cpup != CPU) {
3925 		x86pte_cpu_init(cpup);
3926 		hat_vlp_setup(cpup);
3927 	}
3928 	CPUSET_ATOMIC_ADD(khat_cpuset, cpup->cpu_id);
3929 }
3930 
3931 /*
3932  * HAT part of cpu deletion.
3933  * (currently, we only call this after the cpu is safely passivated.)
3934  */
3935 void
3936 hat_cpu_offline(struct cpu *cpup)
3937 {
3938 	ASSERT(cpup != CPU);
3939 
3940 	CPUSET_ATOMIC_DEL(khat_cpuset, cpup->cpu_id);
3941 	x86pte_cpu_fini(cpup);
3942 	hat_vlp_teardown(cpup);
3943 }
3944 
3945 /*
3946  * Function called after all CPUs are brought online.
3947  * Used to remove low address boot mappings.
3948  */
3949 void
3950 clear_boot_mappings(uintptr_t low, uintptr_t high)
3951 {
3952 	uintptr_t vaddr = low;
3953 	htable_t *ht = NULL;
3954 	level_t level;
3955 	uint_t entry;
3956 	x86pte_t pte;
3957 
3958 	/*
3959 	 * On 1st CPU we can unload the prom mappings, basically we blow away
3960 	 * all virtual mappings under _userlimit.
3961 	 */
3962 	while (vaddr < high) {
3963 		pte = htable_walk(kas.a_hat, &ht, &vaddr, high);
3964 		if (ht == NULL)
3965 			break;
3966 
3967 		level = ht->ht_level;
3968 		entry = htable_va2entry(vaddr, ht);
3969 		ASSERT(level <= mmu.max_page_level);
3970 		ASSERT(PTE_ISPAGE(pte, level));
3971 
3972 		/*
3973 		 * Unload the mapping from the page tables.
3974 		 */
3975 		(void) x86pte_inval(ht, entry, 0, NULL);
3976 		ASSERT(ht->ht_valid_cnt > 0);
3977 		HTABLE_DEC(ht->ht_valid_cnt);
3978 		PGCNT_DEC(ht->ht_hat, ht->ht_level);
3979 
3980 		vaddr += LEVEL_SIZE(ht->ht_level);
3981 	}
3982 	if (ht)
3983 		htable_release(ht);
3984 }
3985 
3986 /*
3987  * Atomically update a new translation for a single page.  If the
3988  * currently installed PTE doesn't match the value we expect to find,
3989  * it's not updated and we return the PTE we found.
3990  *
3991  * If activating nosync or NOWRITE and the page was modified we need to sync
3992  * with the page_t. Also sync with page_t if clearing ref/mod bits.
3993  */
3994 static x86pte_t
3995 hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, x86pte_t new)
3996 {
3997 	page_t		*pp;
3998 	uint_t		rm = 0;
3999 	x86pte_t	replaced;
4000 
4001 	if (PTE_GET(expected, PT_SOFTWARE) < PT_NOSYNC &&
4002 	    PTE_GET(expected, PT_MOD | PT_REF) &&
4003 	    (PTE_GET(new, PT_NOSYNC) || !PTE_GET(new, PT_WRITABLE) ||
4004 	    !PTE_GET(new, PT_MOD | PT_REF))) {
4005 
4006 		ASSERT(!pfn_is_foreign(PTE2PFN(expected, ht->ht_level)));
4007 		pp = page_numtopp_nolock(PTE2PFN(expected, ht->ht_level));
4008 		ASSERT(pp != NULL);
4009 		if (PTE_GET(expected, PT_MOD))
4010 			rm |= P_MOD;
4011 		if (PTE_GET(expected, PT_REF))
4012 			rm |= P_REF;
4013 		PTE_CLR(new, PT_MOD | PT_REF);
4014 	}
4015 
4016 	replaced = x86pte_update(ht, entry, expected, new);
4017 	if (replaced != expected)
4018 		return (replaced);
4019 
4020 	if (rm) {
4021 		/*
4022 		 * sync to all constituent pages of a large page
4023 		 */
4024 		pgcnt_t pgcnt = page_get_pagecnt(ht->ht_level);
4025 		ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt));
4026 		while (pgcnt-- > 0) {
4027 			/*
4028 			 * hat_page_demote() can't decrease
4029 			 * pszc below this mapping size
4030 			 * since large mapping existed after we
4031 			 * took mlist lock.
4032 			 */
4033 			ASSERT(pp->p_szc >= ht->ht_level);
4034 			hat_page_setattr(pp, rm);
4035 			++pp;
4036 		}
4037 	}
4038 
4039 	return (0);
4040 }
4041 
4042 /* ARGSUSED */
4043 void
4044 hat_join_srd(struct hat *hat, vnode_t *evp)
4045 {
4046 }
4047 
4048 /* ARGSUSED */
4049 hat_region_cookie_t
4050 hat_join_region(struct hat *hat,
4051     caddr_t r_saddr,
4052     size_t r_size,
4053     void *r_obj,
4054     u_offset_t r_objoff,
4055     uchar_t r_perm,
4056     uchar_t r_pgszc,
4057     hat_rgn_cb_func_t r_cb_function,
4058     uint_t flags)
4059 {
4060 	panic("No shared region support on x86");
4061 	return (HAT_INVALID_REGION_COOKIE);
4062 }
4063 
4064 /* ARGSUSED */
4065 void
4066 hat_leave_region(struct hat *hat, hat_region_cookie_t rcookie, uint_t flags)
4067 {
4068 	panic("No shared region support on x86");
4069 }
4070 
4071 /* ARGSUSED */
4072 void
4073 hat_dup_region(struct hat *hat, hat_region_cookie_t rcookie)
4074 {
4075 	panic("No shared region support on x86");
4076 }
4077 
4078 
4079 /*
4080  * Kernel Physical Mapping (kpm) facility
4081  *
4082  * Most of the routines needed to support segkpm are almost no-ops on the
4083  * x86 platform.  We map in the entire segment when it is created and leave
4084  * it mapped in, so there is no additional work required to set up and tear
4085  * down individual mappings.  All of these routines were created to support
4086  * SPARC platforms that have to avoid aliasing in their virtually indexed
4087  * caches.
4088  *
4089  * Most of the routines have sanity checks in them (e.g. verifying that the
4090  * passed-in page is locked).  We don't actually care about most of these
4091  * checks on x86, but we leave them in place to identify problems in the
4092  * upper levels.
4093  */
4094 
4095 /*
4096  * Map in a locked page and return the vaddr.
4097  */
4098 /*ARGSUSED*/
4099 caddr_t
4100 hat_kpm_mapin(struct page *pp, struct kpme *kpme)
4101 {
4102 	caddr_t		vaddr;
4103 
4104 #ifdef DEBUG
4105 	if (kpm_enable == 0) {
4106 		cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set\n");
4107 		return ((caddr_t)NULL);
4108 	}
4109 
4110 	if (pp == NULL || PAGE_LOCKED(pp) == 0) {
4111 		cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked\n");
4112 		return ((caddr_t)NULL);
4113 	}
4114 #endif
4115 
4116 	vaddr = hat_kpm_page2va(pp, 1);
4117 
4118 	return (vaddr);
4119 }
4120 
4121 /*
4122  * Mapout a locked page.
4123  */
4124 /*ARGSUSED*/
4125 void
4126 hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr)
4127 {
4128 #ifdef DEBUG
4129 	if (kpm_enable == 0) {
4130 		cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set\n");
4131 		return;
4132 	}
4133 
4134 	if (IS_KPM_ADDR(vaddr) == 0) {
4135 		cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address\n");
4136 		return;
4137 	}
4138 
4139 	if (pp == NULL || PAGE_LOCKED(pp) == 0) {
4140 		cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked\n");
4141 		return;
4142 	}
4143 #endif
4144 }
4145 
4146 /*
4147  * Return the kpm virtual address for a specific pfn
4148  */
4149 caddr_t
4150 hat_kpm_pfn2va(pfn_t pfn)
4151 {
4152 	uintptr_t vaddr = (uintptr_t)kpm_vbase + mmu_ptob(pfn);
4153 
4154 	ASSERT(!pfn_is_foreign(pfn));
4155 	return ((caddr_t)vaddr);
4156 }
4157 
4158 /*
4159  * Return the kpm virtual address for the page at pp.
4160  */
4161 /*ARGSUSED*/
4162 caddr_t
4163 hat_kpm_page2va(struct page *pp, int checkswap)
4164 {
4165 	return (hat_kpm_pfn2va(pp->p_pagenum));
4166 }
4167 
4168 /*
4169  * Return the page frame number for the kpm virtual address vaddr.
4170  */
4171 pfn_t
4172 hat_kpm_va2pfn(caddr_t vaddr)
4173 {
4174 	pfn_t		pfn;
4175 
4176 	ASSERT(IS_KPM_ADDR(vaddr));
4177 
4178 	pfn = (pfn_t)btop(vaddr - kpm_vbase);
4179 
4180 	return (pfn);
4181 }
4182 
4183 
4184 /*
4185  * Return the page for the kpm virtual address vaddr.
4186  */
4187 page_t *
4188 hat_kpm_vaddr2page(caddr_t vaddr)
4189 {
4190 	pfn_t		pfn;
4191 
4192 	ASSERT(IS_KPM_ADDR(vaddr));
4193 
4194 	pfn = hat_kpm_va2pfn(vaddr);
4195 
4196 	return (page_numtopp_nolock(pfn));
4197 }
4198 
4199 /*
4200  * hat_kpm_fault is called from segkpm_fault when we take a page fault on a
4201  * KPM page.  This should never happen on x86
4202  */
4203 int
4204 hat_kpm_fault(hat_t *hat, caddr_t vaddr)
4205 {
4206 	panic("pagefault in seg_kpm.  hat: 0x%p  vaddr: 0x%p", hat, vaddr);
4207 
4208 	return (0);
4209 }
4210 
4211 /*ARGSUSED*/
4212 void
4213 hat_kpm_mseghash_clear(int nentries)
4214 {}
4215 
4216 /*ARGSUSED*/
4217 void
4218 hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp)
4219 {}
4220 
4221 #ifdef __xpv
4222 /*
4223  * There are specific Hypervisor calls to establish and remove mappings
4224  * to grant table references and the privcmd driver. We have to ensure
4225  * that a page table actually exists.
4226  */
4227 void
4228 hat_prepare_mapping(hat_t *hat, caddr_t addr)
4229 {
4230 	ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE));
4231 	(void) htable_create(hat, (uintptr_t)addr, 0, NULL);
4232 }
4233 
4234 void
4235 hat_release_mapping(hat_t *hat, caddr_t addr)
4236 {
4237 	htable_t *ht;
4238 
4239 	ASSERT(IS_P2ALIGNED((uintptr_t)addr, MMU_PAGESIZE));
4240 	ht = htable_lookup(hat, (uintptr_t)addr, 0);
4241 	ASSERT(ht != NULL);
4242 	ASSERT(ht->ht_busy >= 2);
4243 	htable_release(ht);
4244 	htable_release(ht);
4245 }
4246 #endif
4247