xref: /titanic_44/usr/src/uts/sun4v/os/ppage.c (revision fedab560fb18c85777c255ea9f445ffaf6830d30)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
54f4136d2Sjb145095  * Common Development and Distribution License (the "License").
64f4136d2Sjb145095  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
224f4136d2Sjb145095  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate #include <sys/types.h>
297c478bd9Sstevel@tonic-gate #include <sys/systm.h>
307c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
317c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
327c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
337c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
347c478bd9Sstevel@tonic-gate #include <sys/mman.h>
357c478bd9Sstevel@tonic-gate #include <sys/vm.h>
367c478bd9Sstevel@tonic-gate #include <sys/cpu.h>
377c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
387c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
397c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
407c478bd9Sstevel@tonic-gate #include <vm/as.h>
417c478bd9Sstevel@tonic-gate #include <vm/hat.h>
427c478bd9Sstevel@tonic-gate #include <vm/as.h>
437c478bd9Sstevel@tonic-gate #include <vm/page.h>
447c478bd9Sstevel@tonic-gate #include <vm/seg.h>
457c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
467c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h>
477c478bd9Sstevel@tonic-gate #include <sys/debug.h>
487c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h>
497c478bd9Sstevel@tonic-gate 
507c478bd9Sstevel@tonic-gate /*
517c478bd9Sstevel@tonic-gate  * A quick way to generate a cache consistent address to map in a page.
527c478bd9Sstevel@tonic-gate  * users: ppcopy, pagezero, /proc, dev/mem
537c478bd9Sstevel@tonic-gate  *
547c478bd9Sstevel@tonic-gate  * The ppmapin/ppmapout routines provide a quick way of generating a cache
557c478bd9Sstevel@tonic-gate  * consistent address by reserving a given amount of kernel address space.
567c478bd9Sstevel@tonic-gate  * The base is PPMAPBASE and its size is PPMAPSIZE.  This memory is divided
577c478bd9Sstevel@tonic-gate  * into x number of sets, where x is the number of colors for the virtual
587c478bd9Sstevel@tonic-gate  * cache. The number of colors is how many times a page can be mapped
597c478bd9Sstevel@tonic-gate  * simulatenously in the cache.  For direct map caches this translates to
607c478bd9Sstevel@tonic-gate  * the number of pages in the cache.
617c478bd9Sstevel@tonic-gate  * Each set will be assigned a group of virtual pages from the reserved memory
627c478bd9Sstevel@tonic-gate  * depending on its virtual color.
637c478bd9Sstevel@tonic-gate  * When trying to assign a virtual address we will find out the color for the
647c478bd9Sstevel@tonic-gate  * physical page in question (if applicable).  Then we will try to find an
657c478bd9Sstevel@tonic-gate  * available virtual page from the set of the appropiate color.
667c478bd9Sstevel@tonic-gate  */
677c478bd9Sstevel@tonic-gate 
687c478bd9Sstevel@tonic-gate int pp_slots = 4;		/* small default, tuned by cpu module */
697c478bd9Sstevel@tonic-gate 
707c478bd9Sstevel@tonic-gate /* tuned by cpu module, default is "safe" */
717c478bd9Sstevel@tonic-gate int pp_consistent_coloring = PPAGE_STORES_POLLUTE | PPAGE_LOADS_POLLUTE;
727c478bd9Sstevel@tonic-gate 
737c478bd9Sstevel@tonic-gate static caddr_t	ppmap_vaddrs[PPMAPSIZE / MMU_PAGESIZE];
747c478bd9Sstevel@tonic-gate static int	nsets;			/* number of sets */
757c478bd9Sstevel@tonic-gate static int	ppmap_shift;		/* set selector */
767c478bd9Sstevel@tonic-gate 
777c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
787c478bd9Sstevel@tonic-gate #define		MAXCOLORS	16	/* for debug only */
797c478bd9Sstevel@tonic-gate static int	ppalloc_noslot = 0;	/* # of allocations from kernelmap */
80*fedab560Sae112802 static int	align_hits;
817c478bd9Sstevel@tonic-gate static int	pp_allocs;		/* # of ppmapin requests */
827c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
837c478bd9Sstevel@tonic-gate 
847c478bd9Sstevel@tonic-gate /*
857c478bd9Sstevel@tonic-gate  * There are only 64 TLB entries on spitfire, 16 on cheetah
867c478bd9Sstevel@tonic-gate  * (fully-associative TLB) so we allow the cpu module to tune the
877c478bd9Sstevel@tonic-gate  * number to use here via pp_slots.
887c478bd9Sstevel@tonic-gate  */
897c478bd9Sstevel@tonic-gate static struct ppmap_va {
907c478bd9Sstevel@tonic-gate 	caddr_t	ppmap_slots[MAXPP_SLOTS];
917c478bd9Sstevel@tonic-gate } ppmap_va[NCPU];
927c478bd9Sstevel@tonic-gate 
93*fedab560Sae112802 /* prevent compilation with VAC defined */
94*fedab560Sae112802 #ifdef VAC
95*fedab560Sae112802 #error "sun4v ppmapin and ppmapout do not support VAC"
96*fedab560Sae112802 #endif
97*fedab560Sae112802 
987c478bd9Sstevel@tonic-gate void
997c478bd9Sstevel@tonic-gate ppmapinit(void)
1007c478bd9Sstevel@tonic-gate {
101*fedab560Sae112802 	int nset;
1027c478bd9Sstevel@tonic-gate 	caddr_t va;
1037c478bd9Sstevel@tonic-gate 
1047c478bd9Sstevel@tonic-gate 	ASSERT(pp_slots <= MAXPP_SLOTS);
1057c478bd9Sstevel@tonic-gate 
1067c478bd9Sstevel@tonic-gate 	va = (caddr_t)PPMAPBASE;
1077c478bd9Sstevel@tonic-gate 
1087c478bd9Sstevel@tonic-gate 	/*
109*fedab560Sae112802 	 * sun4v does not have a virtual indexed cache and simply
110*fedab560Sae112802 	 * has only one set containing all pages.
1117c478bd9Sstevel@tonic-gate 	 */
1127c478bd9Sstevel@tonic-gate 	nsets = mmu_btop(PPMAPSIZE);
1137c478bd9Sstevel@tonic-gate 	ppmap_shift = MMU_PAGESHIFT;
114*fedab560Sae112802 
1157c478bd9Sstevel@tonic-gate 	for (nset = 0; nset < nsets; nset++) {
116*fedab560Sae112802 		ppmap_vaddrs[nset] =
117*fedab560Sae112802 		    (caddr_t)((uintptr_t)va + (nset * MMU_PAGESIZE));
1187c478bd9Sstevel@tonic-gate 	}
1197c478bd9Sstevel@tonic-gate }
1207c478bd9Sstevel@tonic-gate 
1217c478bd9Sstevel@tonic-gate /*
1227c478bd9Sstevel@tonic-gate  * Allocate a cache consistent virtual address to map a page, pp,
1237c478bd9Sstevel@tonic-gate  * with protection, vprot; and map it in the MMU, using the most
1247c478bd9Sstevel@tonic-gate  * efficient means possible.  The argument avoid is a virtual address
1257c478bd9Sstevel@tonic-gate  * hint which when masked yields an offset into a virtual cache
1267c478bd9Sstevel@tonic-gate  * that should be avoided when allocating an address to map in a
1277c478bd9Sstevel@tonic-gate  * page.  An avoid arg of -1 means you don't care, for instance pagezero.
1287c478bd9Sstevel@tonic-gate  *
1297c478bd9Sstevel@tonic-gate  * machine dependent, depends on virtual address space layout,
1307c478bd9Sstevel@tonic-gate  * understands that all kernel addresses have bit 31 set.
1317c478bd9Sstevel@tonic-gate  *
1327c478bd9Sstevel@tonic-gate  * NOTE: For sun4 platforms the meaning of the hint argument is opposite from
1337c478bd9Sstevel@tonic-gate  * that found in other architectures.  In other architectures the hint
1347c478bd9Sstevel@tonic-gate  * (called avoid) was used to ask ppmapin to NOT use the specified cache color.
1357c478bd9Sstevel@tonic-gate  * This was used to avoid virtual cache trashing in the bcopy.  Unfortunately
1367c478bd9Sstevel@tonic-gate  * in the case of a COW,  this later on caused a cache aliasing conflict.  In
1377c478bd9Sstevel@tonic-gate  * sun4, the bcopy routine uses the block ld/st instructions so we don't have
1387c478bd9Sstevel@tonic-gate  * to worry about virtual cache trashing.  Actually, by using the hint to choose
1397c478bd9Sstevel@tonic-gate  * the right color we can almost guarantee a cache conflict will not occur.
1407c478bd9Sstevel@tonic-gate  */
1417c478bd9Sstevel@tonic-gate 
142*fedab560Sae112802 /*ARGSUSED2*/
1437c478bd9Sstevel@tonic-gate caddr_t
1447c478bd9Sstevel@tonic-gate ppmapin(page_t *pp, uint_t vprot, caddr_t hint)
1457c478bd9Sstevel@tonic-gate {
146*fedab560Sae112802 	int nset;
1477c478bd9Sstevel@tonic-gate 	caddr_t va;
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
1507c478bd9Sstevel@tonic-gate 	pp_allocs++;
1517c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
1527c478bd9Sstevel@tonic-gate 
1537c478bd9Sstevel@tonic-gate 	/*
154*fedab560Sae112802 	 * For sun4v caches are physical caches, we can pick any address
155*fedab560Sae112802 	 * we want.
1567c478bd9Sstevel@tonic-gate 	 */
1577c478bd9Sstevel@tonic-gate 	for (nset = 0; nset < nsets; nset++) {
158*fedab560Sae112802 		va = ppmap_vaddrs[nset];
1597c478bd9Sstevel@tonic-gate 		if (va != NULL) {
1607c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
161*fedab560Sae112802 			align_hits++;
1627c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
163*fedab560Sae112802 			if (casptr(&ppmap_vaddrs[nset], va, NULL) == va) {
1647c478bd9Sstevel@tonic-gate 				hat_memload(kas.a_hat, va, pp,
1657c478bd9Sstevel@tonic-gate 					vprot | HAT_NOSYNC,
1667c478bd9Sstevel@tonic-gate 					HAT_LOAD_LOCK);
1677c478bd9Sstevel@tonic-gate 				return (va);
1687c478bd9Sstevel@tonic-gate 			}
1697c478bd9Sstevel@tonic-gate 		}
1707c478bd9Sstevel@tonic-gate 	}
1717c478bd9Sstevel@tonic-gate 
1727c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
1737c478bd9Sstevel@tonic-gate 	ppalloc_noslot++;
1747c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
1757c478bd9Sstevel@tonic-gate 
1767c478bd9Sstevel@tonic-gate 	/*
1777c478bd9Sstevel@tonic-gate 	 * No free slots; get a random one from the kernel heap area.
1787c478bd9Sstevel@tonic-gate 	 */
1797c478bd9Sstevel@tonic-gate 	va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
1807c478bd9Sstevel@tonic-gate 
1817c478bd9Sstevel@tonic-gate 	hat_memload(kas.a_hat, va, pp, vprot | HAT_NOSYNC, HAT_LOAD_LOCK);
1827c478bd9Sstevel@tonic-gate 
1837c478bd9Sstevel@tonic-gate 	return (va);
1847c478bd9Sstevel@tonic-gate 
1857c478bd9Sstevel@tonic-gate }
1867c478bd9Sstevel@tonic-gate 
1877c478bd9Sstevel@tonic-gate void
1887c478bd9Sstevel@tonic-gate ppmapout(caddr_t va)
1897c478bd9Sstevel@tonic-gate {
190*fedab560Sae112802 	int nset;
1917c478bd9Sstevel@tonic-gate 
1927c478bd9Sstevel@tonic-gate 	if (va >= kernelheap && va < ekernelheap) {
1937c478bd9Sstevel@tonic-gate 		/*
1947c478bd9Sstevel@tonic-gate 		 * Space came from kernelmap, flush the page and
1957c478bd9Sstevel@tonic-gate 		 * return the space.
1967c478bd9Sstevel@tonic-gate 		 */
1977c478bd9Sstevel@tonic-gate 		hat_unload(kas.a_hat, va, PAGESIZE,
1987c478bd9Sstevel@tonic-gate 		    (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK));
1997c478bd9Sstevel@tonic-gate 		vmem_free(heap_arena, va, PAGESIZE);
2007c478bd9Sstevel@tonic-gate 	} else {
2017c478bd9Sstevel@tonic-gate 		/*
2027c478bd9Sstevel@tonic-gate 		 * Space came from ppmap_vaddrs[], give it back.
2037c478bd9Sstevel@tonic-gate 		 */
2047c478bd9Sstevel@tonic-gate 		nset = ((uintptr_t)va >> ppmap_shift) & (nsets - 1);
2057c478bd9Sstevel@tonic-gate 		hat_unload(kas.a_hat, va, PAGESIZE,
2067c478bd9Sstevel@tonic-gate 		    (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK));
2077c478bd9Sstevel@tonic-gate 
208*fedab560Sae112802 		ASSERT(ppmap_vaddrs[nset] == NULL);
209*fedab560Sae112802 		ppmap_vaddrs[nset] = va;
2107c478bd9Sstevel@tonic-gate 	}
2117c478bd9Sstevel@tonic-gate }
2127c478bd9Sstevel@tonic-gate 
2137c478bd9Sstevel@tonic-gate #ifdef DEBUG
2147c478bd9Sstevel@tonic-gate #define	PP_STAT_ADD(stat)	(stat)++
2157c478bd9Sstevel@tonic-gate uint_t pload, ploadfail;
2167c478bd9Sstevel@tonic-gate uint_t ppzero, ppzero_short;
2177c478bd9Sstevel@tonic-gate #else
2187c478bd9Sstevel@tonic-gate #define	PP_STAT_ADD(stat)
2197c478bd9Sstevel@tonic-gate #endif /* DEBUG */
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate static void
2227c478bd9Sstevel@tonic-gate pp_unload_tlb(caddr_t *pslot, caddr_t va)
2237c478bd9Sstevel@tonic-gate {
2247c478bd9Sstevel@tonic-gate 	ASSERT(*pslot == va);
2257c478bd9Sstevel@tonic-gate 
2261e2e7a75Shuah 	vtag_flushpage(va, (uint64_t)ksfmmup);
2277c478bd9Sstevel@tonic-gate 	*pslot = NULL;				/* release the slot */
2287c478bd9Sstevel@tonic-gate }
2297c478bd9Sstevel@tonic-gate 
2307c478bd9Sstevel@tonic-gate /*
2317c478bd9Sstevel@tonic-gate  * Routine to copy kernel pages during relocation.  It will copy one
2327c478bd9Sstevel@tonic-gate  * PAGESIZE page to another PAGESIZE page.  This function may be called
2337c478bd9Sstevel@tonic-gate  * above LOCK_LEVEL so it should not grab any locks.
2347c478bd9Sstevel@tonic-gate  */
2357c478bd9Sstevel@tonic-gate void
2367c478bd9Sstevel@tonic-gate ppcopy_kernel__relocatable(page_t *fm_pp, page_t *to_pp)
2377c478bd9Sstevel@tonic-gate {
2387c478bd9Sstevel@tonic-gate 	uint64_t fm_pa, to_pa;
2397c478bd9Sstevel@tonic-gate 	size_t nbytes;
2407c478bd9Sstevel@tonic-gate 
2417c478bd9Sstevel@tonic-gate 	fm_pa = (uint64_t)(fm_pp->p_pagenum) << MMU_PAGESHIFT;
2427c478bd9Sstevel@tonic-gate 	to_pa = (uint64_t)(to_pp->p_pagenum) << MMU_PAGESHIFT;
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 	nbytes = MMU_PAGESIZE;
2457c478bd9Sstevel@tonic-gate 
2467c478bd9Sstevel@tonic-gate 	for (; nbytes > 0; fm_pa += 32, to_pa += 32, nbytes -= 32)
2477c478bd9Sstevel@tonic-gate 		hw_pa_bcopy32(fm_pa, to_pa);
2487c478bd9Sstevel@tonic-gate }
2497c478bd9Sstevel@tonic-gate 
2507c478bd9Sstevel@tonic-gate /*
2517c478bd9Sstevel@tonic-gate  * Copy the data from the physical page represented by "frompp" to
2527c478bd9Sstevel@tonic-gate  * that represented by "topp".
2537c478bd9Sstevel@tonic-gate  *
2547c478bd9Sstevel@tonic-gate  * Try to use per cpu mapping first, if that fails then call pp_mapin
2557c478bd9Sstevel@tonic-gate  * to load it.
2567c478bd9Sstevel@tonic-gate  */
2577c478bd9Sstevel@tonic-gate void
2587c478bd9Sstevel@tonic-gate ppcopy(page_t *fm_pp, page_t *to_pp)
2597c478bd9Sstevel@tonic-gate {
2604f4136d2Sjb145095 	caddr_t fm_va;
2614f4136d2Sjb145095 	caddr_t to_va;
2624f4136d2Sjb145095 	boolean_t fast;
2637c478bd9Sstevel@tonic-gate 
2644f4136d2Sjb145095 	ASSERT(PAGE_LOCKED(fm_pp));
2654f4136d2Sjb145095 	ASSERT(PAGE_LOCKED(to_pp));
2664f4136d2Sjb145095 
2674f4136d2Sjb145095 	/*
2684f4136d2Sjb145095 	 * Try to map using KPM.  If it fails, fall back to
2694f4136d2Sjb145095 	 * ppmapin/ppmapout.
2704f4136d2Sjb145095 	 */
2714f4136d2Sjb145095 	if ((fm_va = hat_kpm_mapin(fm_pp, NULL)) == NULL ||
2724f4136d2Sjb145095 	    (to_va = hat_kpm_mapin(to_pp, NULL)) == NULL) {
2734f4136d2Sjb145095 		if (fm_va != NULL)
2744f4136d2Sjb145095 			hat_kpm_mapout(fm_pp, NULL, fm_va);
2757c478bd9Sstevel@tonic-gate 		fm_va = ppmapin(fm_pp, PROT_READ, (caddr_t)-1);
2767c478bd9Sstevel@tonic-gate 		to_va = ppmapin(to_pp, PROT_READ | PROT_WRITE, fm_va);
2774f4136d2Sjb145095 		fast = B_FALSE;
2784f4136d2Sjb145095 	} else
2794f4136d2Sjb145095 		fast = B_TRUE;
2804f4136d2Sjb145095 
2817c478bd9Sstevel@tonic-gate 	bcopy(fm_va, to_va, PAGESIZE);
2824f4136d2Sjb145095 
2834f4136d2Sjb145095 	/* Unmap */
2844f4136d2Sjb145095 	if (fast) {
2854f4136d2Sjb145095 		hat_kpm_mapout(fm_pp, NULL, fm_va);
2864f4136d2Sjb145095 		hat_kpm_mapout(to_pp, NULL, to_va);
2874f4136d2Sjb145095 	} else {
2887c478bd9Sstevel@tonic-gate 		ppmapout(fm_va);
2897c478bd9Sstevel@tonic-gate 		ppmapout(to_va);
2907c478bd9Sstevel@tonic-gate 	}
2914f4136d2Sjb145095 }
2927c478bd9Sstevel@tonic-gate 
2937c478bd9Sstevel@tonic-gate /*
2947c478bd9Sstevel@tonic-gate  * Zero the physical page from off to off + len given by `pp'
2957c478bd9Sstevel@tonic-gate  * without changing the reference and modified bits of page.
2967c478bd9Sstevel@tonic-gate  *
2977c478bd9Sstevel@tonic-gate  * Again, we'll try per cpu mapping first.
2987c478bd9Sstevel@tonic-gate  */
2994f4136d2Sjb145095 
3007c478bd9Sstevel@tonic-gate void
3017c478bd9Sstevel@tonic-gate pagezero(page_t *pp, uint_t off, uint_t len)
3027c478bd9Sstevel@tonic-gate {
3037c478bd9Sstevel@tonic-gate 	caddr_t va;
3047c478bd9Sstevel@tonic-gate 	extern int hwblkclr(void *, size_t);
3057c478bd9Sstevel@tonic-gate 	extern int use_hw_bzero;
3064f4136d2Sjb145095 	boolean_t fast;
3077c478bd9Sstevel@tonic-gate 
3087c478bd9Sstevel@tonic-gate 	ASSERT((int)len > 0 && (int)off >= 0 && off + len <= PAGESIZE);
3097c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp));
3107c478bd9Sstevel@tonic-gate 
3117c478bd9Sstevel@tonic-gate 	PP_STAT_ADD(ppzero);
3127c478bd9Sstevel@tonic-gate 
3137c478bd9Sstevel@tonic-gate 	if (len != MMU_PAGESIZE || !use_hw_bzero) {
3147c478bd9Sstevel@tonic-gate 		PP_STAT_ADD(ppzero_short);
3157c478bd9Sstevel@tonic-gate 	}
3167c478bd9Sstevel@tonic-gate 
3177c478bd9Sstevel@tonic-gate 	kpreempt_disable();
3187c478bd9Sstevel@tonic-gate 
3194f4136d2Sjb145095 	/*
3204f4136d2Sjb145095 	 * Try to use KPM.  If that fails, fall back to
3214f4136d2Sjb145095 	 * ppmapin/ppmapout.
3224f4136d2Sjb145095 	 */
3234f4136d2Sjb145095 	fast = B_TRUE;
3244f4136d2Sjb145095 	va = hat_kpm_mapin(pp, NULL);
3254f4136d2Sjb145095 	if (va == NULL) {
3264f4136d2Sjb145095 		fast = B_FALSE;
3277c478bd9Sstevel@tonic-gate 		va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1);
3284f4136d2Sjb145095 	}
3297c478bd9Sstevel@tonic-gate 
3307c478bd9Sstevel@tonic-gate 	if (!use_hw_bzero) {
3317c478bd9Sstevel@tonic-gate 		bzero(va + off, len);
3327c478bd9Sstevel@tonic-gate 		sync_icache(va + off, len);
3337c478bd9Sstevel@tonic-gate 	} else if (hwblkclr(va + off, len)) {
3347c478bd9Sstevel@tonic-gate 		/*
3357c478bd9Sstevel@tonic-gate 		 * We may not have used block commit asi.
3367c478bd9Sstevel@tonic-gate 		 * So flush the I-$ manually
3377c478bd9Sstevel@tonic-gate 		 */
3387c478bd9Sstevel@tonic-gate 		sync_icache(va + off, len);
3397c478bd9Sstevel@tonic-gate 	} else {
3407c478bd9Sstevel@tonic-gate 		/*
3414f4136d2Sjb145095 		 * We have used blk commit, and flushed the I-$.
3424f4136d2Sjb145095 		 * However we still may have an instruction in the
3434f4136d2Sjb145095 		 * pipeline. Only a flush will invalidate that.
3447c478bd9Sstevel@tonic-gate 		 */
3457c478bd9Sstevel@tonic-gate 		doflush(va);
3467c478bd9Sstevel@tonic-gate 	}
3477c478bd9Sstevel@tonic-gate 
3484f4136d2Sjb145095 	if (fast) {
3494f4136d2Sjb145095 		hat_kpm_mapout(pp, NULL, va);
3504f4136d2Sjb145095 	} else {
3517c478bd9Sstevel@tonic-gate 		ppmapout(va);
3524f4136d2Sjb145095 	}
3537c478bd9Sstevel@tonic-gate 	kpreempt_enable();
3547c478bd9Sstevel@tonic-gate }
355