xref: /titanic_51/usr/src/uts/sun4/vm/sfmmu.c (revision 986fd29a0dc13f7608ef7f508f6e700bd7bc2720)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
51e2e7a75Shuah  * Common Development and Distribution License (the "License").
61e2e7a75Shuah  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22bb121940Sdp78419  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate #include <sys/types.h>
297c478bd9Sstevel@tonic-gate #include <vm/hat.h>
307c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h>
317c478bd9Sstevel@tonic-gate #include <vm/page.h>
327c478bd9Sstevel@tonic-gate #include <sys/pte.h>
337c478bd9Sstevel@tonic-gate #include <sys/systm.h>
347c478bd9Sstevel@tonic-gate #include <sys/mman.h>
357c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
367c478bd9Sstevel@tonic-gate #include <sys/machparam.h>
377c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
387c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
397c478bd9Sstevel@tonic-gate #include <sys/mmu.h>
407c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
417c478bd9Sstevel@tonic-gate #include <sys/cpu.h>
427c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
437c478bd9Sstevel@tonic-gate #include <sys/debug.h>
447c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
457c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
467c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
477c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
487c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
497c478bd9Sstevel@tonic-gate #include <vm/as.h>
507c478bd9Sstevel@tonic-gate #include <vm/seg.h>
517c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
527c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
537c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h>
547c478bd9Sstevel@tonic-gate #include <vm/rm.h>
557c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h>
567c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
577c478bd9Sstevel@tonic-gate #include <sys/vm_machparam.h>
587c478bd9Sstevel@tonic-gate #include <sys/promif.h>
597c478bd9Sstevel@tonic-gate #include <sys/prom_isa.h>
607c478bd9Sstevel@tonic-gate #include <sys/prom_plat.h>
617c478bd9Sstevel@tonic-gate #include <sys/prom_debug.h>
627c478bd9Sstevel@tonic-gate #include <sys/privregs.h>
637c478bd9Sstevel@tonic-gate #include <sys/bootconf.h>
647c478bd9Sstevel@tonic-gate #include <sys/memlist.h>
657c478bd9Sstevel@tonic-gate #include <sys/memlist_plat.h>
667c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h>
677c478bd9Sstevel@tonic-gate #include <sys/reboot.h>
687c478bd9Sstevel@tonic-gate #include <sys/kdi.h>
697c478bd9Sstevel@tonic-gate 
707c478bd9Sstevel@tonic-gate /*
717c478bd9Sstevel@tonic-gate  * Static routines
727c478bd9Sstevel@tonic-gate  */
737c478bd9Sstevel@tonic-gate static void	sfmmu_map_prom_mappings(struct translation *, size_t);
747c478bd9Sstevel@tonic-gate static struct translation *read_prom_mappings(size_t *);
757c478bd9Sstevel@tonic-gate static void	sfmmu_reloc_trap_handler(void *, void *, size_t);
767c478bd9Sstevel@tonic-gate 
777c478bd9Sstevel@tonic-gate /*
787c478bd9Sstevel@tonic-gate  * External routines
797c478bd9Sstevel@tonic-gate  */
807c478bd9Sstevel@tonic-gate extern void sfmmu_remap_kernel(void);
817c478bd9Sstevel@tonic-gate extern void sfmmu_patch_utsb(void);
827c478bd9Sstevel@tonic-gate 
837c478bd9Sstevel@tonic-gate /*
847c478bd9Sstevel@tonic-gate  * Global Data:
857c478bd9Sstevel@tonic-gate  */
867c478bd9Sstevel@tonic-gate extern caddr_t	textva, datava;
877c478bd9Sstevel@tonic-gate extern tte_t	ktext_tte, kdata_tte;	/* ttes for kernel text and data */
887c478bd9Sstevel@tonic-gate extern int	enable_bigktsb;
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate uint64_t memsegspa = (uintptr_t)MSEG_NULLPTR_PA; /* memsegs physical linkage */
917c478bd9Sstevel@tonic-gate uint64_t memseg_phash[N_MEM_SLOTS];	/* use physical memseg addresses */
927c478bd9Sstevel@tonic-gate 
937c478bd9Sstevel@tonic-gate int	sfmmu_kern_mapped = 0;
947c478bd9Sstevel@tonic-gate 
957c478bd9Sstevel@tonic-gate /*
967c478bd9Sstevel@tonic-gate  * DMMU primary context register for the kernel context. Machine specific code
977c478bd9Sstevel@tonic-gate  * inserts correct page size codes when necessary
987c478bd9Sstevel@tonic-gate  */
997c478bd9Sstevel@tonic-gate uint64_t kcontextreg = KCONTEXT;
1007c478bd9Sstevel@tonic-gate 
1011e45ea5aSblakej #ifdef DEBUG
1021e45ea5aSblakej static int ndata_middle_hole_detected = 0;
1031e45ea5aSblakej #endif
1041e45ea5aSblakej 
1057c478bd9Sstevel@tonic-gate /* Extern Global Data */
1067c478bd9Sstevel@tonic-gate 
1077c478bd9Sstevel@tonic-gate extern int page_relocate_ready;
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate /*
1107c478bd9Sstevel@tonic-gate  * Controls the logic which enables the use of the
1117c478bd9Sstevel@tonic-gate  * QUAD_LDD_PHYS ASI for TSB accesses.
1127c478bd9Sstevel@tonic-gate  */
1137c478bd9Sstevel@tonic-gate extern int	ktsb_phys;
1147c478bd9Sstevel@tonic-gate 
1157c478bd9Sstevel@tonic-gate /*
1167c478bd9Sstevel@tonic-gate  * Global Routines called from within:
1177c478bd9Sstevel@tonic-gate  *	usr/src/uts/sun4u
1187c478bd9Sstevel@tonic-gate  *	usr/src/uts/sfmmu
1197c478bd9Sstevel@tonic-gate  *	usr/src/uts/sun
1207c478bd9Sstevel@tonic-gate  */
1217c478bd9Sstevel@tonic-gate 
1227c478bd9Sstevel@tonic-gate pfn_t
1237c478bd9Sstevel@tonic-gate va_to_pfn(void *vaddr)
1247c478bd9Sstevel@tonic-gate {
1257c478bd9Sstevel@tonic-gate 	u_longlong_t physaddr;
1267c478bd9Sstevel@tonic-gate 	int mode, valid;
1277c478bd9Sstevel@tonic-gate 
1287c478bd9Sstevel@tonic-gate 	if (tba_taken_over)
1297c478bd9Sstevel@tonic-gate 		return (hat_getpfnum(kas.a_hat, (caddr_t)vaddr));
1307c478bd9Sstevel@tonic-gate 
131bb121940Sdp78419 #if !defined(C_OBP)
132bb121940Sdp78419 	if ((caddr_t)vaddr >= kmem64_base && (caddr_t)vaddr < kmem64_end) {
133bb121940Sdp78419 		if (kmem64_pabase == (uint64_t)-1)
134bb121940Sdp78419 			prom_panic("va_to_pfn: kmem64_pabase not init");
135bb121940Sdp78419 		physaddr = kmem64_pabase + ((caddr_t)vaddr - kmem64_base);
136bb121940Sdp78419 		return ((pfn_t)physaddr >> MMU_PAGESHIFT);
137bb121940Sdp78419 	}
138bb121940Sdp78419 #endif	/* !C_OBP */
139bb121940Sdp78419 
1407c478bd9Sstevel@tonic-gate 	if ((prom_translate_virt(vaddr, &valid, &physaddr, &mode) != -1) &&
1417c478bd9Sstevel@tonic-gate 	    (valid == -1)) {
1427c478bd9Sstevel@tonic-gate 		return ((pfn_t)(physaddr >> MMU_PAGESHIFT));
1437c478bd9Sstevel@tonic-gate 	}
1447c478bd9Sstevel@tonic-gate 	return (PFN_INVALID);
1457c478bd9Sstevel@tonic-gate }
1467c478bd9Sstevel@tonic-gate 
1477c478bd9Sstevel@tonic-gate uint64_t
1487c478bd9Sstevel@tonic-gate va_to_pa(void *vaddr)
1497c478bd9Sstevel@tonic-gate {
1507c478bd9Sstevel@tonic-gate 	pfn_t pfn;
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate 	if ((pfn = va_to_pfn(vaddr)) == PFN_INVALID)
1537c478bd9Sstevel@tonic-gate 		return ((uint64_t)-1);
1547c478bd9Sstevel@tonic-gate 	return (((uint64_t)pfn << MMU_PAGESHIFT) |
1557c478bd9Sstevel@tonic-gate 	    ((uint64_t)vaddr & MMU_PAGEOFFSET));
1567c478bd9Sstevel@tonic-gate }
1577c478bd9Sstevel@tonic-gate 
1587c478bd9Sstevel@tonic-gate void
1597c478bd9Sstevel@tonic-gate hat_kern_setup(void)
1607c478bd9Sstevel@tonic-gate {
1617c478bd9Sstevel@tonic-gate 	struct translation *trans_root;
1627c478bd9Sstevel@tonic-gate 	size_t ntrans_root;
1637c478bd9Sstevel@tonic-gate 	extern void startup_fixup_physavail(void);
1647c478bd9Sstevel@tonic-gate 
1657c478bd9Sstevel@tonic-gate 	/*
1667c478bd9Sstevel@tonic-gate 	 * These are the steps we take to take over the mmu from the prom.
1677c478bd9Sstevel@tonic-gate 	 *
1687c478bd9Sstevel@tonic-gate 	 * (1)	Read the prom's mappings through the translation property.
1697c478bd9Sstevel@tonic-gate 	 * (2)	Remap the kernel text and kernel data with 2 locked 4MB ttes.
1707c478bd9Sstevel@tonic-gate 	 *	Create the the hmeblks for these 2 ttes at this time.
1717c478bd9Sstevel@tonic-gate 	 * (3)	Create hat structures for all other prom mappings.  Since the
1727c478bd9Sstevel@tonic-gate 	 *	kernel text and data hme_blks have already been created we
1737c478bd9Sstevel@tonic-gate 	 *	skip the equivalent prom's mappings.
1747c478bd9Sstevel@tonic-gate 	 * (4)	Initialize the tsb and its corresponding hardware regs.
1757c478bd9Sstevel@tonic-gate 	 * (5)	Take over the trap table (currently in startup).
1767c478bd9Sstevel@tonic-gate 	 * (6)	Up to this point it is possible the prom required some of its
1777c478bd9Sstevel@tonic-gate 	 *	locked tte's.  Now that we own the trap table we remove them.
1787c478bd9Sstevel@tonic-gate 	 */
1797c478bd9Sstevel@tonic-gate 
1807c478bd9Sstevel@tonic-gate 	ktsb_pbase = va_to_pa(ktsb_base);
1817c478bd9Sstevel@tonic-gate 	ktsb4m_pbase = va_to_pa(ktsb4m_base);
1827c478bd9Sstevel@tonic-gate 	PRM_DEBUG(ktsb_pbase);
1837c478bd9Sstevel@tonic-gate 	PRM_DEBUG(ktsb4m_pbase);
1847c478bd9Sstevel@tonic-gate 
1857c478bd9Sstevel@tonic-gate 	sfmmu_patch_ktsb();
1867c478bd9Sstevel@tonic-gate 	sfmmu_patch_utsb();
1877c478bd9Sstevel@tonic-gate 	sfmmu_patch_mmu_asi(ktsb_phys);
1887c478bd9Sstevel@tonic-gate 
1897c478bd9Sstevel@tonic-gate 	sfmmu_init_tsbs();
1907c478bd9Sstevel@tonic-gate 
1917c478bd9Sstevel@tonic-gate 	if (kpm_enable) {
1927c478bd9Sstevel@tonic-gate 		sfmmu_kpm_patch_tlbm();
1937c478bd9Sstevel@tonic-gate 		if (kpm_smallpages == 0) {
1947c478bd9Sstevel@tonic-gate 			sfmmu_kpm_patch_tsbm();
1957c478bd9Sstevel@tonic-gate 		}
1967c478bd9Sstevel@tonic-gate 	}
1977c478bd9Sstevel@tonic-gate 
1987dacfc44Spaulsan 	if (!shctx_on) {
19905d3dc4bSpaulsan 		sfmmu_patch_shctx();
20005d3dc4bSpaulsan 	}
20105d3dc4bSpaulsan 
2027c478bd9Sstevel@tonic-gate 	/*
2037c478bd9Sstevel@tonic-gate 	 * The 8K-indexed kernel TSB space is used to hold
2047c478bd9Sstevel@tonic-gate 	 * translations below...
2057c478bd9Sstevel@tonic-gate 	 */
2067c478bd9Sstevel@tonic-gate 	trans_root = read_prom_mappings(&ntrans_root);
2077c478bd9Sstevel@tonic-gate 	sfmmu_remap_kernel();
2087c478bd9Sstevel@tonic-gate 	startup_fixup_physavail();
2097c478bd9Sstevel@tonic-gate 	mmu_init_kernel_pgsz(kas.a_hat);
2107c478bd9Sstevel@tonic-gate 	sfmmu_map_prom_mappings(trans_root, ntrans_root);
2117c478bd9Sstevel@tonic-gate 
2127c478bd9Sstevel@tonic-gate 	/*
2137c478bd9Sstevel@tonic-gate 	 * We invalidate 8K kernel TSB because we used it in
2147c478bd9Sstevel@tonic-gate 	 * sfmmu_map_prom_mappings()
2157c478bd9Sstevel@tonic-gate 	 */
2167c478bd9Sstevel@tonic-gate 	sfmmu_inv_tsb(ktsb_base, ktsb_sz);
2177c478bd9Sstevel@tonic-gate 	sfmmu_inv_tsb(ktsb4m_base, ktsb4m_sz);
2187c478bd9Sstevel@tonic-gate 
2197c478bd9Sstevel@tonic-gate 	sfmmu_init_ktsbinfo();
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate 
2227c478bd9Sstevel@tonic-gate 	sfmmu_kern_mapped = 1;
2237c478bd9Sstevel@tonic-gate 
2247c478bd9Sstevel@tonic-gate 	/*
2257c478bd9Sstevel@tonic-gate 	 * hments have been created for mapped pages, and thus we're ready
2267c478bd9Sstevel@tonic-gate 	 * for kmdb to start using its own trap table.  It walks the hments
2277c478bd9Sstevel@tonic-gate 	 * to resolve TLB misses, and can't be used until they're ready.
2287c478bd9Sstevel@tonic-gate 	 */
2297c478bd9Sstevel@tonic-gate 	if (boothowto & RB_DEBUG)
2307c478bd9Sstevel@tonic-gate 		kdi_dvec_vmready();
2317c478bd9Sstevel@tonic-gate }
2327c478bd9Sstevel@tonic-gate 
2337c478bd9Sstevel@tonic-gate /*
2347c478bd9Sstevel@tonic-gate  * Macro used below to convert the prom's 32-bit high and low fields into
2357c478bd9Sstevel@tonic-gate  * a value appropriate for the 64-bit kernel.
2367c478bd9Sstevel@tonic-gate  */
2377c478bd9Sstevel@tonic-gate 
2387c478bd9Sstevel@tonic-gate #define	COMBINE(hi, lo) (((uint64_t)(uint32_t)(hi) << 32) | (uint32_t)(lo))
2397c478bd9Sstevel@tonic-gate 
2407c478bd9Sstevel@tonic-gate /*
241bb121940Sdp78419  * Track larges pages used.
242bb121940Sdp78419  * Provides observability for this feature on non-debug kernels.
243bb121940Sdp78419  */
244bb121940Sdp78419 ulong_t map_prom_lpcount[MMU_PAGE_SIZES];
245bb121940Sdp78419 
246bb121940Sdp78419 /*
2477c478bd9Sstevel@tonic-gate  * This function traverses the prom mapping list and creates equivalent
2487c478bd9Sstevel@tonic-gate  * mappings in the sfmmu mapping hash.
2497c478bd9Sstevel@tonic-gate  */
2507c478bd9Sstevel@tonic-gate static void
2517c478bd9Sstevel@tonic-gate sfmmu_map_prom_mappings(struct translation *trans_root, size_t ntrans_root)
2527c478bd9Sstevel@tonic-gate {
2537c478bd9Sstevel@tonic-gate 	struct translation *promt;
2547c478bd9Sstevel@tonic-gate 	tte_t	tte, oldtte, *ttep;
2557c478bd9Sstevel@tonic-gate 	pfn_t	pfn, oldpfn, basepfn;
2567c478bd9Sstevel@tonic-gate 	caddr_t vaddr;
2577c478bd9Sstevel@tonic-gate 	size_t	size, offset;
2587c478bd9Sstevel@tonic-gate 	unsigned long i;
2597c478bd9Sstevel@tonic-gate 	uint_t	attr;
2607c478bd9Sstevel@tonic-gate 	page_t *pp;
2617c478bd9Sstevel@tonic-gate 	extern struct memlist *virt_avail;
262d9aef8afSjesusm 	char buf[256];
2637c478bd9Sstevel@tonic-gate 
2647c478bd9Sstevel@tonic-gate 	ttep = &tte;
2657c478bd9Sstevel@tonic-gate 	for (i = 0, promt = trans_root; i < ntrans_root; i++, promt++) {
2667c478bd9Sstevel@tonic-gate 		ASSERT(promt->tte_hi != 0);
2677c478bd9Sstevel@tonic-gate 		ASSERT32(promt->virt_hi == 0 && promt->size_hi == 0);
2687c478bd9Sstevel@tonic-gate 
269bb121940Sdp78419 		vaddr = (caddr_t)COMBINE(promt->virt_hi, promt->virt_lo);
270bb121940Sdp78419 
2717c478bd9Sstevel@tonic-gate 		/*
2727c478bd9Sstevel@tonic-gate 		 * hack until we get rid of map-for-unix
2737c478bd9Sstevel@tonic-gate 		 */
274bb121940Sdp78419 		if (vaddr < (caddr_t)KERNELBASE)
2757c478bd9Sstevel@tonic-gate 			continue;
2767c478bd9Sstevel@tonic-gate 
2777c478bd9Sstevel@tonic-gate 		ttep->tte_inthi = promt->tte_hi;
2787c478bd9Sstevel@tonic-gate 		ttep->tte_intlo = promt->tte_lo;
2797c478bd9Sstevel@tonic-gate 		attr = PROC_DATA | HAT_NOSYNC;
2807c478bd9Sstevel@tonic-gate #if defined(TTE_IS_GLOBAL)
2817c478bd9Sstevel@tonic-gate 		if (TTE_IS_GLOBAL(ttep)) {
2827c478bd9Sstevel@tonic-gate 			/*
2837c478bd9Sstevel@tonic-gate 			 * The prom better not use global translations
2847c478bd9Sstevel@tonic-gate 			 * because a user process might use the same
2857c478bd9Sstevel@tonic-gate 			 * virtual addresses
2867c478bd9Sstevel@tonic-gate 			 */
287d9aef8afSjesusm 			prom_panic("sfmmu_map_prom_mappings: global"
288d9aef8afSjesusm 			    " translation");
2897c478bd9Sstevel@tonic-gate 			TTE_SET_LOFLAGS(ttep, TTE_GLB_INT, 0);
2907c478bd9Sstevel@tonic-gate 		}
2917c478bd9Sstevel@tonic-gate #endif
2927c478bd9Sstevel@tonic-gate 		if (TTE_IS_LOCKED(ttep)) {
2937c478bd9Sstevel@tonic-gate 			/* clear the lock bits */
2947c478bd9Sstevel@tonic-gate 			TTE_CLR_LOCKED(ttep);
2957c478bd9Sstevel@tonic-gate 		}
2967c478bd9Sstevel@tonic-gate 		attr |= (TTE_IS_VCACHEABLE(ttep)) ? 0 : SFMMU_UNCACHEVTTE;
2977c478bd9Sstevel@tonic-gate 		attr |= (TTE_IS_PCACHEABLE(ttep)) ? 0 : SFMMU_UNCACHEPTTE;
2987c478bd9Sstevel@tonic-gate 		attr |= (TTE_IS_SIDEFFECT(ttep)) ? SFMMU_SIDEFFECT : 0;
2997c478bd9Sstevel@tonic-gate 		attr |= (TTE_IS_IE(ttep)) ? HAT_STRUCTURE_LE : 0;
3007c478bd9Sstevel@tonic-gate 
3017c478bd9Sstevel@tonic-gate 		size = COMBINE(promt->size_hi, promt->size_lo);
3027c478bd9Sstevel@tonic-gate 		offset = 0;
3037c478bd9Sstevel@tonic-gate 		basepfn = TTE_TO_PFN((caddr_t)COMBINE(promt->virt_hi,
3047c478bd9Sstevel@tonic-gate 		    promt->virt_lo), ttep);
3057c478bd9Sstevel@tonic-gate 		while (size) {
3067c478bd9Sstevel@tonic-gate 			vaddr = (caddr_t)(COMBINE(promt->virt_hi,
3077c478bd9Sstevel@tonic-gate 			    promt->virt_lo) + offset);
3087c478bd9Sstevel@tonic-gate 
3097c478bd9Sstevel@tonic-gate 			/*
3107c478bd9Sstevel@tonic-gate 			 * make sure address is not in virt-avail list
3117c478bd9Sstevel@tonic-gate 			 */
3127c478bd9Sstevel@tonic-gate 			if (address_in_memlist(virt_avail, (uint64_t)vaddr,
3137c478bd9Sstevel@tonic-gate 			    size)) {
314d9aef8afSjesusm 				prom_panic("sfmmu_map_prom_mappings:"
315d9aef8afSjesusm 				    " inconsistent translation/avail lists");
3167c478bd9Sstevel@tonic-gate 			}
3177c478bd9Sstevel@tonic-gate 
3187c478bd9Sstevel@tonic-gate 			pfn = basepfn + mmu_btop(offset);
3197c478bd9Sstevel@tonic-gate 			if (pf_is_memory(pfn)) {
3207c478bd9Sstevel@tonic-gate 				if (attr & SFMMU_UNCACHEPTTE) {
321d9aef8afSjesusm 					prom_panic("sfmmu_map_prom_mappings:"
3227c478bd9Sstevel@tonic-gate 					    " uncached prom memory page");
3237c478bd9Sstevel@tonic-gate 				}
3247c478bd9Sstevel@tonic-gate 			} else {
3257c478bd9Sstevel@tonic-gate 				if (!(attr & SFMMU_SIDEFFECT)) {
326d9aef8afSjesusm 					prom_panic("sfmmu_map_prom_mappings:"
327d9aef8afSjesusm 					    " prom i/o page without"
328d9aef8afSjesusm 					    " side-effect");
3297c478bd9Sstevel@tonic-gate 				}
3307c478bd9Sstevel@tonic-gate 			}
331bb121940Sdp78419 
332bb121940Sdp78419 			/*
333bb121940Sdp78419 			 * skip kmem64 area
334bb121940Sdp78419 			 */
335bb121940Sdp78419 			if (vaddr >= kmem64_base &&
336bb121940Sdp78419 			    vaddr < kmem64_aligned_end) {
337bb121940Sdp78419 #if !defined(C_OBP)
338d9aef8afSjesusm 				prom_panic("sfmmu_map_prom_mappings:"
339d9aef8afSjesusm 				    " unexpected kmem64 prom mapping");
340bb121940Sdp78419 #else	/* !C_OBP */
341bb121940Sdp78419 				size_t mapsz;
342bb121940Sdp78419 
343bb121940Sdp78419 				if (ptob(pfn) !=
344bb121940Sdp78419 				    kmem64_pabase + (vaddr - kmem64_base)) {
345d9aef8afSjesusm 					prom_panic("sfmmu_map_prom_mappings:"
346d9aef8afSjesusm 					    " unexpected kmem64 prom mapping");
347bb121940Sdp78419 				}
348bb121940Sdp78419 
349bb121940Sdp78419 				mapsz = kmem64_aligned_end - vaddr;
350bb121940Sdp78419 				if (mapsz >= size) {
351bb121940Sdp78419 					break;
352bb121940Sdp78419 				}
353bb121940Sdp78419 				size -= mapsz;
354bb121940Sdp78419 				offset += mapsz;
355bb121940Sdp78419 				continue;
356bb121940Sdp78419 #endif	/* !C_OBP */
357bb121940Sdp78419 			}
358bb121940Sdp78419 
3597c478bd9Sstevel@tonic-gate 			oldpfn = sfmmu_vatopfn(vaddr, KHATID, &oldtte);
3607c478bd9Sstevel@tonic-gate 			ASSERT(oldpfn != PFN_SUSPENDED);
3617c478bd9Sstevel@tonic-gate 			ASSERT(page_relocate_ready == 0);
3627c478bd9Sstevel@tonic-gate 
3637c478bd9Sstevel@tonic-gate 			if (oldpfn != PFN_INVALID) {
3647c478bd9Sstevel@tonic-gate 				/*
3657c478bd9Sstevel@tonic-gate 				 * mapping already exists.
3667c478bd9Sstevel@tonic-gate 				 * Verify they are equal
3677c478bd9Sstevel@tonic-gate 				 */
3687c478bd9Sstevel@tonic-gate 				if (pfn != oldpfn) {
369d9aef8afSjesusm 					(void) snprintf(buf, sizeof (buf),
370d9aef8afSjesusm 					"sfmmu_map_prom_mappings: mapping"
371d9aef8afSjesusm 					" conflict (va = 0x%p, pfn = 0x%p,"
372d9aef8afSjesusm 					" oldpfn = 0x%p)", (void *)vaddr,
373d9aef8afSjesusm 					    (void *)pfn, (void *)oldpfn);
374d9aef8afSjesusm 					prom_panic(buf);
3757c478bd9Sstevel@tonic-gate 				}
3767c478bd9Sstevel@tonic-gate 				size -= MMU_PAGESIZE;
3777c478bd9Sstevel@tonic-gate 				offset += MMU_PAGESIZE;
3787c478bd9Sstevel@tonic-gate 				continue;
3797c478bd9Sstevel@tonic-gate 			}
3807c478bd9Sstevel@tonic-gate 
3817c478bd9Sstevel@tonic-gate 			pp = page_numtopp_nolock(pfn);
3827c478bd9Sstevel@tonic-gate 			if ((pp != NULL) && PP_ISFREE((page_t *)pp)) {
383d9aef8afSjesusm 				(void) snprintf(buf, sizeof (buf),
384d9aef8afSjesusm 				"sfmmu_map_prom_mappings: prom-mapped"
385d9aef8afSjesusm 				" page (va = 0x%p, pfn = 0x%p) on free list",
386d9aef8afSjesusm 				    (void *)vaddr, (void *)pfn);
387d9aef8afSjesusm 				prom_panic(buf);
3887c478bd9Sstevel@tonic-gate 			}
3897c478bd9Sstevel@tonic-gate 
3907c478bd9Sstevel@tonic-gate 			sfmmu_memtte(ttep, pfn, attr, TTE8K);
3917c478bd9Sstevel@tonic-gate 			sfmmu_tteload(kas.a_hat, ttep, vaddr, pp,
3927c478bd9Sstevel@tonic-gate 			    HAT_LOAD_LOCK | SFMMU_NO_TSBLOAD);
3937c478bd9Sstevel@tonic-gate 			size -= MMU_PAGESIZE;
3947c478bd9Sstevel@tonic-gate 			offset += MMU_PAGESIZE;
3957c478bd9Sstevel@tonic-gate 		}
3967c478bd9Sstevel@tonic-gate 	}
397bb121940Sdp78419 
398bb121940Sdp78419 	/*
399bb121940Sdp78419 	 * We claimed kmem64 from prom, so now we need to load tte.
400bb121940Sdp78419 	 */
401bb121940Sdp78419 	if (kmem64_base != NULL) {
402bb121940Sdp78419 		pgcnt_t pages;
403bb121940Sdp78419 		size_t psize;
404bb121940Sdp78419 		int pszc;
405bb121940Sdp78419 
406bb121940Sdp78419 		pszc = kmem64_szc;
407bb121940Sdp78419 #ifdef sun4u
408bb121940Sdp78419 		if (pszc > TTE8K) {
409bb121940Sdp78419 			pszc = segkmem_lpszc;
410bb121940Sdp78419 		}
411bb121940Sdp78419 #endif	/* sun4u */
412bb121940Sdp78419 		psize = TTEBYTES(pszc);
413bb121940Sdp78419 		pages = btop(psize);
414bb121940Sdp78419 		basepfn = kmem64_pabase >> MMU_PAGESHIFT;
415bb121940Sdp78419 		vaddr = kmem64_base;
416bb121940Sdp78419 		while (vaddr < kmem64_end) {
417bb121940Sdp78419 			sfmmu_memtte(ttep, basepfn,
418bb121940Sdp78419 			    PROC_DATA | HAT_NOSYNC, pszc);
419bb121940Sdp78419 			sfmmu_tteload(kas.a_hat, ttep, vaddr, NULL,
420bb121940Sdp78419 			    HAT_LOAD_LOCK | SFMMU_NO_TSBLOAD);
421bb121940Sdp78419 			vaddr += psize;
422bb121940Sdp78419 			basepfn += pages;
423bb121940Sdp78419 		}
424bb121940Sdp78419 		map_prom_lpcount[pszc] =
425bb121940Sdp78419 		    ((caddr_t)P2ROUNDUP((uintptr_t)kmem64_end, psize) -
426bb121940Sdp78419 		    kmem64_base) >> TTE_PAGE_SHIFT(pszc);
427bb121940Sdp78419 	}
4287c478bd9Sstevel@tonic-gate }
4297c478bd9Sstevel@tonic-gate 
4307c478bd9Sstevel@tonic-gate #undef COMBINE	/* local to previous routine */
4317c478bd9Sstevel@tonic-gate 
4327c478bd9Sstevel@tonic-gate /*
4337c478bd9Sstevel@tonic-gate  * This routine reads in the "translations" property in to a buffer and
4347c478bd9Sstevel@tonic-gate  * returns a pointer to this buffer and the number of translations.
4357c478bd9Sstevel@tonic-gate  */
4367c478bd9Sstevel@tonic-gate static struct translation *
4377c478bd9Sstevel@tonic-gate read_prom_mappings(size_t *ntransrootp)
4387c478bd9Sstevel@tonic-gate {
4397c478bd9Sstevel@tonic-gate 	char *prop = "translations";
4407c478bd9Sstevel@tonic-gate 	size_t translen;
441fa9e4066Sahrens 	pnode_t node;
4427c478bd9Sstevel@tonic-gate 	struct translation *transroot;
4437c478bd9Sstevel@tonic-gate 
4447c478bd9Sstevel@tonic-gate 	/*
4457c478bd9Sstevel@tonic-gate 	 * the "translations" property is associated with the mmu node
4467c478bd9Sstevel@tonic-gate 	 */
447fa9e4066Sahrens 	node = (pnode_t)prom_getphandle(prom_mmu_ihandle());
4487c478bd9Sstevel@tonic-gate 
4497c478bd9Sstevel@tonic-gate 	/*
4507c478bd9Sstevel@tonic-gate 	 * We use the TSB space to read in the prom mappings.  This space
4517c478bd9Sstevel@tonic-gate 	 * is currently not being used because we haven't taken over the
4527c478bd9Sstevel@tonic-gate 	 * trap table yet.  It should be big enough to hold the mappings.
4537c478bd9Sstevel@tonic-gate 	 */
4547c478bd9Sstevel@tonic-gate 	if ((translen = prom_getproplen(node, prop)) == -1)
4557c478bd9Sstevel@tonic-gate 		cmn_err(CE_PANIC, "no translations property");
4567c478bd9Sstevel@tonic-gate 	*ntransrootp = translen / sizeof (*transroot);
4577c478bd9Sstevel@tonic-gate 	translen = roundup(translen, MMU_PAGESIZE);
4587c478bd9Sstevel@tonic-gate 	PRM_DEBUG(translen);
4597c478bd9Sstevel@tonic-gate 	if (translen > TSB_BYTES(ktsb_szcode))
4607c478bd9Sstevel@tonic-gate 		cmn_err(CE_PANIC, "not enough space for translations");
4617c478bd9Sstevel@tonic-gate 
4627c478bd9Sstevel@tonic-gate 	transroot = (struct translation *)ktsb_base;
4637c478bd9Sstevel@tonic-gate 	ASSERT(transroot);
4647c478bd9Sstevel@tonic-gate 	if (prom_getprop(node, prop, (caddr_t)transroot) == -1) {
4657c478bd9Sstevel@tonic-gate 		cmn_err(CE_PANIC, "translations getprop failed");
4667c478bd9Sstevel@tonic-gate 	}
4677c478bd9Sstevel@tonic-gate 	return (transroot);
4687c478bd9Sstevel@tonic-gate }
4697c478bd9Sstevel@tonic-gate 
4707c478bd9Sstevel@tonic-gate /*
4717c478bd9Sstevel@tonic-gate  * Init routine of the nucleus data memory allocator.
4727c478bd9Sstevel@tonic-gate  *
4737c478bd9Sstevel@tonic-gate  * The nucleus data memory allocator is organized in ecache_alignsize'd
4747c478bd9Sstevel@tonic-gate  * memory chunks. Memory allocated by ndata_alloc() will never be freed.
4757c478bd9Sstevel@tonic-gate  *
4767c478bd9Sstevel@tonic-gate  * The ndata argument is used as header of the ndata freelist.
4777c478bd9Sstevel@tonic-gate  * Other freelist nodes are placed in the nucleus memory itself
4787c478bd9Sstevel@tonic-gate  * at the beginning of a free memory chunk. Therefore a freelist
4797c478bd9Sstevel@tonic-gate  * node (struct memlist) must fit into the smallest allocatable
4807c478bd9Sstevel@tonic-gate  * memory chunk (ecache_alignsize bytes).
4817c478bd9Sstevel@tonic-gate  *
4827c478bd9Sstevel@tonic-gate  * The memory interval [base, end] passed to ndata_alloc_init() must be
4837c478bd9Sstevel@tonic-gate  * bzero'd to allow the allocator to return bzero'd memory easily.
4847c478bd9Sstevel@tonic-gate  */
4857c478bd9Sstevel@tonic-gate void
4867c478bd9Sstevel@tonic-gate ndata_alloc_init(struct memlist *ndata, uintptr_t base, uintptr_t end)
4877c478bd9Sstevel@tonic-gate {
4887c478bd9Sstevel@tonic-gate 	ASSERT(sizeof (struct memlist) <= ecache_alignsize);
4897c478bd9Sstevel@tonic-gate 
4907c478bd9Sstevel@tonic-gate 	base = roundup(base, ecache_alignsize);
4917c478bd9Sstevel@tonic-gate 	end = end - end % ecache_alignsize;
4927c478bd9Sstevel@tonic-gate 
4937c478bd9Sstevel@tonic-gate 	ASSERT(base < end);
4947c478bd9Sstevel@tonic-gate 
4957c478bd9Sstevel@tonic-gate 	ndata->address = base;
4967c478bd9Sstevel@tonic-gate 	ndata->size = end - base;
4977c478bd9Sstevel@tonic-gate 	ndata->next = NULL;
4987c478bd9Sstevel@tonic-gate 	ndata->prev = NULL;
4997c478bd9Sstevel@tonic-gate }
5007c478bd9Sstevel@tonic-gate 
5017c478bd9Sstevel@tonic-gate /*
5027c478bd9Sstevel@tonic-gate  * Deliver the size of the largest free memory chunk.
5037c478bd9Sstevel@tonic-gate  */
5047c478bd9Sstevel@tonic-gate size_t
5057c478bd9Sstevel@tonic-gate ndata_maxsize(struct memlist *ndata)
5067c478bd9Sstevel@tonic-gate {
5077c478bd9Sstevel@tonic-gate 	size_t chunksize = ndata->size;
5087c478bd9Sstevel@tonic-gate 
5097c478bd9Sstevel@tonic-gate 	while ((ndata = ndata->next) != NULL) {
5107c478bd9Sstevel@tonic-gate 		if (chunksize < ndata->size)
5117c478bd9Sstevel@tonic-gate 			chunksize = ndata->size;
5127c478bd9Sstevel@tonic-gate 	}
5137c478bd9Sstevel@tonic-gate 
5147c478bd9Sstevel@tonic-gate 	return (chunksize);
5157c478bd9Sstevel@tonic-gate }
5167c478bd9Sstevel@tonic-gate 
5177c478bd9Sstevel@tonic-gate 
5187c478bd9Sstevel@tonic-gate /*
5197c478bd9Sstevel@tonic-gate  * Allocate the last properly aligned memory chunk.
5207c478bd9Sstevel@tonic-gate  * This function is called when no more large nucleus memory chunks
5217c478bd9Sstevel@tonic-gate  * will be allocated.  The remaining free nucleus memory at the end
5227c478bd9Sstevel@tonic-gate  * of the nucleus can be added to the phys_avail list.
5237c478bd9Sstevel@tonic-gate  */
5247c478bd9Sstevel@tonic-gate void *
5251e45ea5aSblakej ndata_extra_base(struct memlist *ndata, size_t alignment, caddr_t endaddr)
5267c478bd9Sstevel@tonic-gate {
5277c478bd9Sstevel@tonic-gate 	uintptr_t base;
5287c478bd9Sstevel@tonic-gate 	size_t wasteage = 0;
5297c478bd9Sstevel@tonic-gate #ifdef	DEBUG
5307c478bd9Sstevel@tonic-gate 	static int called = 0;
5317c478bd9Sstevel@tonic-gate 
5327c478bd9Sstevel@tonic-gate 	if (called++ > 0)
5337c478bd9Sstevel@tonic-gate 		cmn_err(CE_PANIC, "ndata_extra_base() called more than once");
5347c478bd9Sstevel@tonic-gate #endif /* DEBUG */
5357c478bd9Sstevel@tonic-gate 
5367c478bd9Sstevel@tonic-gate 	/*
5377c478bd9Sstevel@tonic-gate 	 * The alignment needs to be a multiple of ecache_alignsize.
5387c478bd9Sstevel@tonic-gate 	 */
5397c478bd9Sstevel@tonic-gate 	ASSERT((alignment % ecache_alignsize) ==  0);
5407c478bd9Sstevel@tonic-gate 
5417c478bd9Sstevel@tonic-gate 	while (ndata->next != NULL) {
5427c478bd9Sstevel@tonic-gate 		wasteage += ndata->size;
5437c478bd9Sstevel@tonic-gate 		ndata = ndata->next;
5447c478bd9Sstevel@tonic-gate 	}
5457c478bd9Sstevel@tonic-gate 
5467c478bd9Sstevel@tonic-gate 	base = roundup(ndata->address, alignment);
5477c478bd9Sstevel@tonic-gate 
5487c478bd9Sstevel@tonic-gate 	if (base >= ndata->address + ndata->size)
5497c478bd9Sstevel@tonic-gate 		return (NULL);
5507c478bd9Sstevel@tonic-gate 
5511e45ea5aSblakej 	if ((caddr_t)(ndata->address + ndata->size) != endaddr) {
5521e45ea5aSblakej #ifdef DEBUG
5531e45ea5aSblakej 		ndata_middle_hole_detected = 1;	/* see if we hit this again */
5541e45ea5aSblakej #endif
5551e45ea5aSblakej 		return (NULL);
5561e45ea5aSblakej 	}
5571e45ea5aSblakej 
5587c478bd9Sstevel@tonic-gate 	if (base == ndata->address) {
5597c478bd9Sstevel@tonic-gate 		if (ndata->prev != NULL)
5607c478bd9Sstevel@tonic-gate 			ndata->prev->next = NULL;
5617c478bd9Sstevel@tonic-gate 		else
5627c478bd9Sstevel@tonic-gate 			ndata->size = 0;
5637c478bd9Sstevel@tonic-gate 
5647c478bd9Sstevel@tonic-gate 		bzero((void *)base, sizeof (struct memlist));
5657c478bd9Sstevel@tonic-gate 
5667c478bd9Sstevel@tonic-gate 	} else {
5677c478bd9Sstevel@tonic-gate 		ndata->size = base - ndata->address;
5687c478bd9Sstevel@tonic-gate 		wasteage += ndata->size;
5697c478bd9Sstevel@tonic-gate 	}
5707c478bd9Sstevel@tonic-gate 	PRM_DEBUG(wasteage);
5717c478bd9Sstevel@tonic-gate 
5727c478bd9Sstevel@tonic-gate 	return ((void *)base);
5737c478bd9Sstevel@tonic-gate }
5747c478bd9Sstevel@tonic-gate 
5757c478bd9Sstevel@tonic-gate /*
5767c478bd9Sstevel@tonic-gate  * Select the best matching buffer, avoid memory fragmentation.
5777c478bd9Sstevel@tonic-gate  */
5787c478bd9Sstevel@tonic-gate static struct memlist *
5797c478bd9Sstevel@tonic-gate ndata_select_chunk(struct memlist *ndata, size_t wanted, size_t alignment)
5807c478bd9Sstevel@tonic-gate {
5817c478bd9Sstevel@tonic-gate 	struct memlist *fnd_below = NULL;
5827c478bd9Sstevel@tonic-gate 	struct memlist *fnd_above = NULL;
5837c478bd9Sstevel@tonic-gate 	struct memlist *fnd_unused = NULL;
5847c478bd9Sstevel@tonic-gate 	struct memlist *frlist;
5857c478bd9Sstevel@tonic-gate 	uintptr_t base;
5867c478bd9Sstevel@tonic-gate 	uintptr_t end;
5877c478bd9Sstevel@tonic-gate 	size_t below;
5887c478bd9Sstevel@tonic-gate 	size_t above;
5897c478bd9Sstevel@tonic-gate 	size_t unused;
5907c478bd9Sstevel@tonic-gate 	size_t best_below = ULONG_MAX;
5917c478bd9Sstevel@tonic-gate 	size_t best_above = ULONG_MAX;
5927c478bd9Sstevel@tonic-gate 	size_t best_unused = ULONG_MAX;
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate 	ASSERT(ndata != NULL);
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate 	/*
5977c478bd9Sstevel@tonic-gate 	 * Look for the best matching buffer, avoid memory fragmentation.
5987c478bd9Sstevel@tonic-gate 	 * The following strategy is used, try to find
5997c478bd9Sstevel@tonic-gate 	 *   1. an exact fitting buffer
6007c478bd9Sstevel@tonic-gate 	 *   2. avoid wasting any space below the buffer, take first
6017c478bd9Sstevel@tonic-gate 	 *	fitting buffer
6027c478bd9Sstevel@tonic-gate 	 *   3. avoid wasting any space above the buffer, take first
6037c478bd9Sstevel@tonic-gate 	 *	fitting buffer
6047c478bd9Sstevel@tonic-gate 	 *   4. avoid wasting space, take first fitting buffer
6057c478bd9Sstevel@tonic-gate 	 *   5. take the last buffer in chain
6067c478bd9Sstevel@tonic-gate 	 */
6077c478bd9Sstevel@tonic-gate 	for (frlist = ndata; frlist != NULL; frlist = frlist->next) {
6087c478bd9Sstevel@tonic-gate 		base = roundup(frlist->address, alignment);
6097c478bd9Sstevel@tonic-gate 		end = roundup(base + wanted, ecache_alignsize);
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate 		if (end > frlist->address + frlist->size)
6127c478bd9Sstevel@tonic-gate 			continue;
6137c478bd9Sstevel@tonic-gate 
6147c478bd9Sstevel@tonic-gate 		below = (base - frlist->address) / ecache_alignsize;
6157c478bd9Sstevel@tonic-gate 		above = (frlist->address + frlist->size - end) /
6167c478bd9Sstevel@tonic-gate 		    ecache_alignsize;
6177c478bd9Sstevel@tonic-gate 		unused = below + above;
6187c478bd9Sstevel@tonic-gate 
6197c478bd9Sstevel@tonic-gate 		if (unused == 0)
6207c478bd9Sstevel@tonic-gate 			return (frlist);
6217c478bd9Sstevel@tonic-gate 
6227c478bd9Sstevel@tonic-gate 		if (frlist->next == NULL)
6237c478bd9Sstevel@tonic-gate 			break;
6247c478bd9Sstevel@tonic-gate 
6257c478bd9Sstevel@tonic-gate 		if (below < best_below) {
6267c478bd9Sstevel@tonic-gate 			best_below = below;
6277c478bd9Sstevel@tonic-gate 			fnd_below = frlist;
6287c478bd9Sstevel@tonic-gate 		}
6297c478bd9Sstevel@tonic-gate 
6307c478bd9Sstevel@tonic-gate 		if (above < best_above) {
6317c478bd9Sstevel@tonic-gate 			best_above = above;
6327c478bd9Sstevel@tonic-gate 			fnd_above = frlist;
6337c478bd9Sstevel@tonic-gate 		}
6347c478bd9Sstevel@tonic-gate 
6357c478bd9Sstevel@tonic-gate 		if (unused < best_unused) {
6367c478bd9Sstevel@tonic-gate 			best_unused = unused;
6377c478bd9Sstevel@tonic-gate 			fnd_unused = frlist;
6387c478bd9Sstevel@tonic-gate 		}
6397c478bd9Sstevel@tonic-gate 	}
6407c478bd9Sstevel@tonic-gate 
6417c478bd9Sstevel@tonic-gate 	if (best_below == 0)
6427c478bd9Sstevel@tonic-gate 		return (fnd_below);
6437c478bd9Sstevel@tonic-gate 	if (best_above == 0)
6447c478bd9Sstevel@tonic-gate 		return (fnd_above);
6457c478bd9Sstevel@tonic-gate 	if (best_unused < ULONG_MAX)
6467c478bd9Sstevel@tonic-gate 		return (fnd_unused);
6477c478bd9Sstevel@tonic-gate 
6487c478bd9Sstevel@tonic-gate 	return (frlist);
6497c478bd9Sstevel@tonic-gate }
6507c478bd9Sstevel@tonic-gate 
6517c478bd9Sstevel@tonic-gate /*
6527c478bd9Sstevel@tonic-gate  * Nucleus data memory allocator.
6537c478bd9Sstevel@tonic-gate  * The granularity of the allocator is ecache_alignsize.
6547c478bd9Sstevel@tonic-gate  * See also comment for ndata_alloc_init().
6557c478bd9Sstevel@tonic-gate  */
6567c478bd9Sstevel@tonic-gate void *
6577c478bd9Sstevel@tonic-gate ndata_alloc(struct memlist *ndata, size_t wanted, size_t alignment)
6587c478bd9Sstevel@tonic-gate {
6597c478bd9Sstevel@tonic-gate 	struct memlist *found;
6607c478bd9Sstevel@tonic-gate 	struct memlist *fnd_above;
6617c478bd9Sstevel@tonic-gate 	uintptr_t base;
6627c478bd9Sstevel@tonic-gate 	uintptr_t end;
6637c478bd9Sstevel@tonic-gate 	size_t below;
6647c478bd9Sstevel@tonic-gate 	size_t above;
6657c478bd9Sstevel@tonic-gate 
6667c478bd9Sstevel@tonic-gate 	/*
6677c478bd9Sstevel@tonic-gate 	 * Look for the best matching buffer, avoid memory fragmentation.
6687c478bd9Sstevel@tonic-gate 	 */
6697c478bd9Sstevel@tonic-gate 	if ((found = ndata_select_chunk(ndata, wanted, alignment)) == NULL)
6707c478bd9Sstevel@tonic-gate 		return (NULL);
6717c478bd9Sstevel@tonic-gate 
6727c478bd9Sstevel@tonic-gate 	/*
6737c478bd9Sstevel@tonic-gate 	 * Allocate the nucleus data buffer.
6747c478bd9Sstevel@tonic-gate 	 */
6757c478bd9Sstevel@tonic-gate 	base = roundup(found->address, alignment);
6767c478bd9Sstevel@tonic-gate 	end = roundup(base + wanted, ecache_alignsize);
6777c478bd9Sstevel@tonic-gate 	ASSERT(end <= found->address + found->size);
6787c478bd9Sstevel@tonic-gate 
6797c478bd9Sstevel@tonic-gate 	below = base - found->address;
6807c478bd9Sstevel@tonic-gate 	above = found->address + found->size - end;
6817c478bd9Sstevel@tonic-gate 	ASSERT(above == 0 || (above % ecache_alignsize) == 0);
6827c478bd9Sstevel@tonic-gate 
6837c478bd9Sstevel@tonic-gate 	if (below >= ecache_alignsize) {
6847c478bd9Sstevel@tonic-gate 		/*
6857c478bd9Sstevel@tonic-gate 		 * There is free memory below the allocated memory chunk.
6867c478bd9Sstevel@tonic-gate 		 */
6877c478bd9Sstevel@tonic-gate 		found->size = below - below % ecache_alignsize;
6887c478bd9Sstevel@tonic-gate 
6897c478bd9Sstevel@tonic-gate 		if (above) {
6907c478bd9Sstevel@tonic-gate 			fnd_above = (struct memlist *)end;
6917c478bd9Sstevel@tonic-gate 			fnd_above->address = end;
6927c478bd9Sstevel@tonic-gate 			fnd_above->size = above;
6937c478bd9Sstevel@tonic-gate 
6947c478bd9Sstevel@tonic-gate 			if ((fnd_above->next = found->next) != NULL)
6957c478bd9Sstevel@tonic-gate 				found->next->prev = fnd_above;
6967c478bd9Sstevel@tonic-gate 			fnd_above->prev = found;
6977c478bd9Sstevel@tonic-gate 			found->next = fnd_above;
6987c478bd9Sstevel@tonic-gate 		}
6997c478bd9Sstevel@tonic-gate 
7007c478bd9Sstevel@tonic-gate 		return ((void *)base);
7017c478bd9Sstevel@tonic-gate 	}
7027c478bd9Sstevel@tonic-gate 
7037c478bd9Sstevel@tonic-gate 	if (found->prev == NULL) {
7047c478bd9Sstevel@tonic-gate 		/*
7057c478bd9Sstevel@tonic-gate 		 * The first chunk (ndata) is selected.
7067c478bd9Sstevel@tonic-gate 		 */
7077c478bd9Sstevel@tonic-gate 		ASSERT(found == ndata);
7087c478bd9Sstevel@tonic-gate 		if (above) {
7097c478bd9Sstevel@tonic-gate 			found->address = end;
7107c478bd9Sstevel@tonic-gate 			found->size = above;
7117c478bd9Sstevel@tonic-gate 		} else if (found->next != NULL) {
7127c478bd9Sstevel@tonic-gate 			found->address = found->next->address;
7137c478bd9Sstevel@tonic-gate 			found->size = found->next->size;
7147c478bd9Sstevel@tonic-gate 			if ((found->next = found->next->next) != NULL)
7157c478bd9Sstevel@tonic-gate 				found->next->prev = found;
7167c478bd9Sstevel@tonic-gate 
7177c478bd9Sstevel@tonic-gate 			bzero((void *)found->address, sizeof (struct memlist));
7187c478bd9Sstevel@tonic-gate 		} else {
7197c478bd9Sstevel@tonic-gate 			found->address = end;
7207c478bd9Sstevel@tonic-gate 			found->size = 0;
7217c478bd9Sstevel@tonic-gate 		}
7227c478bd9Sstevel@tonic-gate 
7237c478bd9Sstevel@tonic-gate 		return ((void *)base);
7247c478bd9Sstevel@tonic-gate 	}
7257c478bd9Sstevel@tonic-gate 
7267c478bd9Sstevel@tonic-gate 	/*
7277c478bd9Sstevel@tonic-gate 	 * Not the first chunk.
7287c478bd9Sstevel@tonic-gate 	 */
7297c478bd9Sstevel@tonic-gate 	if (above) {
7307c478bd9Sstevel@tonic-gate 		fnd_above = (struct memlist *)end;
7317c478bd9Sstevel@tonic-gate 		fnd_above->address = end;
7327c478bd9Sstevel@tonic-gate 		fnd_above->size = above;
7337c478bd9Sstevel@tonic-gate 
7347c478bd9Sstevel@tonic-gate 		if ((fnd_above->next = found->next) != NULL)
7357c478bd9Sstevel@tonic-gate 			fnd_above->next->prev = fnd_above;
7367c478bd9Sstevel@tonic-gate 		fnd_above->prev = found->prev;
7377c478bd9Sstevel@tonic-gate 		found->prev->next = fnd_above;
7387c478bd9Sstevel@tonic-gate 
7397c478bd9Sstevel@tonic-gate 	} else {
7407c478bd9Sstevel@tonic-gate 		if ((found->prev->next = found->next) != NULL)
7417c478bd9Sstevel@tonic-gate 			found->next->prev = found->prev;
7427c478bd9Sstevel@tonic-gate 	}
7437c478bd9Sstevel@tonic-gate 
7447c478bd9Sstevel@tonic-gate 	bzero((void *)found->address, sizeof (struct memlist));
7457c478bd9Sstevel@tonic-gate 
7467c478bd9Sstevel@tonic-gate 	return ((void *)base);
7477c478bd9Sstevel@tonic-gate }
7487c478bd9Sstevel@tonic-gate 
7497c478bd9Sstevel@tonic-gate /*
7507c478bd9Sstevel@tonic-gate  * Size the kernel TSBs based upon the amount of physical
7517c478bd9Sstevel@tonic-gate  * memory in the system.
7527c478bd9Sstevel@tonic-gate  */
7537c478bd9Sstevel@tonic-gate static void
7547c478bd9Sstevel@tonic-gate calc_tsb_sizes(pgcnt_t npages)
7557c478bd9Sstevel@tonic-gate {
7567c478bd9Sstevel@tonic-gate 	PRM_DEBUG(npages);
7577c478bd9Sstevel@tonic-gate 
7587c478bd9Sstevel@tonic-gate 	if (npages <= TSB_FREEMEM_MIN) {
7597c478bd9Sstevel@tonic-gate 		ktsb_szcode = TSB_128K_SZCODE;
7607c478bd9Sstevel@tonic-gate 		enable_bigktsb = 0;
7617c478bd9Sstevel@tonic-gate 	} else if (npages <= TSB_FREEMEM_LARGE / 2) {
7627c478bd9Sstevel@tonic-gate 		ktsb_szcode = TSB_256K_SZCODE;
7637c478bd9Sstevel@tonic-gate 		enable_bigktsb = 0;
7647c478bd9Sstevel@tonic-gate 	} else if (npages <= TSB_FREEMEM_LARGE) {
7657c478bd9Sstevel@tonic-gate 		ktsb_szcode = TSB_512K_SZCODE;
7667c478bd9Sstevel@tonic-gate 		enable_bigktsb = 0;
7677c478bd9Sstevel@tonic-gate 	} else if (npages <= TSB_FREEMEM_LARGE * 2 ||
7687c478bd9Sstevel@tonic-gate 	    enable_bigktsb == 0) {
7697c478bd9Sstevel@tonic-gate 		ktsb_szcode = TSB_1M_SZCODE;
7707c478bd9Sstevel@tonic-gate 		enable_bigktsb = 0;
7717c478bd9Sstevel@tonic-gate 	} else {
7727c478bd9Sstevel@tonic-gate 		ktsb_szcode = highbit(npages - 1);
7737c478bd9Sstevel@tonic-gate 		ktsb_szcode -= TSB_START_SIZE;
7747c478bd9Sstevel@tonic-gate 		ktsb_szcode = MAX(ktsb_szcode, MIN_BIGKTSB_SZCODE);
7757c478bd9Sstevel@tonic-gate 		ktsb_szcode = MIN(ktsb_szcode, MAX_BIGKTSB_SZCODE);
7767c478bd9Sstevel@tonic-gate 	}
7777c478bd9Sstevel@tonic-gate 
7787c478bd9Sstevel@tonic-gate 	/*
7797c478bd9Sstevel@tonic-gate 	 * We choose the TSB to hold kernel 4M mappings to have twice
7807c478bd9Sstevel@tonic-gate 	 * the reach as the primary kernel TSB since this TSB will
7817c478bd9Sstevel@tonic-gate 	 * potentially (currently) be shared by both mappings to all of
782bb121940Sdp78419 	 * physical memory plus user TSBs. If this TSB has to be in nucleus
783bb121940Sdp78419 	 * (only for Spitfire and Cheetah) limit its size to 64K.
7847c478bd9Sstevel@tonic-gate 	 */
785bb121940Sdp78419 	ktsb4m_szcode = highbit((2 * npages) / TTEPAGES(TTE4M) - 1);
786bb121940Sdp78419 	ktsb4m_szcode -= TSB_START_SIZE;
787bb121940Sdp78419 	ktsb4m_szcode = MAX(ktsb4m_szcode, TSB_MIN_SZCODE);
788bb121940Sdp78419 	ktsb4m_szcode = MIN(ktsb4m_szcode, TSB_SOFTSZ_MASK);
789bb121940Sdp78419 	if ((enable_bigktsb == 0 || ktsb_phys == 0) && ktsb4m_szcode >
790bb121940Sdp78419 	    TSB_64K_SZCODE) {
791bb121940Sdp78419 		ktsb4m_szcode = TSB_64K_SZCODE;
792bb121940Sdp78419 		max_bootlp_tteszc = TTE8K;
793bb121940Sdp78419 	}
7947c478bd9Sstevel@tonic-gate 
7957c478bd9Sstevel@tonic-gate 	ktsb_sz = TSB_BYTES(ktsb_szcode);	/* kernel 8K tsb size */
7967c478bd9Sstevel@tonic-gate 	ktsb4m_sz = TSB_BYTES(ktsb4m_szcode);	/* kernel 4M tsb size */
7977c478bd9Sstevel@tonic-gate }
7987c478bd9Sstevel@tonic-gate 
7997c478bd9Sstevel@tonic-gate /*
8007c478bd9Sstevel@tonic-gate  * Allocate kernel TSBs from nucleus data memory.
8017c478bd9Sstevel@tonic-gate  * The function return 0 on success and -1 on failure.
8027c478bd9Sstevel@tonic-gate  */
8037c478bd9Sstevel@tonic-gate int
8047c478bd9Sstevel@tonic-gate ndata_alloc_tsbs(struct memlist *ndata, pgcnt_t npages)
8057c478bd9Sstevel@tonic-gate {
8067c478bd9Sstevel@tonic-gate 	/*
807bb121940Sdp78419 	 * Set ktsb_phys to 1 if the processor supports ASI_QUAD_LDD_PHYS.
808bb121940Sdp78419 	 */
809bb121940Sdp78419 	sfmmu_setup_4lp();
810bb121940Sdp78419 
811bb121940Sdp78419 	/*
8127c478bd9Sstevel@tonic-gate 	 * Size the kernel TSBs based upon the amount of physical
8137c478bd9Sstevel@tonic-gate 	 * memory in the system.
8147c478bd9Sstevel@tonic-gate 	 */
8157c478bd9Sstevel@tonic-gate 	calc_tsb_sizes(npages);
8167c478bd9Sstevel@tonic-gate 
8177c478bd9Sstevel@tonic-gate 	/*
8187c478bd9Sstevel@tonic-gate 	 * Allocate the 8K kernel TSB if it belongs inside the nucleus.
8197c478bd9Sstevel@tonic-gate 	 */
8207c478bd9Sstevel@tonic-gate 	if (enable_bigktsb == 0) {
8217c478bd9Sstevel@tonic-gate 		if ((ktsb_base = ndata_alloc(ndata, ktsb_sz, ktsb_sz)) == NULL)
8227c478bd9Sstevel@tonic-gate 			return (-1);
8237c478bd9Sstevel@tonic-gate 		ASSERT(!((uintptr_t)ktsb_base & (ktsb_sz - 1)));
8247c478bd9Sstevel@tonic-gate 
8257c478bd9Sstevel@tonic-gate 		PRM_DEBUG(ktsb_base);
8267c478bd9Sstevel@tonic-gate 		PRM_DEBUG(ktsb_sz);
8277c478bd9Sstevel@tonic-gate 		PRM_DEBUG(ktsb_szcode);
8287c478bd9Sstevel@tonic-gate 	}
8297c478bd9Sstevel@tonic-gate 
8307c478bd9Sstevel@tonic-gate 	/*
8317c478bd9Sstevel@tonic-gate 	 * Next, allocate 4M kernel TSB from the nucleus since it's small.
8327c478bd9Sstevel@tonic-gate 	 */
833bb121940Sdp78419 	if (ktsb4m_szcode <= TSB_64K_SZCODE) {
834bb121940Sdp78419 
835bb121940Sdp78419 		ktsb4m_base = ndata_alloc(ndata, ktsb4m_sz, ktsb4m_sz);
836bb121940Sdp78419 		if (ktsb4m_base == NULL)
8377c478bd9Sstevel@tonic-gate 			return (-1);
8387c478bd9Sstevel@tonic-gate 		ASSERT(!((uintptr_t)ktsb4m_base & (ktsb4m_sz - 1)));
8397c478bd9Sstevel@tonic-gate 
8407c478bd9Sstevel@tonic-gate 		PRM_DEBUG(ktsb4m_base);
8417c478bd9Sstevel@tonic-gate 		PRM_DEBUG(ktsb4m_sz);
8427c478bd9Sstevel@tonic-gate 		PRM_DEBUG(ktsb4m_szcode);
843bb121940Sdp78419 	}
8447c478bd9Sstevel@tonic-gate 
8457c478bd9Sstevel@tonic-gate 	return (0);
8467c478bd9Sstevel@tonic-gate }
8477c478bd9Sstevel@tonic-gate 
848*986fd29aSsetje size_t
849*986fd29aSsetje calc_hmehash_sz(pgcnt_t npages)
8507c478bd9Sstevel@tonic-gate {
8517c478bd9Sstevel@tonic-gate 	ulong_t hme_buckets;
8527c478bd9Sstevel@tonic-gate 
8537c478bd9Sstevel@tonic-gate 	/*
8547c478bd9Sstevel@tonic-gate 	 * The number of buckets in the hme hash tables
8557c478bd9Sstevel@tonic-gate 	 * is a power of 2 such that the average hash chain length is
8567c478bd9Sstevel@tonic-gate 	 * HMENT_HASHAVELEN.  The number of buckets for the user hash is
8577c478bd9Sstevel@tonic-gate 	 * a function of physical memory and a predefined overmapping factor.
8587c478bd9Sstevel@tonic-gate 	 * The number of buckets for the kernel hash is a function of
8597c478bd9Sstevel@tonic-gate 	 * physical memory only.
8607c478bd9Sstevel@tonic-gate 	 */
8617c478bd9Sstevel@tonic-gate 	hme_buckets = (npages * HMEHASH_FACTOR) /
8627c478bd9Sstevel@tonic-gate 	    (HMENT_HASHAVELEN * (HMEBLK_SPAN(TTE8K) >> MMU_PAGESHIFT));
8637c478bd9Sstevel@tonic-gate 
8647c478bd9Sstevel@tonic-gate 	uhmehash_num = (int)MIN(hme_buckets, MAX_UHME_BUCKETS);
8657c478bd9Sstevel@tonic-gate 
8667c478bd9Sstevel@tonic-gate 	if (uhmehash_num > USER_BUCKETS_THRESHOLD) {
8677c478bd9Sstevel@tonic-gate 		/*
8687c478bd9Sstevel@tonic-gate 		 * if uhmehash_num is not power of 2 round it down to the
8697c478bd9Sstevel@tonic-gate 		 *  next power of 2.
8707c478bd9Sstevel@tonic-gate 		 */
8717c478bd9Sstevel@tonic-gate 		uint_t align = 1 << (highbit(uhmehash_num - 1) - 1);
8727c478bd9Sstevel@tonic-gate 		uhmehash_num = P2ALIGN(uhmehash_num, align);
8737c478bd9Sstevel@tonic-gate 	} else
8747c478bd9Sstevel@tonic-gate 		uhmehash_num = 1 << highbit(uhmehash_num - 1);
8757c478bd9Sstevel@tonic-gate 
8767c478bd9Sstevel@tonic-gate 	hme_buckets = npages / (HMEBLK_SPAN(TTE8K) >> MMU_PAGESHIFT);
8777c478bd9Sstevel@tonic-gate 	khmehash_num = (int)MIN(hme_buckets, MAX_KHME_BUCKETS);
8787c478bd9Sstevel@tonic-gate 	khmehash_num = 1 << highbit(khmehash_num - 1);
8797c478bd9Sstevel@tonic-gate 	khmehash_num = MAX(khmehash_num, MIN_KHME_BUCKETS);
8807c478bd9Sstevel@tonic-gate 
881*986fd29aSsetje 	return ((uhmehash_num + khmehash_num) * sizeof (struct hmehash_bucket));
8827c478bd9Sstevel@tonic-gate }
8837c478bd9Sstevel@tonic-gate 
884*986fd29aSsetje caddr_t
885*986fd29aSsetje alloc_hmehash(caddr_t alloc_base)
886*986fd29aSsetje {
887*986fd29aSsetje 	size_t khmehash_sz, uhmehash_sz;
888*986fd29aSsetje 
889*986fd29aSsetje 	khme_hash = (struct hmehash_bucket *)alloc_base;
890*986fd29aSsetje 	khmehash_sz = khmehash_num * sizeof (struct hmehash_bucket);
891*986fd29aSsetje 	alloc_base += khmehash_sz;
892*986fd29aSsetje 
893*986fd29aSsetje 	uhme_hash = (struct hmehash_bucket *)alloc_base;
894*986fd29aSsetje 	uhmehash_sz = uhmehash_num * sizeof (struct hmehash_bucket);
895*986fd29aSsetje 	alloc_base += uhmehash_sz;
896*986fd29aSsetje 
8977c478bd9Sstevel@tonic-gate 	PRM_DEBUG(khme_hash);
8987c478bd9Sstevel@tonic-gate 	PRM_DEBUG(uhme_hash);
899*986fd29aSsetje 
900*986fd29aSsetje 	return (alloc_base);
901*986fd29aSsetje }
902*986fd29aSsetje 
903*986fd29aSsetje /*
904*986fd29aSsetje  * Allocate hat structs from the nucleus data memory.
905*986fd29aSsetje  */
906*986fd29aSsetje int
907*986fd29aSsetje ndata_alloc_hat(struct memlist *ndata, pgcnt_t npages)
908*986fd29aSsetje {
909*986fd29aSsetje 	size_t	mml_alloc_sz;
910*986fd29aSsetje 	size_t	cb_alloc_sz;
9117c478bd9Sstevel@tonic-gate 
9127c478bd9Sstevel@tonic-gate 	/*
9137c478bd9Sstevel@tonic-gate 	 * For the page mapping list mutex array we allocate one mutex
9147c478bd9Sstevel@tonic-gate 	 * for every 128 pages (1 MB) with a minimum of 64 entries and
9157c478bd9Sstevel@tonic-gate 	 * a maximum of 8K entries. For the initial computation npages
9167c478bd9Sstevel@tonic-gate 	 * is rounded up (ie. 1 << highbit(npages * 1.5 / 128))
9177c478bd9Sstevel@tonic-gate 	 *
9187c478bd9Sstevel@tonic-gate 	 * mml_shift is roughly log2(mml_table_sz) + 3 for MLIST_HASH
9197c478bd9Sstevel@tonic-gate 	 */
9207c478bd9Sstevel@tonic-gate 	mml_table_sz = 1 << highbit((npages * 3) / 256);
9217c478bd9Sstevel@tonic-gate 	if (mml_table_sz < 64)
9227c478bd9Sstevel@tonic-gate 		mml_table_sz = 64;
9237c478bd9Sstevel@tonic-gate 	else if (mml_table_sz > 8192)
9247c478bd9Sstevel@tonic-gate 		mml_table_sz = 8192;
9257c478bd9Sstevel@tonic-gate 	mml_shift = highbit(mml_table_sz) + 3;
9267c478bd9Sstevel@tonic-gate 
9277c478bd9Sstevel@tonic-gate 	PRM_DEBUG(mml_table_sz);
9287c478bd9Sstevel@tonic-gate 	PRM_DEBUG(mml_shift);
9297c478bd9Sstevel@tonic-gate 
9307c478bd9Sstevel@tonic-gate 	mml_alloc_sz = mml_table_sz * sizeof (kmutex_t);
9317c478bd9Sstevel@tonic-gate 
9327c478bd9Sstevel@tonic-gate 	mml_table = ndata_alloc(ndata, mml_alloc_sz, ecache_alignsize);
933*986fd29aSsetje 	if (mml_table == NULL)
934*986fd29aSsetje 		return (-1);
9357c478bd9Sstevel@tonic-gate 	PRM_DEBUG(mml_table);
9367c478bd9Sstevel@tonic-gate 
9377c478bd9Sstevel@tonic-gate 	cb_alloc_sz = sfmmu_max_cb_id * sizeof (struct sfmmu_callback);
9387c478bd9Sstevel@tonic-gate 	PRM_DEBUG(cb_alloc_sz);
9397c478bd9Sstevel@tonic-gate 	sfmmu_cb_table = ndata_alloc(ndata, cb_alloc_sz, ecache_alignsize);
940*986fd29aSsetje 	if (sfmmu_cb_table == NULL)
941*986fd29aSsetje 		return (-1);
9427c478bd9Sstevel@tonic-gate 	PRM_DEBUG(sfmmu_cb_table);
9437c478bd9Sstevel@tonic-gate 
944*986fd29aSsetje 	return (0);
945*986fd29aSsetje }
946*986fd29aSsetje 
947*986fd29aSsetje int
948*986fd29aSsetje ndata_alloc_kpm(struct memlist *ndata, pgcnt_t kpm_npages)
949*986fd29aSsetje {
950*986fd29aSsetje 	size_t	kpmp_alloc_sz;
951*986fd29aSsetje 
9527c478bd9Sstevel@tonic-gate 	/*
9537c478bd9Sstevel@tonic-gate 	 * For the kpm_page mutex array we allocate one mutex every 16
9547c478bd9Sstevel@tonic-gate 	 * kpm pages (64MB). In smallpage mode we allocate one mutex
9557c478bd9Sstevel@tonic-gate 	 * every 8K pages. The minimum is set to 64 entries and the
9567c478bd9Sstevel@tonic-gate 	 * maximum to 8K entries.
9577c478bd9Sstevel@tonic-gate 	 */
9587c478bd9Sstevel@tonic-gate 	if (kpm_smallpages == 0) {
9597c478bd9Sstevel@tonic-gate 		kpmp_shift = highbit(sizeof (kpm_page_t)) - 1;
9607c478bd9Sstevel@tonic-gate 		kpmp_table_sz = 1 << highbit(kpm_npages / 16);
9617c478bd9Sstevel@tonic-gate 		kpmp_table_sz = (kpmp_table_sz < 64) ? 64 :
9627c478bd9Sstevel@tonic-gate 		    ((kpmp_table_sz > 8192) ? 8192 : kpmp_table_sz);
9637c478bd9Sstevel@tonic-gate 		kpmp_alloc_sz = kpmp_table_sz * sizeof (kpm_hlk_t);
9647c478bd9Sstevel@tonic-gate 
9657c478bd9Sstevel@tonic-gate 		kpmp_table = ndata_alloc(ndata, kpmp_alloc_sz,
9667c478bd9Sstevel@tonic-gate 		    ecache_alignsize);
967*986fd29aSsetje 		if (kpmp_table == NULL)
968*986fd29aSsetje 			return (-1);
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate 		PRM_DEBUG(kpmp_table);
9717c478bd9Sstevel@tonic-gate 		PRM_DEBUG(kpmp_table_sz);
9727c478bd9Sstevel@tonic-gate 
9737c478bd9Sstevel@tonic-gate 		kpmp_stable_sz = 0;
9747c478bd9Sstevel@tonic-gate 		kpmp_stable = NULL;
9757c478bd9Sstevel@tonic-gate 	} else {
9767c478bd9Sstevel@tonic-gate 		ASSERT(kpm_pgsz == PAGESIZE);
9777c478bd9Sstevel@tonic-gate 		kpmp_shift = highbit(sizeof (kpm_shlk_t)) + 1;
9787c478bd9Sstevel@tonic-gate 		kpmp_stable_sz = 1 << highbit(kpm_npages / 8192);
9797c478bd9Sstevel@tonic-gate 		kpmp_stable_sz = (kpmp_stable_sz < 64) ? 64 :
9807c478bd9Sstevel@tonic-gate 		    ((kpmp_stable_sz > 8192) ? 8192 : kpmp_stable_sz);
9817c478bd9Sstevel@tonic-gate 		kpmp_alloc_sz = kpmp_stable_sz * sizeof (kpm_shlk_t);
9827c478bd9Sstevel@tonic-gate 
9837c478bd9Sstevel@tonic-gate 		kpmp_stable = ndata_alloc(ndata, kpmp_alloc_sz,
9847c478bd9Sstevel@tonic-gate 		    ecache_alignsize);
985*986fd29aSsetje 		if (kpmp_stable == NULL)
986*986fd29aSsetje 			return (-1);
9877c478bd9Sstevel@tonic-gate 
9887c478bd9Sstevel@tonic-gate 		PRM_DEBUG(kpmp_stable);
9897c478bd9Sstevel@tonic-gate 		PRM_DEBUG(kpmp_stable_sz);
9907c478bd9Sstevel@tonic-gate 
9917c478bd9Sstevel@tonic-gate 		kpmp_table_sz = 0;
9927c478bd9Sstevel@tonic-gate 		kpmp_table = NULL;
9937c478bd9Sstevel@tonic-gate 	}
9947c478bd9Sstevel@tonic-gate 	PRM_DEBUG(kpmp_shift);
9957c478bd9Sstevel@tonic-gate 
9967c478bd9Sstevel@tonic-gate 	return (0);
9977c478bd9Sstevel@tonic-gate }
9987c478bd9Sstevel@tonic-gate 
999bb121940Sdp78419 /*
1000bb121940Sdp78419  * This function bop allocs kernel TSBs.
10017c478bd9Sstevel@tonic-gate  */
10027c478bd9Sstevel@tonic-gate caddr_t
10037c478bd9Sstevel@tonic-gate sfmmu_ktsb_alloc(caddr_t tsbbase)
10047c478bd9Sstevel@tonic-gate {
10057c478bd9Sstevel@tonic-gate 	caddr_t vaddr;
10067c478bd9Sstevel@tonic-gate 
10077c478bd9Sstevel@tonic-gate 	if (enable_bigktsb) {
10087c478bd9Sstevel@tonic-gate 		ktsb_base = (caddr_t)roundup((uintptr_t)tsbbase, ktsb_sz);
1009*986fd29aSsetje 		vaddr = prom_alloc(ktsb_base, ktsb_sz, ktsb_sz);
10107c478bd9Sstevel@tonic-gate 		if (vaddr != ktsb_base)
10117c478bd9Sstevel@tonic-gate 			cmn_err(CE_PANIC, "sfmmu_ktsb_alloc: can't alloc"
1012bb121940Sdp78419 			    " 8K bigktsb");
10137c478bd9Sstevel@tonic-gate 		ktsb_base = vaddr;
10147c478bd9Sstevel@tonic-gate 		tsbbase = ktsb_base + ktsb_sz;
10157c478bd9Sstevel@tonic-gate 		PRM_DEBUG(ktsb_base);
10167c478bd9Sstevel@tonic-gate 		PRM_DEBUG(tsbbase);
10177c478bd9Sstevel@tonic-gate 	}
1018bb121940Sdp78419 
1019bb121940Sdp78419 	if (ktsb4m_szcode > TSB_64K_SZCODE) {
1020bb121940Sdp78419 		ASSERT(ktsb_phys && enable_bigktsb);
1021bb121940Sdp78419 		ktsb4m_base = (caddr_t)roundup((uintptr_t)tsbbase, ktsb4m_sz);
1022bb121940Sdp78419 		vaddr = (caddr_t)BOP_ALLOC(bootops, ktsb4m_base, ktsb4m_sz,
1023bb121940Sdp78419 		    ktsb4m_sz);
1024bb121940Sdp78419 		if (vaddr != ktsb4m_base)
1025bb121940Sdp78419 			cmn_err(CE_PANIC, "sfmmu_ktsb_alloc: can't alloc"
1026bb121940Sdp78419 			    " 4M bigktsb");
1027bb121940Sdp78419 		ktsb4m_base = vaddr;
1028bb121940Sdp78419 		tsbbase = ktsb4m_base + ktsb4m_sz;
1029bb121940Sdp78419 		PRM_DEBUG(ktsb4m_base);
1030bb121940Sdp78419 		PRM_DEBUG(tsbbase);
1031bb121940Sdp78419 	}
10327c478bd9Sstevel@tonic-gate 	return (tsbbase);
10337c478bd9Sstevel@tonic-gate }
10347c478bd9Sstevel@tonic-gate 
10357c478bd9Sstevel@tonic-gate /*
10367c478bd9Sstevel@tonic-gate  * Moves code assembled outside of the trap table into the trap
10377c478bd9Sstevel@tonic-gate  * table taking care to relocate relative branches to code outside
10387c478bd9Sstevel@tonic-gate  * of the trap handler.
10397c478bd9Sstevel@tonic-gate  */
10407c478bd9Sstevel@tonic-gate static void
10417c478bd9Sstevel@tonic-gate sfmmu_reloc_trap_handler(void *tablep, void *start, size_t count)
10427c478bd9Sstevel@tonic-gate {
10437c478bd9Sstevel@tonic-gate 	size_t i;
10447c478bd9Sstevel@tonic-gate 	uint32_t *src;
10457c478bd9Sstevel@tonic-gate 	uint32_t *dst;
10467c478bd9Sstevel@tonic-gate 	uint32_t inst;
10477c478bd9Sstevel@tonic-gate 	int op, op2;
10487c478bd9Sstevel@tonic-gate 	int32_t offset;
10497c478bd9Sstevel@tonic-gate 	int disp;
10507c478bd9Sstevel@tonic-gate 
10517c478bd9Sstevel@tonic-gate 	src = start;
10527c478bd9Sstevel@tonic-gate 	dst = tablep;
10537c478bd9Sstevel@tonic-gate 	offset = src - dst;
10547c478bd9Sstevel@tonic-gate 	for (src = start, i = 0; i < count; i++, src++, dst++) {
10557c478bd9Sstevel@tonic-gate 		inst = *dst = *src;
10567c478bd9Sstevel@tonic-gate 		op = (inst >> 30) & 0x2;
10577c478bd9Sstevel@tonic-gate 		if (op == 1) {
10587c478bd9Sstevel@tonic-gate 			/* call */
10597c478bd9Sstevel@tonic-gate 			disp = ((int32_t)inst << 2) >> 2; /* sign-extend */
10607c478bd9Sstevel@tonic-gate 			if (disp + i >= 0 && disp + i < count)
10617c478bd9Sstevel@tonic-gate 				continue;
10627c478bd9Sstevel@tonic-gate 			disp += offset;
10637c478bd9Sstevel@tonic-gate 			inst = 0x40000000u | (disp & 0x3fffffffu);
10647c478bd9Sstevel@tonic-gate 			*dst = inst;
10657c478bd9Sstevel@tonic-gate 		} else if (op == 0) {
10667c478bd9Sstevel@tonic-gate 			/* branch or sethi */
10677c478bd9Sstevel@tonic-gate 			op2 = (inst >> 22) & 0x7;
10687c478bd9Sstevel@tonic-gate 
10697c478bd9Sstevel@tonic-gate 			switch (op2) {
10707c478bd9Sstevel@tonic-gate 			case 0x3: /* BPr */
10717c478bd9Sstevel@tonic-gate 				disp = (((inst >> 20) & 0x3) << 14) |
10727c478bd9Sstevel@tonic-gate 				    (inst & 0x3fff);
10737c478bd9Sstevel@tonic-gate 				disp = (disp << 16) >> 16; /* sign-extend */
10747c478bd9Sstevel@tonic-gate 				if (disp + i >= 0 && disp + i < count)
10757c478bd9Sstevel@tonic-gate 					continue;
10767c478bd9Sstevel@tonic-gate 				disp += offset;
10777c478bd9Sstevel@tonic-gate 				if (((disp << 16) >> 16) != disp)
10787c478bd9Sstevel@tonic-gate 					cmn_err(CE_PANIC, "bad reloc");
10797c478bd9Sstevel@tonic-gate 				inst &= ~0x303fff;
10807c478bd9Sstevel@tonic-gate 				inst |= (disp & 0x3fff);
10817c478bd9Sstevel@tonic-gate 				inst |= (disp & 0xc000) << 6;
10827c478bd9Sstevel@tonic-gate 				break;
10837c478bd9Sstevel@tonic-gate 
10847c478bd9Sstevel@tonic-gate 			case 0x2: /* Bicc */
10857c478bd9Sstevel@tonic-gate 				disp = ((int32_t)inst << 10) >> 10;
10867c478bd9Sstevel@tonic-gate 				if (disp + i >= 0 && disp + i < count)
10877c478bd9Sstevel@tonic-gate 					continue;
10887c478bd9Sstevel@tonic-gate 				disp += offset;
10897c478bd9Sstevel@tonic-gate 				if (((disp << 10) >> 10) != disp)
10907c478bd9Sstevel@tonic-gate 					cmn_err(CE_PANIC, "bad reloc");
10917c478bd9Sstevel@tonic-gate 				inst &= ~0x3fffff;
10927c478bd9Sstevel@tonic-gate 				inst |= (disp & 0x3fffff);
10937c478bd9Sstevel@tonic-gate 				break;
10947c478bd9Sstevel@tonic-gate 
10957c478bd9Sstevel@tonic-gate 			case 0x1: /* Bpcc */
10967c478bd9Sstevel@tonic-gate 				disp = ((int32_t)inst << 13) >> 13;
10977c478bd9Sstevel@tonic-gate 				if (disp + i >= 0 && disp + i < count)
10987c478bd9Sstevel@tonic-gate 					continue;
10997c478bd9Sstevel@tonic-gate 				disp += offset;
11007c478bd9Sstevel@tonic-gate 				if (((disp << 13) >> 13) != disp)
11017c478bd9Sstevel@tonic-gate 					cmn_err(CE_PANIC, "bad reloc");
11027c478bd9Sstevel@tonic-gate 				inst &= ~0x7ffff;
11037c478bd9Sstevel@tonic-gate 				inst |= (disp & 0x7ffffu);
11047c478bd9Sstevel@tonic-gate 				break;
11057c478bd9Sstevel@tonic-gate 			}
11067c478bd9Sstevel@tonic-gate 			*dst = inst;
11077c478bd9Sstevel@tonic-gate 		}
11087c478bd9Sstevel@tonic-gate 	}
11097c478bd9Sstevel@tonic-gate 	flush_instr_mem(tablep, count * sizeof (uint32_t));
11107c478bd9Sstevel@tonic-gate }
11117c478bd9Sstevel@tonic-gate 
11127c478bd9Sstevel@tonic-gate /*
11137c478bd9Sstevel@tonic-gate  * Routine to allocate a large page to use in the TSB caches.
11147c478bd9Sstevel@tonic-gate  */
11157c478bd9Sstevel@tonic-gate /*ARGSUSED*/
11167c478bd9Sstevel@tonic-gate static page_t *
11177c478bd9Sstevel@tonic-gate sfmmu_tsb_page_create(void *addr, size_t size, int vmflag, void *arg)
11187c478bd9Sstevel@tonic-gate {
11197c478bd9Sstevel@tonic-gate 	int pgflags;
11207c478bd9Sstevel@tonic-gate 
11217c478bd9Sstevel@tonic-gate 	pgflags = PG_EXCL;
11227c478bd9Sstevel@tonic-gate 	if ((vmflag & VM_NOSLEEP) == 0)
11237c478bd9Sstevel@tonic-gate 		pgflags |= PG_WAIT;
11247c478bd9Sstevel@tonic-gate 	if (vmflag & VM_PANIC)
11257c478bd9Sstevel@tonic-gate 		pgflags |= PG_PANIC;
11267c478bd9Sstevel@tonic-gate 	if (vmflag & VM_PUSHPAGE)
11277c478bd9Sstevel@tonic-gate 		pgflags |= PG_PUSHPAGE;
11287c478bd9Sstevel@tonic-gate 
11297c478bd9Sstevel@tonic-gate 	return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
11307c478bd9Sstevel@tonic-gate 	    pgflags, &kvseg, addr, arg));
11317c478bd9Sstevel@tonic-gate }
11327c478bd9Sstevel@tonic-gate 
11337c478bd9Sstevel@tonic-gate /*
11347c478bd9Sstevel@tonic-gate  * Allocate a large page to back the virtual address range
11357c478bd9Sstevel@tonic-gate  * [addr, addr + size).  If addr is NULL, allocate the virtual address
11367c478bd9Sstevel@tonic-gate  * space as well.
11377c478bd9Sstevel@tonic-gate  */
11387c478bd9Sstevel@tonic-gate static void *
11397c478bd9Sstevel@tonic-gate sfmmu_tsb_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
11407c478bd9Sstevel@tonic-gate     uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
11417c478bd9Sstevel@tonic-gate     void *pcarg)
11427c478bd9Sstevel@tonic-gate {
11437c478bd9Sstevel@tonic-gate 	page_t *ppl;
11447c478bd9Sstevel@tonic-gate 	page_t *rootpp;
11457c478bd9Sstevel@tonic-gate 	caddr_t addr = inaddr;
11467c478bd9Sstevel@tonic-gate 	pgcnt_t npages = btopr(size);
11477c478bd9Sstevel@tonic-gate 	page_t **ppa;
11487c478bd9Sstevel@tonic-gate 	int i = 0;
11497c478bd9Sstevel@tonic-gate 
11507c478bd9Sstevel@tonic-gate 	/*
11517c478bd9Sstevel@tonic-gate 	 * Assuming that only TSBs will call this with size > PAGESIZE
11527c478bd9Sstevel@tonic-gate 	 * There is no reason why this couldn't be expanded to 8k pages as
11537c478bd9Sstevel@tonic-gate 	 * well, or other page sizes in the future .... but for now, we
11547c478bd9Sstevel@tonic-gate 	 * only support fixed sized page requests.
11557c478bd9Sstevel@tonic-gate 	 */
11567c478bd9Sstevel@tonic-gate 	if ((inaddr == NULL) && ((addr = vmem_xalloc(vmp, size, size, 0, 0,
11577c478bd9Sstevel@tonic-gate 	    NULL, NULL, vmflag)) == NULL))
11587c478bd9Sstevel@tonic-gate 		return (NULL);
11597c478bd9Sstevel@tonic-gate 
11607c478bd9Sstevel@tonic-gate 	if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
11617c478bd9Sstevel@tonic-gate 		if (inaddr == NULL)
11627c478bd9Sstevel@tonic-gate 			vmem_xfree(vmp, addr, size);
11637c478bd9Sstevel@tonic-gate 		return (NULL);
11647c478bd9Sstevel@tonic-gate 	}
11657c478bd9Sstevel@tonic-gate 
11667c478bd9Sstevel@tonic-gate 	ppl = page_create_func(addr, size, vmflag, pcarg);
11677c478bd9Sstevel@tonic-gate 	if (ppl == NULL) {
11687c478bd9Sstevel@tonic-gate 		if (inaddr == NULL)
11697c478bd9Sstevel@tonic-gate 			vmem_xfree(vmp, addr, size);
11707c478bd9Sstevel@tonic-gate 		page_unresv(npages);
11717c478bd9Sstevel@tonic-gate 		return (NULL);
11727c478bd9Sstevel@tonic-gate 	}
11737c478bd9Sstevel@tonic-gate 
11747c478bd9Sstevel@tonic-gate 	rootpp = ppl;
11757c478bd9Sstevel@tonic-gate 	ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);
11767c478bd9Sstevel@tonic-gate 	while (ppl != NULL) {
11777c478bd9Sstevel@tonic-gate 		page_t *pp = ppl;
11787c478bd9Sstevel@tonic-gate 		ppa[i++] = pp;
11797c478bd9Sstevel@tonic-gate 		page_sub(&ppl, pp);
11807c478bd9Sstevel@tonic-gate 		ASSERT(page_iolock_assert(pp));
11817c478bd9Sstevel@tonic-gate 		page_io_unlock(pp);
11827c478bd9Sstevel@tonic-gate 	}
11837c478bd9Sstevel@tonic-gate 
11847c478bd9Sstevel@tonic-gate 	/*
11857c478bd9Sstevel@tonic-gate 	 * Load the locked entry.  It's OK to preload the entry into
11867c478bd9Sstevel@tonic-gate 	 * the TSB since we now support large mappings in the kernel TSB.
11877c478bd9Sstevel@tonic-gate 	 */
11887c478bd9Sstevel@tonic-gate 	hat_memload_array(kas.a_hat, (caddr_t)rootpp->p_offset, size,
11897c478bd9Sstevel@tonic-gate 	    ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, HAT_LOAD_LOCK);
11907c478bd9Sstevel@tonic-gate 
11917c478bd9Sstevel@tonic-gate 	for (--i; i >= 0; --i) {
11927c478bd9Sstevel@tonic-gate 		(void) page_pp_lock(ppa[i], 0, 1);
11937c478bd9Sstevel@tonic-gate 		page_unlock(ppa[i]);
11947c478bd9Sstevel@tonic-gate 	}
11957c478bd9Sstevel@tonic-gate 
11967c478bd9Sstevel@tonic-gate 	kmem_free(ppa, npages * sizeof (page_t *));
11977c478bd9Sstevel@tonic-gate 	return (addr);
11987c478bd9Sstevel@tonic-gate }
11997c478bd9Sstevel@tonic-gate 
12007c478bd9Sstevel@tonic-gate /* Called to import new spans into the TSB vmem arenas */
12017c478bd9Sstevel@tonic-gate void *
12027c478bd9Sstevel@tonic-gate sfmmu_tsb_segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
12037c478bd9Sstevel@tonic-gate {
12047c478bd9Sstevel@tonic-gate 	lgrp_id_t lgrpid = LGRP_NONE;
12057c478bd9Sstevel@tonic-gate 
12067c478bd9Sstevel@tonic-gate 	if (tsb_lgrp_affinity) {
12077c478bd9Sstevel@tonic-gate 		/*
12087c478bd9Sstevel@tonic-gate 		 * Search for the vmp->lgrpid mapping by brute force;
12097c478bd9Sstevel@tonic-gate 		 * some day vmp will have an lgrp, until then we have
12107c478bd9Sstevel@tonic-gate 		 * to do this the hard way.
12117c478bd9Sstevel@tonic-gate 		 */
12127c478bd9Sstevel@tonic-gate 		for (lgrpid = 0; lgrpid < NLGRPS_MAX &&
1213*986fd29aSsetje 		    vmp != kmem_tsb_default_arena[lgrpid]; lgrpid++)
1214*986fd29aSsetje 			;
12157c478bd9Sstevel@tonic-gate 		if (lgrpid == NLGRPS_MAX)
12167c478bd9Sstevel@tonic-gate 			lgrpid = LGRP_NONE;
12177c478bd9Sstevel@tonic-gate 	}
12187c478bd9Sstevel@tonic-gate 
12197c478bd9Sstevel@tonic-gate 	return (sfmmu_tsb_xalloc(vmp, NULL, size, vmflag, 0,
12207c478bd9Sstevel@tonic-gate 	    sfmmu_tsb_page_create, lgrpid != LGRP_NONE? &lgrpid : NULL));
12217c478bd9Sstevel@tonic-gate }
12227c478bd9Sstevel@tonic-gate 
12237c478bd9Sstevel@tonic-gate /* Called to free spans from the TSB vmem arenas */
12247c478bd9Sstevel@tonic-gate void
12257c478bd9Sstevel@tonic-gate sfmmu_tsb_segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
12267c478bd9Sstevel@tonic-gate {
12277c478bd9Sstevel@tonic-gate 	page_t *pp;
12287c478bd9Sstevel@tonic-gate 	caddr_t addr = inaddr;
12297c478bd9Sstevel@tonic-gate 	caddr_t eaddr;
12307c478bd9Sstevel@tonic-gate 	pgcnt_t npages = btopr(size);
12317c478bd9Sstevel@tonic-gate 	pgcnt_t pgs_left = npages;
12327c478bd9Sstevel@tonic-gate 	page_t *rootpp = NULL;
12337c478bd9Sstevel@tonic-gate 
12347c478bd9Sstevel@tonic-gate 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
12357c478bd9Sstevel@tonic-gate 
12367c478bd9Sstevel@tonic-gate 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
12377c478bd9Sstevel@tonic-gate 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
12387c478bd9Sstevel@tonic-gate 		if (pp == NULL)
12397c478bd9Sstevel@tonic-gate 			panic("sfmmu_tsb_segkmem_free: page not found");
12407c478bd9Sstevel@tonic-gate 
12417c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_EXCL(pp));
12427c478bd9Sstevel@tonic-gate 		page_pp_unlock(pp, 0, 1);
12437c478bd9Sstevel@tonic-gate 
12447c478bd9Sstevel@tonic-gate 		if (rootpp == NULL)
12457c478bd9Sstevel@tonic-gate 			rootpp = pp;
12467c478bd9Sstevel@tonic-gate 		if (--pgs_left == 0) {
12477c478bd9Sstevel@tonic-gate 			/*
12487c478bd9Sstevel@tonic-gate 			 * similar logic to segspt_free_pages, but we know we
12497c478bd9Sstevel@tonic-gate 			 * have one large page.
12507c478bd9Sstevel@tonic-gate 			 */
12517c478bd9Sstevel@tonic-gate 			page_destroy_pages(rootpp);
12527c478bd9Sstevel@tonic-gate 		}
12537c478bd9Sstevel@tonic-gate 	}
12547c478bd9Sstevel@tonic-gate 	page_unresv(npages);
12557c478bd9Sstevel@tonic-gate 
12567c478bd9Sstevel@tonic-gate 	if (vmp != NULL)
12577c478bd9Sstevel@tonic-gate 		vmem_xfree(vmp, inaddr, size);
12587c478bd9Sstevel@tonic-gate }
1259