/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * External routines and data structures */ extern void sfmmu_cache_flushcolor(int, pfn_t); /* * Static routines */ static void sfmmu_set_tlb(void); /* * Global Data: */ caddr_t textva, datava; tte_t ktext_tte, kdata_tte; /* ttes for kernel text and data */ int enable_bigktsb = 1; tte_t bigktsb_ttes[MAX_BIGKTSB_TTES]; int bigktsb_nttes = 0; /* * Controls the logic which enables the use of the * QUAD_LDD_PHYS ASI for TSB accesses. */ int ktsb_phys = 0; /* * This routine remaps the kernel using large ttes * All entries except locked ones will be removed from the tlb. * It assumes that both the text and data segments reside in a separate * 4mb virtual and physical contigous memory chunk. This routine * is only executed by the first cpu. The remaining cpus execute * sfmmu_mp_startup() instead. * XXX It assumes that the start of the text segment is KERNELBASE. It should * actually be based on start. */ void sfmmu_remap_kernel(void) { pfn_t pfn; uint_t attr; int flags; extern char end[]; extern struct as kas; textva = (caddr_t)(KERNELBASE & MMU_PAGEMASK4M); pfn = va_to_pfn(textva); if (pfn == PFN_INVALID) prom_panic("can't find kernel text pfn"); pfn &= TTE_PFNMASK(TTE4M); attr = PROC_TEXT | HAT_NOSYNC; flags = HAT_LOAD_LOCK | SFMMU_NO_TSBLOAD; sfmmu_memtte(&ktext_tte, pfn, attr, TTE4M); /* * We set the lock bit in the tte to lock the translation in * the tlb. Note we cannot lock Panther 32M/256M pages into the tlb. * This note is here to make sure that no one tries to remap the * kernel using 32M or 256M tte's on Panther cpus. */ TTE_SET_LOCKED(&ktext_tte); sfmmu_tteload(kas.a_hat, &ktext_tte, textva, NULL, flags); datava = (caddr_t)((uintptr_t)end & MMU_PAGEMASK4M); pfn = va_to_pfn(datava); if (pfn == PFN_INVALID) prom_panic("can't find kernel data pfn"); pfn &= TTE_PFNMASK(TTE4M); attr = PROC_DATA | HAT_NOSYNC; sfmmu_memtte(&kdata_tte, pfn, attr, TTE4M); /* * We set the lock bit in the tte to lock the translation in * the tlb. We also set the mod bit to avoid taking dirty bit * traps on kernel data. */ TTE_SET_LOCKED(&kdata_tte); TTE_SET_LOFLAGS(&kdata_tte, 0, TTE_HWWR_INT); sfmmu_tteload(kas.a_hat, &kdata_tte, datava, (struct page *)NULL, flags); /* * create bigktsb ttes if necessary. */ if (enable_bigktsb) { int i = 0; caddr_t va = ktsb_base; size_t tsbsz = ktsb_sz; tte_t tte; ASSERT(va >= datava + MMU_PAGESIZE4M); ASSERT(tsbsz >= MMU_PAGESIZE4M); ASSERT(IS_P2ALIGNED(tsbsz, tsbsz)); ASSERT(IS_P2ALIGNED(va, tsbsz)); attr = PROC_DATA | HAT_NOSYNC; while (tsbsz != 0) { ASSERT(i < MAX_BIGKTSB_TTES); pfn = va_to_pfn(va); ASSERT(pfn != PFN_INVALID); ASSERT((pfn & ~TTE_PFNMASK(TTE4M)) == 0); sfmmu_memtte(&tte, pfn, attr, TTE4M); ASSERT(TTE_IS_MOD(&tte)); /* * No need to lock if we use physical addresses. * Since we invalidate the kernel TSB using virtual * addresses, it's an optimization to load them now * so that we won't have to load them later. */ if (!ktsb_phys) { TTE_SET_LOCKED(&tte); } sfmmu_tteload(kas.a_hat, &tte, va, NULL, flags); bigktsb_ttes[i] = tte; va += MMU_PAGESIZE4M; tsbsz -= MMU_PAGESIZE4M; i++; } bigktsb_nttes = i; } sfmmu_set_tlb(); } #ifndef UTSB_PHYS /* * Unmap all references to user TSBs from the TLB of the current processor. */ static void sfmmu_clear_user_tsbs() { caddr_t va; caddr_t end_va; /* Demap all pages in the VA range for the first user TSB */ va = utsb_vabase; end_va = va + tsb_slab_size; while (va < end_va) { vtag_flushpage(va, (uint64_t)ksfmmup); va += MMU_PAGESIZE; } /* Demap all pages in the VA range for the second user TSB */ va = utsb4m_vabase; end_va = va + tsb_slab_size; while (va < end_va) { vtag_flushpage(va, (uint64_t)ksfmmup); va += MMU_PAGESIZE; } } #endif /* UTSB_PHYS */ /* * Setup the kernel's locked tte's */ void sfmmu_set_tlb(void) { uint_t index; struct cpu_node *cpunode; cpunode = &cpunodes[getprocessorid()]; index = cpunode->itlb_size; /* * NOTE: the prom will do an explicit unmap of the VAs from the TLBs * in the following functions before loading the new value into the * TLB. Thus if there was an entry already in the TLB at a different * location, it will get unmapped before we load the entry at the * specified location. */ (void) prom_itlb_load(index - 1, *(uint64_t *)&ktext_tte, textva); index = cpunode->dtlb_size; (void) prom_dtlb_load(index - 1, *(uint64_t *)&kdata_tte, datava); (void) prom_dtlb_load(index - 2, *(uint64_t *)&ktext_tte, textva); index -= 3; #ifndef UTSB_PHYS utsb_dtlb_ttenum = index--; utsb4m_dtlb_ttenum = index--; sfmmu_clear_user_tsbs(); #endif /* UTSB_PHYS */ if (!ktsb_phys && enable_bigktsb) { int i; caddr_t va = ktsb_base; uint64_t tte; ASSERT(bigktsb_nttes <= MAX_BIGKTSB_TTES); for (i = 0; i < bigktsb_nttes; i++) { tte = *(uint64_t *)&bigktsb_ttes[i]; (void) prom_dtlb_load(index, tte, va); va += MMU_PAGESIZE4M; index--; } } dtlb_resv_ttenum = index + 1; } /* * This routine is executed by all other cpus except the first one * at initialization time. It is responsible for taking over the * mmu from the prom. We follow these steps. * Lock the kernel's ttes in the TLB * Initialize the tsb hardware registers * Take over the trap table * Flush the prom's locked entries from the TLB */ void sfmmu_mp_startup(void) { sfmmu_set_tlb(); setwstate(WSTATE_KERN); prom_set_traptable(&trap_table); install_va_to_tte(); } void kdi_tlb_page_lock(caddr_t va, int do_dtlb) { tte_t tte; pfn_t pfn = va_to_pfn(va); tte.tte_inthi = TTE_VALID_INT | TTE_SZ_INT(TTE8K) | TTE_PFN_INTHI(pfn); tte.tte_intlo = TTE_PFN_INTLO(pfn) | TTE_LCK_INT | TTE_CP_INT | TTE_PRIV_INT | TTE_HWWR_INT; vtag_flushpage(va, (uint64_t)ksfmmup); sfmmu_itlb_ld_kva(va, &tte); if (do_dtlb) sfmmu_dtlb_ld_kva(va, &tte); } /*ARGSUSED*/ void kdi_tlb_page_unlock(caddr_t va, int do_dtlb) { vtag_flushpage(va, (uint64_t)ksfmmup); } /* clear user TSB information (applicable to hardware TSB walkers) */ void sfmmu_clear_utsbinfo() { } /*ARGSUSED*/ void sfmmu_setup_tsbinfo(sfmmu_t *sfmmup) { } /* * Invalidate a TSB. If floating point is enabled we use * a fast block-store routine, otherwise we use the old method * of walking the TSB setting each tag to TSBTAG_INVALID. */ void sfmmu_inv_tsb(caddr_t tsb_base, uint_t tsb_bytes) { extern void sfmmu_inv_tsb_fast(caddr_t, uint_t); struct tsbe *tsbaddr; /* CONSTCOND */ if (fpu_exists) { sfmmu_inv_tsb_fast(tsb_base, tsb_bytes); return; } for (tsbaddr = (struct tsbe *)tsb_base; (uintptr_t)tsbaddr < (uintptr_t)(tsb_base + tsb_bytes); tsbaddr++) { tsbaddr->tte_tag.tag_inthi = TSBTAG_INVALID; } if (ktsb_phys && tsb_base == ktsb_base) dcache_flushall(); } /* * Completely flush the D-cache on all cpus. */ void sfmmu_cache_flushall() { int i; for (i = 0; i < CACHE_NUM_COLOR; i++) sfmmu_cache_flushcolor(i, 0); }