160727d8bSWarner Losh /*- 25244eac9SBenno Rice * Copyright (c) 2001 The NetBSD Foundation, Inc. 35244eac9SBenno Rice * All rights reserved. 45244eac9SBenno Rice * 55244eac9SBenno Rice * This code is derived from software contributed to The NetBSD Foundation 65244eac9SBenno Rice * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc. 75244eac9SBenno Rice * 85244eac9SBenno Rice * Redistribution and use in source and binary forms, with or without 95244eac9SBenno Rice * modification, are permitted provided that the following conditions 105244eac9SBenno Rice * are met: 115244eac9SBenno Rice * 1. Redistributions of source code must retain the above copyright 125244eac9SBenno Rice * notice, this list of conditions and the following disclaimer. 135244eac9SBenno Rice * 2. Redistributions in binary form must reproduce the above copyright 145244eac9SBenno Rice * notice, this list of conditions and the following disclaimer in the 155244eac9SBenno Rice * documentation and/or other materials provided with the distribution. 165244eac9SBenno Rice * 175244eac9SBenno Rice * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 185244eac9SBenno Rice * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 195244eac9SBenno Rice * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 205244eac9SBenno Rice * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 215244eac9SBenno Rice * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 225244eac9SBenno Rice * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 235244eac9SBenno Rice * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 245244eac9SBenno Rice * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 255244eac9SBenno Rice * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 265244eac9SBenno Rice * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 275244eac9SBenno Rice * POSSIBILITY OF SUCH DAMAGE. 285244eac9SBenno Rice */ 2960727d8bSWarner Losh /*- 30f9bac91bSBenno Rice * Copyright (C) 1995, 1996 Wolfgang Solfrank. 31f9bac91bSBenno Rice * Copyright (C) 1995, 1996 TooLs GmbH. 32f9bac91bSBenno Rice * All rights reserved. 33f9bac91bSBenno Rice * 34f9bac91bSBenno Rice * Redistribution and use in source and binary forms, with or without 35f9bac91bSBenno Rice * modification, are permitted provided that the following conditions 36f9bac91bSBenno Rice * are met: 37f9bac91bSBenno Rice * 1. Redistributions of source code must retain the above copyright 38f9bac91bSBenno Rice * notice, this list of conditions and the following disclaimer. 39f9bac91bSBenno Rice * 2. Redistributions in binary form must reproduce the above copyright 40f9bac91bSBenno Rice * notice, this list of conditions and the following disclaimer in the 41f9bac91bSBenno Rice * documentation and/or other materials provided with the distribution. 42f9bac91bSBenno Rice * 3. All advertising materials mentioning features or use of this software 43f9bac91bSBenno Rice * must display the following acknowledgement: 44f9bac91bSBenno Rice * This product includes software developed by TooLs GmbH. 45f9bac91bSBenno Rice * 4. The name of TooLs GmbH may not be used to endorse or promote products 46f9bac91bSBenno Rice * derived from this software without specific prior written permission. 47f9bac91bSBenno Rice * 48f9bac91bSBenno Rice * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 49f9bac91bSBenno Rice * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 50f9bac91bSBenno Rice * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 51f9bac91bSBenno Rice * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 52f9bac91bSBenno Rice * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 53f9bac91bSBenno Rice * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 54f9bac91bSBenno Rice * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 55f9bac91bSBenno Rice * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 56f9bac91bSBenno Rice * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 57f9bac91bSBenno Rice * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58f9bac91bSBenno Rice * 59111c77dcSBenno Rice * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $ 60f9bac91bSBenno Rice */ 6160727d8bSWarner Losh /*- 62f9bac91bSBenno Rice * Copyright (C) 2001 Benno Rice. 63f9bac91bSBenno Rice * All rights reserved. 64f9bac91bSBenno Rice * 65f9bac91bSBenno Rice * Redistribution and use in source and binary forms, with or without 66f9bac91bSBenno Rice * modification, are permitted provided that the following conditions 67f9bac91bSBenno Rice * are met: 68f9bac91bSBenno Rice * 1. Redistributions of source code must retain the above copyright 69f9bac91bSBenno Rice * notice, this list of conditions and the following disclaimer. 70f9bac91bSBenno Rice * 2. Redistributions in binary form must reproduce the above copyright 71f9bac91bSBenno Rice * notice, this list of conditions and the following disclaimer in the 72f9bac91bSBenno Rice * documentation and/or other materials provided with the distribution. 73f9bac91bSBenno Rice * 74f9bac91bSBenno Rice * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 75f9bac91bSBenno Rice * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 76f9bac91bSBenno Rice * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 77f9bac91bSBenno Rice * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 78f9bac91bSBenno Rice * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 79f9bac91bSBenno Rice * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 80f9bac91bSBenno Rice * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 81f9bac91bSBenno Rice * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 82f9bac91bSBenno Rice * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 83f9bac91bSBenno Rice * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 84f9bac91bSBenno Rice */ 85f9bac91bSBenno Rice 868368cf8fSDavid E. O'Brien #include <sys/cdefs.h> 878368cf8fSDavid E. O'Brien __FBSDID("$FreeBSD$"); 88f9bac91bSBenno Rice 895244eac9SBenno Rice /* 905244eac9SBenno Rice * Manages physical address maps. 915244eac9SBenno Rice * 925244eac9SBenno Rice * Since the information managed by this module is also stored by the 935244eac9SBenno Rice * logical address mapping module, this module may throw away valid virtual 945244eac9SBenno Rice * to physical mappings at almost any time. However, invalidations of 955244eac9SBenno Rice * mappings must be done as requested. 965244eac9SBenno Rice * 975244eac9SBenno Rice * In order to cope with hardware architectures which make virtual to 985244eac9SBenno Rice * physical map invalidates expensive, this module may delay invalidate 995244eac9SBenno Rice * reduced protection operations until such time as they are actually 1005244eac9SBenno Rice * necessary. This module is given full information as to which processors 1015244eac9SBenno Rice * are currently using which maps, and to when physical maps must be made 1025244eac9SBenno Rice * correct. 1035244eac9SBenno Rice */ 1045244eac9SBenno Rice 105ad7a226fSPeter Wemm #include "opt_kstack_pages.h" 106ad7a226fSPeter Wemm 107f9bac91bSBenno Rice #include <sys/param.h> 1080b27d710SPeter Wemm #include <sys/kernel.h> 109bdb9ab0dSMark Johnston #include <sys/conf.h> 110c47dd3dbSAttilio Rao #include <sys/queue.h> 111c47dd3dbSAttilio Rao #include <sys/cpuset.h> 112bdb9ab0dSMark Johnston #include <sys/kerneldump.h> 1135244eac9SBenno Rice #include <sys/ktr.h> 11494e0b85eSMark Peek #include <sys/lock.h> 1155244eac9SBenno Rice #include <sys/msgbuf.h> 116f9bac91bSBenno Rice #include <sys/mutex.h> 1175244eac9SBenno Rice #include <sys/proc.h> 1183653f5cbSAlan Cox #include <sys/rwlock.h> 119c47dd3dbSAttilio Rao #include <sys/sched.h> 1205244eac9SBenno Rice #include <sys/sysctl.h> 1215244eac9SBenno Rice #include <sys/systm.h> 1225244eac9SBenno Rice #include <sys/vmmeter.h> 1235244eac9SBenno Rice 1245244eac9SBenno Rice #include <dev/ofw/openfirm.h> 125f9bac91bSBenno Rice 126f9bac91bSBenno Rice #include <vm/vm.h> 127f9bac91bSBenno Rice #include <vm/vm_param.h> 128f9bac91bSBenno Rice #include <vm/vm_kern.h> 129f9bac91bSBenno Rice #include <vm/vm_page.h> 130f9bac91bSBenno Rice #include <vm/vm_map.h> 131f9bac91bSBenno Rice #include <vm/vm_object.h> 132f9bac91bSBenno Rice #include <vm/vm_extern.h> 133f9bac91bSBenno Rice #include <vm/vm_pageout.h> 134378862a7SJeff Roberson #include <vm/uma.h> 135f9bac91bSBenno Rice 1367c277971SPeter Grehan #include <machine/cpu.h> 137b40ce02aSNathan Whitehorn #include <machine/platform.h> 138d699b539SMark Peek #include <machine/bat.h> 1395244eac9SBenno Rice #include <machine/frame.h> 1405244eac9SBenno Rice #include <machine/md_var.h> 1415244eac9SBenno Rice #include <machine/psl.h> 142f9bac91bSBenno Rice #include <machine/pte.h> 14312640815SMarcel Moolenaar #include <machine/smp.h> 1445244eac9SBenno Rice #include <machine/sr.h> 14559276937SPeter Grehan #include <machine/mmuvar.h> 146258dbffeSNathan Whitehorn #include <machine/trap.h> 147f9bac91bSBenno Rice 14859276937SPeter Grehan #include "mmu_if.h" 14959276937SPeter Grehan 15059276937SPeter Grehan #define MOEA_DEBUG 151f9bac91bSBenno Rice 1525244eac9SBenno Rice #define TODO panic("%s: not implemented", __func__); 153f9bac91bSBenno Rice 1545244eac9SBenno Rice #define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4)) 1555244eac9SBenno Rice #define VSID_TO_SR(vsid) ((vsid) & 0xf) 1565244eac9SBenno Rice #define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff) 1575244eac9SBenno Rice 1585244eac9SBenno Rice struct ofw_map { 1595244eac9SBenno Rice vm_offset_t om_va; 1605244eac9SBenno Rice vm_size_t om_len; 1615244eac9SBenno Rice vm_offset_t om_pa; 1625244eac9SBenno Rice u_int om_mode; 1635244eac9SBenno Rice }; 164f9bac91bSBenno Rice 165afd9cb6cSJustin Hibbits extern unsigned char _etext[]; 166afd9cb6cSJustin Hibbits extern unsigned char _end[]; 167afd9cb6cSJustin Hibbits 1685244eac9SBenno Rice /* 1695244eac9SBenno Rice * Map of physical memory regions. 1705244eac9SBenno Rice */ 17131c82d03SBenno Rice static struct mem_region *regions; 17231c82d03SBenno Rice static struct mem_region *pregions; 173c3e289e1SNathan Whitehorn static u_int phys_avail_count; 174c3e289e1SNathan Whitehorn static int regions_sz, pregions_sz; 175aa39961eSBenno Rice static struct ofw_map *translations; 1765244eac9SBenno Rice 177f9bac91bSBenno Rice /* 178f489bf21SAlan Cox * Lock for the pteg and pvo tables. 179f489bf21SAlan Cox */ 18059276937SPeter Grehan struct mtx moea_table_mutex; 181e9b5f218SNathan Whitehorn struct mtx moea_vsid_mutex; 182f489bf21SAlan Cox 183e4f72b32SMarcel Moolenaar /* tlbie instruction synchronization */ 184e4f72b32SMarcel Moolenaar static struct mtx tlbie_mtx; 185e4f72b32SMarcel Moolenaar 186f489bf21SAlan Cox /* 1875244eac9SBenno Rice * PTEG data. 188f9bac91bSBenno Rice */ 18959276937SPeter Grehan static struct pteg *moea_pteg_table; 19059276937SPeter Grehan u_int moea_pteg_count; 19159276937SPeter Grehan u_int moea_pteg_mask; 1925244eac9SBenno Rice 1935244eac9SBenno Rice /* 1945244eac9SBenno Rice * PVO data. 1955244eac9SBenno Rice */ 19659276937SPeter Grehan struct pvo_head *moea_pvo_table; /* pvo entries by pteg index */ 19759276937SPeter Grehan struct pvo_head moea_pvo_kunmanaged = 19859276937SPeter Grehan LIST_HEAD_INITIALIZER(moea_pvo_kunmanaged); /* list of unmanaged pages */ 1995244eac9SBenno Rice 200cfedf924SAttilio Rao static struct rwlock_padalign pvh_global_lock; 2013653f5cbSAlan Cox 20259276937SPeter Grehan uma_zone_t moea_upvo_zone; /* zone for pvo entries for unmanaged pages */ 20359276937SPeter Grehan uma_zone_t moea_mpvo_zone; /* zone for pvo entries for managed pages */ 2045244eac9SBenno Rice 2050d290675SBenno Rice #define BPVO_POOL_SIZE 32768 20659276937SPeter Grehan static struct pvo_entry *moea_bpvo_pool; 20759276937SPeter Grehan static int moea_bpvo_pool_index = 0; 2085244eac9SBenno Rice 2095244eac9SBenno Rice #define VSID_NBPW (sizeof(u_int32_t) * 8) 21059276937SPeter Grehan static u_int moea_vsid_bitmap[NPMAPS / VSID_NBPW]; 2115244eac9SBenno Rice 21259276937SPeter Grehan static boolean_t moea_initialized = FALSE; 2135244eac9SBenno Rice 2145244eac9SBenno Rice /* 2155244eac9SBenno Rice * Statistics. 2165244eac9SBenno Rice */ 21759276937SPeter Grehan u_int moea_pte_valid = 0; 21859276937SPeter Grehan u_int moea_pte_overflow = 0; 21959276937SPeter Grehan u_int moea_pte_replacements = 0; 22059276937SPeter Grehan u_int moea_pvo_entries = 0; 22159276937SPeter Grehan u_int moea_pvo_enter_calls = 0; 22259276937SPeter Grehan u_int moea_pvo_remove_calls = 0; 22359276937SPeter Grehan u_int moea_pte_spills = 0; 22459276937SPeter Grehan SYSCTL_INT(_machdep, OID_AUTO, moea_pte_valid, CTLFLAG_RD, &moea_pte_valid, 2255244eac9SBenno Rice 0, ""); 22659276937SPeter Grehan SYSCTL_INT(_machdep, OID_AUTO, moea_pte_overflow, CTLFLAG_RD, 22759276937SPeter Grehan &moea_pte_overflow, 0, ""); 22859276937SPeter Grehan SYSCTL_INT(_machdep, OID_AUTO, moea_pte_replacements, CTLFLAG_RD, 22959276937SPeter Grehan &moea_pte_replacements, 0, ""); 23059276937SPeter Grehan SYSCTL_INT(_machdep, OID_AUTO, moea_pvo_entries, CTLFLAG_RD, &moea_pvo_entries, 2315244eac9SBenno Rice 0, ""); 23259276937SPeter Grehan SYSCTL_INT(_machdep, OID_AUTO, moea_pvo_enter_calls, CTLFLAG_RD, 23359276937SPeter Grehan &moea_pvo_enter_calls, 0, ""); 23459276937SPeter Grehan SYSCTL_INT(_machdep, OID_AUTO, moea_pvo_remove_calls, CTLFLAG_RD, 23559276937SPeter Grehan &moea_pvo_remove_calls, 0, ""); 23659276937SPeter Grehan SYSCTL_INT(_machdep, OID_AUTO, moea_pte_spills, CTLFLAG_RD, 23759276937SPeter Grehan &moea_pte_spills, 0, ""); 2385244eac9SBenno Rice 2395244eac9SBenno Rice /* 24059276937SPeter Grehan * Allocate physical memory for use in moea_bootstrap. 2415244eac9SBenno Rice */ 24259276937SPeter Grehan static vm_offset_t moea_bootstrap_alloc(vm_size_t, u_int); 2435244eac9SBenno Rice 2445244eac9SBenno Rice /* 2455244eac9SBenno Rice * PTE calls. 2465244eac9SBenno Rice */ 24759276937SPeter Grehan static int moea_pte_insert(u_int, struct pte *); 2485244eac9SBenno Rice 2495244eac9SBenno Rice /* 2505244eac9SBenno Rice * PVO calls. 2515244eac9SBenno Rice */ 25259276937SPeter Grehan static int moea_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *, 2530936003eSJustin Hibbits vm_offset_t, vm_paddr_t, u_int, int); 25459276937SPeter Grehan static void moea_pvo_remove(struct pvo_entry *, int); 25559276937SPeter Grehan static struct pvo_entry *moea_pvo_find_va(pmap_t, vm_offset_t, int *); 25659276937SPeter Grehan static struct pte *moea_pvo_to_pte(const struct pvo_entry *, int); 2575244eac9SBenno Rice 2585244eac9SBenno Rice /* 2595244eac9SBenno Rice * Utility routines. 2605244eac9SBenno Rice */ 26139ffa8c1SKonstantin Belousov static int moea_enter_locked(pmap_t, vm_offset_t, vm_page_t, 26239ffa8c1SKonstantin Belousov vm_prot_t, u_int, int8_t); 2630936003eSJustin Hibbits static void moea_syncicache(vm_paddr_t, vm_size_t); 26459276937SPeter Grehan static boolean_t moea_query_bit(vm_page_t, int); 265ce186587SAlan Cox static u_int moea_clear_bit(vm_page_t, int); 26659276937SPeter Grehan static void moea_kremove(mmu_t, vm_offset_t); 26759276937SPeter Grehan int moea_pte_spill(vm_offset_t); 26859276937SPeter Grehan 26959276937SPeter Grehan /* 27059276937SPeter Grehan * Kernel MMU interface 27159276937SPeter Grehan */ 27259276937SPeter Grehan void moea_clear_modify(mmu_t, vm_page_t); 27359276937SPeter Grehan void moea_copy_page(mmu_t, vm_page_t, vm_page_t); 274e8a4a618SKonstantin Belousov void moea_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 275e8a4a618SKonstantin Belousov vm_page_t *mb, vm_offset_t b_offset, int xfersize); 27639ffa8c1SKonstantin Belousov int moea_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, 27739ffa8c1SKonstantin Belousov int8_t); 278ce142d9eSAlan Cox void moea_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 279ce142d9eSAlan Cox vm_prot_t); 2802053c127SStephan Uphoff void moea_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 28159276937SPeter Grehan vm_paddr_t moea_extract(mmu_t, pmap_t, vm_offset_t); 28259276937SPeter Grehan vm_page_t moea_extract_and_hold(mmu_t, pmap_t, vm_offset_t, vm_prot_t); 28359276937SPeter Grehan void moea_init(mmu_t); 28459276937SPeter Grehan boolean_t moea_is_modified(mmu_t, vm_page_t); 285e396eb60SAlan Cox boolean_t moea_is_prefaultable(mmu_t, pmap_t, vm_offset_t); 2867b85f591SAlan Cox boolean_t moea_is_referenced(mmu_t, vm_page_t); 2878d9e6d9fSAlan Cox int moea_ts_referenced(mmu_t, vm_page_t); 28820b79612SRafal Jaworowski vm_offset_t moea_map(mmu_t, vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 28959276937SPeter Grehan boolean_t moea_page_exists_quick(mmu_t, pmap_t, vm_page_t); 29059677d3cSAlan Cox int moea_page_wired_mappings(mmu_t, vm_page_t); 29159276937SPeter Grehan void moea_pinit(mmu_t, pmap_t); 29259276937SPeter Grehan void moea_pinit0(mmu_t, pmap_t); 29359276937SPeter Grehan void moea_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 29459276937SPeter Grehan void moea_qenter(mmu_t, vm_offset_t, vm_page_t *, int); 29559276937SPeter Grehan void moea_qremove(mmu_t, vm_offset_t, int); 29659276937SPeter Grehan void moea_release(mmu_t, pmap_t); 29759276937SPeter Grehan void moea_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t); 29859276937SPeter Grehan void moea_remove_all(mmu_t, vm_page_t); 29978985e42SAlan Cox void moea_remove_write(mmu_t, vm_page_t); 300a844c68fSAlan Cox void moea_unwire(mmu_t, pmap_t, vm_offset_t, vm_offset_t); 30159276937SPeter Grehan void moea_zero_page(mmu_t, vm_page_t); 30259276937SPeter Grehan void moea_zero_page_area(mmu_t, vm_page_t, int, int); 30359276937SPeter Grehan void moea_zero_page_idle(mmu_t, vm_page_t); 30459276937SPeter Grehan void moea_activate(mmu_t, struct thread *); 30559276937SPeter Grehan void moea_deactivate(mmu_t, struct thread *); 3061c96bdd1SNathan Whitehorn void moea_cpu_bootstrap(mmu_t, int); 30759276937SPeter Grehan void moea_bootstrap(mmu_t, vm_offset_t, vm_offset_t); 30820b79612SRafal Jaworowski void *moea_mapdev(mmu_t, vm_paddr_t, vm_size_t); 3090936003eSJustin Hibbits void *moea_mapdev_attr(mmu_t, vm_paddr_t, vm_size_t, vm_memattr_t); 31059276937SPeter Grehan void moea_unmapdev(mmu_t, vm_offset_t, vm_size_t); 31120b79612SRafal Jaworowski vm_paddr_t moea_kextract(mmu_t, vm_offset_t); 3120936003eSJustin Hibbits void moea_kenter_attr(mmu_t, vm_offset_t, vm_paddr_t, vm_memattr_t); 31320b79612SRafal Jaworowski void moea_kenter(mmu_t, vm_offset_t, vm_paddr_t); 314c1f4123bSNathan Whitehorn void moea_page_set_memattr(mmu_t mmu, vm_page_t m, vm_memattr_t ma); 31520b79612SRafal Jaworowski boolean_t moea_dev_direct_mapped(mmu_t, vm_paddr_t, vm_size_t); 3161a4fcaebSMarcel Moolenaar static void moea_sync_icache(mmu_t, pmap_t, vm_offset_t, vm_size_t); 317bdb9ab0dSMark Johnston void moea_dumpsys_map(mmu_t mmu, vm_paddr_t pa, size_t sz, void **va); 318bdb9ab0dSMark Johnston void moea_scan_init(mmu_t mmu); 319713841afSJason A. Harmening vm_offset_t moea_quick_enter_page(mmu_t mmu, vm_page_t m); 320713841afSJason A. Harmening void moea_quick_remove_page(mmu_t mmu, vm_offset_t addr); 32159276937SPeter Grehan 32259276937SPeter Grehan static mmu_method_t moea_methods[] = { 32359276937SPeter Grehan MMUMETHOD(mmu_clear_modify, moea_clear_modify), 32459276937SPeter Grehan MMUMETHOD(mmu_copy_page, moea_copy_page), 325e8a4a618SKonstantin Belousov MMUMETHOD(mmu_copy_pages, moea_copy_pages), 32659276937SPeter Grehan MMUMETHOD(mmu_enter, moea_enter), 327ce142d9eSAlan Cox MMUMETHOD(mmu_enter_object, moea_enter_object), 32859276937SPeter Grehan MMUMETHOD(mmu_enter_quick, moea_enter_quick), 32959276937SPeter Grehan MMUMETHOD(mmu_extract, moea_extract), 33059276937SPeter Grehan MMUMETHOD(mmu_extract_and_hold, moea_extract_and_hold), 33159276937SPeter Grehan MMUMETHOD(mmu_init, moea_init), 33259276937SPeter Grehan MMUMETHOD(mmu_is_modified, moea_is_modified), 333e396eb60SAlan Cox MMUMETHOD(mmu_is_prefaultable, moea_is_prefaultable), 3347b85f591SAlan Cox MMUMETHOD(mmu_is_referenced, moea_is_referenced), 33559276937SPeter Grehan MMUMETHOD(mmu_ts_referenced, moea_ts_referenced), 33659276937SPeter Grehan MMUMETHOD(mmu_map, moea_map), 33759276937SPeter Grehan MMUMETHOD(mmu_page_exists_quick,moea_page_exists_quick), 33859677d3cSAlan Cox MMUMETHOD(mmu_page_wired_mappings,moea_page_wired_mappings), 33959276937SPeter Grehan MMUMETHOD(mmu_pinit, moea_pinit), 34059276937SPeter Grehan MMUMETHOD(mmu_pinit0, moea_pinit0), 34159276937SPeter Grehan MMUMETHOD(mmu_protect, moea_protect), 34259276937SPeter Grehan MMUMETHOD(mmu_qenter, moea_qenter), 34359276937SPeter Grehan MMUMETHOD(mmu_qremove, moea_qremove), 34459276937SPeter Grehan MMUMETHOD(mmu_release, moea_release), 34559276937SPeter Grehan MMUMETHOD(mmu_remove, moea_remove), 34659276937SPeter Grehan MMUMETHOD(mmu_remove_all, moea_remove_all), 34778985e42SAlan Cox MMUMETHOD(mmu_remove_write, moea_remove_write), 3481a4fcaebSMarcel Moolenaar MMUMETHOD(mmu_sync_icache, moea_sync_icache), 349a844c68fSAlan Cox MMUMETHOD(mmu_unwire, moea_unwire), 35059276937SPeter Grehan MMUMETHOD(mmu_zero_page, moea_zero_page), 35159276937SPeter Grehan MMUMETHOD(mmu_zero_page_area, moea_zero_page_area), 35259276937SPeter Grehan MMUMETHOD(mmu_zero_page_idle, moea_zero_page_idle), 35359276937SPeter Grehan MMUMETHOD(mmu_activate, moea_activate), 35459276937SPeter Grehan MMUMETHOD(mmu_deactivate, moea_deactivate), 355c1f4123bSNathan Whitehorn MMUMETHOD(mmu_page_set_memattr, moea_page_set_memattr), 356713841afSJason A. Harmening MMUMETHOD(mmu_quick_enter_page, moea_quick_enter_page), 357713841afSJason A. Harmening MMUMETHOD(mmu_quick_remove_page, moea_quick_remove_page), 35859276937SPeter Grehan 35959276937SPeter Grehan /* Internal interfaces */ 36059276937SPeter Grehan MMUMETHOD(mmu_bootstrap, moea_bootstrap), 3611c96bdd1SNathan Whitehorn MMUMETHOD(mmu_cpu_bootstrap, moea_cpu_bootstrap), 362c1f4123bSNathan Whitehorn MMUMETHOD(mmu_mapdev_attr, moea_mapdev_attr), 36359276937SPeter Grehan MMUMETHOD(mmu_mapdev, moea_mapdev), 36459276937SPeter Grehan MMUMETHOD(mmu_unmapdev, moea_unmapdev), 36559276937SPeter Grehan MMUMETHOD(mmu_kextract, moea_kextract), 36659276937SPeter Grehan MMUMETHOD(mmu_kenter, moea_kenter), 367c1f4123bSNathan Whitehorn MMUMETHOD(mmu_kenter_attr, moea_kenter_attr), 36859276937SPeter Grehan MMUMETHOD(mmu_dev_direct_mapped,moea_dev_direct_mapped), 369bdb9ab0dSMark Johnston MMUMETHOD(mmu_scan_init, moea_scan_init), 370afd9cb6cSJustin Hibbits MMUMETHOD(mmu_dumpsys_map, moea_dumpsys_map), 37159276937SPeter Grehan 37259276937SPeter Grehan { 0, 0 } 37359276937SPeter Grehan }; 37459276937SPeter Grehan 37533529b98SPeter Grehan MMU_DEF(oea_mmu, MMU_TYPE_OEA, moea_methods, 0); 37633529b98SPeter Grehan 377c1f4123bSNathan Whitehorn static __inline uint32_t 3780936003eSJustin Hibbits moea_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 379c1f4123bSNathan Whitehorn { 380c1f4123bSNathan Whitehorn uint32_t pte_lo; 381c1f4123bSNathan Whitehorn int i; 382c1f4123bSNathan Whitehorn 383c1f4123bSNathan Whitehorn if (ma != VM_MEMATTR_DEFAULT) { 384c1f4123bSNathan Whitehorn switch (ma) { 385c1f4123bSNathan Whitehorn case VM_MEMATTR_UNCACHEABLE: 386c1f4123bSNathan Whitehorn return (PTE_I | PTE_G); 38754ac2713SJustin Hibbits case VM_MEMATTR_CACHEABLE: 38854ac2713SJustin Hibbits return (PTE_M); 389c1f4123bSNathan Whitehorn case VM_MEMATTR_WRITE_COMBINING: 390c1f4123bSNathan Whitehorn case VM_MEMATTR_WRITE_BACK: 391c1f4123bSNathan Whitehorn case VM_MEMATTR_PREFETCHABLE: 392c1f4123bSNathan Whitehorn return (PTE_I); 393c1f4123bSNathan Whitehorn case VM_MEMATTR_WRITE_THROUGH: 394c1f4123bSNathan Whitehorn return (PTE_W | PTE_M); 395c1f4123bSNathan Whitehorn } 396c1f4123bSNathan Whitehorn } 397c1f4123bSNathan Whitehorn 398c1f4123bSNathan Whitehorn /* 399c1f4123bSNathan Whitehorn * Assume the page is cache inhibited and access is guarded unless 400c1f4123bSNathan Whitehorn * it's in our available memory array. 401c1f4123bSNathan Whitehorn */ 402c1f4123bSNathan Whitehorn pte_lo = PTE_I | PTE_G; 403c1f4123bSNathan Whitehorn for (i = 0; i < pregions_sz; i++) { 404c1f4123bSNathan Whitehorn if ((pa >= pregions[i].mr_start) && 405c1f4123bSNathan Whitehorn (pa < (pregions[i].mr_start + pregions[i].mr_size))) { 406c1f4123bSNathan Whitehorn pte_lo = PTE_M; 407c1f4123bSNathan Whitehorn break; 408c1f4123bSNathan Whitehorn } 409c1f4123bSNathan Whitehorn } 410c1f4123bSNathan Whitehorn 411c1f4123bSNathan Whitehorn return pte_lo; 412c1f4123bSNathan Whitehorn } 41359276937SPeter Grehan 414e4f72b32SMarcel Moolenaar static void 415e4f72b32SMarcel Moolenaar tlbie(vm_offset_t va) 416e4f72b32SMarcel Moolenaar { 417e4f72b32SMarcel Moolenaar 418e4f72b32SMarcel Moolenaar mtx_lock_spin(&tlbie_mtx); 41994363f53SNathan Whitehorn __asm __volatile("ptesync"); 420e4f72b32SMarcel Moolenaar __asm __volatile("tlbie %0" :: "r"(va)); 42194363f53SNathan Whitehorn __asm __volatile("eieio; tlbsync; ptesync"); 422e4f72b32SMarcel Moolenaar mtx_unlock_spin(&tlbie_mtx); 423e4f72b32SMarcel Moolenaar } 424e4f72b32SMarcel Moolenaar 425e4f72b32SMarcel Moolenaar static void 426e4f72b32SMarcel Moolenaar tlbia(void) 427e4f72b32SMarcel Moolenaar { 428e4f72b32SMarcel Moolenaar vm_offset_t va; 429e4f72b32SMarcel Moolenaar 430e4f72b32SMarcel Moolenaar for (va = 0; va < 0x00040000; va += 0x00001000) { 431e4f72b32SMarcel Moolenaar __asm __volatile("tlbie %0" :: "r"(va)); 432e4f72b32SMarcel Moolenaar powerpc_sync(); 433e4f72b32SMarcel Moolenaar } 434e4f72b32SMarcel Moolenaar __asm __volatile("tlbsync"); 435e4f72b32SMarcel Moolenaar powerpc_sync(); 436e4f72b32SMarcel Moolenaar } 4375244eac9SBenno Rice 4385244eac9SBenno Rice static __inline int 4395244eac9SBenno Rice va_to_sr(u_int *sr, vm_offset_t va) 4405244eac9SBenno Rice { 4415244eac9SBenno Rice return (sr[(uintptr_t)va >> ADDR_SR_SHFT]); 4425244eac9SBenno Rice } 4435244eac9SBenno Rice 4445244eac9SBenno Rice static __inline u_int 4455244eac9SBenno Rice va_to_pteg(u_int sr, vm_offset_t addr) 4465244eac9SBenno Rice { 4475244eac9SBenno Rice u_int hash; 4485244eac9SBenno Rice 4495244eac9SBenno Rice hash = (sr & SR_VSID_MASK) ^ (((u_int)addr & ADDR_PIDX) >> 4505244eac9SBenno Rice ADDR_PIDX_SHFT); 45159276937SPeter Grehan return (hash & moea_pteg_mask); 4525244eac9SBenno Rice } 4535244eac9SBenno Rice 4545244eac9SBenno Rice static __inline struct pvo_head * 4555244eac9SBenno Rice vm_page_to_pvoh(vm_page_t m) 456f9bac91bSBenno Rice { 457f9bac91bSBenno Rice 4585244eac9SBenno Rice return (&m->md.mdpg_pvoh); 459f9bac91bSBenno Rice } 460f9bac91bSBenno Rice 461f9bac91bSBenno Rice static __inline void 46259276937SPeter Grehan moea_attr_clear(vm_page_t m, int ptebit) 463f9bac91bSBenno Rice { 464f9bac91bSBenno Rice 4653653f5cbSAlan Cox rw_assert(&pvh_global_lock, RA_WLOCKED); 4665244eac9SBenno Rice m->md.mdpg_attrs &= ~ptebit; 4675244eac9SBenno Rice } 4685244eac9SBenno Rice 4695244eac9SBenno Rice static __inline int 47059276937SPeter Grehan moea_attr_fetch(vm_page_t m) 4715244eac9SBenno Rice { 4725244eac9SBenno Rice 4735244eac9SBenno Rice return (m->md.mdpg_attrs); 474f9bac91bSBenno Rice } 475f9bac91bSBenno Rice 476f9bac91bSBenno Rice static __inline void 47759276937SPeter Grehan moea_attr_save(vm_page_t m, int ptebit) 478f9bac91bSBenno Rice { 479f9bac91bSBenno Rice 4803653f5cbSAlan Cox rw_assert(&pvh_global_lock, RA_WLOCKED); 4815244eac9SBenno Rice m->md.mdpg_attrs |= ptebit; 482f9bac91bSBenno Rice } 483f9bac91bSBenno Rice 484f9bac91bSBenno Rice static __inline int 48559276937SPeter Grehan moea_pte_compare(const struct pte *pt, const struct pte *pvo_pt) 486f9bac91bSBenno Rice { 4875244eac9SBenno Rice if (pt->pte_hi == pvo_pt->pte_hi) 4885244eac9SBenno Rice return (1); 489f9bac91bSBenno Rice 4905244eac9SBenno Rice return (0); 491f9bac91bSBenno Rice } 492f9bac91bSBenno Rice 493f9bac91bSBenno Rice static __inline int 49459276937SPeter Grehan moea_pte_match(struct pte *pt, u_int sr, vm_offset_t va, int which) 495f9bac91bSBenno Rice { 4965244eac9SBenno Rice return (pt->pte_hi & ~PTE_VALID) == 4975244eac9SBenno Rice (((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 4985244eac9SBenno Rice ((va >> ADDR_API_SHFT) & PTE_API) | which); 499f9bac91bSBenno Rice } 500f9bac91bSBenno Rice 5015244eac9SBenno Rice static __inline void 50259276937SPeter Grehan moea_pte_create(struct pte *pt, u_int sr, vm_offset_t va, u_int pte_lo) 503f9bac91bSBenno Rice { 504d644a0b7SAlan Cox 505d644a0b7SAlan Cox mtx_assert(&moea_table_mutex, MA_OWNED); 506d644a0b7SAlan Cox 507f9bac91bSBenno Rice /* 5085244eac9SBenno Rice * Construct a PTE. Default to IMB initially. Valid bit only gets 5095244eac9SBenno Rice * set when the real pte is set in memory. 510f9bac91bSBenno Rice * 511f9bac91bSBenno Rice * Note: Don't set the valid bit for correct operation of tlb update. 512f9bac91bSBenno Rice */ 5135244eac9SBenno Rice pt->pte_hi = ((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 5145244eac9SBenno Rice (((va & ADDR_PIDX) >> ADDR_API_SHFT) & PTE_API); 5155244eac9SBenno Rice pt->pte_lo = pte_lo; 516f9bac91bSBenno Rice } 517f9bac91bSBenno Rice 5185244eac9SBenno Rice static __inline void 51959276937SPeter Grehan moea_pte_synch(struct pte *pt, struct pte *pvo_pt) 520f9bac91bSBenno Rice { 521f9bac91bSBenno Rice 522d644a0b7SAlan Cox mtx_assert(&moea_table_mutex, MA_OWNED); 5235244eac9SBenno Rice pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF | PTE_CHG); 524f9bac91bSBenno Rice } 525f9bac91bSBenno Rice 5265244eac9SBenno Rice static __inline void 52759276937SPeter Grehan moea_pte_clear(struct pte *pt, vm_offset_t va, int ptebit) 528f9bac91bSBenno Rice { 5295244eac9SBenno Rice 530d644a0b7SAlan Cox mtx_assert(&moea_table_mutex, MA_OWNED); 531d644a0b7SAlan Cox 5325244eac9SBenno Rice /* 5335244eac9SBenno Rice * As shown in Section 7.6.3.2.3 5345244eac9SBenno Rice */ 5355244eac9SBenno Rice pt->pte_lo &= ~ptebit; 536e4f72b32SMarcel Moolenaar tlbie(va); 5375244eac9SBenno Rice } 5385244eac9SBenno Rice 5395244eac9SBenno Rice static __inline void 54059276937SPeter Grehan moea_pte_set(struct pte *pt, struct pte *pvo_pt) 5415244eac9SBenno Rice { 5425244eac9SBenno Rice 543d644a0b7SAlan Cox mtx_assert(&moea_table_mutex, MA_OWNED); 5445244eac9SBenno Rice pvo_pt->pte_hi |= PTE_VALID; 5455244eac9SBenno Rice 5465244eac9SBenno Rice /* 5475244eac9SBenno Rice * Update the PTE as defined in section 7.6.3.1. 548804d1cc1SJustin Hibbits * Note that the REF/CHG bits are from pvo_pt and thus should have 5495244eac9SBenno Rice * been saved so this routine can restore them (if desired). 5505244eac9SBenno Rice */ 5515244eac9SBenno Rice pt->pte_lo = pvo_pt->pte_lo; 552e4f72b32SMarcel Moolenaar powerpc_sync(); 5535244eac9SBenno Rice pt->pte_hi = pvo_pt->pte_hi; 554e4f72b32SMarcel Moolenaar powerpc_sync(); 55559276937SPeter Grehan moea_pte_valid++; 5565244eac9SBenno Rice } 5575244eac9SBenno Rice 5585244eac9SBenno Rice static __inline void 55959276937SPeter Grehan moea_pte_unset(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 5605244eac9SBenno Rice { 5615244eac9SBenno Rice 562d644a0b7SAlan Cox mtx_assert(&moea_table_mutex, MA_OWNED); 5635244eac9SBenno Rice pvo_pt->pte_hi &= ~PTE_VALID; 5645244eac9SBenno Rice 5655244eac9SBenno Rice /* 5665244eac9SBenno Rice * Force the reg & chg bits back into the PTEs. 5675244eac9SBenno Rice */ 568e4f72b32SMarcel Moolenaar powerpc_sync(); 5695244eac9SBenno Rice 5705244eac9SBenno Rice /* 5715244eac9SBenno Rice * Invalidate the pte. 5725244eac9SBenno Rice */ 5735244eac9SBenno Rice pt->pte_hi &= ~PTE_VALID; 5745244eac9SBenno Rice 575e4f72b32SMarcel Moolenaar tlbie(va); 5765244eac9SBenno Rice 5775244eac9SBenno Rice /* 5785244eac9SBenno Rice * Save the reg & chg bits. 5795244eac9SBenno Rice */ 58059276937SPeter Grehan moea_pte_synch(pt, pvo_pt); 58159276937SPeter Grehan moea_pte_valid--; 5825244eac9SBenno Rice } 5835244eac9SBenno Rice 5845244eac9SBenno Rice static __inline void 58559276937SPeter Grehan moea_pte_change(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 5865244eac9SBenno Rice { 5875244eac9SBenno Rice 5885244eac9SBenno Rice /* 5895244eac9SBenno Rice * Invalidate the PTE 5905244eac9SBenno Rice */ 59159276937SPeter Grehan moea_pte_unset(pt, pvo_pt, va); 59259276937SPeter Grehan moea_pte_set(pt, pvo_pt); 593f9bac91bSBenno Rice } 594f9bac91bSBenno Rice 595f9bac91bSBenno Rice /* 5965244eac9SBenno Rice * Quick sort callout for comparing memory regions. 597f9bac91bSBenno Rice */ 5985244eac9SBenno Rice static int om_cmp(const void *a, const void *b); 5995244eac9SBenno Rice 6005244eac9SBenno Rice static int 6015244eac9SBenno Rice om_cmp(const void *a, const void *b) 6025244eac9SBenno Rice { 6035244eac9SBenno Rice const struct ofw_map *mapa; 6045244eac9SBenno Rice const struct ofw_map *mapb; 6055244eac9SBenno Rice 6065244eac9SBenno Rice mapa = a; 6075244eac9SBenno Rice mapb = b; 6085244eac9SBenno Rice if (mapa->om_pa < mapb->om_pa) 6095244eac9SBenno Rice return (-1); 6105244eac9SBenno Rice else if (mapa->om_pa > mapb->om_pa) 6115244eac9SBenno Rice return (1); 6125244eac9SBenno Rice else 6135244eac9SBenno Rice return (0); 614f9bac91bSBenno Rice } 615f9bac91bSBenno Rice 616f9bac91bSBenno Rice void 6171c96bdd1SNathan Whitehorn moea_cpu_bootstrap(mmu_t mmup, int ap) 61812640815SMarcel Moolenaar { 61912640815SMarcel Moolenaar u_int sdr; 62012640815SMarcel Moolenaar int i; 62112640815SMarcel Moolenaar 62212640815SMarcel Moolenaar if (ap) { 623e4f72b32SMarcel Moolenaar powerpc_sync(); 62412640815SMarcel Moolenaar __asm __volatile("mtdbatu 0,%0" :: "r"(battable[0].batu)); 62512640815SMarcel Moolenaar __asm __volatile("mtdbatl 0,%0" :: "r"(battable[0].batl)); 62612640815SMarcel Moolenaar isync(); 62712640815SMarcel Moolenaar __asm __volatile("mtibatu 0,%0" :: "r"(battable[0].batu)); 62812640815SMarcel Moolenaar __asm __volatile("mtibatl 0,%0" :: "r"(battable[0].batl)); 62912640815SMarcel Moolenaar isync(); 63012640815SMarcel Moolenaar } 63112640815SMarcel Moolenaar 63201d8aa0dSMarcel Moolenaar __asm __volatile("mtdbatu 1,%0" :: "r"(battable[8].batu)); 63301d8aa0dSMarcel Moolenaar __asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl)); 63412640815SMarcel Moolenaar isync(); 63512640815SMarcel Moolenaar 63601d8aa0dSMarcel Moolenaar __asm __volatile("mtibatu 1,%0" :: "r"(0)); 63701d8aa0dSMarcel Moolenaar __asm __volatile("mtdbatu 2,%0" :: "r"(0)); 63801d8aa0dSMarcel Moolenaar __asm __volatile("mtibatu 2,%0" :: "r"(0)); 63901d8aa0dSMarcel Moolenaar __asm __volatile("mtdbatu 3,%0" :: "r"(0)); 64001d8aa0dSMarcel Moolenaar __asm __volatile("mtibatu 3,%0" :: "r"(0)); 64112640815SMarcel Moolenaar isync(); 64212640815SMarcel Moolenaar 64312640815SMarcel Moolenaar for (i = 0; i < 16; i++) 644fe3b4685SNathan Whitehorn mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]); 645e4f72b32SMarcel Moolenaar powerpc_sync(); 64612640815SMarcel Moolenaar 64712640815SMarcel Moolenaar sdr = (u_int)moea_pteg_table | (moea_pteg_mask >> 10); 64812640815SMarcel Moolenaar __asm __volatile("mtsdr1 %0" :: "r"(sdr)); 64912640815SMarcel Moolenaar isync(); 65012640815SMarcel Moolenaar 65186c1fb4cSMarcel Moolenaar tlbia(); 65212640815SMarcel Moolenaar } 65312640815SMarcel Moolenaar 65412640815SMarcel Moolenaar void 65559276937SPeter Grehan moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 656f9bac91bSBenno Rice { 65731c82d03SBenno Rice ihandle_t mmui; 6585244eac9SBenno Rice phandle_t chosen, mmu; 6595244eac9SBenno Rice int sz; 6605244eac9SBenno Rice int i, j; 661e2f6d6e2SPeter Grehan vm_size_t size, physsz, hwphyssz; 6625244eac9SBenno Rice vm_offset_t pa, va, off; 66350c202c5SJeff Roberson void *dpcpu; 664976cc697SNathan Whitehorn register_t msr; 665f9bac91bSBenno Rice 666f9bac91bSBenno Rice /* 66732bc7846SPeter Grehan * Set up BAT0 to map the lowest 256 MB area 6680d290675SBenno Rice */ 6690d290675SBenno Rice battable[0x0].batl = BATL(0x00000000, BAT_M, BAT_PP_RW); 6700d290675SBenno Rice battable[0x0].batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs); 6710d290675SBenno Rice 6720d290675SBenno Rice /* 6730d290675SBenno Rice * Map PCI memory space. 6740d290675SBenno Rice */ 6750d290675SBenno Rice battable[0x8].batl = BATL(0x80000000, BAT_I|BAT_G, BAT_PP_RW); 6760d290675SBenno Rice battable[0x8].batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs); 6770d290675SBenno Rice 6780d290675SBenno Rice battable[0x9].batl = BATL(0x90000000, BAT_I|BAT_G, BAT_PP_RW); 6790d290675SBenno Rice battable[0x9].batu = BATU(0x90000000, BAT_BL_256M, BAT_Vs); 6800d290675SBenno Rice 6810d290675SBenno Rice battable[0xa].batl = BATL(0xa0000000, BAT_I|BAT_G, BAT_PP_RW); 6820d290675SBenno Rice battable[0xa].batu = BATU(0xa0000000, BAT_BL_256M, BAT_Vs); 6830d290675SBenno Rice 6840d290675SBenno Rice battable[0xb].batl = BATL(0xb0000000, BAT_I|BAT_G, BAT_PP_RW); 6850d290675SBenno Rice battable[0xb].batu = BATU(0xb0000000, BAT_BL_256M, BAT_Vs); 6860d290675SBenno Rice 6870d290675SBenno Rice /* 6880d290675SBenno Rice * Map obio devices. 6890d290675SBenno Rice */ 6900d290675SBenno Rice battable[0xf].batl = BATL(0xf0000000, BAT_I|BAT_G, BAT_PP_RW); 6910d290675SBenno Rice battable[0xf].batu = BATU(0xf0000000, BAT_BL_256M, BAT_Vs); 6920d290675SBenno Rice 6930d290675SBenno Rice /* 6945244eac9SBenno Rice * Use an IBAT and a DBAT to map the bottom segment of memory 695976cc697SNathan Whitehorn * where we are. Turn off instruction relocation temporarily 696976cc697SNathan Whitehorn * to prevent faults while reprogramming the IBAT. 697f9bac91bSBenno Rice */ 698976cc697SNathan Whitehorn msr = mfmsr(); 699976cc697SNathan Whitehorn mtmsr(msr & ~PSL_IR); 70059276937SPeter Grehan __asm (".balign 32; \n" 70172ed3108SPeter Grehan "mtibatu 0,%0; mtibatl 0,%1; isync; \n" 7025d64cf91SPeter Grehan "mtdbatu 0,%0; mtdbatl 0,%1; isync" 70312640815SMarcel Moolenaar :: "r"(battable[0].batu), "r"(battable[0].batl)); 704976cc697SNathan Whitehorn mtmsr(msr); 7050d290675SBenno Rice 7060d290675SBenno Rice /* map pci space */ 70712640815SMarcel Moolenaar __asm __volatile("mtdbatu 1,%0" :: "r"(battable[8].batu)); 70812640815SMarcel Moolenaar __asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl)); 70912640815SMarcel Moolenaar isync(); 710f9bac91bSBenno Rice 7111c96bdd1SNathan Whitehorn /* set global direct map flag */ 7121c96bdd1SNathan Whitehorn hw_direct_map = 1; 7131c96bdd1SNathan Whitehorn 71431c82d03SBenno Rice mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 71559276937SPeter Grehan CTR0(KTR_PMAP, "moea_bootstrap: physical memory"); 71631c82d03SBenno Rice 71731c82d03SBenno Rice for (i = 0; i < pregions_sz; i++) { 71832bc7846SPeter Grehan vm_offset_t pa; 71932bc7846SPeter Grehan vm_offset_t end; 72032bc7846SPeter Grehan 72131c82d03SBenno Rice CTR3(KTR_PMAP, "physregion: %#x - %#x (%#x)", 72231c82d03SBenno Rice pregions[i].mr_start, 72331c82d03SBenno Rice pregions[i].mr_start + pregions[i].mr_size, 72431c82d03SBenno Rice pregions[i].mr_size); 72532bc7846SPeter Grehan /* 72632bc7846SPeter Grehan * Install entries into the BAT table to allow all 72732bc7846SPeter Grehan * of physmem to be convered by on-demand BAT entries. 72832bc7846SPeter Grehan * The loop will sometimes set the same battable element 72932bc7846SPeter Grehan * twice, but that's fine since they won't be used for 73032bc7846SPeter Grehan * a while yet. 73132bc7846SPeter Grehan */ 73232bc7846SPeter Grehan pa = pregions[i].mr_start & 0xf0000000; 73332bc7846SPeter Grehan end = pregions[i].mr_start + pregions[i].mr_size; 73432bc7846SPeter Grehan do { 73532bc7846SPeter Grehan u_int n = pa >> ADDR_SR_SHFT; 73632bc7846SPeter Grehan 73732bc7846SPeter Grehan battable[n].batl = BATL(pa, BAT_M, BAT_PP_RW); 73832bc7846SPeter Grehan battable[n].batu = BATU(pa, BAT_BL_256M, BAT_Vs); 73932bc7846SPeter Grehan pa += SEGMENT_LENGTH; 74032bc7846SPeter Grehan } while (pa < end); 74131c82d03SBenno Rice } 74231c82d03SBenno Rice 74331c82d03SBenno Rice if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz) 74459276937SPeter Grehan panic("moea_bootstrap: phys_avail too small"); 74597f7cde4SNathan Whitehorn 7465244eac9SBenno Rice phys_avail_count = 0; 747d2c1f576SBenno Rice physsz = 0; 748b0c21309SPeter Grehan hwphyssz = 0; 749b0c21309SPeter Grehan TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 75031c82d03SBenno Rice for (i = 0, j = 0; i < regions_sz; i++, j += 2) { 7515244eac9SBenno Rice CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start, 7525244eac9SBenno Rice regions[i].mr_start + regions[i].mr_size, 7535244eac9SBenno Rice regions[i].mr_size); 754e2f6d6e2SPeter Grehan if (hwphyssz != 0 && 755e2f6d6e2SPeter Grehan (physsz + regions[i].mr_size) >= hwphyssz) { 756e2f6d6e2SPeter Grehan if (physsz < hwphyssz) { 757e2f6d6e2SPeter Grehan phys_avail[j] = regions[i].mr_start; 758e2f6d6e2SPeter Grehan phys_avail[j + 1] = regions[i].mr_start + 759e2f6d6e2SPeter Grehan hwphyssz - physsz; 760e2f6d6e2SPeter Grehan physsz = hwphyssz; 761e2f6d6e2SPeter Grehan phys_avail_count++; 762e2f6d6e2SPeter Grehan } 763e2f6d6e2SPeter Grehan break; 764e2f6d6e2SPeter Grehan } 7655244eac9SBenno Rice phys_avail[j] = regions[i].mr_start; 7665244eac9SBenno Rice phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 7675244eac9SBenno Rice phys_avail_count++; 768d2c1f576SBenno Rice physsz += regions[i].mr_size; 769f9bac91bSBenno Rice } 770e347e23bSNathan Whitehorn 771e347e23bSNathan Whitehorn /* Check for overlap with the kernel and exception vectors */ 772e347e23bSNathan Whitehorn for (j = 0; j < 2*phys_avail_count; j+=2) { 773e347e23bSNathan Whitehorn if (phys_avail[j] < EXC_LAST) 774e347e23bSNathan Whitehorn phys_avail[j] += EXC_LAST; 775e347e23bSNathan Whitehorn 776e347e23bSNathan Whitehorn if (kernelstart >= phys_avail[j] && 777e347e23bSNathan Whitehorn kernelstart < phys_avail[j+1]) { 778e347e23bSNathan Whitehorn if (kernelend < phys_avail[j+1]) { 779e347e23bSNathan Whitehorn phys_avail[2*phys_avail_count] = 780e347e23bSNathan Whitehorn (kernelend & ~PAGE_MASK) + PAGE_SIZE; 781e347e23bSNathan Whitehorn phys_avail[2*phys_avail_count + 1] = 782e347e23bSNathan Whitehorn phys_avail[j+1]; 783e347e23bSNathan Whitehorn phys_avail_count++; 784e347e23bSNathan Whitehorn } 785e347e23bSNathan Whitehorn 786e347e23bSNathan Whitehorn phys_avail[j+1] = kernelstart & ~PAGE_MASK; 787e347e23bSNathan Whitehorn } 788e347e23bSNathan Whitehorn 789e347e23bSNathan Whitehorn if (kernelend >= phys_avail[j] && 790e347e23bSNathan Whitehorn kernelend < phys_avail[j+1]) { 791e347e23bSNathan Whitehorn if (kernelstart > phys_avail[j]) { 792e347e23bSNathan Whitehorn phys_avail[2*phys_avail_count] = phys_avail[j]; 793e347e23bSNathan Whitehorn phys_avail[2*phys_avail_count + 1] = 794e347e23bSNathan Whitehorn kernelstart & ~PAGE_MASK; 795e347e23bSNathan Whitehorn phys_avail_count++; 796e347e23bSNathan Whitehorn } 797e347e23bSNathan Whitehorn 798e347e23bSNathan Whitehorn phys_avail[j] = (kernelend & ~PAGE_MASK) + PAGE_SIZE; 799e347e23bSNathan Whitehorn } 800e347e23bSNathan Whitehorn } 801e347e23bSNathan Whitehorn 802d2c1f576SBenno Rice physmem = btoc(physsz); 803f9bac91bSBenno Rice 804f9bac91bSBenno Rice /* 8055244eac9SBenno Rice * Allocate PTEG table. 806f9bac91bSBenno Rice */ 8075244eac9SBenno Rice #ifdef PTEGCOUNT 80859276937SPeter Grehan moea_pteg_count = PTEGCOUNT; 8095244eac9SBenno Rice #else 81059276937SPeter Grehan moea_pteg_count = 0x1000; 811f9bac91bSBenno Rice 81259276937SPeter Grehan while (moea_pteg_count < physmem) 81359276937SPeter Grehan moea_pteg_count <<= 1; 814f9bac91bSBenno Rice 81559276937SPeter Grehan moea_pteg_count >>= 1; 8165244eac9SBenno Rice #endif /* PTEGCOUNT */ 817f9bac91bSBenno Rice 81859276937SPeter Grehan size = moea_pteg_count * sizeof(struct pteg); 81959276937SPeter Grehan CTR2(KTR_PMAP, "moea_bootstrap: %d PTEGs, %d bytes", moea_pteg_count, 8205244eac9SBenno Rice size); 82159276937SPeter Grehan moea_pteg_table = (struct pteg *)moea_bootstrap_alloc(size, size); 82259276937SPeter Grehan CTR1(KTR_PMAP, "moea_bootstrap: PTEG table at %p", moea_pteg_table); 82359276937SPeter Grehan bzero((void *)moea_pteg_table, moea_pteg_count * sizeof(struct pteg)); 82459276937SPeter Grehan moea_pteg_mask = moea_pteg_count - 1; 825f9bac91bSBenno Rice 8265244eac9SBenno Rice /* 827864bc520SBenno Rice * Allocate pv/overflow lists. 8285244eac9SBenno Rice */ 82959276937SPeter Grehan size = sizeof(struct pvo_head) * moea_pteg_count; 83059276937SPeter Grehan moea_pvo_table = (struct pvo_head *)moea_bootstrap_alloc(size, 8315244eac9SBenno Rice PAGE_SIZE); 83259276937SPeter Grehan CTR1(KTR_PMAP, "moea_bootstrap: PVO table at %p", moea_pvo_table); 83359276937SPeter Grehan for (i = 0; i < moea_pteg_count; i++) 83459276937SPeter Grehan LIST_INIT(&moea_pvo_table[i]); 8355244eac9SBenno Rice 8365244eac9SBenno Rice /* 837f489bf21SAlan Cox * Initialize the lock that synchronizes access to the pteg and pvo 838f489bf21SAlan Cox * tables. 839f489bf21SAlan Cox */ 840d644a0b7SAlan Cox mtx_init(&moea_table_mutex, "pmap table", NULL, MTX_DEF | 841d644a0b7SAlan Cox MTX_RECURSE); 842e9b5f218SNathan Whitehorn mtx_init(&moea_vsid_mutex, "VSID table", NULL, MTX_DEF); 843f489bf21SAlan Cox 844e4f72b32SMarcel Moolenaar mtx_init(&tlbie_mtx, "tlbie", NULL, MTX_SPIN); 845e4f72b32SMarcel Moolenaar 846f489bf21SAlan Cox /* 8475244eac9SBenno Rice * Initialise the unmanaged pvo pool. 8485244eac9SBenno Rice */ 84959276937SPeter Grehan moea_bpvo_pool = (struct pvo_entry *)moea_bootstrap_alloc( 8500d290675SBenno Rice BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0); 85159276937SPeter Grehan moea_bpvo_pool_index = 0; 8525244eac9SBenno Rice 8535244eac9SBenno Rice /* 8545244eac9SBenno Rice * Make sure kernel vsid is allocated as well as VSID 0. 8555244eac9SBenno Rice */ 85659276937SPeter Grehan moea_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW] 8575244eac9SBenno Rice |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 85859276937SPeter Grehan moea_vsid_bitmap[0] |= 1; 8595244eac9SBenno Rice 8605244eac9SBenno Rice /* 861fe3b4685SNathan Whitehorn * Initialize the kernel pmap (which is statically allocated). 8625244eac9SBenno Rice */ 863fe3b4685SNathan Whitehorn PMAP_LOCK_INIT(kernel_pmap); 864fe3b4685SNathan Whitehorn for (i = 0; i < 16; i++) 865fe3b4685SNathan Whitehorn kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i; 866c47dd3dbSAttilio Rao CPU_FILL(&kernel_pmap->pm_active); 867ccc4a5c7SNathan Whitehorn RB_INIT(&kernel_pmap->pmap_pvo); 868fe3b4685SNathan Whitehorn 869fe3b4685SNathan Whitehorn /* 8703653f5cbSAlan Cox * Initialize the global pv list lock. 8713653f5cbSAlan Cox */ 8723653f5cbSAlan Cox rw_init(&pvh_global_lock, "pmap pv global"); 8733653f5cbSAlan Cox 8743653f5cbSAlan Cox /* 875fe3b4685SNathan Whitehorn * Set up the Open Firmware mappings 876fe3b4685SNathan Whitehorn */ 877e347e23bSNathan Whitehorn chosen = OF_finddevice("/chosen"); 878e347e23bSNathan Whitehorn if (chosen != -1 && OF_getprop(chosen, "mmu", &mmui, 4) != -1 && 879e347e23bSNathan Whitehorn (mmu = OF_instance_to_package(mmui)) != -1 && 880e347e23bSNathan Whitehorn (sz = OF_getproplen(mmu, "translations")) != -1) { 881aa39961eSBenno Rice translations = NULL; 8826cc1cdf4SPeter Grehan for (i = 0; phys_avail[i] != 0; i += 2) { 8836cc1cdf4SPeter Grehan if (phys_avail[i + 1] >= sz) { 884aa39961eSBenno Rice translations = (struct ofw_map *)phys_avail[i]; 8856cc1cdf4SPeter Grehan break; 8866cc1cdf4SPeter Grehan } 887aa39961eSBenno Rice } 888aa39961eSBenno Rice if (translations == NULL) 88959276937SPeter Grehan panic("moea_bootstrap: no space to copy translations"); 8905244eac9SBenno Rice bzero(translations, sz); 8915244eac9SBenno Rice if (OF_getprop(mmu, "translations", translations, sz) == -1) 89259276937SPeter Grehan panic("moea_bootstrap: can't get ofw translations"); 89359276937SPeter Grehan CTR0(KTR_PMAP, "moea_bootstrap: translations"); 89431c82d03SBenno Rice sz /= sizeof(*translations); 8955244eac9SBenno Rice qsort(translations, sz, sizeof (*translations), om_cmp); 896ed1e1e2aSNathan Whitehorn for (i = 0; i < sz; i++) { 8975244eac9SBenno Rice CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x", 8985244eac9SBenno Rice translations[i].om_pa, translations[i].om_va, 8995244eac9SBenno Rice translations[i].om_len); 9005244eac9SBenno Rice 90132bc7846SPeter Grehan /* 902e347e23bSNathan Whitehorn * If the mapping is 1:1, let the RAM and device 903e347e23bSNathan Whitehorn * on-demand BAT tables take care of the translation. 90432bc7846SPeter Grehan */ 90532bc7846SPeter Grehan if (translations[i].om_va == translations[i].om_pa) 90632bc7846SPeter Grehan continue; 9075244eac9SBenno Rice 90832bc7846SPeter Grehan /* Enter the pages */ 909e347e23bSNathan Whitehorn for (off = 0; off < translations[i].om_len; 910e347e23bSNathan Whitehorn off += PAGE_SIZE) 911fe3b4685SNathan Whitehorn moea_kenter(mmup, translations[i].om_va + off, 912fe3b4685SNathan Whitehorn translations[i].om_pa + off); 913f9bac91bSBenno Rice } 914e347e23bSNathan Whitehorn } 915014ffa99SMarcel Moolenaar 916014ffa99SMarcel Moolenaar /* 917014ffa99SMarcel Moolenaar * Calculate the last available physical address. 918014ffa99SMarcel Moolenaar */ 919014ffa99SMarcel Moolenaar for (i = 0; phys_avail[i + 2] != 0; i += 2) 920014ffa99SMarcel Moolenaar ; 921014ffa99SMarcel Moolenaar Maxmem = powerpc_btop(phys_avail[i + 1]); 9225244eac9SBenno Rice 9231c96bdd1SNathan Whitehorn moea_cpu_bootstrap(mmup,0); 9245244eac9SBenno Rice 9255244eac9SBenno Rice pmap_bootstrapped++; 926014ffa99SMarcel Moolenaar 927014ffa99SMarcel Moolenaar /* 928014ffa99SMarcel Moolenaar * Set the start and end of kva. 929014ffa99SMarcel Moolenaar */ 930014ffa99SMarcel Moolenaar virtual_avail = VM_MIN_KERNEL_ADDRESS; 931ab739706SNathan Whitehorn virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; 932014ffa99SMarcel Moolenaar 933014ffa99SMarcel Moolenaar /* 934014ffa99SMarcel Moolenaar * Allocate a kernel stack with a guard page for thread0 and map it 935014ffa99SMarcel Moolenaar * into the kernel page map. 936014ffa99SMarcel Moolenaar */ 937edc82223SKonstantin Belousov pa = moea_bootstrap_alloc(kstack_pages * PAGE_SIZE, PAGE_SIZE); 938014ffa99SMarcel Moolenaar va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 939edc82223SKonstantin Belousov virtual_avail = va + kstack_pages * PAGE_SIZE; 940014ffa99SMarcel Moolenaar CTR2(KTR_PMAP, "moea_bootstrap: kstack0 at %#x (%#x)", pa, va); 941014ffa99SMarcel Moolenaar thread0.td_kstack = va; 942edc82223SKonstantin Belousov thread0.td_kstack_pages = kstack_pages; 943edc82223SKonstantin Belousov for (i = 0; i < kstack_pages; i++) { 944c2ede4b3SMartin Blapp moea_kenter(mmup, va, pa); 945014ffa99SMarcel Moolenaar pa += PAGE_SIZE; 946014ffa99SMarcel Moolenaar va += PAGE_SIZE; 947014ffa99SMarcel Moolenaar } 948014ffa99SMarcel Moolenaar 949014ffa99SMarcel Moolenaar /* 950014ffa99SMarcel Moolenaar * Allocate virtual address space for the message buffer. 951014ffa99SMarcel Moolenaar */ 9524053b05bSSergey Kandaurov pa = msgbuf_phys = moea_bootstrap_alloc(msgbufsize, PAGE_SIZE); 953014ffa99SMarcel Moolenaar msgbufp = (struct msgbuf *)virtual_avail; 954014ffa99SMarcel Moolenaar va = virtual_avail; 9554053b05bSSergey Kandaurov virtual_avail += round_page(msgbufsize); 956014ffa99SMarcel Moolenaar while (va < virtual_avail) { 957c2ede4b3SMartin Blapp moea_kenter(mmup, va, pa); 958014ffa99SMarcel Moolenaar pa += PAGE_SIZE; 959014ffa99SMarcel Moolenaar va += PAGE_SIZE; 960014ffa99SMarcel Moolenaar } 96150c202c5SJeff Roberson 96250c202c5SJeff Roberson /* 96350c202c5SJeff Roberson * Allocate virtual address space for the dynamic percpu area. 96450c202c5SJeff Roberson */ 96550c202c5SJeff Roberson pa = moea_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE); 96650c202c5SJeff Roberson dpcpu = (void *)virtual_avail; 96750c202c5SJeff Roberson va = virtual_avail; 96850c202c5SJeff Roberson virtual_avail += DPCPU_SIZE; 96950c202c5SJeff Roberson while (va < virtual_avail) { 970c2ede4b3SMartin Blapp moea_kenter(mmup, va, pa); 97150c202c5SJeff Roberson pa += PAGE_SIZE; 97250c202c5SJeff Roberson va += PAGE_SIZE; 97350c202c5SJeff Roberson } 97450c202c5SJeff Roberson dpcpu_init(dpcpu, 0); 9755244eac9SBenno Rice } 9765244eac9SBenno Rice 9775244eac9SBenno Rice /* 9785244eac9SBenno Rice * Activate a user pmap. The pmap must be activated before it's address 9795244eac9SBenno Rice * space can be accessed in any way. 980f9bac91bSBenno Rice */ 981f9bac91bSBenno Rice void 98259276937SPeter Grehan moea_activate(mmu_t mmu, struct thread *td) 983f9bac91bSBenno Rice { 9848207b362SBenno Rice pmap_t pm, pmr; 985f9bac91bSBenno Rice 986f9bac91bSBenno Rice /* 98732bc7846SPeter Grehan * Load all the data we need up front to encourage the compiler to 9885244eac9SBenno Rice * not issue any loads while we have interrupts disabled below. 989f9bac91bSBenno Rice */ 9905244eac9SBenno Rice pm = &td->td_proc->p_vmspace->vm_pmap; 99152a7870dSNathan Whitehorn pmr = pm->pmap_phys; 9928207b362SBenno Rice 993c7c2767eSAttilio Rao CPU_SET(PCPU_GET(cpuid), &pm->pm_active); 9948207b362SBenno Rice PCPU_SET(curpmap, pmr); 995d1295abdSNathan Whitehorn 996d1295abdSNathan Whitehorn mtsrin(USER_SR << ADDR_SR_SHFT, td->td_pcb->pcb_cpu.aim.usr_vsid); 997ac6ba8bdSBenno Rice } 998ac6ba8bdSBenno Rice 999ac6ba8bdSBenno Rice void 100059276937SPeter Grehan moea_deactivate(mmu_t mmu, struct thread *td) 1001ac6ba8bdSBenno Rice { 1002ac6ba8bdSBenno Rice pmap_t pm; 1003ac6ba8bdSBenno Rice 1004ac6ba8bdSBenno Rice pm = &td->td_proc->p_vmspace->vm_pmap; 1005c7c2767eSAttilio Rao CPU_CLR(PCPU_GET(cpuid), &pm->pm_active); 10068207b362SBenno Rice PCPU_SET(curpmap, NULL); 1007f9bac91bSBenno Rice } 1008f9bac91bSBenno Rice 1009f9bac91bSBenno Rice void 1010a844c68fSAlan Cox moea_unwire(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1011a844c68fSAlan Cox { 1012a844c68fSAlan Cox struct pvo_entry key, *pvo; 1013a844c68fSAlan Cox 1014a844c68fSAlan Cox PMAP_LOCK(pm); 1015a844c68fSAlan Cox key.pvo_vaddr = sva; 1016a844c68fSAlan Cox for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 1017a844c68fSAlan Cox pvo != NULL && PVO_VADDR(pvo) < eva; 1018a844c68fSAlan Cox pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { 1019a844c68fSAlan Cox if ((pvo->pvo_vaddr & PVO_WIRED) == 0) 1020a844c68fSAlan Cox panic("moea_unwire: pvo %p is missing PVO_WIRED", pvo); 1021a844c68fSAlan Cox pvo->pvo_vaddr &= ~PVO_WIRED; 1022a844c68fSAlan Cox pm->pm_stats.wired_count--; 1023a844c68fSAlan Cox } 1024a844c68fSAlan Cox PMAP_UNLOCK(pm); 1025a844c68fSAlan Cox } 1026a844c68fSAlan Cox 1027a844c68fSAlan Cox void 102859276937SPeter Grehan moea_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst) 1029f9bac91bSBenno Rice { 103025e2288dSBenno Rice vm_offset_t dst; 103125e2288dSBenno Rice vm_offset_t src; 103225e2288dSBenno Rice 103325e2288dSBenno Rice dst = VM_PAGE_TO_PHYS(mdst); 103425e2288dSBenno Rice src = VM_PAGE_TO_PHYS(msrc); 103525e2288dSBenno Rice 1036e3c2930dSNathan Whitehorn bcopy((void *)src, (void *)dst, PAGE_SIZE); 1037f9bac91bSBenno Rice } 1038111c77dcSBenno Rice 1039e8a4a618SKonstantin Belousov void 1040e8a4a618SKonstantin Belousov moea_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 1041e8a4a618SKonstantin Belousov vm_page_t *mb, vm_offset_t b_offset, int xfersize) 1042e8a4a618SKonstantin Belousov { 1043e8a4a618SKonstantin Belousov void *a_cp, *b_cp; 1044e8a4a618SKonstantin Belousov vm_offset_t a_pg_offset, b_pg_offset; 1045e8a4a618SKonstantin Belousov int cnt; 1046e8a4a618SKonstantin Belousov 1047e8a4a618SKonstantin Belousov while (xfersize > 0) { 1048e8a4a618SKonstantin Belousov a_pg_offset = a_offset & PAGE_MASK; 1049e8a4a618SKonstantin Belousov cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 1050e8a4a618SKonstantin Belousov a_cp = (char *)VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]) + 1051e8a4a618SKonstantin Belousov a_pg_offset; 1052e8a4a618SKonstantin Belousov b_pg_offset = b_offset & PAGE_MASK; 1053e8a4a618SKonstantin Belousov cnt = min(cnt, PAGE_SIZE - b_pg_offset); 1054e8a4a618SKonstantin Belousov b_cp = (char *)VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]) + 1055e8a4a618SKonstantin Belousov b_pg_offset; 1056e8a4a618SKonstantin Belousov bcopy(a_cp, b_cp, cnt); 1057e8a4a618SKonstantin Belousov a_offset += cnt; 1058e8a4a618SKonstantin Belousov b_offset += cnt; 1059e8a4a618SKonstantin Belousov xfersize -= cnt; 1060e8a4a618SKonstantin Belousov } 1061e8a4a618SKonstantin Belousov } 1062e8a4a618SKonstantin Belousov 1063111c77dcSBenno Rice /* 10645244eac9SBenno Rice * Zero a page of physical memory by temporarily mapping it into the tlb. 10655244eac9SBenno Rice */ 10665244eac9SBenno Rice void 106759276937SPeter Grehan moea_zero_page(mmu_t mmu, vm_page_t m) 10685244eac9SBenno Rice { 1069fe938c08SJustin Hibbits vm_offset_t off, pa = VM_PAGE_TO_PHYS(m); 10705244eac9SBenno Rice 1071fe938c08SJustin Hibbits for (off = 0; off < PAGE_SIZE; off += cacheline_size) 1072fe938c08SJustin Hibbits __asm __volatile("dcbz 0,%0" :: "r"(pa + off)); 10735244eac9SBenno Rice } 10745244eac9SBenno Rice 10755244eac9SBenno Rice void 107659276937SPeter Grehan moea_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size) 10775244eac9SBenno Rice { 10783495845eSBenno Rice vm_offset_t pa = VM_PAGE_TO_PHYS(m); 10795b43c63dSMarcel Moolenaar void *va = (void *)(pa + off); 10803495845eSBenno Rice 10815b43c63dSMarcel Moolenaar bzero(va, size); 10825244eac9SBenno Rice } 10835244eac9SBenno Rice 1084a58b3a68SPeter Wemm void 108559276937SPeter Grehan moea_zero_page_idle(mmu_t mmu, vm_page_t m) 1086a58b3a68SPeter Wemm { 1087a58b3a68SPeter Wemm 1088fe938c08SJustin Hibbits moea_zero_page(mmu, m); 1089a58b3a68SPeter Wemm } 1090a58b3a68SPeter Wemm 1091713841afSJason A. Harmening vm_offset_t 1092713841afSJason A. Harmening moea_quick_enter_page(mmu_t mmu, vm_page_t m) 1093713841afSJason A. Harmening { 1094713841afSJason A. Harmening 1095713841afSJason A. Harmening return (VM_PAGE_TO_PHYS(m)); 1096713841afSJason A. Harmening } 1097713841afSJason A. Harmening 1098713841afSJason A. Harmening void 1099713841afSJason A. Harmening moea_quick_remove_page(mmu_t mmu, vm_offset_t addr) 1100713841afSJason A. Harmening { 1101713841afSJason A. Harmening } 1102713841afSJason A. Harmening 11035244eac9SBenno Rice /* 11045244eac9SBenno Rice * Map the given physical page at the specified virtual address in the 11055244eac9SBenno Rice * target pmap with the protection requested. If specified the page 11065244eac9SBenno Rice * will be wired down. 11075244eac9SBenno Rice */ 110839ffa8c1SKonstantin Belousov int 110959276937SPeter Grehan moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 111039ffa8c1SKonstantin Belousov u_int flags, int8_t psind) 11115244eac9SBenno Rice { 111239ffa8c1SKonstantin Belousov int error; 1113ce142d9eSAlan Cox 111439ffa8c1SKonstantin Belousov for (;;) { 11153653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 1116ce142d9eSAlan Cox PMAP_LOCK(pmap); 111739ffa8c1SKonstantin Belousov error = moea_enter_locked(pmap, va, m, prot, flags, psind); 11183653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 1119ce142d9eSAlan Cox PMAP_UNLOCK(pmap); 112039ffa8c1SKonstantin Belousov if (error != ENOMEM) 112139ffa8c1SKonstantin Belousov return (KERN_SUCCESS); 112239ffa8c1SKonstantin Belousov if ((flags & PMAP_ENTER_NOSLEEP) != 0) 112339ffa8c1SKonstantin Belousov return (KERN_RESOURCE_SHORTAGE); 112439ffa8c1SKonstantin Belousov VM_OBJECT_ASSERT_UNLOCKED(m->object); 112539ffa8c1SKonstantin Belousov VM_WAIT; 112639ffa8c1SKonstantin Belousov } 1127ce142d9eSAlan Cox } 1128ce142d9eSAlan Cox 1129ce142d9eSAlan Cox /* 1130ce142d9eSAlan Cox * Map the given physical page at the specified virtual address in the 1131ce142d9eSAlan Cox * target pmap with the protection requested. If specified the page 1132ce142d9eSAlan Cox * will be wired down. 1133ce142d9eSAlan Cox * 1134f26bcf99SAlan Cox * The global pvh and pmap must be locked. 1135ce142d9eSAlan Cox */ 113639ffa8c1SKonstantin Belousov static int 1137ce142d9eSAlan Cox moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 113839ffa8c1SKonstantin Belousov u_int flags, int8_t psind __unused) 1139ce142d9eSAlan Cox { 11405244eac9SBenno Rice struct pvo_head *pvo_head; 1141378862a7SJeff Roberson uma_zone_t zone; 114257bd5cceSNathan Whitehorn u_int pte_lo, pvo_flags; 11435244eac9SBenno Rice int error; 11445244eac9SBenno Rice 1145081b8e20SAlan Cox if (pmap_bootstrapped) 1146081b8e20SAlan Cox rw_assert(&pvh_global_lock, RA_WLOCKED); 1147081b8e20SAlan Cox PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1148081b8e20SAlan Cox if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) 1149081b8e20SAlan Cox VM_OBJECT_ASSERT_LOCKED(m->object); 1150081b8e20SAlan Cox 1151081b8e20SAlan Cox if ((m->oflags & VPO_UNMANAGED) != 0 || !moea_initialized) { 115259276937SPeter Grehan pvo_head = &moea_pvo_kunmanaged; 115359276937SPeter Grehan zone = moea_upvo_zone; 11545244eac9SBenno Rice pvo_flags = 0; 11555244eac9SBenno Rice } else { 115603b6e025SPeter Grehan pvo_head = vm_page_to_pvoh(m); 115759276937SPeter Grehan zone = moea_mpvo_zone; 11585244eac9SBenno Rice pvo_flags = PVO_MANAGED; 11595244eac9SBenno Rice } 11604dba5df1SPeter Grehan 1161cd6a97f0SNathan Whitehorn pte_lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m)); 11625244eac9SBenno Rice 116344b8bd66SAlan Cox if (prot & VM_PROT_WRITE) { 11645244eac9SBenno Rice pte_lo |= PTE_BW; 11652368a371SAlan Cox if (pmap_bootstrapped && 1166d98d0ce2SKonstantin Belousov (m->oflags & VPO_UNMANAGED) == 0) 11673407fefeSKonstantin Belousov vm_page_aflag_set(m, PGA_WRITEABLE); 116844b8bd66SAlan Cox } else 11695244eac9SBenno Rice pte_lo |= PTE_BR; 11705244eac9SBenno Rice 117139ffa8c1SKonstantin Belousov if ((flags & PMAP_ENTER_WIRED) != 0) 11725244eac9SBenno Rice pvo_flags |= PVO_WIRED; 11735244eac9SBenno Rice 117459276937SPeter Grehan error = moea_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), 11758207b362SBenno Rice pte_lo, pvo_flags); 11765244eac9SBenno Rice 11778207b362SBenno Rice /* 117857bd5cceSNathan Whitehorn * Flush the real page from the instruction cache. This has be done 117957bd5cceSNathan Whitehorn * for all user mappings to prevent information leakage via the 1180805bee55SNathan Whitehorn * instruction cache. moea_pvo_enter() returns ENOENT for the first 1181805bee55SNathan Whitehorn * mapping for a page. 11828207b362SBenno Rice */ 1183805bee55SNathan Whitehorn if (pmap != kernel_pmap && error == ENOENT && 1184805bee55SNathan Whitehorn (pte_lo & (PTE_I | PTE_G)) == 0) 118559276937SPeter Grehan moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); 118639ffa8c1SKonstantin Belousov 118739ffa8c1SKonstantin Belousov return (error); 1188ce142d9eSAlan Cox } 1189ce142d9eSAlan Cox 1190ce142d9eSAlan Cox /* 1191ce142d9eSAlan Cox * Maps a sequence of resident pages belonging to the same object. 1192ce142d9eSAlan Cox * The sequence begins with the given page m_start. This page is 1193ce142d9eSAlan Cox * mapped at the given virtual address start. Each subsequent page is 1194ce142d9eSAlan Cox * mapped at a virtual address that is offset from start by the same 1195ce142d9eSAlan Cox * amount as the page is offset from m_start within the object. The 1196ce142d9eSAlan Cox * last page in the sequence is the page with the largest offset from 1197ce142d9eSAlan Cox * m_start that can be mapped at a virtual address less than the given 1198ce142d9eSAlan Cox * virtual address end. Not every virtual page between start and end 1199ce142d9eSAlan Cox * is mapped; only those for which a resident page exists with the 1200ce142d9eSAlan Cox * corresponding offset from m_start are mapped. 1201ce142d9eSAlan Cox */ 1202ce142d9eSAlan Cox void 1203ce142d9eSAlan Cox moea_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end, 1204ce142d9eSAlan Cox vm_page_t m_start, vm_prot_t prot) 1205ce142d9eSAlan Cox { 1206ce142d9eSAlan Cox vm_page_t m; 1207ce142d9eSAlan Cox vm_pindex_t diff, psize; 1208ce142d9eSAlan Cox 12099af6d512SAttilio Rao VM_OBJECT_ASSERT_LOCKED(m_start->object); 12109af6d512SAttilio Rao 1211ce142d9eSAlan Cox psize = atop(end - start); 1212ce142d9eSAlan Cox m = m_start; 12133653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 1214ce142d9eSAlan Cox PMAP_LOCK(pm); 1215ce142d9eSAlan Cox while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1216ce142d9eSAlan Cox moea_enter_locked(pm, start + ptoa(diff), m, prot & 121739ffa8c1SKonstantin Belousov (VM_PROT_READ | VM_PROT_EXECUTE), 0, 0); 1218ce142d9eSAlan Cox m = TAILQ_NEXT(m, listq); 1219ce142d9eSAlan Cox } 12203653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 1221ce142d9eSAlan Cox PMAP_UNLOCK(pm); 12225244eac9SBenno Rice } 12235244eac9SBenno Rice 12242053c127SStephan Uphoff void 122559276937SPeter Grehan moea_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m, 12262053c127SStephan Uphoff vm_prot_t prot) 1227dca96f1aSAlan Cox { 1228dca96f1aSAlan Cox 12293653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 1230ce142d9eSAlan Cox PMAP_LOCK(pm); 1231ce142d9eSAlan Cox moea_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 123239ffa8c1SKonstantin Belousov 0, 0); 12333653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 1234ce142d9eSAlan Cox PMAP_UNLOCK(pm); 1235dca96f1aSAlan Cox } 1236dca96f1aSAlan Cox 123756b09388SAlan Cox vm_paddr_t 123859276937SPeter Grehan moea_extract(mmu_t mmu, pmap_t pm, vm_offset_t va) 12395244eac9SBenno Rice { 12400f92104cSBenno Rice struct pvo_entry *pvo; 124148d0b1a0SAlan Cox vm_paddr_t pa; 12420f92104cSBenno Rice 124348d0b1a0SAlan Cox PMAP_LOCK(pm); 124459276937SPeter Grehan pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 124548d0b1a0SAlan Cox if (pvo == NULL) 124648d0b1a0SAlan Cox pa = 0; 124748d0b1a0SAlan Cox else 124852a7870dSNathan Whitehorn pa = (pvo->pvo_pte.pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF); 124948d0b1a0SAlan Cox PMAP_UNLOCK(pm); 125048d0b1a0SAlan Cox return (pa); 12515244eac9SBenno Rice } 12525244eac9SBenno Rice 12535244eac9SBenno Rice /* 125484792e72SPeter Grehan * Atomically extract and hold the physical page with the given 125584792e72SPeter Grehan * pmap and virtual address pair if that mapping permits the given 125684792e72SPeter Grehan * protection. 125784792e72SPeter Grehan */ 125884792e72SPeter Grehan vm_page_t 125959276937SPeter Grehan moea_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot) 126084792e72SPeter Grehan { 1261ab50a262SAlan Cox struct pvo_entry *pvo; 126284792e72SPeter Grehan vm_page_t m; 12632965a453SKip Macy vm_paddr_t pa; 126484792e72SPeter Grehan 126584792e72SPeter Grehan m = NULL; 12662965a453SKip Macy pa = 0; 126748d0b1a0SAlan Cox PMAP_LOCK(pmap); 12682965a453SKip Macy retry: 126959276937SPeter Grehan pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); 127052a7870dSNathan Whitehorn if (pvo != NULL && (pvo->pvo_pte.pte.pte_hi & PTE_VALID) && 127152a7870dSNathan Whitehorn ((pvo->pvo_pte.pte.pte_lo & PTE_PP) == PTE_RW || 1272ab50a262SAlan Cox (prot & VM_PROT_WRITE) == 0)) { 12732965a453SKip Macy if (vm_page_pa_tryrelock(pmap, pvo->pvo_pte.pte.pte_lo & PTE_RPGN, &pa)) 12742965a453SKip Macy goto retry; 127552a7870dSNathan Whitehorn m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte.pte_lo & PTE_RPGN); 127684792e72SPeter Grehan vm_page_hold(m); 127784792e72SPeter Grehan } 12782965a453SKip Macy PA_UNLOCK_COND(pa); 127948d0b1a0SAlan Cox PMAP_UNLOCK(pmap); 128084792e72SPeter Grehan return (m); 128184792e72SPeter Grehan } 128284792e72SPeter Grehan 12835244eac9SBenno Rice void 128459276937SPeter Grehan moea_init(mmu_t mmu) 12855244eac9SBenno Rice { 12865244eac9SBenno Rice 128759276937SPeter Grehan moea_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), 12880ee6dbd7SPeter Grehan NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 12890ee6dbd7SPeter Grehan UMA_ZONE_VM | UMA_ZONE_NOFREE); 129059276937SPeter Grehan moea_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry), 12910ee6dbd7SPeter Grehan NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 12920ee6dbd7SPeter Grehan UMA_ZONE_VM | UMA_ZONE_NOFREE); 129359276937SPeter Grehan moea_initialized = TRUE; 12945244eac9SBenno Rice } 12955244eac9SBenno Rice 12965244eac9SBenno Rice boolean_t 12977b85f591SAlan Cox moea_is_referenced(mmu_t mmu, vm_page_t m) 12987b85f591SAlan Cox { 12998d9e6d9fSAlan Cox boolean_t rv; 13007b85f591SAlan Cox 1301d98d0ce2SKonstantin Belousov KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1302c46b90e9SAlan Cox ("moea_is_referenced: page %p is not managed", m)); 13038d9e6d9fSAlan Cox rw_wlock(&pvh_global_lock); 13048d9e6d9fSAlan Cox rv = moea_query_bit(m, PTE_REF); 13058d9e6d9fSAlan Cox rw_wunlock(&pvh_global_lock); 13068d9e6d9fSAlan Cox return (rv); 13077b85f591SAlan Cox } 13087b85f591SAlan Cox 13097b85f591SAlan Cox boolean_t 131059276937SPeter Grehan moea_is_modified(mmu_t mmu, vm_page_t m) 13115244eac9SBenno Rice { 13128d9e6d9fSAlan Cox boolean_t rv; 13130f92104cSBenno Rice 1314d98d0ce2SKonstantin Belousov KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1315567e51e1SAlan Cox ("moea_is_modified: page %p is not managed", m)); 1316567e51e1SAlan Cox 1317567e51e1SAlan Cox /* 1318c7aebda8SAttilio Rao * If the page is not exclusive busied, then PGA_WRITEABLE cannot be 13193407fefeSKonstantin Belousov * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 1320567e51e1SAlan Cox * is clear, no PTEs can have PTE_CHG set. 1321567e51e1SAlan Cox */ 132289f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(m->object); 1323c7aebda8SAttilio Rao if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) 13240f92104cSBenno Rice return (FALSE); 13258d9e6d9fSAlan Cox rw_wlock(&pvh_global_lock); 13268d9e6d9fSAlan Cox rv = moea_query_bit(m, PTE_CHG); 13278d9e6d9fSAlan Cox rw_wunlock(&pvh_global_lock); 13288d9e6d9fSAlan Cox return (rv); 1329566526a9SAlan Cox } 1330566526a9SAlan Cox 1331e396eb60SAlan Cox boolean_t 1332e396eb60SAlan Cox moea_is_prefaultable(mmu_t mmu, pmap_t pmap, vm_offset_t va) 1333e396eb60SAlan Cox { 1334e396eb60SAlan Cox struct pvo_entry *pvo; 1335e396eb60SAlan Cox boolean_t rv; 1336e396eb60SAlan Cox 1337e396eb60SAlan Cox PMAP_LOCK(pmap); 1338e396eb60SAlan Cox pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); 1339e396eb60SAlan Cox rv = pvo == NULL || (pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0; 1340e396eb60SAlan Cox PMAP_UNLOCK(pmap); 1341e396eb60SAlan Cox return (rv); 1342e396eb60SAlan Cox } 1343e396eb60SAlan Cox 13445244eac9SBenno Rice void 134559276937SPeter Grehan moea_clear_modify(mmu_t mmu, vm_page_t m) 134603b6e025SPeter Grehan { 134703b6e025SPeter Grehan 1348d98d0ce2SKonstantin Belousov KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1349567e51e1SAlan Cox ("moea_clear_modify: page %p is not managed", m)); 135089f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(m->object); 1351c7aebda8SAttilio Rao KASSERT(!vm_page_xbusied(m), 1352c7aebda8SAttilio Rao ("moea_clear_modify: page %p is exclusive busy", m)); 1353567e51e1SAlan Cox 1354567e51e1SAlan Cox /* 13553407fefeSKonstantin Belousov * If the page is not PGA_WRITEABLE, then no PTEs can have PTE_CHG 1356567e51e1SAlan Cox * set. If the object containing the page is locked and the page is 1357c7aebda8SAttilio Rao * not exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 1358567e51e1SAlan Cox */ 13593407fefeSKonstantin Belousov if ((m->aflags & PGA_WRITEABLE) == 0) 136003b6e025SPeter Grehan return; 13618d9e6d9fSAlan Cox rw_wlock(&pvh_global_lock); 1362ce186587SAlan Cox moea_clear_bit(m, PTE_CHG); 13638d9e6d9fSAlan Cox rw_wunlock(&pvh_global_lock); 13645244eac9SBenno Rice } 13655244eac9SBenno Rice 13667f3a4093SMike Silbersack /* 136778985e42SAlan Cox * Clear the write and modified bits in each of the given page's mappings. 136878985e42SAlan Cox */ 136978985e42SAlan Cox void 137078985e42SAlan Cox moea_remove_write(mmu_t mmu, vm_page_t m) 137178985e42SAlan Cox { 137278985e42SAlan Cox struct pvo_entry *pvo; 137378985e42SAlan Cox struct pte *pt; 137478985e42SAlan Cox pmap_t pmap; 137578985e42SAlan Cox u_int lo; 137678985e42SAlan Cox 1377d98d0ce2SKonstantin Belousov KASSERT((m->oflags & VPO_UNMANAGED) == 0, 13789ab6032fSAlan Cox ("moea_remove_write: page %p is not managed", m)); 13799ab6032fSAlan Cox 13809ab6032fSAlan Cox /* 1381c7aebda8SAttilio Rao * If the page is not exclusive busied, then PGA_WRITEABLE cannot be 1382c7aebda8SAttilio Rao * set by another thread while the object is locked. Thus, 1383c7aebda8SAttilio Rao * if PGA_WRITEABLE is clear, no page table entries need updating. 13849ab6032fSAlan Cox */ 138589f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(m->object); 1386c7aebda8SAttilio Rao if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) 138778985e42SAlan Cox return; 13883653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 138978985e42SAlan Cox lo = moea_attr_fetch(m); 1390e4f72b32SMarcel Moolenaar powerpc_sync(); 139178985e42SAlan Cox LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 139278985e42SAlan Cox pmap = pvo->pvo_pmap; 139378985e42SAlan Cox PMAP_LOCK(pmap); 139452a7870dSNathan Whitehorn if ((pvo->pvo_pte.pte.pte_lo & PTE_PP) != PTE_BR) { 139578985e42SAlan Cox pt = moea_pvo_to_pte(pvo, -1); 139652a7870dSNathan Whitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_PP; 139752a7870dSNathan Whitehorn pvo->pvo_pte.pte.pte_lo |= PTE_BR; 139878985e42SAlan Cox if (pt != NULL) { 139952a7870dSNathan Whitehorn moea_pte_synch(pt, &pvo->pvo_pte.pte); 140052a7870dSNathan Whitehorn lo |= pvo->pvo_pte.pte.pte_lo; 140152a7870dSNathan Whitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_CHG; 140252a7870dSNathan Whitehorn moea_pte_change(pt, &pvo->pvo_pte.pte, 140378985e42SAlan Cox pvo->pvo_vaddr); 140478985e42SAlan Cox mtx_unlock(&moea_table_mutex); 140578985e42SAlan Cox } 140678985e42SAlan Cox } 140778985e42SAlan Cox PMAP_UNLOCK(pmap); 140878985e42SAlan Cox } 140978985e42SAlan Cox if ((lo & PTE_CHG) != 0) { 141078985e42SAlan Cox moea_attr_clear(m, PTE_CHG); 141178985e42SAlan Cox vm_page_dirty(m); 141278985e42SAlan Cox } 14133407fefeSKonstantin Belousov vm_page_aflag_clear(m, PGA_WRITEABLE); 14143653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 141578985e42SAlan Cox } 141678985e42SAlan Cox 141778985e42SAlan Cox /* 141859276937SPeter Grehan * moea_ts_referenced: 14197f3a4093SMike Silbersack * 14207f3a4093SMike Silbersack * Return a count of reference bits for a page, clearing those bits. 14217f3a4093SMike Silbersack * It is not necessary for every reference bit to be cleared, but it 14227f3a4093SMike Silbersack * is necessary that 0 only be returned when there are truly no 14237f3a4093SMike Silbersack * reference bits set. 14247f3a4093SMike Silbersack * 14257f3a4093SMike Silbersack * XXX: The exact number of bits to check and clear is a matter that 14267f3a4093SMike Silbersack * should be tested and standardized at some point in the future for 14277f3a4093SMike Silbersack * optimal aging of shared pages. 14287f3a4093SMike Silbersack */ 14298d9e6d9fSAlan Cox int 143059276937SPeter Grehan moea_ts_referenced(mmu_t mmu, vm_page_t m) 14315244eac9SBenno Rice { 14328d9e6d9fSAlan Cox int count; 143303b6e025SPeter Grehan 1434d98d0ce2SKonstantin Belousov KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1435ce186587SAlan Cox ("moea_ts_referenced: page %p is not managed", m)); 14368d9e6d9fSAlan Cox rw_wlock(&pvh_global_lock); 14378d9e6d9fSAlan Cox count = moea_clear_bit(m, PTE_REF); 14388d9e6d9fSAlan Cox rw_wunlock(&pvh_global_lock); 14398d9e6d9fSAlan Cox return (count); 14405244eac9SBenno Rice } 14415244eac9SBenno Rice 14425244eac9SBenno Rice /* 1443c1f4123bSNathan Whitehorn * Modify the WIMG settings of all mappings for a page. 1444c1f4123bSNathan Whitehorn */ 1445c1f4123bSNathan Whitehorn void 1446c1f4123bSNathan Whitehorn moea_page_set_memattr(mmu_t mmu, vm_page_t m, vm_memattr_t ma) 1447c1f4123bSNathan Whitehorn { 1448c1f4123bSNathan Whitehorn struct pvo_entry *pvo; 1449cd6a97f0SNathan Whitehorn struct pvo_head *pvo_head; 1450c1f4123bSNathan Whitehorn struct pte *pt; 1451c1f4123bSNathan Whitehorn pmap_t pmap; 1452c1f4123bSNathan Whitehorn u_int lo; 1453c1f4123bSNathan Whitehorn 1454d98d0ce2SKonstantin Belousov if ((m->oflags & VPO_UNMANAGED) != 0) { 1455cd6a97f0SNathan Whitehorn m->md.mdpg_cache_attrs = ma; 1456cd6a97f0SNathan Whitehorn return; 1457cd6a97f0SNathan Whitehorn } 1458cd6a97f0SNathan Whitehorn 14593653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 1460cd6a97f0SNathan Whitehorn pvo_head = vm_page_to_pvoh(m); 1461c1f4123bSNathan Whitehorn lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), ma); 1462cd6a97f0SNathan Whitehorn 1463cd6a97f0SNathan Whitehorn LIST_FOREACH(pvo, pvo_head, pvo_vlink) { 1464c1f4123bSNathan Whitehorn pmap = pvo->pvo_pmap; 1465c1f4123bSNathan Whitehorn PMAP_LOCK(pmap); 1466c1f4123bSNathan Whitehorn pt = moea_pvo_to_pte(pvo, -1); 1467c1f4123bSNathan Whitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_WIMG; 1468c1f4123bSNathan Whitehorn pvo->pvo_pte.pte.pte_lo |= lo; 1469c1f4123bSNathan Whitehorn if (pt != NULL) { 1470c1f4123bSNathan Whitehorn moea_pte_change(pt, &pvo->pvo_pte.pte, 1471c1f4123bSNathan Whitehorn pvo->pvo_vaddr); 1472c1f4123bSNathan Whitehorn if (pvo->pvo_pmap == kernel_pmap) 1473c1f4123bSNathan Whitehorn isync(); 1474c1f4123bSNathan Whitehorn } 1475c1f4123bSNathan Whitehorn mtx_unlock(&moea_table_mutex); 1476c1f4123bSNathan Whitehorn PMAP_UNLOCK(pmap); 1477c1f4123bSNathan Whitehorn } 1478c1f4123bSNathan Whitehorn m->md.mdpg_cache_attrs = ma; 14793653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 1480c1f4123bSNathan Whitehorn } 1481c1f4123bSNathan Whitehorn 1482c1f4123bSNathan Whitehorn /* 14835244eac9SBenno Rice * Map a wired page into kernel virtual address space. 14845244eac9SBenno Rice */ 14855244eac9SBenno Rice void 148620b79612SRafal Jaworowski moea_kenter(mmu_t mmu, vm_offset_t va, vm_paddr_t pa) 14875244eac9SBenno Rice { 1488c1f4123bSNathan Whitehorn 1489c1f4123bSNathan Whitehorn moea_kenter_attr(mmu, va, pa, VM_MEMATTR_DEFAULT); 1490c1f4123bSNathan Whitehorn } 1491c1f4123bSNathan Whitehorn 1492c1f4123bSNathan Whitehorn void 14930936003eSJustin Hibbits moea_kenter_attr(mmu_t mmu, vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 1494c1f4123bSNathan Whitehorn { 14955244eac9SBenno Rice u_int pte_lo; 14965244eac9SBenno Rice int error; 14975244eac9SBenno Rice 14985244eac9SBenno Rice #if 0 14995244eac9SBenno Rice if (va < VM_MIN_KERNEL_ADDRESS) 150059276937SPeter Grehan panic("moea_kenter: attempt to enter non-kernel address %#x", 15015244eac9SBenno Rice va); 15025244eac9SBenno Rice #endif 15035244eac9SBenno Rice 1504c1f4123bSNathan Whitehorn pte_lo = moea_calc_wimg(pa, ma); 15055244eac9SBenno Rice 15064711f8d7SAlan Cox PMAP_LOCK(kernel_pmap); 150759276937SPeter Grehan error = moea_pvo_enter(kernel_pmap, moea_upvo_zone, 150859276937SPeter Grehan &moea_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED); 15095244eac9SBenno Rice 15105244eac9SBenno Rice if (error != 0 && error != ENOENT) 151159276937SPeter Grehan panic("moea_kenter: failed to enter va %#x pa %#x: %d", va, 15125244eac9SBenno Rice pa, error); 15135244eac9SBenno Rice 15144711f8d7SAlan Cox PMAP_UNLOCK(kernel_pmap); 15155244eac9SBenno Rice } 15165244eac9SBenno Rice 1517e79f59e8SBenno Rice /* 1518e79f59e8SBenno Rice * Extract the physical page address associated with the given kernel virtual 1519e79f59e8SBenno Rice * address. 1520e79f59e8SBenno Rice */ 152120b79612SRafal Jaworowski vm_paddr_t 152259276937SPeter Grehan moea_kextract(mmu_t mmu, vm_offset_t va) 15235244eac9SBenno Rice { 1524e79f59e8SBenno Rice struct pvo_entry *pvo; 152548d0b1a0SAlan Cox vm_paddr_t pa; 1526e79f59e8SBenno Rice 15270efd0097SPeter Grehan /* 152852a7870dSNathan Whitehorn * Allow direct mappings on 32-bit OEA 15290efd0097SPeter Grehan */ 15300efd0097SPeter Grehan if (va < VM_MIN_KERNEL_ADDRESS) { 15310efd0097SPeter Grehan return (va); 15320efd0097SPeter Grehan } 15330efd0097SPeter Grehan 153448d0b1a0SAlan Cox PMAP_LOCK(kernel_pmap); 153559276937SPeter Grehan pvo = moea_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL); 153659276937SPeter Grehan KASSERT(pvo != NULL, ("moea_kextract: no addr found")); 153752a7870dSNathan Whitehorn pa = (pvo->pvo_pte.pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF); 153848d0b1a0SAlan Cox PMAP_UNLOCK(kernel_pmap); 153948d0b1a0SAlan Cox return (pa); 1540e79f59e8SBenno Rice } 1541e79f59e8SBenno Rice 154288afb2a3SBenno Rice /* 154388afb2a3SBenno Rice * Remove a wired page from kernel virtual address space. 154488afb2a3SBenno Rice */ 15455244eac9SBenno Rice void 154659276937SPeter Grehan moea_kremove(mmu_t mmu, vm_offset_t va) 15475244eac9SBenno Rice { 154888afb2a3SBenno Rice 154959276937SPeter Grehan moea_remove(mmu, kernel_pmap, va, va + PAGE_SIZE); 15505244eac9SBenno Rice } 15515244eac9SBenno Rice 15525244eac9SBenno Rice /* 15535244eac9SBenno Rice * Map a range of physical addresses into kernel virtual address space. 15545244eac9SBenno Rice * 15555244eac9SBenno Rice * The value passed in *virt is a suggested virtual address for the mapping. 15565244eac9SBenno Rice * Architectures which can support a direct-mapped physical to virtual region 15575244eac9SBenno Rice * can return the appropriate address within that region, leaving '*virt' 15585244eac9SBenno Rice * unchanged. We cannot and therefore do not; *virt is updated with the 15595244eac9SBenno Rice * first usable address after the mapped region. 15605244eac9SBenno Rice */ 15615244eac9SBenno Rice vm_offset_t 156220b79612SRafal Jaworowski moea_map(mmu_t mmu, vm_offset_t *virt, vm_paddr_t pa_start, 156320b79612SRafal Jaworowski vm_paddr_t pa_end, int prot) 15645244eac9SBenno Rice { 15655244eac9SBenno Rice vm_offset_t sva, va; 15665244eac9SBenno Rice 15675244eac9SBenno Rice sva = *virt; 15685244eac9SBenno Rice va = sva; 15695244eac9SBenno Rice for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE) 157059276937SPeter Grehan moea_kenter(mmu, va, pa_start); 15715244eac9SBenno Rice *virt = va; 15725244eac9SBenno Rice return (sva); 15735244eac9SBenno Rice } 15745244eac9SBenno Rice 15755244eac9SBenno Rice /* 15767f3a4093SMike Silbersack * Returns true if the pmap's pv is one of the first 15777f3a4093SMike Silbersack * 16 pvs linked to from this page. This count may 15787f3a4093SMike Silbersack * be changed upwards or downwards in the future; it 15797f3a4093SMike Silbersack * is only necessary that true be returned for a small 15807f3a4093SMike Silbersack * subset of pmaps for proper page aging. 15817f3a4093SMike Silbersack */ 15825244eac9SBenno Rice boolean_t 158359276937SPeter Grehan moea_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) 15845244eac9SBenno Rice { 158503b6e025SPeter Grehan int loops; 158603b6e025SPeter Grehan struct pvo_entry *pvo; 1587ce186587SAlan Cox boolean_t rv; 158803b6e025SPeter Grehan 1589d98d0ce2SKonstantin Belousov KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1590ce186587SAlan Cox ("moea_page_exists_quick: page %p is not managed", m)); 159103b6e025SPeter Grehan loops = 0; 1592ce186587SAlan Cox rv = FALSE; 15933653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 159403b6e025SPeter Grehan LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 1595ce186587SAlan Cox if (pvo->pvo_pmap == pmap) { 1596ce186587SAlan Cox rv = TRUE; 1597ce186587SAlan Cox break; 1598ce186587SAlan Cox } 159903b6e025SPeter Grehan if (++loops >= 16) 160003b6e025SPeter Grehan break; 160103b6e025SPeter Grehan } 16023653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 1603ce186587SAlan Cox return (rv); 16045244eac9SBenno Rice } 16055244eac9SBenno Rice 160659677d3cSAlan Cox /* 160759677d3cSAlan Cox * Return the number of managed mappings to the given physical page 160859677d3cSAlan Cox * that are wired. 160959677d3cSAlan Cox */ 161059677d3cSAlan Cox int 161159677d3cSAlan Cox moea_page_wired_mappings(mmu_t mmu, vm_page_t m) 161259677d3cSAlan Cox { 161359677d3cSAlan Cox struct pvo_entry *pvo; 161459677d3cSAlan Cox int count; 161559677d3cSAlan Cox 161659677d3cSAlan Cox count = 0; 1617d98d0ce2SKonstantin Belousov if ((m->oflags & VPO_UNMANAGED) != 0) 161859677d3cSAlan Cox return (count); 16193653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 162059677d3cSAlan Cox LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) 162159677d3cSAlan Cox if ((pvo->pvo_vaddr & PVO_WIRED) != 0) 162259677d3cSAlan Cox count++; 16233653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 162459677d3cSAlan Cox return (count); 162559677d3cSAlan Cox } 162659677d3cSAlan Cox 162759276937SPeter Grehan static u_int moea_vsidcontext; 16285244eac9SBenno Rice 16295244eac9SBenno Rice void 163059276937SPeter Grehan moea_pinit(mmu_t mmu, pmap_t pmap) 16315244eac9SBenno Rice { 16325244eac9SBenno Rice int i, mask; 16335244eac9SBenno Rice u_int entropy; 16345244eac9SBenno Rice 163559276937SPeter Grehan KASSERT((int)pmap < VM_MIN_KERNEL_ADDRESS, ("moea_pinit: virt pmap")); 1636ccc4a5c7SNathan Whitehorn RB_INIT(&pmap->pmap_pvo); 16374daf20b2SPeter Grehan 16385244eac9SBenno Rice entropy = 0; 16395244eac9SBenno Rice __asm __volatile("mftb %0" : "=r"(entropy)); 16405244eac9SBenno Rice 164152a7870dSNathan Whitehorn if ((pmap->pmap_phys = (pmap_t)moea_kextract(mmu, (vm_offset_t)pmap)) 164252a7870dSNathan Whitehorn == NULL) { 164352a7870dSNathan Whitehorn pmap->pmap_phys = pmap; 164452a7870dSNathan Whitehorn } 164552a7870dSNathan Whitehorn 164652a7870dSNathan Whitehorn 1647e9b5f218SNathan Whitehorn mtx_lock(&moea_vsid_mutex); 16485244eac9SBenno Rice /* 16495244eac9SBenno Rice * Allocate some segment registers for this pmap. 16505244eac9SBenno Rice */ 16515244eac9SBenno Rice for (i = 0; i < NPMAPS; i += VSID_NBPW) { 16525244eac9SBenno Rice u_int hash, n; 16535244eac9SBenno Rice 16545244eac9SBenno Rice /* 16555244eac9SBenno Rice * Create a new value by mutiplying by a prime and adding in 16565244eac9SBenno Rice * entropy from the timebase register. This is to make the 16575244eac9SBenno Rice * VSID more random so that the PT hash function collides 16585244eac9SBenno Rice * less often. (Note that the prime casues gcc to do shifts 16595244eac9SBenno Rice * instead of a multiply.) 16605244eac9SBenno Rice */ 166159276937SPeter Grehan moea_vsidcontext = (moea_vsidcontext * 0x1105) + entropy; 166259276937SPeter Grehan hash = moea_vsidcontext & (NPMAPS - 1); 16635244eac9SBenno Rice if (hash == 0) /* 0 is special, avoid it */ 16645244eac9SBenno Rice continue; 16655244eac9SBenno Rice n = hash >> 5; 16665244eac9SBenno Rice mask = 1 << (hash & (VSID_NBPW - 1)); 166759276937SPeter Grehan hash = (moea_vsidcontext & 0xfffff); 166859276937SPeter Grehan if (moea_vsid_bitmap[n] & mask) { /* collision? */ 16695244eac9SBenno Rice /* anything free in this bucket? */ 167059276937SPeter Grehan if (moea_vsid_bitmap[n] == 0xffffffff) { 167159276937SPeter Grehan entropy = (moea_vsidcontext >> 20); 16725244eac9SBenno Rice continue; 16735244eac9SBenno Rice } 16740dfddf6eSNathan Whitehorn i = ffs(~moea_vsid_bitmap[n]) - 1; 16755244eac9SBenno Rice mask = 1 << i; 1676*d9c9c81cSPedro F. Giffuni hash &= rounddown2(0xfffff, VSID_NBPW); 16775244eac9SBenno Rice hash |= i; 16785244eac9SBenno Rice } 167946e93cbbSNathan Whitehorn KASSERT(!(moea_vsid_bitmap[n] & mask), 168046e93cbbSNathan Whitehorn ("Allocating in-use VSID group %#x\n", hash)); 168159276937SPeter Grehan moea_vsid_bitmap[n] |= mask; 16825244eac9SBenno Rice for (i = 0; i < 16; i++) 16835244eac9SBenno Rice pmap->pm_sr[i] = VSID_MAKE(i, hash); 1684e9b5f218SNathan Whitehorn mtx_unlock(&moea_vsid_mutex); 16855244eac9SBenno Rice return; 16865244eac9SBenno Rice } 16875244eac9SBenno Rice 1688e9b5f218SNathan Whitehorn mtx_unlock(&moea_vsid_mutex); 168959276937SPeter Grehan panic("moea_pinit: out of segments"); 16905244eac9SBenno Rice } 16915244eac9SBenno Rice 16925244eac9SBenno Rice /* 16935244eac9SBenno Rice * Initialize the pmap associated with process 0. 16945244eac9SBenno Rice */ 16955244eac9SBenno Rice void 169659276937SPeter Grehan moea_pinit0(mmu_t mmu, pmap_t pm) 16975244eac9SBenno Rice { 16985244eac9SBenno Rice 1699e68c64f0SKonstantin Belousov PMAP_LOCK_INIT(pm); 170059276937SPeter Grehan moea_pinit(mmu, pm); 17015244eac9SBenno Rice bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 17025244eac9SBenno Rice } 17035244eac9SBenno Rice 1704e79f59e8SBenno Rice /* 1705e79f59e8SBenno Rice * Set the physical protection on the specified range of this map as requested. 1706e79f59e8SBenno Rice */ 17075244eac9SBenno Rice void 170859276937SPeter Grehan moea_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, 170959276937SPeter Grehan vm_prot_t prot) 17105244eac9SBenno Rice { 1711ccc4a5c7SNathan Whitehorn struct pvo_entry *pvo, *tpvo, key; 1712e79f59e8SBenno Rice struct pte *pt; 1713e79f59e8SBenno Rice 1714e79f59e8SBenno Rice KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 171559276937SPeter Grehan ("moea_protect: non current pmap")); 1716e79f59e8SBenno Rice 1717e79f59e8SBenno Rice if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 171859276937SPeter Grehan moea_remove(mmu, pm, sva, eva); 1719e79f59e8SBenno Rice return; 1720e79f59e8SBenno Rice } 1721e79f59e8SBenno Rice 17223653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 172348d0b1a0SAlan Cox PMAP_LOCK(pm); 1724ccc4a5c7SNathan Whitehorn key.pvo_vaddr = sva; 1725ccc4a5c7SNathan Whitehorn for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 1726ccc4a5c7SNathan Whitehorn pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { 1727ccc4a5c7SNathan Whitehorn tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); 1728e79f59e8SBenno Rice 1729e79f59e8SBenno Rice /* 1730e79f59e8SBenno Rice * Grab the PTE pointer before we diddle with the cached PTE 1731e79f59e8SBenno Rice * copy. 1732e79f59e8SBenno Rice */ 1733ccc4a5c7SNathan Whitehorn pt = moea_pvo_to_pte(pvo, -1); 1734e79f59e8SBenno Rice /* 1735e79f59e8SBenno Rice * Change the protection of the page. 1736e79f59e8SBenno Rice */ 173752a7870dSNathan Whitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_PP; 173852a7870dSNathan Whitehorn pvo->pvo_pte.pte.pte_lo |= PTE_BR; 1739e79f59e8SBenno Rice 1740e79f59e8SBenno Rice /* 1741e79f59e8SBenno Rice * If the PVO is in the page table, update that pte as well. 1742e79f59e8SBenno Rice */ 1743d644a0b7SAlan Cox if (pt != NULL) { 174452a7870dSNathan Whitehorn moea_pte_change(pt, &pvo->pvo_pte.pte, pvo->pvo_vaddr); 1745d644a0b7SAlan Cox mtx_unlock(&moea_table_mutex); 1746d644a0b7SAlan Cox } 1747e79f59e8SBenno Rice } 17483653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 174948d0b1a0SAlan Cox PMAP_UNLOCK(pm); 17505244eac9SBenno Rice } 17515244eac9SBenno Rice 175288afb2a3SBenno Rice /* 175388afb2a3SBenno Rice * Map a list of wired pages into kernel virtual address space. This is 175488afb2a3SBenno Rice * intended for temporary mappings which do not need page modification or 175588afb2a3SBenno Rice * references recorded. Existing mappings in the region are overwritten. 175688afb2a3SBenno Rice */ 17575244eac9SBenno Rice void 175859276937SPeter Grehan moea_qenter(mmu_t mmu, vm_offset_t sva, vm_page_t *m, int count) 17595244eac9SBenno Rice { 176003b6e025SPeter Grehan vm_offset_t va; 17615244eac9SBenno Rice 176203b6e025SPeter Grehan va = sva; 176303b6e025SPeter Grehan while (count-- > 0) { 176459276937SPeter Grehan moea_kenter(mmu, va, VM_PAGE_TO_PHYS(*m)); 176503b6e025SPeter Grehan va += PAGE_SIZE; 176603b6e025SPeter Grehan m++; 176703b6e025SPeter Grehan } 17685244eac9SBenno Rice } 17695244eac9SBenno Rice 177088afb2a3SBenno Rice /* 177188afb2a3SBenno Rice * Remove page mappings from kernel virtual address space. Intended for 177259276937SPeter Grehan * temporary mappings entered by moea_qenter. 177388afb2a3SBenno Rice */ 17745244eac9SBenno Rice void 177559276937SPeter Grehan moea_qremove(mmu_t mmu, vm_offset_t sva, int count) 17765244eac9SBenno Rice { 177703b6e025SPeter Grehan vm_offset_t va; 177888afb2a3SBenno Rice 177903b6e025SPeter Grehan va = sva; 178003b6e025SPeter Grehan while (count-- > 0) { 178159276937SPeter Grehan moea_kremove(mmu, va); 178203b6e025SPeter Grehan va += PAGE_SIZE; 178303b6e025SPeter Grehan } 17845244eac9SBenno Rice } 17855244eac9SBenno Rice 17865244eac9SBenno Rice void 178759276937SPeter Grehan moea_release(mmu_t mmu, pmap_t pmap) 17885244eac9SBenno Rice { 178932bc7846SPeter Grehan int idx, mask; 179032bc7846SPeter Grehan 179132bc7846SPeter Grehan /* 179232bc7846SPeter Grehan * Free segment register's VSID 179332bc7846SPeter Grehan */ 179432bc7846SPeter Grehan if (pmap->pm_sr[0] == 0) 179559276937SPeter Grehan panic("moea_release"); 179632bc7846SPeter Grehan 1797e9b5f218SNathan Whitehorn mtx_lock(&moea_vsid_mutex); 179832bc7846SPeter Grehan idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1); 179932bc7846SPeter Grehan mask = 1 << (idx % VSID_NBPW); 180032bc7846SPeter Grehan idx /= VSID_NBPW; 180159276937SPeter Grehan moea_vsid_bitmap[idx] &= ~mask; 1802e9b5f218SNathan Whitehorn mtx_unlock(&moea_vsid_mutex); 18035244eac9SBenno Rice } 18045244eac9SBenno Rice 180588afb2a3SBenno Rice /* 180688afb2a3SBenno Rice * Remove the given range of addresses from the specified map. 180788afb2a3SBenno Rice */ 18085244eac9SBenno Rice void 180959276937SPeter Grehan moea_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) 18105244eac9SBenno Rice { 1811ccc4a5c7SNathan Whitehorn struct pvo_entry *pvo, *tpvo, key; 181288afb2a3SBenno Rice 18133653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 181448d0b1a0SAlan Cox PMAP_LOCK(pm); 1815ccc4a5c7SNathan Whitehorn key.pvo_vaddr = sva; 1816ccc4a5c7SNathan Whitehorn for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 1817ccc4a5c7SNathan Whitehorn pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { 1818ccc4a5c7SNathan Whitehorn tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); 1819598d99ddSNathan Whitehorn moea_pvo_remove(pvo, -1); 1820598d99ddSNathan Whitehorn } 182148d0b1a0SAlan Cox PMAP_UNLOCK(pm); 18223653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 18235244eac9SBenno Rice } 18245244eac9SBenno Rice 1825e79f59e8SBenno Rice /* 182659276937SPeter Grehan * Remove physical page from all pmaps in which it resides. moea_pvo_remove() 182703b6e025SPeter Grehan * will reflect changes in pte's back to the vm_page. 182803b6e025SPeter Grehan */ 182903b6e025SPeter Grehan void 183059276937SPeter Grehan moea_remove_all(mmu_t mmu, vm_page_t m) 183103b6e025SPeter Grehan { 183203b6e025SPeter Grehan struct pvo_head *pvo_head; 183303b6e025SPeter Grehan struct pvo_entry *pvo, *next_pvo; 183448d0b1a0SAlan Cox pmap_t pmap; 183503b6e025SPeter Grehan 18363653f5cbSAlan Cox rw_wlock(&pvh_global_lock); 183703b6e025SPeter Grehan pvo_head = vm_page_to_pvoh(m); 183803b6e025SPeter Grehan for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 183903b6e025SPeter Grehan next_pvo = LIST_NEXT(pvo, pvo_vlink); 184003b6e025SPeter Grehan 184148d0b1a0SAlan Cox pmap = pvo->pvo_pmap; 184248d0b1a0SAlan Cox PMAP_LOCK(pmap); 184359276937SPeter Grehan moea_pvo_remove(pvo, -1); 184448d0b1a0SAlan Cox PMAP_UNLOCK(pmap); 184503b6e025SPeter Grehan } 18468d9e6d9fSAlan Cox if ((m->aflags & PGA_WRITEABLE) && moea_query_bit(m, PTE_CHG)) { 1847c668b5b4SNathan Whitehorn moea_attr_clear(m, PTE_CHG); 1848062c8f4cSNathan Whitehorn vm_page_dirty(m); 1849062c8f4cSNathan Whitehorn } 18503407fefeSKonstantin Belousov vm_page_aflag_clear(m, PGA_WRITEABLE); 18513653f5cbSAlan Cox rw_wunlock(&pvh_global_lock); 185203b6e025SPeter Grehan } 185303b6e025SPeter Grehan 185403b6e025SPeter Grehan /* 18555244eac9SBenno Rice * Allocate a physical page of memory directly from the phys_avail map. 185659276937SPeter Grehan * Can only be called from moea_bootstrap before avail start and end are 18575244eac9SBenno Rice * calculated. 18585244eac9SBenno Rice */ 18595244eac9SBenno Rice static vm_offset_t 186059276937SPeter Grehan moea_bootstrap_alloc(vm_size_t size, u_int align) 18615244eac9SBenno Rice { 18625244eac9SBenno Rice vm_offset_t s, e; 18635244eac9SBenno Rice int i, j; 18645244eac9SBenno Rice 18655244eac9SBenno Rice size = round_page(size); 18665244eac9SBenno Rice for (i = 0; phys_avail[i + 1] != 0; i += 2) { 18675244eac9SBenno Rice if (align != 0) 1868*d9c9c81cSPedro F. Giffuni s = roundup2(phys_avail[i], align); 18695244eac9SBenno Rice else 18705244eac9SBenno Rice s = phys_avail[i]; 18715244eac9SBenno Rice e = s + size; 18725244eac9SBenno Rice 18735244eac9SBenno Rice if (s < phys_avail[i] || e > phys_avail[i + 1]) 18745244eac9SBenno Rice continue; 18755244eac9SBenno Rice 18765244eac9SBenno Rice if (s == phys_avail[i]) { 18775244eac9SBenno Rice phys_avail[i] += size; 18785244eac9SBenno Rice } else if (e == phys_avail[i + 1]) { 18795244eac9SBenno Rice phys_avail[i + 1] -= size; 18805244eac9SBenno Rice } else { 18815244eac9SBenno Rice for (j = phys_avail_count * 2; j > i; j -= 2) { 18825244eac9SBenno Rice phys_avail[j] = phys_avail[j - 2]; 18835244eac9SBenno Rice phys_avail[j + 1] = phys_avail[j - 1]; 18845244eac9SBenno Rice } 18855244eac9SBenno Rice 18865244eac9SBenno Rice phys_avail[i + 3] = phys_avail[i + 1]; 18875244eac9SBenno Rice phys_avail[i + 1] = s; 18885244eac9SBenno Rice phys_avail[i + 2] = e; 18895244eac9SBenno Rice phys_avail_count++; 18905244eac9SBenno Rice } 18915244eac9SBenno Rice 18925244eac9SBenno Rice return (s); 18935244eac9SBenno Rice } 189459276937SPeter Grehan panic("moea_bootstrap_alloc: could not allocate memory"); 18955244eac9SBenno Rice } 18965244eac9SBenno Rice 18975244eac9SBenno Rice static void 18980936003eSJustin Hibbits moea_syncicache(vm_paddr_t pa, vm_size_t len) 18995244eac9SBenno Rice { 19005244eac9SBenno Rice __syncicache((void *)pa, len); 19015244eac9SBenno Rice } 19025244eac9SBenno Rice 19035244eac9SBenno Rice static int 190459276937SPeter Grehan moea_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head, 19050936003eSJustin Hibbits vm_offset_t va, vm_paddr_t pa, u_int pte_lo, int flags) 19065244eac9SBenno Rice { 19075244eac9SBenno Rice struct pvo_entry *pvo; 19085244eac9SBenno Rice u_int sr; 19095244eac9SBenno Rice int first; 19105244eac9SBenno Rice u_int ptegidx; 19115244eac9SBenno Rice int i; 191232bc7846SPeter Grehan int bootstrap; 19135244eac9SBenno Rice 191459276937SPeter Grehan moea_pvo_enter_calls++; 19158207b362SBenno Rice first = 0; 191632bc7846SPeter Grehan bootstrap = 0; 191732bc7846SPeter Grehan 19185244eac9SBenno Rice /* 19195244eac9SBenno Rice * Compute the PTE Group index. 19205244eac9SBenno Rice */ 19215244eac9SBenno Rice va &= ~ADDR_POFF; 19225244eac9SBenno Rice sr = va_to_sr(pm->pm_sr, va); 19235244eac9SBenno Rice ptegidx = va_to_pteg(sr, va); 19245244eac9SBenno Rice 19255244eac9SBenno Rice /* 19265244eac9SBenno Rice * Remove any existing mapping for this page. Reuse the pvo entry if 19275244eac9SBenno Rice * there is a mapping. 19285244eac9SBenno Rice */ 192959276937SPeter Grehan mtx_lock(&moea_table_mutex); 193059276937SPeter Grehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { 19315244eac9SBenno Rice if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 193252a7870dSNathan Whitehorn if ((pvo->pvo_pte.pte.pte_lo & PTE_RPGN) == pa && 193352a7870dSNathan Whitehorn (pvo->pvo_pte.pte.pte_lo & PTE_PP) == 1934fafc7362SBenno Rice (pte_lo & PTE_PP)) { 1935add03590SAlan Cox /* 1936add03590SAlan Cox * The PTE is not changing. Instead, this may 1937add03590SAlan Cox * be a request to change the mapping's wired 1938add03590SAlan Cox * attribute. 1939add03590SAlan Cox */ 194059276937SPeter Grehan mtx_unlock(&moea_table_mutex); 1941add03590SAlan Cox if ((flags & PVO_WIRED) != 0 && 1942add03590SAlan Cox (pvo->pvo_vaddr & PVO_WIRED) == 0) { 1943add03590SAlan Cox pvo->pvo_vaddr |= PVO_WIRED; 1944add03590SAlan Cox pm->pm_stats.wired_count++; 1945add03590SAlan Cox } else if ((flags & PVO_WIRED) == 0 && 1946add03590SAlan Cox (pvo->pvo_vaddr & PVO_WIRED) != 0) { 1947add03590SAlan Cox pvo->pvo_vaddr &= ~PVO_WIRED; 1948add03590SAlan Cox pm->pm_stats.wired_count--; 1949add03590SAlan Cox } 195049f8f727SBenno Rice return (0); 1951fafc7362SBenno Rice } 195259276937SPeter Grehan moea_pvo_remove(pvo, -1); 19535244eac9SBenno Rice break; 19545244eac9SBenno Rice } 19555244eac9SBenno Rice } 19565244eac9SBenno Rice 19575244eac9SBenno Rice /* 19585244eac9SBenno Rice * If we aren't overwriting a mapping, try to allocate. 19595244eac9SBenno Rice */ 196059276937SPeter Grehan if (moea_initialized) { 1961378862a7SJeff Roberson pvo = uma_zalloc(zone, M_NOWAIT); 196249f8f727SBenno Rice } else { 196359276937SPeter Grehan if (moea_bpvo_pool_index >= BPVO_POOL_SIZE) { 196459276937SPeter Grehan panic("moea_enter: bpvo pool exhausted, %d, %d, %d", 196559276937SPeter Grehan moea_bpvo_pool_index, BPVO_POOL_SIZE, 19660d290675SBenno Rice BPVO_POOL_SIZE * sizeof(struct pvo_entry)); 196749f8f727SBenno Rice } 196859276937SPeter Grehan pvo = &moea_bpvo_pool[moea_bpvo_pool_index]; 196959276937SPeter Grehan moea_bpvo_pool_index++; 197032bc7846SPeter Grehan bootstrap = 1; 197149f8f727SBenno Rice } 19725244eac9SBenno Rice 19735244eac9SBenno Rice if (pvo == NULL) { 197459276937SPeter Grehan mtx_unlock(&moea_table_mutex); 19755244eac9SBenno Rice return (ENOMEM); 19765244eac9SBenno Rice } 19775244eac9SBenno Rice 197859276937SPeter Grehan moea_pvo_entries++; 19795244eac9SBenno Rice pvo->pvo_vaddr = va; 19805244eac9SBenno Rice pvo->pvo_pmap = pm; 198159276937SPeter Grehan LIST_INSERT_HEAD(&moea_pvo_table[ptegidx], pvo, pvo_olink); 19825244eac9SBenno Rice pvo->pvo_vaddr &= ~ADDR_POFF; 19835244eac9SBenno Rice if (flags & PVO_WIRED) 19845244eac9SBenno Rice pvo->pvo_vaddr |= PVO_WIRED; 198559276937SPeter Grehan if (pvo_head != &moea_pvo_kunmanaged) 19865244eac9SBenno Rice pvo->pvo_vaddr |= PVO_MANAGED; 198732bc7846SPeter Grehan if (bootstrap) 198832bc7846SPeter Grehan pvo->pvo_vaddr |= PVO_BOOTSTRAP; 19894dba5df1SPeter Grehan 199052a7870dSNathan Whitehorn moea_pte_create(&pvo->pvo_pte.pte, sr, va, pa | pte_lo); 19915244eac9SBenno Rice 19925244eac9SBenno Rice /* 1993598d99ddSNathan Whitehorn * Add to pmap list 1994598d99ddSNathan Whitehorn */ 1995ccc4a5c7SNathan Whitehorn RB_INSERT(pvo_tree, &pm->pmap_pvo, pvo); 1996598d99ddSNathan Whitehorn 1997598d99ddSNathan Whitehorn /* 19985244eac9SBenno Rice * Remember if the list was empty and therefore will be the first 19995244eac9SBenno Rice * item. 20005244eac9SBenno Rice */ 20018207b362SBenno Rice if (LIST_FIRST(pvo_head) == NULL) 20028207b362SBenno Rice first = 1; 20035244eac9SBenno Rice LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 20044dba5df1SPeter Grehan 2005bfc30490SAlan Cox if (pvo->pvo_vaddr & PVO_WIRED) 2006c3d11d22SAlan Cox pm->pm_stats.wired_count++; 2007c3d11d22SAlan Cox pm->pm_stats.resident_count++; 20085244eac9SBenno Rice 200952a7870dSNathan Whitehorn i = moea_pte_insert(ptegidx, &pvo->pvo_pte.pte); 2010804d1cc1SJustin Hibbits KASSERT(i < 8, ("Invalid PTE index")); 20115244eac9SBenno Rice if (i >= 0) { 20125244eac9SBenno Rice PVO_PTEGIDX_SET(pvo, i); 20135244eac9SBenno Rice } else { 201459276937SPeter Grehan panic("moea_pvo_enter: overflow"); 201559276937SPeter Grehan moea_pte_overflow++; 20165244eac9SBenno Rice } 201759276937SPeter Grehan mtx_unlock(&moea_table_mutex); 20184dba5df1SPeter Grehan 20195244eac9SBenno Rice return (first ? ENOENT : 0); 20205244eac9SBenno Rice } 20215244eac9SBenno Rice 20225244eac9SBenno Rice static void 202359276937SPeter Grehan moea_pvo_remove(struct pvo_entry *pvo, int pteidx) 20245244eac9SBenno Rice { 20255244eac9SBenno Rice struct pte *pt; 20265244eac9SBenno Rice 20275244eac9SBenno Rice /* 20285244eac9SBenno Rice * If there is an active pte entry, we need to deactivate it (and 20295244eac9SBenno Rice * save the ref & cfg bits). 20305244eac9SBenno Rice */ 203159276937SPeter Grehan pt = moea_pvo_to_pte(pvo, pteidx); 20325244eac9SBenno Rice if (pt != NULL) { 203352a7870dSNathan Whitehorn moea_pte_unset(pt, &pvo->pvo_pte.pte, pvo->pvo_vaddr); 2034d644a0b7SAlan Cox mtx_unlock(&moea_table_mutex); 20355244eac9SBenno Rice PVO_PTEGIDX_CLR(pvo); 20365244eac9SBenno Rice } else { 203759276937SPeter Grehan moea_pte_overflow--; 20385244eac9SBenno Rice } 20395244eac9SBenno Rice 20405244eac9SBenno Rice /* 20415244eac9SBenno Rice * Update our statistics. 20425244eac9SBenno Rice */ 20435244eac9SBenno Rice pvo->pvo_pmap->pm_stats.resident_count--; 2044bfc30490SAlan Cox if (pvo->pvo_vaddr & PVO_WIRED) 20455244eac9SBenno Rice pvo->pvo_pmap->pm_stats.wired_count--; 20465244eac9SBenno Rice 20475244eac9SBenno Rice /* 20485244eac9SBenno Rice * Save the REF/CHG bits into their cache if the page is managed. 20495244eac9SBenno Rice */ 2050d98d0ce2SKonstantin Belousov if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED) { 20515244eac9SBenno Rice struct vm_page *pg; 20525244eac9SBenno Rice 205352a7870dSNathan Whitehorn pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte.pte_lo & PTE_RPGN); 20545244eac9SBenno Rice if (pg != NULL) { 205552a7870dSNathan Whitehorn moea_attr_save(pg, pvo->pvo_pte.pte.pte_lo & 20565244eac9SBenno Rice (PTE_REF | PTE_CHG)); 20575244eac9SBenno Rice } 20585244eac9SBenno Rice } 20595244eac9SBenno Rice 20605244eac9SBenno Rice /* 2061598d99ddSNathan Whitehorn * Remove this PVO from the PV and pmap lists. 20625244eac9SBenno Rice */ 20635244eac9SBenno Rice LIST_REMOVE(pvo, pvo_vlink); 2064ccc4a5c7SNathan Whitehorn RB_REMOVE(pvo_tree, &pvo->pvo_pmap->pmap_pvo, pvo); 20655244eac9SBenno Rice 20665244eac9SBenno Rice /* 20675244eac9SBenno Rice * Remove this from the overflow list and return it to the pool 20685244eac9SBenno Rice * if we aren't going to reuse it. 20695244eac9SBenno Rice */ 20705244eac9SBenno Rice LIST_REMOVE(pvo, pvo_olink); 207149f8f727SBenno Rice if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP)) 207259276937SPeter Grehan uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? moea_mpvo_zone : 207359276937SPeter Grehan moea_upvo_zone, pvo); 207459276937SPeter Grehan moea_pvo_entries--; 207559276937SPeter Grehan moea_pvo_remove_calls++; 20765244eac9SBenno Rice } 20775244eac9SBenno Rice 20785244eac9SBenno Rice static __inline int 207959276937SPeter Grehan moea_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 20805244eac9SBenno Rice { 20815244eac9SBenno Rice int pteidx; 20825244eac9SBenno Rice 20835244eac9SBenno Rice /* 20845244eac9SBenno Rice * We can find the actual pte entry without searching by grabbing 20855244eac9SBenno Rice * the PTEG index from 3 unused bits in pte_lo[11:9] and by 20865244eac9SBenno Rice * noticing the HID bit. 20875244eac9SBenno Rice */ 20885244eac9SBenno Rice pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 208952a7870dSNathan Whitehorn if (pvo->pvo_pte.pte.pte_hi & PTE_HID) 209059276937SPeter Grehan pteidx ^= moea_pteg_mask * 8; 20915244eac9SBenno Rice 20925244eac9SBenno Rice return (pteidx); 20935244eac9SBenno Rice } 20945244eac9SBenno Rice 20955244eac9SBenno Rice static struct pvo_entry * 209659276937SPeter Grehan moea_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p) 20975244eac9SBenno Rice { 20985244eac9SBenno Rice struct pvo_entry *pvo; 20995244eac9SBenno Rice int ptegidx; 21005244eac9SBenno Rice u_int sr; 21015244eac9SBenno Rice 21025244eac9SBenno Rice va &= ~ADDR_POFF; 21035244eac9SBenno Rice sr = va_to_sr(pm->pm_sr, va); 21045244eac9SBenno Rice ptegidx = va_to_pteg(sr, va); 21055244eac9SBenno Rice 210659276937SPeter Grehan mtx_lock(&moea_table_mutex); 210759276937SPeter Grehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { 21085244eac9SBenno Rice if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 21095244eac9SBenno Rice if (pteidx_p) 211059276937SPeter Grehan *pteidx_p = moea_pvo_pte_index(pvo, ptegidx); 2111f489bf21SAlan Cox break; 21125244eac9SBenno Rice } 21135244eac9SBenno Rice } 211459276937SPeter Grehan mtx_unlock(&moea_table_mutex); 21155244eac9SBenno Rice 2116f489bf21SAlan Cox return (pvo); 21175244eac9SBenno Rice } 21185244eac9SBenno Rice 21195244eac9SBenno Rice static struct pte * 212059276937SPeter Grehan moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 21215244eac9SBenno Rice { 21225244eac9SBenno Rice struct pte *pt; 21235244eac9SBenno Rice 21245244eac9SBenno Rice /* 21255244eac9SBenno Rice * If we haven't been supplied the ptegidx, calculate it. 21265244eac9SBenno Rice */ 21275244eac9SBenno Rice if (pteidx == -1) { 21285244eac9SBenno Rice int ptegidx; 21295244eac9SBenno Rice u_int sr; 21305244eac9SBenno Rice 21315244eac9SBenno Rice sr = va_to_sr(pvo->pvo_pmap->pm_sr, pvo->pvo_vaddr); 21325244eac9SBenno Rice ptegidx = va_to_pteg(sr, pvo->pvo_vaddr); 213359276937SPeter Grehan pteidx = moea_pvo_pte_index(pvo, ptegidx); 21345244eac9SBenno Rice } 21355244eac9SBenno Rice 213659276937SPeter Grehan pt = &moea_pteg_table[pteidx >> 3].pt[pteidx & 7]; 2137d644a0b7SAlan Cox mtx_lock(&moea_table_mutex); 21385244eac9SBenno Rice 213952a7870dSNathan Whitehorn if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 214059276937SPeter Grehan panic("moea_pvo_to_pte: pvo %p has valid pte in pvo but no " 21415244eac9SBenno Rice "valid pte index", pvo); 21425244eac9SBenno Rice } 21435244eac9SBenno Rice 214452a7870dSNathan Whitehorn if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 214559276937SPeter Grehan panic("moea_pvo_to_pte: pvo %p has valid pte index in pvo " 21465244eac9SBenno Rice "pvo but no valid pte", pvo); 21475244eac9SBenno Rice } 21485244eac9SBenno Rice 214952a7870dSNathan Whitehorn if ((pt->pte_hi ^ (pvo->pvo_pte.pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 215052a7870dSNathan Whitehorn if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0) { 215159276937SPeter Grehan panic("moea_pvo_to_pte: pvo %p has valid pte in " 215259276937SPeter Grehan "moea_pteg_table %p but invalid in pvo", pvo, pt); 21535244eac9SBenno Rice } 21545244eac9SBenno Rice 215552a7870dSNathan Whitehorn if (((pt->pte_lo ^ pvo->pvo_pte.pte.pte_lo) & ~(PTE_CHG|PTE_REF)) 21565244eac9SBenno Rice != 0) { 215759276937SPeter Grehan panic("moea_pvo_to_pte: pvo %p pte does not match " 215859276937SPeter Grehan "pte %p in moea_pteg_table", pvo, pt); 21595244eac9SBenno Rice } 21605244eac9SBenno Rice 2161d644a0b7SAlan Cox mtx_assert(&moea_table_mutex, MA_OWNED); 21625244eac9SBenno Rice return (pt); 21635244eac9SBenno Rice } 21645244eac9SBenno Rice 216552a7870dSNathan Whitehorn if (pvo->pvo_pte.pte.pte_hi & PTE_VALID) { 216659276937SPeter Grehan panic("moea_pvo_to_pte: pvo %p has invalid pte %p in " 2167804d1cc1SJustin Hibbits "moea_pteg_table but valid in pvo: %8x, %8x", pvo, pt, pvo->pvo_pte.pte.pte_hi, pt->pte_hi); 21685244eac9SBenno Rice } 21695244eac9SBenno Rice 2170d644a0b7SAlan Cox mtx_unlock(&moea_table_mutex); 21715244eac9SBenno Rice return (NULL); 21725244eac9SBenno Rice } 21735244eac9SBenno Rice 21745244eac9SBenno Rice /* 21755244eac9SBenno Rice * XXX: THIS STUFF SHOULD BE IN pte.c? 21765244eac9SBenno Rice */ 21775244eac9SBenno Rice int 217859276937SPeter Grehan moea_pte_spill(vm_offset_t addr) 21795244eac9SBenno Rice { 21805244eac9SBenno Rice struct pvo_entry *source_pvo, *victim_pvo; 21815244eac9SBenno Rice struct pvo_entry *pvo; 21825244eac9SBenno Rice int ptegidx, i, j; 21835244eac9SBenno Rice u_int sr; 21845244eac9SBenno Rice struct pteg *pteg; 21855244eac9SBenno Rice struct pte *pt; 21865244eac9SBenno Rice 218759276937SPeter Grehan moea_pte_spills++; 21885244eac9SBenno Rice 2189d080d5fdSBenno Rice sr = mfsrin(addr); 21905244eac9SBenno Rice ptegidx = va_to_pteg(sr, addr); 21915244eac9SBenno Rice 21925244eac9SBenno Rice /* 21935244eac9SBenno Rice * Have to substitute some entry. Use the primary hash for this. 21945244eac9SBenno Rice * Use low bits of timebase as random generator. 21955244eac9SBenno Rice */ 219659276937SPeter Grehan pteg = &moea_pteg_table[ptegidx]; 219759276937SPeter Grehan mtx_lock(&moea_table_mutex); 21985244eac9SBenno Rice __asm __volatile("mftb %0" : "=r"(i)); 21995244eac9SBenno Rice i &= 7; 22005244eac9SBenno Rice pt = &pteg->pt[i]; 22015244eac9SBenno Rice 22025244eac9SBenno Rice source_pvo = NULL; 22035244eac9SBenno Rice victim_pvo = NULL; 220459276937SPeter Grehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { 22055244eac9SBenno Rice /* 22065244eac9SBenno Rice * We need to find a pvo entry for this address. 22075244eac9SBenno Rice */ 22085244eac9SBenno Rice if (source_pvo == NULL && 220952a7870dSNathan Whitehorn moea_pte_match(&pvo->pvo_pte.pte, sr, addr, 221052a7870dSNathan Whitehorn pvo->pvo_pte.pte.pte_hi & PTE_HID)) { 22115244eac9SBenno Rice /* 22125244eac9SBenno Rice * Now found an entry to be spilled into the pteg. 22135244eac9SBenno Rice * The PTE is now valid, so we know it's active. 22145244eac9SBenno Rice */ 221552a7870dSNathan Whitehorn j = moea_pte_insert(ptegidx, &pvo->pvo_pte.pte); 22165244eac9SBenno Rice 22175244eac9SBenno Rice if (j >= 0) { 22185244eac9SBenno Rice PVO_PTEGIDX_SET(pvo, j); 221959276937SPeter Grehan moea_pte_overflow--; 222059276937SPeter Grehan mtx_unlock(&moea_table_mutex); 22215244eac9SBenno Rice return (1); 22225244eac9SBenno Rice } 22235244eac9SBenno Rice 22245244eac9SBenno Rice source_pvo = pvo; 22255244eac9SBenno Rice 22265244eac9SBenno Rice if (victim_pvo != NULL) 22275244eac9SBenno Rice break; 22285244eac9SBenno Rice } 22295244eac9SBenno Rice 22305244eac9SBenno Rice /* 22315244eac9SBenno Rice * We also need the pvo entry of the victim we are replacing 22325244eac9SBenno Rice * so save the R & C bits of the PTE. 22335244eac9SBenno Rice */ 22345244eac9SBenno Rice if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 223552a7870dSNathan Whitehorn moea_pte_compare(pt, &pvo->pvo_pte.pte)) { 22365244eac9SBenno Rice victim_pvo = pvo; 22375244eac9SBenno Rice if (source_pvo != NULL) 22385244eac9SBenno Rice break; 22395244eac9SBenno Rice } 22405244eac9SBenno Rice } 22415244eac9SBenno Rice 2242f489bf21SAlan Cox if (source_pvo == NULL) { 224359276937SPeter Grehan mtx_unlock(&moea_table_mutex); 22445244eac9SBenno Rice return (0); 2245f489bf21SAlan Cox } 22465244eac9SBenno Rice 22475244eac9SBenno Rice if (victim_pvo == NULL) { 22485244eac9SBenno Rice if ((pt->pte_hi & PTE_HID) == 0) 224959276937SPeter Grehan panic("moea_pte_spill: victim p-pte (%p) has no pvo" 22505244eac9SBenno Rice "entry", pt); 22515244eac9SBenno Rice 22525244eac9SBenno Rice /* 22535244eac9SBenno Rice * If this is a secondary PTE, we need to search it's primary 22545244eac9SBenno Rice * pvo bucket for the matching PVO. 22555244eac9SBenno Rice */ 225659276937SPeter Grehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx ^ moea_pteg_mask], 22575244eac9SBenno Rice pvo_olink) { 22585244eac9SBenno Rice /* 22595244eac9SBenno Rice * We also need the pvo entry of the victim we are 22605244eac9SBenno Rice * replacing so save the R & C bits of the PTE. 22615244eac9SBenno Rice */ 226252a7870dSNathan Whitehorn if (moea_pte_compare(pt, &pvo->pvo_pte.pte)) { 22635244eac9SBenno Rice victim_pvo = pvo; 22645244eac9SBenno Rice break; 22655244eac9SBenno Rice } 22665244eac9SBenno Rice } 22675244eac9SBenno Rice 22685244eac9SBenno Rice if (victim_pvo == NULL) 226959276937SPeter Grehan panic("moea_pte_spill: victim s-pte (%p) has no pvo" 22705244eac9SBenno Rice "entry", pt); 22715244eac9SBenno Rice } 22725244eac9SBenno Rice 22735244eac9SBenno Rice /* 22745244eac9SBenno Rice * We are invalidating the TLB entry for the EA we are replacing even 22755244eac9SBenno Rice * though it's valid. If we don't, we lose any ref/chg bit changes 22765244eac9SBenno Rice * contained in the TLB entry. 22775244eac9SBenno Rice */ 227852a7870dSNathan Whitehorn source_pvo->pvo_pte.pte.pte_hi &= ~PTE_HID; 22795244eac9SBenno Rice 228052a7870dSNathan Whitehorn moea_pte_unset(pt, &victim_pvo->pvo_pte.pte, victim_pvo->pvo_vaddr); 228152a7870dSNathan Whitehorn moea_pte_set(pt, &source_pvo->pvo_pte.pte); 22825244eac9SBenno Rice 22835244eac9SBenno Rice PVO_PTEGIDX_CLR(victim_pvo); 22845244eac9SBenno Rice PVO_PTEGIDX_SET(source_pvo, i); 228559276937SPeter Grehan moea_pte_replacements++; 22865244eac9SBenno Rice 228759276937SPeter Grehan mtx_unlock(&moea_table_mutex); 22885244eac9SBenno Rice return (1); 22895244eac9SBenno Rice } 22905244eac9SBenno Rice 2291804d1cc1SJustin Hibbits static __inline struct pvo_entry * 2292804d1cc1SJustin Hibbits moea_pte_spillable_ident(u_int ptegidx) 2293804d1cc1SJustin Hibbits { 2294804d1cc1SJustin Hibbits struct pte *pt; 2295804d1cc1SJustin Hibbits struct pvo_entry *pvo_walk, *pvo = NULL; 2296804d1cc1SJustin Hibbits 2297804d1cc1SJustin Hibbits LIST_FOREACH(pvo_walk, &moea_pvo_table[ptegidx], pvo_olink) { 2298804d1cc1SJustin Hibbits if (pvo_walk->pvo_vaddr & PVO_WIRED) 2299804d1cc1SJustin Hibbits continue; 2300804d1cc1SJustin Hibbits 2301804d1cc1SJustin Hibbits if (!(pvo_walk->pvo_pte.pte.pte_hi & PTE_VALID)) 2302804d1cc1SJustin Hibbits continue; 2303804d1cc1SJustin Hibbits 2304804d1cc1SJustin Hibbits pt = moea_pvo_to_pte(pvo_walk, -1); 2305804d1cc1SJustin Hibbits 2306804d1cc1SJustin Hibbits if (pt == NULL) 2307804d1cc1SJustin Hibbits continue; 2308804d1cc1SJustin Hibbits 2309804d1cc1SJustin Hibbits pvo = pvo_walk; 2310804d1cc1SJustin Hibbits 2311804d1cc1SJustin Hibbits mtx_unlock(&moea_table_mutex); 2312804d1cc1SJustin Hibbits if (!(pt->pte_lo & PTE_REF)) 2313804d1cc1SJustin Hibbits return (pvo_walk); 2314804d1cc1SJustin Hibbits } 2315804d1cc1SJustin Hibbits 2316804d1cc1SJustin Hibbits return (pvo); 2317804d1cc1SJustin Hibbits } 2318804d1cc1SJustin Hibbits 23195244eac9SBenno Rice static int 232059276937SPeter Grehan moea_pte_insert(u_int ptegidx, struct pte *pvo_pt) 23215244eac9SBenno Rice { 23225244eac9SBenno Rice struct pte *pt; 2323804d1cc1SJustin Hibbits struct pvo_entry *victim_pvo; 23245244eac9SBenno Rice int i; 2325804d1cc1SJustin Hibbits int victim_idx; 2326804d1cc1SJustin Hibbits u_int pteg_bkpidx = ptegidx; 23275244eac9SBenno Rice 2328d644a0b7SAlan Cox mtx_assert(&moea_table_mutex, MA_OWNED); 2329d644a0b7SAlan Cox 23305244eac9SBenno Rice /* 23315244eac9SBenno Rice * First try primary hash. 23325244eac9SBenno Rice */ 233359276937SPeter Grehan for (pt = moea_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 23345244eac9SBenno Rice if ((pt->pte_hi & PTE_VALID) == 0) { 23355244eac9SBenno Rice pvo_pt->pte_hi &= ~PTE_HID; 233659276937SPeter Grehan moea_pte_set(pt, pvo_pt); 23375244eac9SBenno Rice return (i); 23385244eac9SBenno Rice } 23395244eac9SBenno Rice } 23405244eac9SBenno Rice 23415244eac9SBenno Rice /* 23425244eac9SBenno Rice * Now try secondary hash. 23435244eac9SBenno Rice */ 234459276937SPeter Grehan ptegidx ^= moea_pteg_mask; 2345bd8e6f87SPeter Grehan 234659276937SPeter Grehan for (pt = moea_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 23475244eac9SBenno Rice if ((pt->pte_hi & PTE_VALID) == 0) { 23485244eac9SBenno Rice pvo_pt->pte_hi |= PTE_HID; 234959276937SPeter Grehan moea_pte_set(pt, pvo_pt); 23505244eac9SBenno Rice return (i); 23515244eac9SBenno Rice } 23525244eac9SBenno Rice } 23535244eac9SBenno Rice 2354804d1cc1SJustin Hibbits /* Try again, but this time try to force a PTE out. */ 2355804d1cc1SJustin Hibbits ptegidx = pteg_bkpidx; 2356804d1cc1SJustin Hibbits 2357804d1cc1SJustin Hibbits victim_pvo = moea_pte_spillable_ident(ptegidx); 2358804d1cc1SJustin Hibbits if (victim_pvo == NULL) { 2359804d1cc1SJustin Hibbits ptegidx ^= moea_pteg_mask; 2360804d1cc1SJustin Hibbits victim_pvo = moea_pte_spillable_ident(ptegidx); 2361804d1cc1SJustin Hibbits } 2362804d1cc1SJustin Hibbits 2363804d1cc1SJustin Hibbits if (victim_pvo == NULL) { 236459276937SPeter Grehan panic("moea_pte_insert: overflow"); 23655244eac9SBenno Rice return (-1); 23665244eac9SBenno Rice } 23675244eac9SBenno Rice 2368804d1cc1SJustin Hibbits victim_idx = moea_pvo_pte_index(victim_pvo, ptegidx); 2369804d1cc1SJustin Hibbits 2370804d1cc1SJustin Hibbits if (pteg_bkpidx == ptegidx) 2371804d1cc1SJustin Hibbits pvo_pt->pte_hi &= ~PTE_HID; 2372804d1cc1SJustin Hibbits else 2373804d1cc1SJustin Hibbits pvo_pt->pte_hi |= PTE_HID; 2374804d1cc1SJustin Hibbits 2375804d1cc1SJustin Hibbits /* 2376804d1cc1SJustin Hibbits * Synchronize the sacrifice PTE with its PVO, then mark both 2377804d1cc1SJustin Hibbits * invalid. The PVO will be reused when/if the VM system comes 2378804d1cc1SJustin Hibbits * here after a fault. 2379804d1cc1SJustin Hibbits */ 2380804d1cc1SJustin Hibbits pt = &moea_pteg_table[victim_idx >> 3].pt[victim_idx & 7]; 2381804d1cc1SJustin Hibbits 2382804d1cc1SJustin Hibbits if (pt->pte_hi != victim_pvo->pvo_pte.pte.pte_hi) 2383804d1cc1SJustin Hibbits panic("Victim PVO doesn't match PTE! PVO: %8x, PTE: %8x", victim_pvo->pvo_pte.pte.pte_hi, pt->pte_hi); 2384804d1cc1SJustin Hibbits 2385804d1cc1SJustin Hibbits /* 2386804d1cc1SJustin Hibbits * Set the new PTE. 2387804d1cc1SJustin Hibbits */ 2388804d1cc1SJustin Hibbits moea_pte_unset(pt, &victim_pvo->pvo_pte.pte, victim_pvo->pvo_vaddr); 2389804d1cc1SJustin Hibbits PVO_PTEGIDX_CLR(victim_pvo); 2390804d1cc1SJustin Hibbits moea_pte_overflow++; 2391804d1cc1SJustin Hibbits moea_pte_set(pt, pvo_pt); 2392804d1cc1SJustin Hibbits 2393804d1cc1SJustin Hibbits return (victim_idx & 7); 2394804d1cc1SJustin Hibbits } 2395804d1cc1SJustin Hibbits 23965244eac9SBenno Rice static boolean_t 239759276937SPeter Grehan moea_query_bit(vm_page_t m, int ptebit) 23985244eac9SBenno Rice { 23995244eac9SBenno Rice struct pvo_entry *pvo; 24005244eac9SBenno Rice struct pte *pt; 24015244eac9SBenno Rice 24028d9e6d9fSAlan Cox rw_assert(&pvh_global_lock, RA_WLOCKED); 240359276937SPeter Grehan if (moea_attr_fetch(m) & ptebit) 24045244eac9SBenno Rice return (TRUE); 24055244eac9SBenno Rice 24065244eac9SBenno Rice LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 24075244eac9SBenno Rice 24085244eac9SBenno Rice /* 24095244eac9SBenno Rice * See if we saved the bit off. If so, cache it and return 24105244eac9SBenno Rice * success. 24115244eac9SBenno Rice */ 241252a7870dSNathan Whitehorn if (pvo->pvo_pte.pte.pte_lo & ptebit) { 241359276937SPeter Grehan moea_attr_save(m, ptebit); 24145244eac9SBenno Rice return (TRUE); 24155244eac9SBenno Rice } 24165244eac9SBenno Rice } 24175244eac9SBenno Rice 24185244eac9SBenno Rice /* 24195244eac9SBenno Rice * No luck, now go through the hard part of looking at the PTEs 24205244eac9SBenno Rice * themselves. Sync so that any pending REF/CHG bits are flushed to 24215244eac9SBenno Rice * the PTEs. 24225244eac9SBenno Rice */ 2423e4f72b32SMarcel Moolenaar powerpc_sync(); 24245244eac9SBenno Rice LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 24255244eac9SBenno Rice 24265244eac9SBenno Rice /* 24275244eac9SBenno Rice * See if this pvo has a valid PTE. if so, fetch the 24285244eac9SBenno Rice * REF/CHG bits from the valid PTE. If the appropriate 24295244eac9SBenno Rice * ptebit is set, cache it and return success. 24305244eac9SBenno Rice */ 243159276937SPeter Grehan pt = moea_pvo_to_pte(pvo, -1); 24325244eac9SBenno Rice if (pt != NULL) { 243352a7870dSNathan Whitehorn moea_pte_synch(pt, &pvo->pvo_pte.pte); 2434d644a0b7SAlan Cox mtx_unlock(&moea_table_mutex); 243552a7870dSNathan Whitehorn if (pvo->pvo_pte.pte.pte_lo & ptebit) { 243659276937SPeter Grehan moea_attr_save(m, ptebit); 24375244eac9SBenno Rice return (TRUE); 24385244eac9SBenno Rice } 24395244eac9SBenno Rice } 24405244eac9SBenno Rice } 24415244eac9SBenno Rice 24424f7daed0SAndrew Gallatin return (FALSE); 24435244eac9SBenno Rice } 24445244eac9SBenno Rice 244503b6e025SPeter Grehan static u_int 2446ce186587SAlan Cox moea_clear_bit(vm_page_t m, int ptebit) 24475244eac9SBenno Rice { 244803b6e025SPeter Grehan u_int count; 24495244eac9SBenno Rice struct pvo_entry *pvo; 24505244eac9SBenno Rice struct pte *pt; 2451ce186587SAlan Cox 24528d9e6d9fSAlan Cox rw_assert(&pvh_global_lock, RA_WLOCKED); 24535244eac9SBenno Rice 24545244eac9SBenno Rice /* 24555244eac9SBenno Rice * Clear the cached value. 24565244eac9SBenno Rice */ 245759276937SPeter Grehan moea_attr_clear(m, ptebit); 24585244eac9SBenno Rice 24595244eac9SBenno Rice /* 24605244eac9SBenno Rice * Sync so that any pending REF/CHG bits are flushed to the PTEs (so 24615244eac9SBenno Rice * we can reset the right ones). note that since the pvo entries and 24625244eac9SBenno Rice * list heads are accessed via BAT0 and are never placed in the page 24635244eac9SBenno Rice * table, we don't have to worry about further accesses setting the 24645244eac9SBenno Rice * REF/CHG bits. 24655244eac9SBenno Rice */ 2466e4f72b32SMarcel Moolenaar powerpc_sync(); 24675244eac9SBenno Rice 24685244eac9SBenno Rice /* 24695244eac9SBenno Rice * For each pvo entry, clear the pvo's ptebit. If this pvo has a 24705244eac9SBenno Rice * valid pte clear the ptebit from the valid pte. 24715244eac9SBenno Rice */ 247203b6e025SPeter Grehan count = 0; 24735244eac9SBenno Rice LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 247459276937SPeter Grehan pt = moea_pvo_to_pte(pvo, -1); 24755244eac9SBenno Rice if (pt != NULL) { 247652a7870dSNathan Whitehorn moea_pte_synch(pt, &pvo->pvo_pte.pte); 247752a7870dSNathan Whitehorn if (pvo->pvo_pte.pte.pte_lo & ptebit) { 247803b6e025SPeter Grehan count++; 247959276937SPeter Grehan moea_pte_clear(pt, PVO_VADDR(pvo), ptebit); 24805244eac9SBenno Rice } 2481d644a0b7SAlan Cox mtx_unlock(&moea_table_mutex); 248203b6e025SPeter Grehan } 248352a7870dSNathan Whitehorn pvo->pvo_pte.pte.pte_lo &= ~ptebit; 24845244eac9SBenno Rice } 24855244eac9SBenno Rice 248603b6e025SPeter Grehan return (count); 2487bdf71f56SBenno Rice } 24888bbfa33aSBenno Rice 24898bbfa33aSBenno Rice /* 249032bc7846SPeter Grehan * Return true if the physical range is encompassed by the battable[idx] 249132bc7846SPeter Grehan */ 249232bc7846SPeter Grehan static int 24930936003eSJustin Hibbits moea_bat_mapped(int idx, vm_paddr_t pa, vm_size_t size) 249432bc7846SPeter Grehan { 249532bc7846SPeter Grehan u_int prot; 249632bc7846SPeter Grehan u_int32_t start; 249732bc7846SPeter Grehan u_int32_t end; 249832bc7846SPeter Grehan u_int32_t bat_ble; 249932bc7846SPeter Grehan 250032bc7846SPeter Grehan /* 250132bc7846SPeter Grehan * Return immediately if not a valid mapping 250232bc7846SPeter Grehan */ 2503c4bcebedSNathan Whitehorn if (!(battable[idx].batu & BAT_Vs)) 250432bc7846SPeter Grehan return (EINVAL); 250532bc7846SPeter Grehan 250632bc7846SPeter Grehan /* 250732bc7846SPeter Grehan * The BAT entry must be cache-inhibited, guarded, and r/w 250832bc7846SPeter Grehan * so it can function as an i/o page 250932bc7846SPeter Grehan */ 251032bc7846SPeter Grehan prot = battable[idx].batl & (BAT_I|BAT_G|BAT_PP_RW); 251132bc7846SPeter Grehan if (prot != (BAT_I|BAT_G|BAT_PP_RW)) 251232bc7846SPeter Grehan return (EPERM); 251332bc7846SPeter Grehan 251432bc7846SPeter Grehan /* 251532bc7846SPeter Grehan * The address should be within the BAT range. Assume that the 251632bc7846SPeter Grehan * start address in the BAT has the correct alignment (thus 251732bc7846SPeter Grehan * not requiring masking) 251832bc7846SPeter Grehan */ 251932bc7846SPeter Grehan start = battable[idx].batl & BAT_PBS; 252032bc7846SPeter Grehan bat_ble = (battable[idx].batu & ~(BAT_EBS)) | 0x03; 252132bc7846SPeter Grehan end = start | (bat_ble << 15) | 0x7fff; 252232bc7846SPeter Grehan 252332bc7846SPeter Grehan if ((pa < start) || ((pa + size) > end)) 252432bc7846SPeter Grehan return (ERANGE); 252532bc7846SPeter Grehan 252632bc7846SPeter Grehan return (0); 252732bc7846SPeter Grehan } 252832bc7846SPeter Grehan 252959276937SPeter Grehan boolean_t 253020b79612SRafal Jaworowski moea_dev_direct_mapped(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 2531c0763d37SSuleiman Souhlal { 2532c0763d37SSuleiman Souhlal int i; 2533c0763d37SSuleiman Souhlal 2534c0763d37SSuleiman Souhlal /* 2535c0763d37SSuleiman Souhlal * This currently does not work for entries that 2536c0763d37SSuleiman Souhlal * overlap 256M BAT segments. 2537c0763d37SSuleiman Souhlal */ 2538c0763d37SSuleiman Souhlal 2539c0763d37SSuleiman Souhlal for(i = 0; i < 16; i++) 254059276937SPeter Grehan if (moea_bat_mapped(i, pa, size) == 0) 2541c0763d37SSuleiman Souhlal return (0); 2542c0763d37SSuleiman Souhlal 2543c0763d37SSuleiman Souhlal return (EFAULT); 2544c0763d37SSuleiman Souhlal } 254532bc7846SPeter Grehan 254632bc7846SPeter Grehan /* 25478bbfa33aSBenno Rice * Map a set of physical memory pages into the kernel virtual 25488bbfa33aSBenno Rice * address space. Return a pointer to where it is mapped. This 25498bbfa33aSBenno Rice * routine is intended to be used for mapping device memory, 25508bbfa33aSBenno Rice * NOT real memory. 25518bbfa33aSBenno Rice */ 25528bbfa33aSBenno Rice void * 255320b79612SRafal Jaworowski moea_mapdev(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 25548bbfa33aSBenno Rice { 2555c1f4123bSNathan Whitehorn 2556c1f4123bSNathan Whitehorn return (moea_mapdev_attr(mmu, pa, size, VM_MEMATTR_DEFAULT)); 2557c1f4123bSNathan Whitehorn } 2558c1f4123bSNathan Whitehorn 2559c1f4123bSNathan Whitehorn void * 25600936003eSJustin Hibbits moea_mapdev_attr(mmu_t mmu, vm_paddr_t pa, vm_size_t size, vm_memattr_t ma) 2561c1f4123bSNathan Whitehorn { 256232bc7846SPeter Grehan vm_offset_t va, tmpva, ppa, offset; 256332bc7846SPeter Grehan int i; 25648bbfa33aSBenno Rice 256532bc7846SPeter Grehan ppa = trunc_page(pa); 25668bbfa33aSBenno Rice offset = pa & PAGE_MASK; 25678bbfa33aSBenno Rice size = roundup(offset + size, PAGE_SIZE); 25688bbfa33aSBenno Rice 256932bc7846SPeter Grehan /* 257032bc7846SPeter Grehan * If the physical address lies within a valid BAT table entry, 257132bc7846SPeter Grehan * return the 1:1 mapping. This currently doesn't work 257232bc7846SPeter Grehan * for regions that overlap 256M BAT segments. 257332bc7846SPeter Grehan */ 257432bc7846SPeter Grehan for (i = 0; i < 16; i++) { 257559276937SPeter Grehan if (moea_bat_mapped(i, pa, size) == 0) 257632bc7846SPeter Grehan return ((void *) pa); 257732bc7846SPeter Grehan } 257832bc7846SPeter Grehan 25795df87b21SJeff Roberson va = kva_alloc(size); 25808bbfa33aSBenno Rice if (!va) 258159276937SPeter Grehan panic("moea_mapdev: Couldn't alloc kernel virtual memory"); 25828bbfa33aSBenno Rice 25838bbfa33aSBenno Rice for (tmpva = va; size > 0;) { 2584c1f4123bSNathan Whitehorn moea_kenter_attr(mmu, tmpva, ppa, ma); 2585e4f72b32SMarcel Moolenaar tlbie(tmpva); 25868bbfa33aSBenno Rice size -= PAGE_SIZE; 25878bbfa33aSBenno Rice tmpva += PAGE_SIZE; 258832bc7846SPeter Grehan ppa += PAGE_SIZE; 25898bbfa33aSBenno Rice } 25908bbfa33aSBenno Rice 25918bbfa33aSBenno Rice return ((void *)(va + offset)); 25928bbfa33aSBenno Rice } 25938bbfa33aSBenno Rice 25948bbfa33aSBenno Rice void 259559276937SPeter Grehan moea_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size) 25968bbfa33aSBenno Rice { 25978bbfa33aSBenno Rice vm_offset_t base, offset; 25988bbfa33aSBenno Rice 259932bc7846SPeter Grehan /* 260032bc7846SPeter Grehan * If this is outside kernel virtual space, then it's a 260132bc7846SPeter Grehan * battable entry and doesn't require unmapping 260232bc7846SPeter Grehan */ 2603ab739706SNathan Whitehorn if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= virtual_end)) { 26048bbfa33aSBenno Rice base = trunc_page(va); 26058bbfa33aSBenno Rice offset = va & PAGE_MASK; 26068bbfa33aSBenno Rice size = roundup(offset + size, PAGE_SIZE); 26075df87b21SJeff Roberson kva_free(base, size); 26088bbfa33aSBenno Rice } 260932bc7846SPeter Grehan } 26101a4fcaebSMarcel Moolenaar 26111a4fcaebSMarcel Moolenaar static void 26121a4fcaebSMarcel Moolenaar moea_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) 26131a4fcaebSMarcel Moolenaar { 26141a4fcaebSMarcel Moolenaar struct pvo_entry *pvo; 26151a4fcaebSMarcel Moolenaar vm_offset_t lim; 26161a4fcaebSMarcel Moolenaar vm_paddr_t pa; 26171a4fcaebSMarcel Moolenaar vm_size_t len; 26181a4fcaebSMarcel Moolenaar 26191a4fcaebSMarcel Moolenaar PMAP_LOCK(pm); 26201a4fcaebSMarcel Moolenaar while (sz > 0) { 26211a4fcaebSMarcel Moolenaar lim = round_page(va); 26221a4fcaebSMarcel Moolenaar len = MIN(lim - va, sz); 26231a4fcaebSMarcel Moolenaar pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 26241a4fcaebSMarcel Moolenaar if (pvo != NULL) { 26251a4fcaebSMarcel Moolenaar pa = (pvo->pvo_pte.pte.pte_lo & PTE_RPGN) | 26261a4fcaebSMarcel Moolenaar (va & ADDR_POFF); 26271a4fcaebSMarcel Moolenaar moea_syncicache(pa, len); 26281a4fcaebSMarcel Moolenaar } 26291a4fcaebSMarcel Moolenaar va += len; 26301a4fcaebSMarcel Moolenaar sz -= len; 26311a4fcaebSMarcel Moolenaar } 26321a4fcaebSMarcel Moolenaar PMAP_UNLOCK(pm); 26331a4fcaebSMarcel Moolenaar } 2634afd9cb6cSJustin Hibbits 2635bdb9ab0dSMark Johnston void 2636bdb9ab0dSMark Johnston moea_dumpsys_map(mmu_t mmu, vm_paddr_t pa, size_t sz, void **va) 2637afd9cb6cSJustin Hibbits { 2638bdb9ab0dSMark Johnston 2639bdb9ab0dSMark Johnston *va = (void *)pa; 2640afd9cb6cSJustin Hibbits } 2641afd9cb6cSJustin Hibbits 2642bdb9ab0dSMark Johnston extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1]; 2643bdb9ab0dSMark Johnston 2644bdb9ab0dSMark Johnston void 2645bdb9ab0dSMark Johnston moea_scan_init(mmu_t mmu) 2646afd9cb6cSJustin Hibbits { 2647afd9cb6cSJustin Hibbits struct pvo_entry *pvo; 2648afd9cb6cSJustin Hibbits vm_offset_t va; 2649bdb9ab0dSMark Johnston int i; 2650afd9cb6cSJustin Hibbits 2651bdb9ab0dSMark Johnston if (!do_minidump) { 2652bdb9ab0dSMark Johnston /* Initialize phys. segments for dumpsys(). */ 2653bdb9ab0dSMark Johnston memset(&dump_map, 0, sizeof(dump_map)); 2654bdb9ab0dSMark Johnston mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 2655bdb9ab0dSMark Johnston for (i = 0; i < pregions_sz; i++) { 2656bdb9ab0dSMark Johnston dump_map[i].pa_start = pregions[i].mr_start; 2657bdb9ab0dSMark Johnston dump_map[i].pa_size = pregions[i].mr_size; 2658afd9cb6cSJustin Hibbits } 2659bdb9ab0dSMark Johnston return; 2660bdb9ab0dSMark Johnston } 2661bdb9ab0dSMark Johnston 2662bdb9ab0dSMark Johnston /* Virtual segments for minidumps: */ 2663bdb9ab0dSMark Johnston memset(&dump_map, 0, sizeof(dump_map)); 2664bdb9ab0dSMark Johnston 2665bdb9ab0dSMark Johnston /* 1st: kernel .data and .bss. */ 2666bdb9ab0dSMark Johnston dump_map[0].pa_start = trunc_page((uintptr_t)_etext); 2667bdb9ab0dSMark Johnston dump_map[0].pa_size = 2668bdb9ab0dSMark Johnston round_page((uintptr_t)_end) - dump_map[0].pa_start; 2669bdb9ab0dSMark Johnston 2670afd9cb6cSJustin Hibbits /* 2nd: msgbuf and tables (see pmap_bootstrap()). */ 2671bdb9ab0dSMark Johnston dump_map[1].pa_start = (vm_paddr_t)msgbufp->msg_ptr; 2672bdb9ab0dSMark Johnston dump_map[1].pa_size = round_page(msgbufp->msg_size); 2673bdb9ab0dSMark Johnston 2674afd9cb6cSJustin Hibbits /* 3rd: kernel VM. */ 2675bdb9ab0dSMark Johnston va = dump_map[1].pa_start + dump_map[1].pa_size; 2676afd9cb6cSJustin Hibbits /* Find start of next chunk (from va). */ 2677afd9cb6cSJustin Hibbits while (va < virtual_end) { 2678afd9cb6cSJustin Hibbits /* Don't dump the buffer cache. */ 2679bdb9ab0dSMark Johnston if (va >= kmi.buffer_sva && va < kmi.buffer_eva) { 2680afd9cb6cSJustin Hibbits va = kmi.buffer_eva; 2681afd9cb6cSJustin Hibbits continue; 2682afd9cb6cSJustin Hibbits } 2683bdb9ab0dSMark Johnston pvo = moea_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL); 2684bdb9ab0dSMark Johnston if (pvo != NULL && (pvo->pvo_pte.pte.pte_hi & PTE_VALID)) 2685afd9cb6cSJustin Hibbits break; 2686afd9cb6cSJustin Hibbits va += PAGE_SIZE; 2687afd9cb6cSJustin Hibbits } 2688afd9cb6cSJustin Hibbits if (va < virtual_end) { 2689bdb9ab0dSMark Johnston dump_map[2].pa_start = va; 2690afd9cb6cSJustin Hibbits va += PAGE_SIZE; 2691afd9cb6cSJustin Hibbits /* Find last page in chunk. */ 2692afd9cb6cSJustin Hibbits while (va < virtual_end) { 2693afd9cb6cSJustin Hibbits /* Don't run into the buffer cache. */ 2694afd9cb6cSJustin Hibbits if (va == kmi.buffer_sva) 2695afd9cb6cSJustin Hibbits break; 2696bdb9ab0dSMark Johnston pvo = moea_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, 2697bdb9ab0dSMark Johnston NULL); 2698afd9cb6cSJustin Hibbits if (pvo == NULL || 2699afd9cb6cSJustin Hibbits !(pvo->pvo_pte.pte.pte_hi & PTE_VALID)) 2700afd9cb6cSJustin Hibbits break; 2701afd9cb6cSJustin Hibbits va += PAGE_SIZE; 2702afd9cb6cSJustin Hibbits } 2703bdb9ab0dSMark Johnston dump_map[2].pa_size = va - dump_map[2].pa_start; 2704afd9cb6cSJustin Hibbits } 2705afd9cb6cSJustin Hibbits } 2706