170d64ceaSPaul Mackerras /* 270d64ceaSPaul Mackerras * PowerPC version 370d64ceaSPaul Mackerras * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 470d64ceaSPaul Mackerras * 570d64ceaSPaul Mackerras * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 670d64ceaSPaul Mackerras * and Cort Dougan (PReP) (cort@cs.nmt.edu) 770d64ceaSPaul Mackerras * Copyright (C) 1996 Paul Mackerras 870d64ceaSPaul Mackerras * Amiga/APUS changes by Jesper Skov (jskov@cygnus.co.uk). 970d64ceaSPaul Mackerras * 1070d64ceaSPaul Mackerras * Derived from "arch/i386/mm/init.c" 1170d64ceaSPaul Mackerras * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 1270d64ceaSPaul Mackerras * 1370d64ceaSPaul Mackerras * Dave Engebretsen <engebret@us.ibm.com> 1470d64ceaSPaul Mackerras * Rework for PPC64 port. 1570d64ceaSPaul Mackerras * 1670d64ceaSPaul Mackerras * This program is free software; you can redistribute it and/or 1770d64ceaSPaul Mackerras * modify it under the terms of the GNU General Public License 1870d64ceaSPaul Mackerras * as published by the Free Software Foundation; either version 1970d64ceaSPaul Mackerras * 2 of the License, or (at your option) any later version. 2070d64ceaSPaul Mackerras * 2170d64ceaSPaul Mackerras */ 2270d64ceaSPaul Mackerras 2387655ff2SBenjamin Herrenschmidt #undef DEBUG 2487655ff2SBenjamin Herrenschmidt 2570d64ceaSPaul Mackerras #include <linux/config.h> 2670d64ceaSPaul Mackerras #include <linux/signal.h> 2770d64ceaSPaul Mackerras #include <linux/sched.h> 2870d64ceaSPaul Mackerras #include <linux/kernel.h> 2970d64ceaSPaul Mackerras #include <linux/errno.h> 3070d64ceaSPaul Mackerras #include <linux/string.h> 3170d64ceaSPaul Mackerras #include <linux/types.h> 3270d64ceaSPaul Mackerras #include <linux/mman.h> 3370d64ceaSPaul Mackerras #include <linux/mm.h> 3470d64ceaSPaul Mackerras #include <linux/swap.h> 3570d64ceaSPaul Mackerras #include <linux/stddef.h> 3670d64ceaSPaul Mackerras #include <linux/vmalloc.h> 3770d64ceaSPaul Mackerras #include <linux/init.h> 3870d64ceaSPaul Mackerras #include <linux/delay.h> 3970d64ceaSPaul Mackerras #include <linux/bootmem.h> 4070d64ceaSPaul Mackerras #include <linux/highmem.h> 4170d64ceaSPaul Mackerras #include <linux/idr.h> 4270d64ceaSPaul Mackerras #include <linux/nodemask.h> 4370d64ceaSPaul Mackerras #include <linux/module.h> 4470d64ceaSPaul Mackerras 4570d64ceaSPaul Mackerras #include <asm/pgalloc.h> 4670d64ceaSPaul Mackerras #include <asm/page.h> 4770d64ceaSPaul Mackerras #include <asm/prom.h> 4870d64ceaSPaul Mackerras #include <asm/lmb.h> 4970d64ceaSPaul Mackerras #include <asm/rtas.h> 5070d64ceaSPaul Mackerras #include <asm/io.h> 5170d64ceaSPaul Mackerras #include <asm/mmu_context.h> 5270d64ceaSPaul Mackerras #include <asm/pgtable.h> 5370d64ceaSPaul Mackerras #include <asm/mmu.h> 5470d64ceaSPaul Mackerras #include <asm/uaccess.h> 5570d64ceaSPaul Mackerras #include <asm/smp.h> 5670d64ceaSPaul Mackerras #include <asm/machdep.h> 5770d64ceaSPaul Mackerras #include <asm/tlb.h> 5870d64ceaSPaul Mackerras #include <asm/eeh.h> 5970d64ceaSPaul Mackerras #include <asm/processor.h> 6070d64ceaSPaul Mackerras #include <asm/mmzone.h> 6170d64ceaSPaul Mackerras #include <asm/cputable.h> 6270d64ceaSPaul Mackerras #include <asm/sections.h> 6370d64ceaSPaul Mackerras #include <asm/system.h> 6470d64ceaSPaul Mackerras #include <asm/iommu.h> 6570d64ceaSPaul Mackerras #include <asm/abs_addr.h> 6670d64ceaSPaul Mackerras #include <asm/vdso.h> 67*800fc3eeSDavid Gibson 68*800fc3eeSDavid Gibson #include "mmu_decl.h" 6970d64ceaSPaul Mackerras 7087655ff2SBenjamin Herrenschmidt #ifdef DEBUG 7187655ff2SBenjamin Herrenschmidt #define DBG(fmt...) printk(fmt) 7287655ff2SBenjamin Herrenschmidt #else 7387655ff2SBenjamin Herrenschmidt #define DBG(fmt...) 7487655ff2SBenjamin Herrenschmidt #endif 7587655ff2SBenjamin Herrenschmidt 7670d64ceaSPaul Mackerras #if PGTABLE_RANGE > USER_VSID_RANGE 7770d64ceaSPaul Mackerras #warning Limited user VSID range means pagetable space is wasted 7870d64ceaSPaul Mackerras #endif 7970d64ceaSPaul Mackerras 8070d64ceaSPaul Mackerras #if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE) 8170d64ceaSPaul Mackerras #warning TASK_SIZE is smaller than it needs to be. 8270d64ceaSPaul Mackerras #endif 8370d64ceaSPaul Mackerras 8470d64ceaSPaul Mackerras /* max amount of RAM to use */ 8570d64ceaSPaul Mackerras unsigned long __max_memory; 8670d64ceaSPaul Mackerras 8770d64ceaSPaul Mackerras /* info on what we think the IO hole is */ 8870d64ceaSPaul Mackerras unsigned long io_hole_start; 8970d64ceaSPaul Mackerras unsigned long io_hole_size; 9070d64ceaSPaul Mackerras 9170d64ceaSPaul Mackerras /* 9270d64ceaSPaul Mackerras * Do very early mm setup. 9370d64ceaSPaul Mackerras */ 9470d64ceaSPaul Mackerras void __init mm_init_ppc64(void) 9570d64ceaSPaul Mackerras { 9670d64ceaSPaul Mackerras #ifndef CONFIG_PPC_ISERIES 9770d64ceaSPaul Mackerras unsigned long i; 9870d64ceaSPaul Mackerras #endif 9970d64ceaSPaul Mackerras 10070d64ceaSPaul Mackerras ppc64_boot_msg(0x100, "MM Init"); 10170d64ceaSPaul Mackerras 10270d64ceaSPaul Mackerras /* This is the story of the IO hole... please, keep seated, 10370d64ceaSPaul Mackerras * unfortunately, we are out of oxygen masks at the moment. 10470d64ceaSPaul Mackerras * So we need some rough way to tell where your big IO hole 10570d64ceaSPaul Mackerras * is. On pmac, it's between 2G and 4G, on POWER3, it's around 10670d64ceaSPaul Mackerras * that area as well, on POWER4 we don't have one, etc... 10770d64ceaSPaul Mackerras * We need that as a "hint" when sizing the TCE table on POWER3 10870d64ceaSPaul Mackerras * So far, the simplest way that seem work well enough for us it 10970d64ceaSPaul Mackerras * to just assume that the first discontinuity in our physical 11070d64ceaSPaul Mackerras * RAM layout is the IO hole. That may not be correct in the future 11170d64ceaSPaul Mackerras * (and isn't on iSeries but then we don't care ;) 11270d64ceaSPaul Mackerras */ 11370d64ceaSPaul Mackerras 11470d64ceaSPaul Mackerras #ifndef CONFIG_PPC_ISERIES 11570d64ceaSPaul Mackerras for (i = 1; i < lmb.memory.cnt; i++) { 11670d64ceaSPaul Mackerras unsigned long base, prevbase, prevsize; 11770d64ceaSPaul Mackerras 11870d64ceaSPaul Mackerras prevbase = lmb.memory.region[i-1].base; 11970d64ceaSPaul Mackerras prevsize = lmb.memory.region[i-1].size; 12070d64ceaSPaul Mackerras base = lmb.memory.region[i].base; 12170d64ceaSPaul Mackerras if (base > (prevbase + prevsize)) { 12270d64ceaSPaul Mackerras io_hole_start = prevbase + prevsize; 12370d64ceaSPaul Mackerras io_hole_size = base - (prevbase + prevsize); 12470d64ceaSPaul Mackerras break; 12570d64ceaSPaul Mackerras } 12670d64ceaSPaul Mackerras } 12770d64ceaSPaul Mackerras #endif /* CONFIG_PPC_ISERIES */ 12870d64ceaSPaul Mackerras if (io_hole_start) 12970d64ceaSPaul Mackerras printk("IO Hole assumed to be %lx -> %lx\n", 13070d64ceaSPaul Mackerras io_hole_start, io_hole_start + io_hole_size - 1); 13170d64ceaSPaul Mackerras 13270d64ceaSPaul Mackerras ppc64_boot_msg(0x100, "MM Init Done"); 13370d64ceaSPaul Mackerras } 13470d64ceaSPaul Mackerras 13570d64ceaSPaul Mackerras void free_initmem(void) 13670d64ceaSPaul Mackerras { 13770d64ceaSPaul Mackerras unsigned long addr; 13870d64ceaSPaul Mackerras 13970d64ceaSPaul Mackerras addr = (unsigned long)__init_begin; 14070d64ceaSPaul Mackerras for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) { 14170d64ceaSPaul Mackerras memset((void *)addr, 0xcc, PAGE_SIZE); 14270d64ceaSPaul Mackerras ClearPageReserved(virt_to_page(addr)); 14370d64ceaSPaul Mackerras set_page_count(virt_to_page(addr), 1); 14470d64ceaSPaul Mackerras free_page(addr); 14570d64ceaSPaul Mackerras totalram_pages++; 14670d64ceaSPaul Mackerras } 14770d64ceaSPaul Mackerras printk ("Freeing unused kernel memory: %luk freed\n", 14870d64ceaSPaul Mackerras ((unsigned long)__init_end - (unsigned long)__init_begin) >> 10); 14970d64ceaSPaul Mackerras } 15070d64ceaSPaul Mackerras 15170d64ceaSPaul Mackerras #ifdef CONFIG_BLK_DEV_INITRD 15270d64ceaSPaul Mackerras void free_initrd_mem(unsigned long start, unsigned long end) 15370d64ceaSPaul Mackerras { 15470d64ceaSPaul Mackerras if (start < end) 15570d64ceaSPaul Mackerras printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10); 15670d64ceaSPaul Mackerras for (; start < end; start += PAGE_SIZE) { 15770d64ceaSPaul Mackerras ClearPageReserved(virt_to_page(start)); 15870d64ceaSPaul Mackerras set_page_count(virt_to_page(start), 1); 15970d64ceaSPaul Mackerras free_page(start); 16070d64ceaSPaul Mackerras totalram_pages++; 16170d64ceaSPaul Mackerras } 16270d64ceaSPaul Mackerras } 16370d64ceaSPaul Mackerras #endif 16470d64ceaSPaul Mackerras 16570d64ceaSPaul Mackerras static struct kcore_list kcore_vmem; 16670d64ceaSPaul Mackerras 16770d64ceaSPaul Mackerras static int __init setup_kcore(void) 16870d64ceaSPaul Mackerras { 16970d64ceaSPaul Mackerras int i; 17070d64ceaSPaul Mackerras 17170d64ceaSPaul Mackerras for (i=0; i < lmb.memory.cnt; i++) { 17270d64ceaSPaul Mackerras unsigned long base, size; 17370d64ceaSPaul Mackerras struct kcore_list *kcore_mem; 17470d64ceaSPaul Mackerras 17570d64ceaSPaul Mackerras base = lmb.memory.region[i].base; 17670d64ceaSPaul Mackerras size = lmb.memory.region[i].size; 17770d64ceaSPaul Mackerras 17870d64ceaSPaul Mackerras /* GFP_ATOMIC to avoid might_sleep warnings during boot */ 17970d64ceaSPaul Mackerras kcore_mem = kmalloc(sizeof(struct kcore_list), GFP_ATOMIC); 18070d64ceaSPaul Mackerras if (!kcore_mem) 18170d64ceaSPaul Mackerras panic("mem_init: kmalloc failed\n"); 18270d64ceaSPaul Mackerras 18370d64ceaSPaul Mackerras kclist_add(kcore_mem, __va(base), size); 18470d64ceaSPaul Mackerras } 18570d64ceaSPaul Mackerras 18670d64ceaSPaul Mackerras kclist_add(&kcore_vmem, (void *)VMALLOC_START, VMALLOC_END-VMALLOC_START); 18770d64ceaSPaul Mackerras 18870d64ceaSPaul Mackerras return 0; 18970d64ceaSPaul Mackerras } 19070d64ceaSPaul Mackerras module_init(setup_kcore); 19170d64ceaSPaul Mackerras 19270d64ceaSPaul Mackerras static void zero_ctor(void *addr, kmem_cache_t *cache, unsigned long flags) 19370d64ceaSPaul Mackerras { 19470d64ceaSPaul Mackerras memset(addr, 0, kmem_cache_size(cache)); 19570d64ceaSPaul Mackerras } 19670d64ceaSPaul Mackerras 1973c726f8dSBenjamin Herrenschmidt #ifdef CONFIG_PPC_64K_PAGES 19887655ff2SBenjamin Herrenschmidt static const unsigned int pgtable_cache_size[3] = { 19987655ff2SBenjamin Herrenschmidt PTE_TABLE_SIZE, PMD_TABLE_SIZE, PGD_TABLE_SIZE 2003c726f8dSBenjamin Herrenschmidt }; 2013c726f8dSBenjamin Herrenschmidt static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = { 20287655ff2SBenjamin Herrenschmidt "pte_pmd_cache", "pmd_cache", "pgd_cache", 2033c726f8dSBenjamin Herrenschmidt }; 2043c726f8dSBenjamin Herrenschmidt #else 20587655ff2SBenjamin Herrenschmidt static const unsigned int pgtable_cache_size[2] = { 20670d64ceaSPaul Mackerras PTE_TABLE_SIZE, PMD_TABLE_SIZE 20770d64ceaSPaul Mackerras }; 20870d64ceaSPaul Mackerras static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = { 20970d64ceaSPaul Mackerras "pgd_pte_cache", "pud_pmd_cache", 21070d64ceaSPaul Mackerras }; 2113c726f8dSBenjamin Herrenschmidt #endif /* CONFIG_PPC_64K_PAGES */ 21270d64ceaSPaul Mackerras 21370d64ceaSPaul Mackerras kmem_cache_t *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)]; 21470d64ceaSPaul Mackerras 21570d64ceaSPaul Mackerras void pgtable_cache_init(void) 21670d64ceaSPaul Mackerras { 21770d64ceaSPaul Mackerras int i; 21870d64ceaSPaul Mackerras 21970d64ceaSPaul Mackerras for (i = 0; i < ARRAY_SIZE(pgtable_cache_size); i++) { 22070d64ceaSPaul Mackerras int size = pgtable_cache_size[i]; 22170d64ceaSPaul Mackerras const char *name = pgtable_cache_name[i]; 22270d64ceaSPaul Mackerras 22387655ff2SBenjamin Herrenschmidt DBG("Allocating page table cache %s (#%d) " 22487655ff2SBenjamin Herrenschmidt "for size: %08x...\n", name, i, size); 22570d64ceaSPaul Mackerras pgtable_cache[i] = kmem_cache_create(name, 22670d64ceaSPaul Mackerras size, size, 2273c726f8dSBenjamin Herrenschmidt SLAB_HWCACHE_ALIGN | 2283c726f8dSBenjamin Herrenschmidt SLAB_MUST_HWCACHE_ALIGN, 22970d64ceaSPaul Mackerras zero_ctor, 23070d64ceaSPaul Mackerras NULL); 23170d64ceaSPaul Mackerras if (! pgtable_cache[i]) 23270d64ceaSPaul Mackerras panic("pgtable_cache_init(): could not create %s!\n", 23370d64ceaSPaul Mackerras name); 23470d64ceaSPaul Mackerras } 23570d64ceaSPaul Mackerras } 236