170d64ceaSPaul Mackerras /* 270d64ceaSPaul Mackerras * PowerPC version 370d64ceaSPaul Mackerras * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 470d64ceaSPaul Mackerras * 570d64ceaSPaul Mackerras * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 670d64ceaSPaul Mackerras * and Cort Dougan (PReP) (cort@cs.nmt.edu) 770d64ceaSPaul Mackerras * Copyright (C) 1996 Paul Mackerras 870d64ceaSPaul Mackerras * 970d64ceaSPaul Mackerras * Derived from "arch/i386/mm/init.c" 1070d64ceaSPaul Mackerras * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 1170d64ceaSPaul Mackerras * 1270d64ceaSPaul Mackerras * Dave Engebretsen <engebret@us.ibm.com> 1370d64ceaSPaul Mackerras * Rework for PPC64 port. 1470d64ceaSPaul Mackerras * 1570d64ceaSPaul Mackerras * This program is free software; you can redistribute it and/or 1670d64ceaSPaul Mackerras * modify it under the terms of the GNU General Public License 1770d64ceaSPaul Mackerras * as published by the Free Software Foundation; either version 1870d64ceaSPaul Mackerras * 2 of the License, or (at your option) any later version. 1970d64ceaSPaul Mackerras * 2070d64ceaSPaul Mackerras */ 2170d64ceaSPaul Mackerras 2270d64ceaSPaul Mackerras #include <linux/signal.h> 2370d64ceaSPaul Mackerras #include <linux/sched.h> 2470d64ceaSPaul Mackerras #include <linux/kernel.h> 2570d64ceaSPaul Mackerras #include <linux/errno.h> 2670d64ceaSPaul Mackerras #include <linux/string.h> 2770d64ceaSPaul Mackerras #include <linux/types.h> 2870d64ceaSPaul Mackerras #include <linux/mman.h> 2970d64ceaSPaul Mackerras #include <linux/mm.h> 3070d64ceaSPaul Mackerras #include <linux/swap.h> 3170d64ceaSPaul Mackerras #include <linux/stddef.h> 3270d64ceaSPaul Mackerras #include <linux/vmalloc.h> 3370d64ceaSPaul Mackerras #include <linux/init.h> 3470d64ceaSPaul Mackerras #include <linux/delay.h> 3570d64ceaSPaul Mackerras #include <linux/bootmem.h> 3670d64ceaSPaul Mackerras #include <linux/highmem.h> 3770d64ceaSPaul Mackerras #include <linux/idr.h> 3870d64ceaSPaul Mackerras #include <linux/nodemask.h> 3970d64ceaSPaul Mackerras #include <linux/module.h> 40c9cf5528SRandy Dunlap #include <linux/poison.h> 41d9b2b2a2SDavid S. Miller #include <linux/lmb.h> 4270d64ceaSPaul Mackerras 4370d64ceaSPaul Mackerras #include <asm/pgalloc.h> 4470d64ceaSPaul Mackerras #include <asm/page.h> 4570d64ceaSPaul Mackerras #include <asm/prom.h> 4670d64ceaSPaul Mackerras #include <asm/rtas.h> 4770d64ceaSPaul Mackerras #include <asm/io.h> 4870d64ceaSPaul Mackerras #include <asm/mmu_context.h> 4970d64ceaSPaul Mackerras #include <asm/pgtable.h> 5070d64ceaSPaul Mackerras #include <asm/mmu.h> 5170d64ceaSPaul Mackerras #include <asm/uaccess.h> 5270d64ceaSPaul Mackerras #include <asm/smp.h> 5370d64ceaSPaul Mackerras #include <asm/machdep.h> 5470d64ceaSPaul Mackerras #include <asm/tlb.h> 5570d64ceaSPaul Mackerras #include <asm/eeh.h> 5670d64ceaSPaul Mackerras #include <asm/processor.h> 5770d64ceaSPaul Mackerras #include <asm/mmzone.h> 5870d64ceaSPaul Mackerras #include <asm/cputable.h> 5970d64ceaSPaul Mackerras #include <asm/sections.h> 6070d64ceaSPaul Mackerras #include <asm/system.h> 6170d64ceaSPaul Mackerras #include <asm/iommu.h> 6270d64ceaSPaul Mackerras #include <asm/abs_addr.h> 6370d64ceaSPaul Mackerras #include <asm/vdso.h> 64800fc3eeSDavid Gibson 65800fc3eeSDavid Gibson #include "mmu_decl.h" 6670d64ceaSPaul Mackerras 6770d64ceaSPaul Mackerras #if PGTABLE_RANGE > USER_VSID_RANGE 6870d64ceaSPaul Mackerras #warning Limited user VSID range means pagetable space is wasted 6970d64ceaSPaul Mackerras #endif 7070d64ceaSPaul Mackerras 7170d64ceaSPaul Mackerras #if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE) 7270d64ceaSPaul Mackerras #warning TASK_SIZE is smaller than it needs to be. 7370d64ceaSPaul Mackerras #endif 7470d64ceaSPaul Mackerras 7570d64ceaSPaul Mackerras /* max amount of RAM to use */ 7670d64ceaSPaul Mackerras unsigned long __max_memory; 7770d64ceaSPaul Mackerras 78*d7917ba7SKumar Gala phys_addr_t memstart_addr; 79*d7917ba7SKumar Gala 8070d64ceaSPaul Mackerras void free_initmem(void) 8170d64ceaSPaul Mackerras { 8270d64ceaSPaul Mackerras unsigned long addr; 8370d64ceaSPaul Mackerras 8470d64ceaSPaul Mackerras addr = (unsigned long)__init_begin; 8570d64ceaSPaul Mackerras for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) { 86c9cf5528SRandy Dunlap memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 8770d64ceaSPaul Mackerras ClearPageReserved(virt_to_page(addr)); 887835e98bSNick Piggin init_page_count(virt_to_page(addr)); 8970d64ceaSPaul Mackerras free_page(addr); 9070d64ceaSPaul Mackerras totalram_pages++; 9170d64ceaSPaul Mackerras } 9270d64ceaSPaul Mackerras printk ("Freeing unused kernel memory: %luk freed\n", 9370d64ceaSPaul Mackerras ((unsigned long)__init_end - (unsigned long)__init_begin) >> 10); 9470d64ceaSPaul Mackerras } 9570d64ceaSPaul Mackerras 9670d64ceaSPaul Mackerras #ifdef CONFIG_BLK_DEV_INITRD 9770d64ceaSPaul Mackerras void free_initrd_mem(unsigned long start, unsigned long end) 9870d64ceaSPaul Mackerras { 9970d64ceaSPaul Mackerras if (start < end) 10070d64ceaSPaul Mackerras printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10); 10170d64ceaSPaul Mackerras for (; start < end; start += PAGE_SIZE) { 10270d64ceaSPaul Mackerras ClearPageReserved(virt_to_page(start)); 1037835e98bSNick Piggin init_page_count(virt_to_page(start)); 10470d64ceaSPaul Mackerras free_page(start); 10570d64ceaSPaul Mackerras totalram_pages++; 10670d64ceaSPaul Mackerras } 10770d64ceaSPaul Mackerras } 10870d64ceaSPaul Mackerras #endif 10970d64ceaSPaul Mackerras 110df174e3bSEd Swarthout #ifdef CONFIG_PROC_KCORE 11170d64ceaSPaul Mackerras static struct kcore_list kcore_vmem; 11270d64ceaSPaul Mackerras 11370d64ceaSPaul Mackerras static int __init setup_kcore(void) 11470d64ceaSPaul Mackerras { 11570d64ceaSPaul Mackerras int i; 11670d64ceaSPaul Mackerras 11770d64ceaSPaul Mackerras for (i=0; i < lmb.memory.cnt; i++) { 11870d64ceaSPaul Mackerras unsigned long base, size; 11970d64ceaSPaul Mackerras struct kcore_list *kcore_mem; 12070d64ceaSPaul Mackerras 12170d64ceaSPaul Mackerras base = lmb.memory.region[i].base; 12270d64ceaSPaul Mackerras size = lmb.memory.region[i].size; 12370d64ceaSPaul Mackerras 12470d64ceaSPaul Mackerras /* GFP_ATOMIC to avoid might_sleep warnings during boot */ 12570d64ceaSPaul Mackerras kcore_mem = kmalloc(sizeof(struct kcore_list), GFP_ATOMIC); 12670d64ceaSPaul Mackerras if (!kcore_mem) 127e48b1b45SHarvey Harrison panic("%s: kmalloc failed\n", __func__); 12870d64ceaSPaul Mackerras 12970d64ceaSPaul Mackerras kclist_add(kcore_mem, __va(base), size); 13070d64ceaSPaul Mackerras } 13170d64ceaSPaul Mackerras 13270d64ceaSPaul Mackerras kclist_add(&kcore_vmem, (void *)VMALLOC_START, VMALLOC_END-VMALLOC_START); 13370d64ceaSPaul Mackerras 13470d64ceaSPaul Mackerras return 0; 13570d64ceaSPaul Mackerras } 13670d64ceaSPaul Mackerras module_init(setup_kcore); 137df174e3bSEd Swarthout #endif 13870d64ceaSPaul Mackerras 1394ba9b9d0SChristoph Lameter static void zero_ctor(struct kmem_cache *cache, void *addr) 14070d64ceaSPaul Mackerras { 14170d64ceaSPaul Mackerras memset(addr, 0, kmem_cache_size(cache)); 14270d64ceaSPaul Mackerras } 14370d64ceaSPaul Mackerras 14487655ff2SBenjamin Herrenschmidt static const unsigned int pgtable_cache_size[2] = { 145517e2263SHugh Dickins PGD_TABLE_SIZE, PMD_TABLE_SIZE 14670d64ceaSPaul Mackerras }; 14770d64ceaSPaul Mackerras static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = { 148517e2263SHugh Dickins #ifdef CONFIG_PPC_64K_PAGES 149517e2263SHugh Dickins "pgd_cache", "pmd_cache", 150517e2263SHugh Dickins #else 151517e2263SHugh Dickins "pgd_cache", "pud_pmd_cache", 1523c726f8dSBenjamin Herrenschmidt #endif /* CONFIG_PPC_64K_PAGES */ 153517e2263SHugh Dickins }; 15470d64ceaSPaul Mackerras 155f10a04c0SDavid Gibson #ifdef CONFIG_HUGETLB_PAGE 156f10a04c0SDavid Gibson /* Hugepages need one extra cache, initialized in hugetlbpage.c. We 157f10a04c0SDavid Gibson * can't put into the tables above, because HPAGE_SHIFT is not compile 158f10a04c0SDavid Gibson * time constant. */ 159e18b890bSChristoph Lameter struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)+1]; 160f10a04c0SDavid Gibson #else 161e18b890bSChristoph Lameter struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)]; 162f10a04c0SDavid Gibson #endif 16370d64ceaSPaul Mackerras 16470d64ceaSPaul Mackerras void pgtable_cache_init(void) 16570d64ceaSPaul Mackerras { 16670d64ceaSPaul Mackerras int i; 16770d64ceaSPaul Mackerras 16870d64ceaSPaul Mackerras for (i = 0; i < ARRAY_SIZE(pgtable_cache_size); i++) { 16970d64ceaSPaul Mackerras int size = pgtable_cache_size[i]; 17070d64ceaSPaul Mackerras const char *name = pgtable_cache_name[i]; 17170d64ceaSPaul Mackerras 1726548d83aSStephen Rothwell pr_debug("Allocating page table cache %s (#%d) " 17387655ff2SBenjamin Herrenschmidt "for size: %08x...\n", name, i, size); 17470d64ceaSPaul Mackerras pgtable_cache[i] = kmem_cache_create(name, 17570d64ceaSPaul Mackerras size, size, 1760e6b9c98SAkinobu Mita SLAB_PANIC, 17720c2df83SPaul Mundt zero_ctor); 17870d64ceaSPaul Mackerras } 17970d64ceaSPaul Mackerras } 180d29eff7bSAndy Whitcroft 181d29eff7bSAndy Whitcroft #ifdef CONFIG_SPARSEMEM_VMEMMAP 182d29eff7bSAndy Whitcroft /* 183d29eff7bSAndy Whitcroft * Given an address within the vmemmap, determine the pfn of the page that 184d29eff7bSAndy Whitcroft * represents the start of the section it is within. Note that we have to 185d29eff7bSAndy Whitcroft * do this by hand as the proffered address may not be correctly aligned. 186d29eff7bSAndy Whitcroft * Subtraction of non-aligned pointers produces undefined results. 187d29eff7bSAndy Whitcroft */ 188d29eff7bSAndy Whitcroft unsigned long __meminit vmemmap_section_start(unsigned long page) 189d29eff7bSAndy Whitcroft { 190d29eff7bSAndy Whitcroft unsigned long offset = page - ((unsigned long)(vmemmap)); 191d29eff7bSAndy Whitcroft 192d29eff7bSAndy Whitcroft /* Return the pfn of the start of the section. */ 193d29eff7bSAndy Whitcroft return (offset / sizeof(struct page)) & PAGE_SECTION_MASK; 194d29eff7bSAndy Whitcroft } 195d29eff7bSAndy Whitcroft 196d29eff7bSAndy Whitcroft /* 197d29eff7bSAndy Whitcroft * Check if this vmemmap page is already initialised. If any section 198d29eff7bSAndy Whitcroft * which overlaps this vmemmap page is initialised then this page is 199d29eff7bSAndy Whitcroft * initialised already. 200d29eff7bSAndy Whitcroft */ 201d29eff7bSAndy Whitcroft int __meminit vmemmap_populated(unsigned long start, int page_size) 202d29eff7bSAndy Whitcroft { 203d29eff7bSAndy Whitcroft unsigned long end = start + page_size; 204d29eff7bSAndy Whitcroft 205d29eff7bSAndy Whitcroft for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page))) 206d29eff7bSAndy Whitcroft if (pfn_valid(vmemmap_section_start(start))) 207d29eff7bSAndy Whitcroft return 1; 208d29eff7bSAndy Whitcroft 209d29eff7bSAndy Whitcroft return 0; 210d29eff7bSAndy Whitcroft } 211d29eff7bSAndy Whitcroft 212d29eff7bSAndy Whitcroft int __meminit vmemmap_populate(struct page *start_page, 213d29eff7bSAndy Whitcroft unsigned long nr_pages, int node) 214d29eff7bSAndy Whitcroft { 215d29eff7bSAndy Whitcroft unsigned long mode_rw; 216d29eff7bSAndy Whitcroft unsigned long start = (unsigned long)start_page; 217d29eff7bSAndy Whitcroft unsigned long end = (unsigned long)(start_page + nr_pages); 218d29eff7bSAndy Whitcroft unsigned long page_size = 1 << mmu_psize_defs[mmu_linear_psize].shift; 219d29eff7bSAndy Whitcroft 220d29eff7bSAndy Whitcroft mode_rw = _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX; 221d29eff7bSAndy Whitcroft 222d29eff7bSAndy Whitcroft /* Align to the page size of the linear mapping. */ 223d29eff7bSAndy Whitcroft start = _ALIGN_DOWN(start, page_size); 224d29eff7bSAndy Whitcroft 225d29eff7bSAndy Whitcroft for (; start < end; start += page_size) { 226d29eff7bSAndy Whitcroft int mapped; 227d29eff7bSAndy Whitcroft void *p; 228d29eff7bSAndy Whitcroft 229d29eff7bSAndy Whitcroft if (vmemmap_populated(start, page_size)) 230d29eff7bSAndy Whitcroft continue; 231d29eff7bSAndy Whitcroft 232d29eff7bSAndy Whitcroft p = vmemmap_alloc_block(page_size, node); 233d29eff7bSAndy Whitcroft if (!p) 234d29eff7bSAndy Whitcroft return -ENOMEM; 235d29eff7bSAndy Whitcroft 2366548d83aSStephen Rothwell pr_debug("vmemmap %08lx allocated at %p, physical %08lx.\n", 2376548d83aSStephen Rothwell start, p, __pa(p)); 238d29eff7bSAndy Whitcroft 239d29eff7bSAndy Whitcroft mapped = htab_bolt_mapping(start, start + page_size, 240e95206abSAnton Blanchard __pa(p), mode_rw, mmu_linear_psize, 241e95206abSAnton Blanchard mmu_kernel_ssize); 242d29eff7bSAndy Whitcroft BUG_ON(mapped < 0); 243d29eff7bSAndy Whitcroft } 244d29eff7bSAndy Whitcroft 245d29eff7bSAndy Whitcroft return 0; 246d29eff7bSAndy Whitcroft } 247d29eff7bSAndy Whitcroft #endif 248