1ad757b6aSThomas Gleixner /* 2ad757b6aSThomas Gleixner * linux/arch/i386/mm/init.c 3ad757b6aSThomas Gleixner * 4ad757b6aSThomas Gleixner * Copyright (C) 1995 Linus Torvalds 5ad757b6aSThomas Gleixner * 6ad757b6aSThomas Gleixner * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 7ad757b6aSThomas Gleixner */ 8ad757b6aSThomas Gleixner 9ad757b6aSThomas Gleixner #include <linux/module.h> 10ad757b6aSThomas Gleixner #include <linux/signal.h> 11ad757b6aSThomas Gleixner #include <linux/sched.h> 12ad757b6aSThomas Gleixner #include <linux/kernel.h> 13ad757b6aSThomas Gleixner #include <linux/errno.h> 14ad757b6aSThomas Gleixner #include <linux/string.h> 15ad757b6aSThomas Gleixner #include <linux/types.h> 16ad757b6aSThomas Gleixner #include <linux/ptrace.h> 17ad757b6aSThomas Gleixner #include <linux/mman.h> 18ad757b6aSThomas Gleixner #include <linux/mm.h> 19ad757b6aSThomas Gleixner #include <linux/hugetlb.h> 20ad757b6aSThomas Gleixner #include <linux/swap.h> 21ad757b6aSThomas Gleixner #include <linux/smp.h> 22ad757b6aSThomas Gleixner #include <linux/init.h> 23ad757b6aSThomas Gleixner #include <linux/highmem.h> 24ad757b6aSThomas Gleixner #include <linux/pagemap.h> 25ad757b6aSThomas Gleixner #include <linux/pfn.h> 26ad757b6aSThomas Gleixner #include <linux/poison.h> 27ad757b6aSThomas Gleixner #include <linux/bootmem.h> 28ad757b6aSThomas Gleixner #include <linux/slab.h> 29ad757b6aSThomas Gleixner #include <linux/proc_fs.h> 30ad757b6aSThomas Gleixner #include <linux/efi.h> 31ad757b6aSThomas Gleixner #include <linux/memory_hotplug.h> 32ad757b6aSThomas Gleixner #include <linux/initrd.h> 33ad757b6aSThomas Gleixner #include <linux/cpumask.h> 34ad757b6aSThomas Gleixner 35ad757b6aSThomas Gleixner #include <asm/processor.h> 36ad757b6aSThomas Gleixner #include <asm/system.h> 37ad757b6aSThomas Gleixner #include <asm/uaccess.h> 38ad757b6aSThomas Gleixner #include <asm/pgtable.h> 39ad757b6aSThomas Gleixner #include <asm/dma.h> 40ad757b6aSThomas Gleixner #include <asm/fixmap.h> 41ad757b6aSThomas Gleixner #include <asm/e820.h> 42ad757b6aSThomas Gleixner #include <asm/apic.h> 43ad757b6aSThomas Gleixner #include <asm/tlb.h> 44ad757b6aSThomas Gleixner #include <asm/tlbflush.h> 45ad757b6aSThomas Gleixner #include <asm/sections.h> 46ad757b6aSThomas Gleixner #include <asm/paravirt.h> 47ad757b6aSThomas Gleixner 48ad757b6aSThomas Gleixner unsigned int __VMALLOC_RESERVE = 128 << 20; 49ad757b6aSThomas Gleixner 50ad757b6aSThomas Gleixner DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 51ad757b6aSThomas Gleixner unsigned long highstart_pfn, highend_pfn; 52ad757b6aSThomas Gleixner 53ad757b6aSThomas Gleixner static int noinline do_test_wp_bit(void); 54ad757b6aSThomas Gleixner 55ad757b6aSThomas Gleixner /* 56ad757b6aSThomas Gleixner * Creates a middle page table and puts a pointer to it in the 57ad757b6aSThomas Gleixner * given global directory entry. This only returns the gd entry 58ad757b6aSThomas Gleixner * in non-PAE compilation mode, since the middle layer is folded. 59ad757b6aSThomas Gleixner */ 60ad757b6aSThomas Gleixner static pmd_t * __init one_md_table_init(pgd_t *pgd) 61ad757b6aSThomas Gleixner { 62ad757b6aSThomas Gleixner pud_t *pud; 63ad757b6aSThomas Gleixner pmd_t *pmd_table; 64ad757b6aSThomas Gleixner 65ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 66ad757b6aSThomas Gleixner if (!(pgd_val(*pgd) & _PAGE_PRESENT)) { 67ad757b6aSThomas Gleixner pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE); 68ad757b6aSThomas Gleixner 69ad757b6aSThomas Gleixner paravirt_alloc_pd(__pa(pmd_table) >> PAGE_SHIFT); 70ad757b6aSThomas Gleixner set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); 71ad757b6aSThomas Gleixner pud = pud_offset(pgd, 0); 72ad757b6aSThomas Gleixner if (pmd_table != pmd_offset(pud, 0)) 73ad757b6aSThomas Gleixner BUG(); 74ad757b6aSThomas Gleixner } 75ad757b6aSThomas Gleixner #endif 76ad757b6aSThomas Gleixner pud = pud_offset(pgd, 0); 77ad757b6aSThomas Gleixner pmd_table = pmd_offset(pud, 0); 78ad757b6aSThomas Gleixner return pmd_table; 79ad757b6aSThomas Gleixner } 80ad757b6aSThomas Gleixner 81ad757b6aSThomas Gleixner /* 82ad757b6aSThomas Gleixner * Create a page table and place a pointer to it in a middle page 83ad757b6aSThomas Gleixner * directory entry. 84ad757b6aSThomas Gleixner */ 85ad757b6aSThomas Gleixner static pte_t * __init one_page_table_init(pmd_t *pmd) 86ad757b6aSThomas Gleixner { 87ad757b6aSThomas Gleixner if (!(pmd_val(*pmd) & _PAGE_PRESENT)) { 88509a80c4SIngo Molnar pte_t *page_table = NULL; 89509a80c4SIngo Molnar 90509a80c4SIngo Molnar #ifdef CONFIG_DEBUG_PAGEALLOC 91509a80c4SIngo Molnar page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE); 92509a80c4SIngo Molnar #endif 93509a80c4SIngo Molnar if (!page_table) 94509a80c4SIngo Molnar page_table = 95509a80c4SIngo Molnar (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); 96ad757b6aSThomas Gleixner 97ad757b6aSThomas Gleixner paravirt_alloc_pt(&init_mm, __pa(page_table) >> PAGE_SHIFT); 98ad757b6aSThomas Gleixner set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE)); 99ad757b6aSThomas Gleixner BUG_ON(page_table != pte_offset_kernel(pmd, 0)); 100ad757b6aSThomas Gleixner } 101ad757b6aSThomas Gleixner 102ad757b6aSThomas Gleixner return pte_offset_kernel(pmd, 0); 103ad757b6aSThomas Gleixner } 104ad757b6aSThomas Gleixner 105ad757b6aSThomas Gleixner /* 106ad757b6aSThomas Gleixner * This function initializes a certain range of kernel virtual memory 107ad757b6aSThomas Gleixner * with new bootmem page tables, everywhere page tables are missing in 108ad757b6aSThomas Gleixner * the given range. 109ad757b6aSThomas Gleixner */ 110ad757b6aSThomas Gleixner 111ad757b6aSThomas Gleixner /* 112ad757b6aSThomas Gleixner * NOTE: The pagetables are allocated contiguous on the physical space 113ad757b6aSThomas Gleixner * so we can cache the place of the first one and move around without 114ad757b6aSThomas Gleixner * checking the pgd every time. 115ad757b6aSThomas Gleixner */ 116ad757b6aSThomas Gleixner static void __init page_table_range_init (unsigned long start, unsigned long end, pgd_t *pgd_base) 117ad757b6aSThomas Gleixner { 118ad757b6aSThomas Gleixner pgd_t *pgd; 119ad757b6aSThomas Gleixner pmd_t *pmd; 120ad757b6aSThomas Gleixner int pgd_idx, pmd_idx; 121ad757b6aSThomas Gleixner unsigned long vaddr; 122ad757b6aSThomas Gleixner 123ad757b6aSThomas Gleixner vaddr = start; 124ad757b6aSThomas Gleixner pgd_idx = pgd_index(vaddr); 125ad757b6aSThomas Gleixner pmd_idx = pmd_index(vaddr); 126ad757b6aSThomas Gleixner pgd = pgd_base + pgd_idx; 127ad757b6aSThomas Gleixner 128ad757b6aSThomas Gleixner for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { 129ad757b6aSThomas Gleixner pmd = one_md_table_init(pgd); 130ad757b6aSThomas Gleixner pmd = pmd + pmd_index(vaddr); 131ad757b6aSThomas Gleixner for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end); pmd++, pmd_idx++) { 132ad757b6aSThomas Gleixner one_page_table_init(pmd); 133ad757b6aSThomas Gleixner 134ad757b6aSThomas Gleixner vaddr += PMD_SIZE; 135ad757b6aSThomas Gleixner } 136ad757b6aSThomas Gleixner pmd_idx = 0; 137ad757b6aSThomas Gleixner } 138ad757b6aSThomas Gleixner } 139ad757b6aSThomas Gleixner 140ad757b6aSThomas Gleixner static inline int is_kernel_text(unsigned long addr) 141ad757b6aSThomas Gleixner { 142ad757b6aSThomas Gleixner if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end) 143ad757b6aSThomas Gleixner return 1; 144ad757b6aSThomas Gleixner return 0; 145ad757b6aSThomas Gleixner } 146ad757b6aSThomas Gleixner 147ad757b6aSThomas Gleixner /* 148ad757b6aSThomas Gleixner * This maps the physical memory to kernel virtual address space, a total 149ad757b6aSThomas Gleixner * of max_low_pfn pages, by creating page tables starting from address 150ad757b6aSThomas Gleixner * PAGE_OFFSET. 151ad757b6aSThomas Gleixner */ 152ad757b6aSThomas Gleixner static void __init kernel_physical_mapping_init(pgd_t *pgd_base) 153ad757b6aSThomas Gleixner { 154ad757b6aSThomas Gleixner unsigned long pfn; 155ad757b6aSThomas Gleixner pgd_t *pgd; 156ad757b6aSThomas Gleixner pmd_t *pmd; 157ad757b6aSThomas Gleixner pte_t *pte; 158ad757b6aSThomas Gleixner int pgd_idx, pmd_idx, pte_ofs; 159ad757b6aSThomas Gleixner 160ad757b6aSThomas Gleixner pgd_idx = pgd_index(PAGE_OFFSET); 161ad757b6aSThomas Gleixner pgd = pgd_base + pgd_idx; 162ad757b6aSThomas Gleixner pfn = 0; 163ad757b6aSThomas Gleixner 164ad757b6aSThomas Gleixner for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { 165ad757b6aSThomas Gleixner pmd = one_md_table_init(pgd); 166ad757b6aSThomas Gleixner if (pfn >= max_low_pfn) 167ad757b6aSThomas Gleixner continue; 168*f3f20de8SJeremy Fitzhardinge for (pmd_idx = 0; 169*f3f20de8SJeremy Fitzhardinge pmd_idx < PTRS_PER_PMD && pfn < max_low_pfn; 170*f3f20de8SJeremy Fitzhardinge pmd++, pmd_idx++) { 171ad757b6aSThomas Gleixner unsigned int address = pfn * PAGE_SIZE + PAGE_OFFSET; 172ad757b6aSThomas Gleixner 173*f3f20de8SJeremy Fitzhardinge /* Map with big pages if possible, otherwise 174*f3f20de8SJeremy Fitzhardinge create normal page tables. */ 175ad757b6aSThomas Gleixner if (cpu_has_pse) { 176*f3f20de8SJeremy Fitzhardinge unsigned int address2; 177*f3f20de8SJeremy Fitzhardinge pgprot_t prot = PAGE_KERNEL_LARGE; 178*f3f20de8SJeremy Fitzhardinge 179*f3f20de8SJeremy Fitzhardinge address2 = (pfn + PTRS_PER_PTE - 1) * PAGE_SIZE + 180*f3f20de8SJeremy Fitzhardinge PAGE_OFFSET + PAGE_SIZE-1; 181*f3f20de8SJeremy Fitzhardinge 182*f3f20de8SJeremy Fitzhardinge if (is_kernel_text(address) || 183*f3f20de8SJeremy Fitzhardinge is_kernel_text(address2)) 184*f3f20de8SJeremy Fitzhardinge prot = PAGE_KERNEL_LARGE_EXEC; 185*f3f20de8SJeremy Fitzhardinge 186*f3f20de8SJeremy Fitzhardinge set_pmd(pmd, pfn_pmd(pfn, prot)); 187ad757b6aSThomas Gleixner 188ad757b6aSThomas Gleixner pfn += PTRS_PER_PTE; 189ad757b6aSThomas Gleixner } else { 190ad757b6aSThomas Gleixner pte = one_page_table_init(pmd); 191ad757b6aSThomas Gleixner 192ad757b6aSThomas Gleixner for (pte_ofs = 0; 193ad757b6aSThomas Gleixner pte_ofs < PTRS_PER_PTE && pfn < max_low_pfn; 194ad757b6aSThomas Gleixner pte++, pfn++, pte_ofs++, address += PAGE_SIZE) { 195*f3f20de8SJeremy Fitzhardinge pgprot_t prot = PAGE_KERNEL; 196*f3f20de8SJeremy Fitzhardinge 197ad757b6aSThomas Gleixner if (is_kernel_text(address)) 198*f3f20de8SJeremy Fitzhardinge prot = PAGE_KERNEL_EXEC; 199*f3f20de8SJeremy Fitzhardinge 200*f3f20de8SJeremy Fitzhardinge set_pte(pte, pfn_pte(pfn, prot)); 201ad757b6aSThomas Gleixner } 202ad757b6aSThomas Gleixner } 203ad757b6aSThomas Gleixner } 204ad757b6aSThomas Gleixner } 205ad757b6aSThomas Gleixner } 206ad757b6aSThomas Gleixner 207ad757b6aSThomas Gleixner static inline int page_kills_ppro(unsigned long pagenr) 208ad757b6aSThomas Gleixner { 209ad757b6aSThomas Gleixner if (pagenr >= 0x70000 && pagenr <= 0x7003F) 210ad757b6aSThomas Gleixner return 1; 211ad757b6aSThomas Gleixner return 0; 212ad757b6aSThomas Gleixner } 213ad757b6aSThomas Gleixner 214ad757b6aSThomas Gleixner int page_is_ram(unsigned long pagenr) 215ad757b6aSThomas Gleixner { 216ad757b6aSThomas Gleixner int i; 217ad757b6aSThomas Gleixner unsigned long addr, end; 218ad757b6aSThomas Gleixner 219ad757b6aSThomas Gleixner if (efi_enabled) { 220ad757b6aSThomas Gleixner efi_memory_desc_t *md; 221ad757b6aSThomas Gleixner void *p; 222ad757b6aSThomas Gleixner 223ad757b6aSThomas Gleixner for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 224ad757b6aSThomas Gleixner md = p; 225ad757b6aSThomas Gleixner if (!is_available_memory(md)) 226ad757b6aSThomas Gleixner continue; 227ad757b6aSThomas Gleixner addr = (md->phys_addr+PAGE_SIZE-1) >> PAGE_SHIFT; 228ad757b6aSThomas Gleixner end = (md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT)) >> PAGE_SHIFT; 229ad757b6aSThomas Gleixner 230ad757b6aSThomas Gleixner if ((pagenr >= addr) && (pagenr < end)) 231ad757b6aSThomas Gleixner return 1; 232ad757b6aSThomas Gleixner } 233ad757b6aSThomas Gleixner return 0; 234ad757b6aSThomas Gleixner } 235ad757b6aSThomas Gleixner 236ad757b6aSThomas Gleixner for (i = 0; i < e820.nr_map; i++) { 237ad757b6aSThomas Gleixner 238ad757b6aSThomas Gleixner if (e820.map[i].type != E820_RAM) /* not usable memory */ 239ad757b6aSThomas Gleixner continue; 240ad757b6aSThomas Gleixner /* 241ad757b6aSThomas Gleixner * !!!FIXME!!! Some BIOSen report areas as RAM that 242ad757b6aSThomas Gleixner * are not. Notably the 640->1Mb area. We need a sanity 243ad757b6aSThomas Gleixner * check here. 244ad757b6aSThomas Gleixner */ 245ad757b6aSThomas Gleixner addr = (e820.map[i].addr+PAGE_SIZE-1) >> PAGE_SHIFT; 246ad757b6aSThomas Gleixner end = (e820.map[i].addr+e820.map[i].size) >> PAGE_SHIFT; 247ad757b6aSThomas Gleixner if ((pagenr >= addr) && (pagenr < end)) 248ad757b6aSThomas Gleixner return 1; 249ad757b6aSThomas Gleixner } 250ad757b6aSThomas Gleixner return 0; 251ad757b6aSThomas Gleixner } 252ad757b6aSThomas Gleixner 253ad757b6aSThomas Gleixner #ifdef CONFIG_HIGHMEM 254ad757b6aSThomas Gleixner pte_t *kmap_pte; 255ad757b6aSThomas Gleixner pgprot_t kmap_prot; 256ad757b6aSThomas Gleixner 257ad757b6aSThomas Gleixner #define kmap_get_fixmap_pte(vaddr) \ 258ad757b6aSThomas Gleixner pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), vaddr), (vaddr)), (vaddr)) 259ad757b6aSThomas Gleixner 260ad757b6aSThomas Gleixner static void __init kmap_init(void) 261ad757b6aSThomas Gleixner { 262ad757b6aSThomas Gleixner unsigned long kmap_vstart; 263ad757b6aSThomas Gleixner 264ad757b6aSThomas Gleixner /* cache the first kmap pte */ 265ad757b6aSThomas Gleixner kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); 266ad757b6aSThomas Gleixner kmap_pte = kmap_get_fixmap_pte(kmap_vstart); 267ad757b6aSThomas Gleixner 268ad757b6aSThomas Gleixner kmap_prot = PAGE_KERNEL; 269ad757b6aSThomas Gleixner } 270ad757b6aSThomas Gleixner 271ad757b6aSThomas Gleixner static void __init permanent_kmaps_init(pgd_t *pgd_base) 272ad757b6aSThomas Gleixner { 273ad757b6aSThomas Gleixner pgd_t *pgd; 274ad757b6aSThomas Gleixner pud_t *pud; 275ad757b6aSThomas Gleixner pmd_t *pmd; 276ad757b6aSThomas Gleixner pte_t *pte; 277ad757b6aSThomas Gleixner unsigned long vaddr; 278ad757b6aSThomas Gleixner 279ad757b6aSThomas Gleixner vaddr = PKMAP_BASE; 280ad757b6aSThomas Gleixner page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base); 281ad757b6aSThomas Gleixner 282ad757b6aSThomas Gleixner pgd = swapper_pg_dir + pgd_index(vaddr); 283ad757b6aSThomas Gleixner pud = pud_offset(pgd, vaddr); 284ad757b6aSThomas Gleixner pmd = pmd_offset(pud, vaddr); 285ad757b6aSThomas Gleixner pte = pte_offset_kernel(pmd, vaddr); 286ad757b6aSThomas Gleixner pkmap_page_table = pte; 287ad757b6aSThomas Gleixner } 288ad757b6aSThomas Gleixner 289ad757b6aSThomas Gleixner static void __meminit free_new_highpage(struct page *page) 290ad757b6aSThomas Gleixner { 291ad757b6aSThomas Gleixner init_page_count(page); 292ad757b6aSThomas Gleixner __free_page(page); 293ad757b6aSThomas Gleixner totalhigh_pages++; 294ad757b6aSThomas Gleixner } 295ad757b6aSThomas Gleixner 296ad757b6aSThomas Gleixner void __init add_one_highpage_init(struct page *page, int pfn, int bad_ppro) 297ad757b6aSThomas Gleixner { 298ad757b6aSThomas Gleixner if (page_is_ram(pfn) && !(bad_ppro && page_kills_ppro(pfn))) { 299ad757b6aSThomas Gleixner ClearPageReserved(page); 300ad757b6aSThomas Gleixner free_new_highpage(page); 301ad757b6aSThomas Gleixner } else 302ad757b6aSThomas Gleixner SetPageReserved(page); 303ad757b6aSThomas Gleixner } 304ad757b6aSThomas Gleixner 305ad757b6aSThomas Gleixner static int __meminit add_one_highpage_hotplug(struct page *page, unsigned long pfn) 306ad757b6aSThomas Gleixner { 307ad757b6aSThomas Gleixner free_new_highpage(page); 308ad757b6aSThomas Gleixner totalram_pages++; 309ad757b6aSThomas Gleixner #ifdef CONFIG_FLATMEM 310ad757b6aSThomas Gleixner max_mapnr = max(pfn, max_mapnr); 311ad757b6aSThomas Gleixner #endif 312ad757b6aSThomas Gleixner num_physpages++; 313ad757b6aSThomas Gleixner return 0; 314ad757b6aSThomas Gleixner } 315ad757b6aSThomas Gleixner 316ad757b6aSThomas Gleixner /* 317ad757b6aSThomas Gleixner * Not currently handling the NUMA case. 318ad757b6aSThomas Gleixner * Assuming single node and all memory that 319ad757b6aSThomas Gleixner * has been added dynamically that would be 320ad757b6aSThomas Gleixner * onlined here is in HIGHMEM 321ad757b6aSThomas Gleixner */ 322ad757b6aSThomas Gleixner void __meminit online_page(struct page *page) 323ad757b6aSThomas Gleixner { 324ad757b6aSThomas Gleixner ClearPageReserved(page); 325ad757b6aSThomas Gleixner add_one_highpage_hotplug(page, page_to_pfn(page)); 326ad757b6aSThomas Gleixner } 327ad757b6aSThomas Gleixner 328ad757b6aSThomas Gleixner 329ad757b6aSThomas Gleixner #ifdef CONFIG_NUMA 330ad757b6aSThomas Gleixner extern void set_highmem_pages_init(int); 331ad757b6aSThomas Gleixner #else 332ad757b6aSThomas Gleixner static void __init set_highmem_pages_init(int bad_ppro) 333ad757b6aSThomas Gleixner { 334ad757b6aSThomas Gleixner int pfn; 33523be8c7dSIngo Molnar for (pfn = highstart_pfn; pfn < highend_pfn; pfn++) { 33623be8c7dSIngo Molnar /* 33723be8c7dSIngo Molnar * Holes under sparsemem might not have no mem_map[]: 33823be8c7dSIngo Molnar */ 33923be8c7dSIngo Molnar if (pfn_valid(pfn)) 340ad757b6aSThomas Gleixner add_one_highpage_init(pfn_to_page(pfn), pfn, bad_ppro); 34123be8c7dSIngo Molnar } 342ad757b6aSThomas Gleixner totalram_pages += totalhigh_pages; 343ad757b6aSThomas Gleixner } 344ad757b6aSThomas Gleixner #endif /* CONFIG_FLATMEM */ 345ad757b6aSThomas Gleixner 346ad757b6aSThomas Gleixner #else 347ad757b6aSThomas Gleixner #define kmap_init() do { } while (0) 348ad757b6aSThomas Gleixner #define permanent_kmaps_init(pgd_base) do { } while (0) 349ad757b6aSThomas Gleixner #define set_highmem_pages_init(bad_ppro) do { } while (0) 350ad757b6aSThomas Gleixner #endif /* CONFIG_HIGHMEM */ 351ad757b6aSThomas Gleixner 352ad757b6aSThomas Gleixner unsigned long long __PAGE_KERNEL = _PAGE_KERNEL; 353ad757b6aSThomas Gleixner EXPORT_SYMBOL(__PAGE_KERNEL); 354ad757b6aSThomas Gleixner unsigned long long __PAGE_KERNEL_EXEC = _PAGE_KERNEL_EXEC; 355ad757b6aSThomas Gleixner 356ad757b6aSThomas Gleixner #ifdef CONFIG_NUMA 357ad757b6aSThomas Gleixner extern void __init remap_numa_kva(void); 358ad757b6aSThomas Gleixner #else 359ad757b6aSThomas Gleixner #define remap_numa_kva() do {} while (0) 360ad757b6aSThomas Gleixner #endif 361ad757b6aSThomas Gleixner 362ad757b6aSThomas Gleixner void __init native_pagetable_setup_start(pgd_t *base) 363ad757b6aSThomas Gleixner { 364ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 365ad757b6aSThomas Gleixner int i; 366ad757b6aSThomas Gleixner 367ad757b6aSThomas Gleixner /* 368ad757b6aSThomas Gleixner * Init entries of the first-level page table to the 369ad757b6aSThomas Gleixner * zero page, if they haven't already been set up. 370ad757b6aSThomas Gleixner * 371ad757b6aSThomas Gleixner * In a normal native boot, we'll be running on a 372ad757b6aSThomas Gleixner * pagetable rooted in swapper_pg_dir, but not in PAE 373ad757b6aSThomas Gleixner * mode, so this will end up clobbering the mappings 374ad757b6aSThomas Gleixner * for the lower 24Mbytes of the address space, 375ad757b6aSThomas Gleixner * without affecting the kernel address space. 376ad757b6aSThomas Gleixner */ 377ad757b6aSThomas Gleixner for (i = 0; i < USER_PTRS_PER_PGD; i++) 378ad757b6aSThomas Gleixner set_pgd(&base[i], 379ad757b6aSThomas Gleixner __pgd(__pa(empty_zero_page) | _PAGE_PRESENT)); 380ad757b6aSThomas Gleixner 381ad757b6aSThomas Gleixner /* Make sure kernel address space is empty so that a pagetable 382ad757b6aSThomas Gleixner will be allocated for it. */ 383ad757b6aSThomas Gleixner memset(&base[USER_PTRS_PER_PGD], 0, 384ad757b6aSThomas Gleixner KERNEL_PGD_PTRS * sizeof(pgd_t)); 385ad757b6aSThomas Gleixner #else 386ad757b6aSThomas Gleixner paravirt_alloc_pd(__pa(swapper_pg_dir) >> PAGE_SHIFT); 387ad757b6aSThomas Gleixner #endif 388ad757b6aSThomas Gleixner } 389ad757b6aSThomas Gleixner 390ad757b6aSThomas Gleixner void __init native_pagetable_setup_done(pgd_t *base) 391ad757b6aSThomas Gleixner { 392ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 393ad757b6aSThomas Gleixner /* 394ad757b6aSThomas Gleixner * Add low memory identity-mappings - SMP needs it when 395ad757b6aSThomas Gleixner * starting up on an AP from real-mode. In the non-PAE 396ad757b6aSThomas Gleixner * case we already have these mappings through head.S. 397ad757b6aSThomas Gleixner * All user-space mappings are explicitly cleared after 398ad757b6aSThomas Gleixner * SMP startup. 399ad757b6aSThomas Gleixner */ 400ad757b6aSThomas Gleixner set_pgd(&base[0], base[USER_PTRS_PER_PGD]); 401ad757b6aSThomas Gleixner #endif 402ad757b6aSThomas Gleixner } 403ad757b6aSThomas Gleixner 404ad757b6aSThomas Gleixner /* 405ad757b6aSThomas Gleixner * Build a proper pagetable for the kernel mappings. Up until this 406ad757b6aSThomas Gleixner * point, we've been running on some set of pagetables constructed by 407ad757b6aSThomas Gleixner * the boot process. 408ad757b6aSThomas Gleixner * 409ad757b6aSThomas Gleixner * If we're booting on native hardware, this will be a pagetable 410ad757b6aSThomas Gleixner * constructed in arch/i386/kernel/head.S, and not running in PAE mode 411ad757b6aSThomas Gleixner * (even if we'll end up running in PAE). The root of the pagetable 412ad757b6aSThomas Gleixner * will be swapper_pg_dir. 413ad757b6aSThomas Gleixner * 414ad757b6aSThomas Gleixner * If we're booting paravirtualized under a hypervisor, then there are 415ad757b6aSThomas Gleixner * more options: we may already be running PAE, and the pagetable may 416ad757b6aSThomas Gleixner * or may not be based in swapper_pg_dir. In any case, 417ad757b6aSThomas Gleixner * paravirt_pagetable_setup_start() will set up swapper_pg_dir 418ad757b6aSThomas Gleixner * appropriately for the rest of the initialization to work. 419ad757b6aSThomas Gleixner * 420ad757b6aSThomas Gleixner * In general, pagetable_init() assumes that the pagetable may already 421ad757b6aSThomas Gleixner * be partially populated, and so it avoids stomping on any existing 422ad757b6aSThomas Gleixner * mappings. 423ad757b6aSThomas Gleixner */ 424ad757b6aSThomas Gleixner static void __init pagetable_init (void) 425ad757b6aSThomas Gleixner { 426ad757b6aSThomas Gleixner unsigned long vaddr, end; 427ad757b6aSThomas Gleixner pgd_t *pgd_base = swapper_pg_dir; 428ad757b6aSThomas Gleixner 429ad757b6aSThomas Gleixner paravirt_pagetable_setup_start(pgd_base); 430ad757b6aSThomas Gleixner 431ad757b6aSThomas Gleixner /* Enable PSE if available */ 432ad757b6aSThomas Gleixner if (cpu_has_pse) 433ad757b6aSThomas Gleixner set_in_cr4(X86_CR4_PSE); 434ad757b6aSThomas Gleixner 435ad757b6aSThomas Gleixner /* Enable PGE if available */ 436ad757b6aSThomas Gleixner if (cpu_has_pge) { 437ad757b6aSThomas Gleixner set_in_cr4(X86_CR4_PGE); 438ad757b6aSThomas Gleixner __PAGE_KERNEL |= _PAGE_GLOBAL; 439ad757b6aSThomas Gleixner __PAGE_KERNEL_EXEC |= _PAGE_GLOBAL; 440ad757b6aSThomas Gleixner } 441ad757b6aSThomas Gleixner 442ad757b6aSThomas Gleixner kernel_physical_mapping_init(pgd_base); 443ad757b6aSThomas Gleixner remap_numa_kva(); 444ad757b6aSThomas Gleixner 445ad757b6aSThomas Gleixner /* 446ad757b6aSThomas Gleixner * Fixed mappings, only the page table structure has to be 447ad757b6aSThomas Gleixner * created - mappings will be set by set_fixmap(): 448ad757b6aSThomas Gleixner */ 449ad757b6aSThomas Gleixner vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; 450ad757b6aSThomas Gleixner end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK; 451ad757b6aSThomas Gleixner page_table_range_init(vaddr, end, pgd_base); 452ad757b6aSThomas Gleixner 453ad757b6aSThomas Gleixner permanent_kmaps_init(pgd_base); 454ad757b6aSThomas Gleixner 455ad757b6aSThomas Gleixner paravirt_pagetable_setup_done(pgd_base); 456ad757b6aSThomas Gleixner } 457ad757b6aSThomas Gleixner 458ad757b6aSThomas Gleixner #if defined(CONFIG_HIBERNATION) || defined(CONFIG_ACPI) 459ad757b6aSThomas Gleixner /* 460ad757b6aSThomas Gleixner * Swap suspend & friends need this for resume because things like the intel-agp 461ad757b6aSThomas Gleixner * driver might have split up a kernel 4MB mapping. 462ad757b6aSThomas Gleixner */ 463ad757b6aSThomas Gleixner char __nosavedata swsusp_pg_dir[PAGE_SIZE] 464ad757b6aSThomas Gleixner __attribute__ ((aligned (PAGE_SIZE))); 465ad757b6aSThomas Gleixner 466ad757b6aSThomas Gleixner static inline void save_pg_dir(void) 467ad757b6aSThomas Gleixner { 468ad757b6aSThomas Gleixner memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE); 469ad757b6aSThomas Gleixner } 470ad757b6aSThomas Gleixner #else 471ad757b6aSThomas Gleixner static inline void save_pg_dir(void) 472ad757b6aSThomas Gleixner { 473ad757b6aSThomas Gleixner } 474ad757b6aSThomas Gleixner #endif 475ad757b6aSThomas Gleixner 476ad757b6aSThomas Gleixner void zap_low_mappings (void) 477ad757b6aSThomas Gleixner { 478ad757b6aSThomas Gleixner int i; 479ad757b6aSThomas Gleixner 480ad757b6aSThomas Gleixner save_pg_dir(); 481ad757b6aSThomas Gleixner 482ad757b6aSThomas Gleixner /* 483ad757b6aSThomas Gleixner * Zap initial low-memory mappings. 484ad757b6aSThomas Gleixner * 485ad757b6aSThomas Gleixner * Note that "pgd_clear()" doesn't do it for 486ad757b6aSThomas Gleixner * us, because pgd_clear() is a no-op on i386. 487ad757b6aSThomas Gleixner */ 488ad757b6aSThomas Gleixner for (i = 0; i < USER_PTRS_PER_PGD; i++) 489ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 490ad757b6aSThomas Gleixner set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page))); 491ad757b6aSThomas Gleixner #else 492ad757b6aSThomas Gleixner set_pgd(swapper_pg_dir+i, __pgd(0)); 493ad757b6aSThomas Gleixner #endif 494ad757b6aSThomas Gleixner flush_tlb_all(); 495ad757b6aSThomas Gleixner } 496ad757b6aSThomas Gleixner 497ad757b6aSThomas Gleixner int nx_enabled = 0; 498ad757b6aSThomas Gleixner 499ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 500ad757b6aSThomas Gleixner 501ad757b6aSThomas Gleixner static int disable_nx __initdata = 0; 502ad757b6aSThomas Gleixner u64 __supported_pte_mask __read_mostly = ~_PAGE_NX; 503ad757b6aSThomas Gleixner EXPORT_SYMBOL_GPL(__supported_pte_mask); 504ad757b6aSThomas Gleixner 505ad757b6aSThomas Gleixner /* 506ad757b6aSThomas Gleixner * noexec = on|off 507ad757b6aSThomas Gleixner * 508ad757b6aSThomas Gleixner * Control non executable mappings. 509ad757b6aSThomas Gleixner * 510ad757b6aSThomas Gleixner * on Enable 511ad757b6aSThomas Gleixner * off Disable 512ad757b6aSThomas Gleixner */ 513ad757b6aSThomas Gleixner static int __init noexec_setup(char *str) 514ad757b6aSThomas Gleixner { 515ad757b6aSThomas Gleixner if (!str || !strcmp(str, "on")) { 516ad757b6aSThomas Gleixner if (cpu_has_nx) { 517ad757b6aSThomas Gleixner __supported_pte_mask |= _PAGE_NX; 518ad757b6aSThomas Gleixner disable_nx = 0; 519ad757b6aSThomas Gleixner } 520ad757b6aSThomas Gleixner } else if (!strcmp(str,"off")) { 521ad757b6aSThomas Gleixner disable_nx = 1; 522ad757b6aSThomas Gleixner __supported_pte_mask &= ~_PAGE_NX; 523ad757b6aSThomas Gleixner } else 524ad757b6aSThomas Gleixner return -EINVAL; 525ad757b6aSThomas Gleixner 526ad757b6aSThomas Gleixner return 0; 527ad757b6aSThomas Gleixner } 528ad757b6aSThomas Gleixner early_param("noexec", noexec_setup); 529ad757b6aSThomas Gleixner 530ad757b6aSThomas Gleixner static void __init set_nx(void) 531ad757b6aSThomas Gleixner { 532ad757b6aSThomas Gleixner unsigned int v[4], l, h; 533ad757b6aSThomas Gleixner 534ad757b6aSThomas Gleixner if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) { 535ad757b6aSThomas Gleixner cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]); 536ad757b6aSThomas Gleixner if ((v[3] & (1 << 20)) && !disable_nx) { 537ad757b6aSThomas Gleixner rdmsr(MSR_EFER, l, h); 538ad757b6aSThomas Gleixner l |= EFER_NX; 539ad757b6aSThomas Gleixner wrmsr(MSR_EFER, l, h); 540ad757b6aSThomas Gleixner nx_enabled = 1; 541ad757b6aSThomas Gleixner __supported_pte_mask |= _PAGE_NX; 542ad757b6aSThomas Gleixner } 543ad757b6aSThomas Gleixner } 544ad757b6aSThomas Gleixner } 545ad757b6aSThomas Gleixner 546ad757b6aSThomas Gleixner /* 547ad757b6aSThomas Gleixner * Enables/disables executability of a given kernel page and 548ad757b6aSThomas Gleixner * returns the previous setting. 549ad757b6aSThomas Gleixner */ 550ad757b6aSThomas Gleixner int __init set_kernel_exec(unsigned long vaddr, int enable) 551ad757b6aSThomas Gleixner { 552ad757b6aSThomas Gleixner pte_t *pte; 553ad757b6aSThomas Gleixner int ret = 1; 554ad757b6aSThomas Gleixner 555ad757b6aSThomas Gleixner if (!nx_enabled) 556ad757b6aSThomas Gleixner goto out; 557ad757b6aSThomas Gleixner 558ad757b6aSThomas Gleixner pte = lookup_address(vaddr); 559ad757b6aSThomas Gleixner BUG_ON(!pte); 560ad757b6aSThomas Gleixner 561ad757b6aSThomas Gleixner if (!pte_exec_kernel(*pte)) 562ad757b6aSThomas Gleixner ret = 0; 563ad757b6aSThomas Gleixner 564ad757b6aSThomas Gleixner if (enable) 565ad757b6aSThomas Gleixner pte->pte_high &= ~(1 << (_PAGE_BIT_NX - 32)); 566ad757b6aSThomas Gleixner else 567ad757b6aSThomas Gleixner pte->pte_high |= 1 << (_PAGE_BIT_NX - 32); 568ad757b6aSThomas Gleixner pte_update_defer(&init_mm, vaddr, pte); 569ad757b6aSThomas Gleixner __flush_tlb_all(); 570ad757b6aSThomas Gleixner out: 571ad757b6aSThomas Gleixner return ret; 572ad757b6aSThomas Gleixner } 573ad757b6aSThomas Gleixner 574ad757b6aSThomas Gleixner #endif 575ad757b6aSThomas Gleixner 576ad757b6aSThomas Gleixner /* 577ad757b6aSThomas Gleixner * paging_init() sets up the page tables - note that the first 8MB are 578ad757b6aSThomas Gleixner * already mapped by head.S. 579ad757b6aSThomas Gleixner * 580ad757b6aSThomas Gleixner * This routines also unmaps the page at virtual kernel address 0, so 581ad757b6aSThomas Gleixner * that we can trap those pesky NULL-reference errors in the kernel. 582ad757b6aSThomas Gleixner */ 583ad757b6aSThomas Gleixner void __init paging_init(void) 584ad757b6aSThomas Gleixner { 585ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 586ad757b6aSThomas Gleixner set_nx(); 587ad757b6aSThomas Gleixner if (nx_enabled) 588ad757b6aSThomas Gleixner printk("NX (Execute Disable) protection: active\n"); 589ad757b6aSThomas Gleixner #endif 590ad757b6aSThomas Gleixner 591ad757b6aSThomas Gleixner pagetable_init(); 592ad757b6aSThomas Gleixner 593ad757b6aSThomas Gleixner load_cr3(swapper_pg_dir); 594ad757b6aSThomas Gleixner 595ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 596ad757b6aSThomas Gleixner /* 597ad757b6aSThomas Gleixner * We will bail out later - printk doesn't work right now so 598ad757b6aSThomas Gleixner * the user would just see a hanging kernel. 599ad757b6aSThomas Gleixner */ 600ad757b6aSThomas Gleixner if (cpu_has_pae) 601ad757b6aSThomas Gleixner set_in_cr4(X86_CR4_PAE); 602ad757b6aSThomas Gleixner #endif 603ad757b6aSThomas Gleixner __flush_tlb_all(); 604ad757b6aSThomas Gleixner 605ad757b6aSThomas Gleixner kmap_init(); 606ad757b6aSThomas Gleixner } 607ad757b6aSThomas Gleixner 608ad757b6aSThomas Gleixner /* 609ad757b6aSThomas Gleixner * Test if the WP bit works in supervisor mode. It isn't supported on 386's 610ad757b6aSThomas Gleixner * and also on some strange 486's (NexGen etc.). All 586+'s are OK. This 611ad757b6aSThomas Gleixner * used to involve black magic jumps to work around some nasty CPU bugs, 612ad757b6aSThomas Gleixner * but fortunately the switch to using exceptions got rid of all that. 613ad757b6aSThomas Gleixner */ 614ad757b6aSThomas Gleixner 615ad757b6aSThomas Gleixner static void __init test_wp_bit(void) 616ad757b6aSThomas Gleixner { 617ad757b6aSThomas Gleixner printk("Checking if this processor honours the WP bit even in supervisor mode... "); 618ad757b6aSThomas Gleixner 619ad757b6aSThomas Gleixner /* Any page-aligned address will do, the test is non-destructive */ 620ad757b6aSThomas Gleixner __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY); 621ad757b6aSThomas Gleixner boot_cpu_data.wp_works_ok = do_test_wp_bit(); 622ad757b6aSThomas Gleixner clear_fixmap(FIX_WP_TEST); 623ad757b6aSThomas Gleixner 624ad757b6aSThomas Gleixner if (!boot_cpu_data.wp_works_ok) { 625ad757b6aSThomas Gleixner printk("No.\n"); 626ad757b6aSThomas Gleixner #ifdef CONFIG_X86_WP_WORKS_OK 627ad757b6aSThomas Gleixner panic("This kernel doesn't support CPU's with broken WP. Recompile it for a 386!"); 628ad757b6aSThomas Gleixner #endif 629ad757b6aSThomas Gleixner } else { 630ad757b6aSThomas Gleixner printk("Ok.\n"); 631ad757b6aSThomas Gleixner } 632ad757b6aSThomas Gleixner } 633ad757b6aSThomas Gleixner 634ad757b6aSThomas Gleixner static struct kcore_list kcore_mem, kcore_vmalloc; 635ad757b6aSThomas Gleixner 636ad757b6aSThomas Gleixner void __init mem_init(void) 637ad757b6aSThomas Gleixner { 638ad757b6aSThomas Gleixner extern int ppro_with_ram_bug(void); 639ad757b6aSThomas Gleixner int codesize, reservedpages, datasize, initsize; 640ad757b6aSThomas Gleixner int tmp; 641ad757b6aSThomas Gleixner int bad_ppro; 642ad757b6aSThomas Gleixner 643ad757b6aSThomas Gleixner #ifdef CONFIG_FLATMEM 644ad757b6aSThomas Gleixner BUG_ON(!mem_map); 645ad757b6aSThomas Gleixner #endif 646ad757b6aSThomas Gleixner 647ad757b6aSThomas Gleixner bad_ppro = ppro_with_ram_bug(); 648ad757b6aSThomas Gleixner 649ad757b6aSThomas Gleixner #ifdef CONFIG_HIGHMEM 650ad757b6aSThomas Gleixner /* check that fixmap and pkmap do not overlap */ 651ad757b6aSThomas Gleixner if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) { 652ad757b6aSThomas Gleixner printk(KERN_ERR "fixmap and kmap areas overlap - this will crash\n"); 653ad757b6aSThomas Gleixner printk(KERN_ERR "pkstart: %lxh pkend: %lxh fixstart %lxh\n", 654ad757b6aSThomas Gleixner PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, FIXADDR_START); 655ad757b6aSThomas Gleixner BUG(); 656ad757b6aSThomas Gleixner } 657ad757b6aSThomas Gleixner #endif 658ad757b6aSThomas Gleixner 659ad757b6aSThomas Gleixner /* this will put all low memory onto the freelists */ 660ad757b6aSThomas Gleixner totalram_pages += free_all_bootmem(); 661ad757b6aSThomas Gleixner 662ad757b6aSThomas Gleixner reservedpages = 0; 663ad757b6aSThomas Gleixner for (tmp = 0; tmp < max_low_pfn; tmp++) 664ad757b6aSThomas Gleixner /* 665ad757b6aSThomas Gleixner * Only count reserved RAM pages 666ad757b6aSThomas Gleixner */ 667ad757b6aSThomas Gleixner if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp))) 668ad757b6aSThomas Gleixner reservedpages++; 669ad757b6aSThomas Gleixner 670ad757b6aSThomas Gleixner set_highmem_pages_init(bad_ppro); 671ad757b6aSThomas Gleixner 672ad757b6aSThomas Gleixner codesize = (unsigned long) &_etext - (unsigned long) &_text; 673ad757b6aSThomas Gleixner datasize = (unsigned long) &_edata - (unsigned long) &_etext; 674ad757b6aSThomas Gleixner initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; 675ad757b6aSThomas Gleixner 676ad757b6aSThomas Gleixner kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT); 677ad757b6aSThomas Gleixner kclist_add(&kcore_vmalloc, (void *)VMALLOC_START, 678ad757b6aSThomas Gleixner VMALLOC_END-VMALLOC_START); 679ad757b6aSThomas Gleixner 680ad757b6aSThomas Gleixner printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, %dk reserved, %dk data, %dk init, %ldk highmem)\n", 681ad757b6aSThomas Gleixner (unsigned long) nr_free_pages() << (PAGE_SHIFT-10), 682ad757b6aSThomas Gleixner num_physpages << (PAGE_SHIFT-10), 683ad757b6aSThomas Gleixner codesize >> 10, 684ad757b6aSThomas Gleixner reservedpages << (PAGE_SHIFT-10), 685ad757b6aSThomas Gleixner datasize >> 10, 686ad757b6aSThomas Gleixner initsize >> 10, 687ad757b6aSThomas Gleixner (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)) 688ad757b6aSThomas Gleixner ); 689ad757b6aSThomas Gleixner 690ad757b6aSThomas Gleixner #if 1 /* double-sanity-check paranoia */ 691ad757b6aSThomas Gleixner printk("virtual kernel memory layout:\n" 692ad757b6aSThomas Gleixner " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" 693ad757b6aSThomas Gleixner #ifdef CONFIG_HIGHMEM 694ad757b6aSThomas Gleixner " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n" 695ad757b6aSThomas Gleixner #endif 696ad757b6aSThomas Gleixner " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" 697ad757b6aSThomas Gleixner " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" 698ad757b6aSThomas Gleixner " .init : 0x%08lx - 0x%08lx (%4ld kB)\n" 699ad757b6aSThomas Gleixner " .data : 0x%08lx - 0x%08lx (%4ld kB)\n" 700ad757b6aSThomas Gleixner " .text : 0x%08lx - 0x%08lx (%4ld kB)\n", 701ad757b6aSThomas Gleixner FIXADDR_START, FIXADDR_TOP, 702ad757b6aSThomas Gleixner (FIXADDR_TOP - FIXADDR_START) >> 10, 703ad757b6aSThomas Gleixner 704ad757b6aSThomas Gleixner #ifdef CONFIG_HIGHMEM 705ad757b6aSThomas Gleixner PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, 706ad757b6aSThomas Gleixner (LAST_PKMAP*PAGE_SIZE) >> 10, 707ad757b6aSThomas Gleixner #endif 708ad757b6aSThomas Gleixner 709ad757b6aSThomas Gleixner VMALLOC_START, VMALLOC_END, 710ad757b6aSThomas Gleixner (VMALLOC_END - VMALLOC_START) >> 20, 711ad757b6aSThomas Gleixner 712ad757b6aSThomas Gleixner (unsigned long)__va(0), (unsigned long)high_memory, 713ad757b6aSThomas Gleixner ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20, 714ad757b6aSThomas Gleixner 715ad757b6aSThomas Gleixner (unsigned long)&__init_begin, (unsigned long)&__init_end, 716ad757b6aSThomas Gleixner ((unsigned long)&__init_end - (unsigned long)&__init_begin) >> 10, 717ad757b6aSThomas Gleixner 718ad757b6aSThomas Gleixner (unsigned long)&_etext, (unsigned long)&_edata, 719ad757b6aSThomas Gleixner ((unsigned long)&_edata - (unsigned long)&_etext) >> 10, 720ad757b6aSThomas Gleixner 721ad757b6aSThomas Gleixner (unsigned long)&_text, (unsigned long)&_etext, 722ad757b6aSThomas Gleixner ((unsigned long)&_etext - (unsigned long)&_text) >> 10); 723ad757b6aSThomas Gleixner 724ad757b6aSThomas Gleixner #ifdef CONFIG_HIGHMEM 725ad757b6aSThomas Gleixner BUG_ON(PKMAP_BASE+LAST_PKMAP*PAGE_SIZE > FIXADDR_START); 726ad757b6aSThomas Gleixner BUG_ON(VMALLOC_END > PKMAP_BASE); 727ad757b6aSThomas Gleixner #endif 728ad757b6aSThomas Gleixner BUG_ON(VMALLOC_START > VMALLOC_END); 729ad757b6aSThomas Gleixner BUG_ON((unsigned long)high_memory > VMALLOC_START); 730ad757b6aSThomas Gleixner #endif /* double-sanity-check paranoia */ 731ad757b6aSThomas Gleixner 732ad757b6aSThomas Gleixner #ifdef CONFIG_X86_PAE 733ad757b6aSThomas Gleixner if (!cpu_has_pae) 734ad757b6aSThomas Gleixner panic("cannot execute a PAE-enabled kernel on a PAE-less CPU!"); 735ad757b6aSThomas Gleixner #endif 736ad757b6aSThomas Gleixner if (boot_cpu_data.wp_works_ok < 0) 737ad757b6aSThomas Gleixner test_wp_bit(); 738ad757b6aSThomas Gleixner 739ad757b6aSThomas Gleixner /* 740ad757b6aSThomas Gleixner * Subtle. SMP is doing it's boot stuff late (because it has to 741ad757b6aSThomas Gleixner * fork idle threads) - but it also needs low mappings for the 742ad757b6aSThomas Gleixner * protected-mode entry to work. We zap these entries only after 743ad757b6aSThomas Gleixner * the WP-bit has been tested. 744ad757b6aSThomas Gleixner */ 745ad757b6aSThomas Gleixner #ifndef CONFIG_SMP 746ad757b6aSThomas Gleixner zap_low_mappings(); 747ad757b6aSThomas Gleixner #endif 748ad757b6aSThomas Gleixner } 749ad757b6aSThomas Gleixner 750ad757b6aSThomas Gleixner #ifdef CONFIG_MEMORY_HOTPLUG 751ad757b6aSThomas Gleixner int arch_add_memory(int nid, u64 start, u64 size) 752ad757b6aSThomas Gleixner { 753ad757b6aSThomas Gleixner struct pglist_data *pgdata = NODE_DATA(nid); 754ad757b6aSThomas Gleixner struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM; 755ad757b6aSThomas Gleixner unsigned long start_pfn = start >> PAGE_SHIFT; 756ad757b6aSThomas Gleixner unsigned long nr_pages = size >> PAGE_SHIFT; 757ad757b6aSThomas Gleixner 758ad757b6aSThomas Gleixner return __add_pages(zone, start_pfn, nr_pages); 759ad757b6aSThomas Gleixner } 760ad757b6aSThomas Gleixner 761ad757b6aSThomas Gleixner #endif 762ad757b6aSThomas Gleixner 763ad757b6aSThomas Gleixner struct kmem_cache *pmd_cache; 764ad757b6aSThomas Gleixner 765ad757b6aSThomas Gleixner void __init pgtable_cache_init(void) 766ad757b6aSThomas Gleixner { 7674f817847SJeremy Fitzhardinge if (PTRS_PER_PMD > 1) 768ad757b6aSThomas Gleixner pmd_cache = kmem_cache_create("pmd", 769ad757b6aSThomas Gleixner PTRS_PER_PMD*sizeof(pmd_t), 770ad757b6aSThomas Gleixner PTRS_PER_PMD*sizeof(pmd_t), 771ad757b6aSThomas Gleixner SLAB_PANIC, 772ad757b6aSThomas Gleixner pmd_ctor); 773ad757b6aSThomas Gleixner } 774ad757b6aSThomas Gleixner 775ad757b6aSThomas Gleixner /* 776ad757b6aSThomas Gleixner * This function cannot be __init, since exceptions don't work in that 777ad757b6aSThomas Gleixner * section. Put this after the callers, so that it cannot be inlined. 778ad757b6aSThomas Gleixner */ 779ad757b6aSThomas Gleixner static int noinline do_test_wp_bit(void) 780ad757b6aSThomas Gleixner { 781ad757b6aSThomas Gleixner char tmp_reg; 782ad757b6aSThomas Gleixner int flag; 783ad757b6aSThomas Gleixner 784ad757b6aSThomas Gleixner __asm__ __volatile__( 785ad757b6aSThomas Gleixner " movb %0,%1 \n" 786ad757b6aSThomas Gleixner "1: movb %1,%0 \n" 787ad757b6aSThomas Gleixner " xorl %2,%2 \n" 788ad757b6aSThomas Gleixner "2: \n" 789ad757b6aSThomas Gleixner ".section __ex_table,\"a\"\n" 790ad757b6aSThomas Gleixner " .align 4 \n" 791ad757b6aSThomas Gleixner " .long 1b,2b \n" 792ad757b6aSThomas Gleixner ".previous \n" 793ad757b6aSThomas Gleixner :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)), 794ad757b6aSThomas Gleixner "=q" (tmp_reg), 795ad757b6aSThomas Gleixner "=r" (flag) 796ad757b6aSThomas Gleixner :"2" (1) 797ad757b6aSThomas Gleixner :"memory"); 798ad757b6aSThomas Gleixner 799ad757b6aSThomas Gleixner return flag; 800ad757b6aSThomas Gleixner } 801ad757b6aSThomas Gleixner 802ad757b6aSThomas Gleixner #ifdef CONFIG_DEBUG_RODATA 803ad757b6aSThomas Gleixner 804ad757b6aSThomas Gleixner void mark_rodata_ro(void) 805ad757b6aSThomas Gleixner { 806ad757b6aSThomas Gleixner unsigned long start = PFN_ALIGN(_text); 807ad757b6aSThomas Gleixner unsigned long size = PFN_ALIGN(_etext) - start; 808ad757b6aSThomas Gleixner 809ad757b6aSThomas Gleixner #ifndef CONFIG_KPROBES 810ad757b6aSThomas Gleixner #ifdef CONFIG_HOTPLUG_CPU 811ad757b6aSThomas Gleixner /* It must still be possible to apply SMP alternatives. */ 812ad757b6aSThomas Gleixner if (num_possible_cpus() <= 1) 813ad757b6aSThomas Gleixner #endif 814ad757b6aSThomas Gleixner { 815ad757b6aSThomas Gleixner change_page_attr(virt_to_page(start), 816ad757b6aSThomas Gleixner size >> PAGE_SHIFT, PAGE_KERNEL_RX); 817ad757b6aSThomas Gleixner printk("Write protecting the kernel text: %luk\n", size >> 10); 818ad757b6aSThomas Gleixner } 819ad757b6aSThomas Gleixner #endif 820ad757b6aSThomas Gleixner start += size; 821ad757b6aSThomas Gleixner size = (unsigned long)__end_rodata - start; 822ad757b6aSThomas Gleixner change_page_attr(virt_to_page(start), 823ad757b6aSThomas Gleixner size >> PAGE_SHIFT, PAGE_KERNEL_RO); 824ad757b6aSThomas Gleixner printk("Write protecting the kernel read-only data: %luk\n", 825ad757b6aSThomas Gleixner size >> 10); 826ad757b6aSThomas Gleixner 827ad757b6aSThomas Gleixner /* 828ad757b6aSThomas Gleixner * change_page_attr() requires a global_flush_tlb() call after it. 829ad757b6aSThomas Gleixner * We do this after the printk so that if something went wrong in the 830ad757b6aSThomas Gleixner * change, the printk gets out at least to give a better debug hint 831ad757b6aSThomas Gleixner * of who is the culprit. 832ad757b6aSThomas Gleixner */ 833ad757b6aSThomas Gleixner global_flush_tlb(); 834ad757b6aSThomas Gleixner } 835ad757b6aSThomas Gleixner #endif 836ad757b6aSThomas Gleixner 837ad757b6aSThomas Gleixner void free_init_pages(char *what, unsigned long begin, unsigned long end) 838ad757b6aSThomas Gleixner { 839ad757b6aSThomas Gleixner unsigned long addr; 840ad757b6aSThomas Gleixner 841ad757b6aSThomas Gleixner for (addr = begin; addr < end; addr += PAGE_SIZE) { 842ad757b6aSThomas Gleixner ClearPageReserved(virt_to_page(addr)); 843ad757b6aSThomas Gleixner init_page_count(virt_to_page(addr)); 844ad757b6aSThomas Gleixner memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 845ad757b6aSThomas Gleixner free_page(addr); 846ad757b6aSThomas Gleixner totalram_pages++; 847ad757b6aSThomas Gleixner } 848ad757b6aSThomas Gleixner printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 849ad757b6aSThomas Gleixner } 850ad757b6aSThomas Gleixner 851ad757b6aSThomas Gleixner void free_initmem(void) 852ad757b6aSThomas Gleixner { 853ad757b6aSThomas Gleixner free_init_pages("unused kernel memory", 854ad757b6aSThomas Gleixner (unsigned long)(&__init_begin), 855ad757b6aSThomas Gleixner (unsigned long)(&__init_end)); 856ad757b6aSThomas Gleixner } 857ad757b6aSThomas Gleixner 858ad757b6aSThomas Gleixner #ifdef CONFIG_BLK_DEV_INITRD 859ad757b6aSThomas Gleixner void free_initrd_mem(unsigned long start, unsigned long end) 860ad757b6aSThomas Gleixner { 861ad757b6aSThomas Gleixner free_init_pages("initrd memory", start, end); 862ad757b6aSThomas Gleixner } 863ad757b6aSThomas Gleixner #endif 864ad757b6aSThomas Gleixner 865