1 #include <linux/initrd.h> 2 #include <linux/ioport.h> 3 #include <linux/swap.h> 4 5 #include <asm/cacheflush.h> 6 #include <asm/e820.h> 7 #include <asm/init.h> 8 #include <asm/page.h> 9 #include <asm/page_types.h> 10 #include <asm/sections.h> 11 #include <asm/setup.h> 12 #include <asm/system.h> 13 #include <asm/tlbflush.h> 14 #include <asm/tlb.h> 15 #include <asm/proto.h> 16 17 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 18 19 unsigned long __initdata e820_table_start; 20 unsigned long __meminitdata e820_table_end; 21 unsigned long __meminitdata e820_table_top; 22 23 int after_bootmem; 24 25 int direct_gbpages 26 #ifdef CONFIG_DIRECT_GBPAGES 27 = 1 28 #endif 29 ; 30 31 static void __init find_early_table_space(unsigned long end, int use_pse, 32 int use_gbpages) 33 { 34 unsigned long puds, pmds, ptes, tables, start; 35 36 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; 37 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); 38 39 if (use_gbpages) { 40 unsigned long extra; 41 42 extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT); 43 pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT; 44 } else 45 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; 46 47 tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE); 48 49 if (use_pse) { 50 unsigned long extra; 51 52 extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT); 53 #ifdef CONFIG_X86_32 54 extra += PMD_SIZE; 55 #endif 56 ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; 57 } else 58 ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; 59 60 tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE); 61 62 #ifdef CONFIG_X86_32 63 /* for fixmap */ 64 tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE); 65 #endif 66 67 /* 68 * RED-PEN putting page tables only on node 0 could 69 * cause a hotspot and fill up ZONE_DMA. The page tables 70 * need roughly 0.5KB per GB. 71 */ 72 #ifdef CONFIG_X86_32 73 start = 0x7000; 74 #else 75 start = 0x8000; 76 #endif 77 e820_table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT, 78 tables, PAGE_SIZE); 79 if (e820_table_start == -1UL) 80 panic("Cannot find space for the kernel page tables"); 81 82 e820_table_start >>= PAGE_SHIFT; 83 e820_table_end = e820_table_start; 84 e820_table_top = e820_table_start + (tables >> PAGE_SHIFT); 85 86 printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n", 87 end, e820_table_start << PAGE_SHIFT, e820_table_top << PAGE_SHIFT); 88 } 89 90 struct map_range { 91 unsigned long start; 92 unsigned long end; 93 unsigned page_size_mask; 94 }; 95 96 #ifdef CONFIG_X86_32 97 #define NR_RANGE_MR 3 98 #else /* CONFIG_X86_64 */ 99 #define NR_RANGE_MR 5 100 #endif 101 102 static int __meminit save_mr(struct map_range *mr, int nr_range, 103 unsigned long start_pfn, unsigned long end_pfn, 104 unsigned long page_size_mask) 105 { 106 if (start_pfn < end_pfn) { 107 if (nr_range >= NR_RANGE_MR) 108 panic("run out of range for init_memory_mapping\n"); 109 mr[nr_range].start = start_pfn<<PAGE_SHIFT; 110 mr[nr_range].end = end_pfn<<PAGE_SHIFT; 111 mr[nr_range].page_size_mask = page_size_mask; 112 nr_range++; 113 } 114 115 return nr_range; 116 } 117 118 /* 119 * Setup the direct mapping of the physical memory at PAGE_OFFSET. 120 * This runs before bootmem is initialized and gets pages directly from 121 * the physical memory. To access them they are temporarily mapped. 122 */ 123 unsigned long __init_refok init_memory_mapping(unsigned long start, 124 unsigned long end) 125 { 126 unsigned long page_size_mask = 0; 127 unsigned long start_pfn, end_pfn; 128 unsigned long ret = 0; 129 unsigned long pos; 130 131 struct map_range mr[NR_RANGE_MR]; 132 int nr_range, i; 133 int use_pse, use_gbpages; 134 135 printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end); 136 137 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK) 138 /* 139 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 140 * This will simplify cpa(), which otherwise needs to support splitting 141 * large pages into small in interrupt context, etc. 142 */ 143 use_pse = use_gbpages = 0; 144 #else 145 use_pse = cpu_has_pse; 146 use_gbpages = direct_gbpages; 147 #endif 148 149 set_nx(); 150 if (nx_enabled) 151 printk(KERN_INFO "NX (Execute Disable) protection: active\n"); 152 153 /* Enable PSE if available */ 154 if (cpu_has_pse) 155 set_in_cr4(X86_CR4_PSE); 156 157 /* Enable PGE if available */ 158 if (cpu_has_pge) { 159 set_in_cr4(X86_CR4_PGE); 160 __supported_pte_mask |= _PAGE_GLOBAL; 161 } 162 163 if (use_gbpages) 164 page_size_mask |= 1 << PG_LEVEL_1G; 165 if (use_pse) 166 page_size_mask |= 1 << PG_LEVEL_2M; 167 168 memset(mr, 0, sizeof(mr)); 169 nr_range = 0; 170 171 /* head if not big page alignment ? */ 172 start_pfn = start >> PAGE_SHIFT; 173 pos = start_pfn << PAGE_SHIFT; 174 #ifdef CONFIG_X86_32 175 /* 176 * Don't use a large page for the first 2/4MB of memory 177 * because there are often fixed size MTRRs in there 178 * and overlapping MTRRs into large pages can cause 179 * slowdowns. 180 */ 181 if (pos == 0) 182 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT); 183 else 184 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 185 << (PMD_SHIFT - PAGE_SHIFT); 186 #else /* CONFIG_X86_64 */ 187 end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT) 188 << (PMD_SHIFT - PAGE_SHIFT); 189 #endif 190 if (end_pfn > (end >> PAGE_SHIFT)) 191 end_pfn = end >> PAGE_SHIFT; 192 if (start_pfn < end_pfn) { 193 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 194 pos = end_pfn << PAGE_SHIFT; 195 } 196 197 /* big page (2M) range */ 198 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 199 << (PMD_SHIFT - PAGE_SHIFT); 200 #ifdef CONFIG_X86_32 201 end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 202 #else /* CONFIG_X86_64 */ 203 end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 204 << (PUD_SHIFT - PAGE_SHIFT); 205 if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT))) 206 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)); 207 #endif 208 209 if (start_pfn < end_pfn) { 210 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 211 page_size_mask & (1<<PG_LEVEL_2M)); 212 pos = end_pfn << PAGE_SHIFT; 213 } 214 215 #ifdef CONFIG_X86_64 216 /* big page (1G) range */ 217 start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 218 << (PUD_SHIFT - PAGE_SHIFT); 219 end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); 220 if (start_pfn < end_pfn) { 221 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 222 page_size_mask & 223 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 224 pos = end_pfn << PAGE_SHIFT; 225 } 226 227 /* tail is not big page (1G) alignment */ 228 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 229 << (PMD_SHIFT - PAGE_SHIFT); 230 end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 231 if (start_pfn < end_pfn) { 232 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 233 page_size_mask & (1<<PG_LEVEL_2M)); 234 pos = end_pfn << PAGE_SHIFT; 235 } 236 #endif 237 238 /* tail is not big page (2M) alignment */ 239 start_pfn = pos>>PAGE_SHIFT; 240 end_pfn = end>>PAGE_SHIFT; 241 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 242 243 /* try to merge same page size and continuous */ 244 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 245 unsigned long old_start; 246 if (mr[i].end != mr[i+1].start || 247 mr[i].page_size_mask != mr[i+1].page_size_mask) 248 continue; 249 /* move it */ 250 old_start = mr[i].start; 251 memmove(&mr[i], &mr[i+1], 252 (nr_range - 1 - i) * sizeof(struct map_range)); 253 mr[i--].start = old_start; 254 nr_range--; 255 } 256 257 for (i = 0; i < nr_range; i++) 258 printk(KERN_DEBUG " %010lx - %010lx page %s\n", 259 mr[i].start, mr[i].end, 260 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 261 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 262 263 /* 264 * Find space for the kernel direct mapping tables. 265 * 266 * Later we should allocate these tables in the local node of the 267 * memory mapped. Unfortunately this is done currently before the 268 * nodes are discovered. 269 */ 270 if (!after_bootmem) 271 find_early_table_space(end, use_pse, use_gbpages); 272 273 #ifdef CONFIG_X86_32 274 for (i = 0; i < nr_range; i++) 275 kernel_physical_mapping_init(mr[i].start, mr[i].end, 276 mr[i].page_size_mask); 277 ret = end; 278 #else /* CONFIG_X86_64 */ 279 for (i = 0; i < nr_range; i++) 280 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 281 mr[i].page_size_mask); 282 #endif 283 284 #ifdef CONFIG_X86_32 285 early_ioremap_page_table_range_init(); 286 287 load_cr3(swapper_pg_dir); 288 #endif 289 290 #ifdef CONFIG_X86_64 291 if (!after_bootmem && !start) { 292 pud_t *pud; 293 pmd_t *pmd; 294 295 mmu_cr4_features = read_cr4(); 296 297 /* 298 * _brk_end cannot change anymore, but it and _end may be 299 * located on different 2M pages. cleanup_highmap(), however, 300 * can only consider _end when it runs, so destroy any 301 * mappings beyond _brk_end here. 302 */ 303 pud = pud_offset(pgd_offset_k(_brk_end), _brk_end); 304 pmd = pmd_offset(pud, _brk_end - 1); 305 while (++pmd <= pmd_offset(pud, (unsigned long)_end - 1)) 306 pmd_clear(pmd); 307 } 308 #endif 309 __flush_tlb_all(); 310 311 if (!after_bootmem && e820_table_end > e820_table_start) 312 reserve_early(e820_table_start << PAGE_SHIFT, 313 e820_table_end << PAGE_SHIFT, "PGTABLE"); 314 315 if (!after_bootmem) 316 early_memtest(start, end); 317 318 return ret >> PAGE_SHIFT; 319 } 320 321 322 /* 323 * devmem_is_allowed() checks to see if /dev/mem access to a certain address 324 * is valid. The argument is a physical page number. 325 * 326 * 327 * On x86, access has to be given to the first megabyte of ram because that area 328 * contains bios code and data regions used by X and dosemu and similar apps. 329 * Access has to be given to non-kernel-ram areas as well, these contain the PCI 330 * mmio resources as well as potential bios/acpi data regions. 331 */ 332 int devmem_is_allowed(unsigned long pagenr) 333 { 334 if (pagenr <= 256) 335 return 1; 336 if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 337 return 0; 338 if (!page_is_ram(pagenr)) 339 return 1; 340 return 0; 341 } 342 343 void free_init_pages(char *what, unsigned long begin, unsigned long end) 344 { 345 unsigned long addr = begin; 346 347 if (addr >= end) 348 return; 349 350 /* 351 * If debugging page accesses then do not free this memory but 352 * mark them not present - any buggy init-section access will 353 * create a kernel page fault: 354 */ 355 #ifdef CONFIG_DEBUG_PAGEALLOC 356 printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n", 357 begin, PAGE_ALIGN(end)); 358 set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 359 #else 360 /* 361 * We just marked the kernel text read only above, now that 362 * we are going to free part of that, we need to make that 363 * writeable first. 364 */ 365 set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 366 367 printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 368 369 for (; addr < end; addr += PAGE_SIZE) { 370 ClearPageReserved(virt_to_page(addr)); 371 init_page_count(virt_to_page(addr)); 372 memset((void *)(addr & ~(PAGE_SIZE-1)), 373 POISON_FREE_INITMEM, PAGE_SIZE); 374 free_page(addr); 375 totalram_pages++; 376 } 377 #endif 378 } 379 380 void free_initmem(void) 381 { 382 free_init_pages("unused kernel memory", 383 (unsigned long)(&__init_begin), 384 (unsigned long)(&__init_end)); 385 } 386 387 #ifdef CONFIG_BLK_DEV_INITRD 388 void free_initrd_mem(unsigned long start, unsigned long end) 389 { 390 free_init_pages("initrd memory", start, end); 391 } 392 #endif 393