1 /* 2 * linux/arch/arm/mm/mmu.c 3 * 4 * Copyright (C) 1995-2005 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/module.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/init.h> 14 #include <linux/mman.h> 15 #include <linux/nodemask.h> 16 #include <linux/memblock.h> 17 #include <linux/fs.h> 18 #include <linux/vmalloc.h> 19 #include <linux/sizes.h> 20 21 #include <asm/cp15.h> 22 #include <asm/cputype.h> 23 #include <asm/sections.h> 24 #include <asm/cachetype.h> 25 #include <asm/setup.h> 26 #include <asm/smp_plat.h> 27 #include <asm/tlb.h> 28 #include <asm/highmem.h> 29 #include <asm/system_info.h> 30 #include <asm/traps.h> 31 32 #include <asm/mach/arch.h> 33 #include <asm/mach/map.h> 34 #include <asm/mach/pci.h> 35 36 #include "mm.h" 37 38 /* 39 * empty_zero_page is a special page that is used for 40 * zero-initialized data and COW. 41 */ 42 struct page *empty_zero_page; 43 EXPORT_SYMBOL(empty_zero_page); 44 45 /* 46 * The pmd table for the upper-most set of pages. 47 */ 48 pmd_t *top_pmd; 49 50 #define CPOLICY_UNCACHED 0 51 #define CPOLICY_BUFFERED 1 52 #define CPOLICY_WRITETHROUGH 2 53 #define CPOLICY_WRITEBACK 3 54 #define CPOLICY_WRITEALLOC 4 55 56 static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK; 57 static unsigned int ecc_mask __initdata = 0; 58 pgprot_t pgprot_user; 59 pgprot_t pgprot_kernel; 60 61 EXPORT_SYMBOL(pgprot_user); 62 EXPORT_SYMBOL(pgprot_kernel); 63 64 struct cachepolicy { 65 const char policy[16]; 66 unsigned int cr_mask; 67 pmdval_t pmd; 68 pteval_t pte; 69 }; 70 71 static struct cachepolicy cache_policies[] __initdata = { 72 { 73 .policy = "uncached", 74 .cr_mask = CR_W|CR_C, 75 .pmd = PMD_SECT_UNCACHED, 76 .pte = L_PTE_MT_UNCACHED, 77 }, { 78 .policy = "buffered", 79 .cr_mask = CR_C, 80 .pmd = PMD_SECT_BUFFERED, 81 .pte = L_PTE_MT_BUFFERABLE, 82 }, { 83 .policy = "writethrough", 84 .cr_mask = 0, 85 .pmd = PMD_SECT_WT, 86 .pte = L_PTE_MT_WRITETHROUGH, 87 }, { 88 .policy = "writeback", 89 .cr_mask = 0, 90 .pmd = PMD_SECT_WB, 91 .pte = L_PTE_MT_WRITEBACK, 92 }, { 93 .policy = "writealloc", 94 .cr_mask = 0, 95 .pmd = PMD_SECT_WBWA, 96 .pte = L_PTE_MT_WRITEALLOC, 97 } 98 }; 99 100 /* 101 * These are useful for identifying cache coherency 102 * problems by allowing the cache or the cache and 103 * writebuffer to be turned off. (Note: the write 104 * buffer should not be on and the cache off). 105 */ 106 static int __init early_cachepolicy(char *p) 107 { 108 int i; 109 110 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) { 111 int len = strlen(cache_policies[i].policy); 112 113 if (memcmp(p, cache_policies[i].policy, len) == 0) { 114 cachepolicy = i; 115 cr_alignment &= ~cache_policies[i].cr_mask; 116 cr_no_alignment &= ~cache_policies[i].cr_mask; 117 break; 118 } 119 } 120 if (i == ARRAY_SIZE(cache_policies)) 121 printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n"); 122 /* 123 * This restriction is partly to do with the way we boot; it is 124 * unpredictable to have memory mapped using two different sets of 125 * memory attributes (shared, type, and cache attribs). We can not 126 * change these attributes once the initial assembly has setup the 127 * page tables. 128 */ 129 if (cpu_architecture() >= CPU_ARCH_ARMv6) { 130 printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n"); 131 cachepolicy = CPOLICY_WRITEBACK; 132 } 133 flush_cache_all(); 134 set_cr(cr_alignment); 135 return 0; 136 } 137 early_param("cachepolicy", early_cachepolicy); 138 139 static int __init early_nocache(char *__unused) 140 { 141 char *p = "buffered"; 142 printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p); 143 early_cachepolicy(p); 144 return 0; 145 } 146 early_param("nocache", early_nocache); 147 148 static int __init early_nowrite(char *__unused) 149 { 150 char *p = "uncached"; 151 printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p); 152 early_cachepolicy(p); 153 return 0; 154 } 155 early_param("nowb", early_nowrite); 156 157 #ifndef CONFIG_ARM_LPAE 158 static int __init early_ecc(char *p) 159 { 160 if (memcmp(p, "on", 2) == 0) 161 ecc_mask = PMD_PROTECTION; 162 else if (memcmp(p, "off", 3) == 0) 163 ecc_mask = 0; 164 return 0; 165 } 166 early_param("ecc", early_ecc); 167 #endif 168 169 static int __init noalign_setup(char *__unused) 170 { 171 cr_alignment &= ~CR_A; 172 cr_no_alignment &= ~CR_A; 173 set_cr(cr_alignment); 174 return 1; 175 } 176 __setup("noalign", noalign_setup); 177 178 #ifndef CONFIG_SMP 179 void adjust_cr(unsigned long mask, unsigned long set) 180 { 181 unsigned long flags; 182 183 mask &= ~CR_A; 184 185 set &= mask; 186 187 local_irq_save(flags); 188 189 cr_no_alignment = (cr_no_alignment & ~mask) | set; 190 cr_alignment = (cr_alignment & ~mask) | set; 191 192 set_cr((get_cr() & ~mask) | set); 193 194 local_irq_restore(flags); 195 } 196 #endif 197 198 #define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN 199 #define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITE 200 201 static struct mem_type mem_types[] = { 202 [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */ 203 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED | 204 L_PTE_SHARED, 205 .prot_l1 = PMD_TYPE_TABLE, 206 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S, 207 .domain = DOMAIN_IO, 208 }, 209 [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */ 210 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED, 211 .prot_l1 = PMD_TYPE_TABLE, 212 .prot_sect = PROT_SECT_DEVICE, 213 .domain = DOMAIN_IO, 214 }, 215 [MT_DEVICE_CACHED] = { /* ioremap_cached */ 216 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED, 217 .prot_l1 = PMD_TYPE_TABLE, 218 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_WB, 219 .domain = DOMAIN_IO, 220 }, 221 [MT_DEVICE_WC] = { /* ioremap_wc */ 222 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC, 223 .prot_l1 = PMD_TYPE_TABLE, 224 .prot_sect = PROT_SECT_DEVICE, 225 .domain = DOMAIN_IO, 226 }, 227 [MT_UNCACHED] = { 228 .prot_pte = PROT_PTE_DEVICE, 229 .prot_l1 = PMD_TYPE_TABLE, 230 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, 231 .domain = DOMAIN_IO, 232 }, 233 [MT_CACHECLEAN] = { 234 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, 235 .domain = DOMAIN_KERNEL, 236 }, 237 #ifndef CONFIG_ARM_LPAE 238 [MT_MINICLEAN] = { 239 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE, 240 .domain = DOMAIN_KERNEL, 241 }, 242 #endif 243 [MT_LOW_VECTORS] = { 244 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 245 L_PTE_RDONLY, 246 .prot_l1 = PMD_TYPE_TABLE, 247 .domain = DOMAIN_USER, 248 }, 249 [MT_HIGH_VECTORS] = { 250 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 251 L_PTE_USER | L_PTE_RDONLY, 252 .prot_l1 = PMD_TYPE_TABLE, 253 .domain = DOMAIN_USER, 254 }, 255 [MT_MEMORY] = { 256 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY, 257 .prot_l1 = PMD_TYPE_TABLE, 258 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE, 259 .domain = DOMAIN_KERNEL, 260 }, 261 [MT_ROM] = { 262 .prot_sect = PMD_TYPE_SECT, 263 .domain = DOMAIN_KERNEL, 264 }, 265 [MT_MEMORY_NONCACHED] = { 266 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 267 L_PTE_MT_BUFFERABLE, 268 .prot_l1 = PMD_TYPE_TABLE, 269 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE, 270 .domain = DOMAIN_KERNEL, 271 }, 272 [MT_MEMORY_DTCM] = { 273 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 274 L_PTE_XN, 275 .prot_l1 = PMD_TYPE_TABLE, 276 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, 277 .domain = DOMAIN_KERNEL, 278 }, 279 [MT_MEMORY_ITCM] = { 280 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY, 281 .prot_l1 = PMD_TYPE_TABLE, 282 .domain = DOMAIN_KERNEL, 283 }, 284 [MT_MEMORY_SO] = { 285 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 286 L_PTE_MT_UNCACHED, 287 .prot_l1 = PMD_TYPE_TABLE, 288 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S | 289 PMD_SECT_UNCACHED | PMD_SECT_XN, 290 .domain = DOMAIN_KERNEL, 291 }, 292 [MT_MEMORY_DMA_READY] = { 293 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY, 294 .prot_l1 = PMD_TYPE_TABLE, 295 .domain = DOMAIN_KERNEL, 296 }, 297 }; 298 299 const struct mem_type *get_mem_type(unsigned int type) 300 { 301 return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL; 302 } 303 EXPORT_SYMBOL(get_mem_type); 304 305 /* 306 * Adjust the PMD section entries according to the CPU in use. 307 */ 308 static void __init build_mem_type_table(void) 309 { 310 struct cachepolicy *cp; 311 unsigned int cr = get_cr(); 312 pteval_t user_pgprot, kern_pgprot, vecs_pgprot; 313 int cpu_arch = cpu_architecture(); 314 int i; 315 316 if (cpu_arch < CPU_ARCH_ARMv6) { 317 #if defined(CONFIG_CPU_DCACHE_DISABLE) 318 if (cachepolicy > CPOLICY_BUFFERED) 319 cachepolicy = CPOLICY_BUFFERED; 320 #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH) 321 if (cachepolicy > CPOLICY_WRITETHROUGH) 322 cachepolicy = CPOLICY_WRITETHROUGH; 323 #endif 324 } 325 if (cpu_arch < CPU_ARCH_ARMv5) { 326 if (cachepolicy >= CPOLICY_WRITEALLOC) 327 cachepolicy = CPOLICY_WRITEBACK; 328 ecc_mask = 0; 329 } 330 if (is_smp()) 331 cachepolicy = CPOLICY_WRITEALLOC; 332 333 /* 334 * Strip out features not present on earlier architectures. 335 * Pre-ARMv5 CPUs don't have TEX bits. Pre-ARMv6 CPUs or those 336 * without extended page tables don't have the 'Shared' bit. 337 */ 338 if (cpu_arch < CPU_ARCH_ARMv5) 339 for (i = 0; i < ARRAY_SIZE(mem_types); i++) 340 mem_types[i].prot_sect &= ~PMD_SECT_TEX(7); 341 if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3()) 342 for (i = 0; i < ARRAY_SIZE(mem_types); i++) 343 mem_types[i].prot_sect &= ~PMD_SECT_S; 344 345 /* 346 * ARMv5 and lower, bit 4 must be set for page tables (was: cache 347 * "update-able on write" bit on ARM610). However, Xscale and 348 * Xscale3 require this bit to be cleared. 349 */ 350 if (cpu_is_xscale() || cpu_is_xsc3()) { 351 for (i = 0; i < ARRAY_SIZE(mem_types); i++) { 352 mem_types[i].prot_sect &= ~PMD_BIT4; 353 mem_types[i].prot_l1 &= ~PMD_BIT4; 354 } 355 } else if (cpu_arch < CPU_ARCH_ARMv6) { 356 for (i = 0; i < ARRAY_SIZE(mem_types); i++) { 357 if (mem_types[i].prot_l1) 358 mem_types[i].prot_l1 |= PMD_BIT4; 359 if (mem_types[i].prot_sect) 360 mem_types[i].prot_sect |= PMD_BIT4; 361 } 362 } 363 364 /* 365 * Mark the device areas according to the CPU/architecture. 366 */ 367 if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) { 368 if (!cpu_is_xsc3()) { 369 /* 370 * Mark device regions on ARMv6+ as execute-never 371 * to prevent speculative instruction fetches. 372 */ 373 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN; 374 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN; 375 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN; 376 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN; 377 } 378 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) { 379 /* 380 * For ARMv7 with TEX remapping, 381 * - shared device is SXCB=1100 382 * - nonshared device is SXCB=0100 383 * - write combine device mem is SXCB=0001 384 * (Uncached Normal memory) 385 */ 386 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1); 387 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1); 388 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE; 389 } else if (cpu_is_xsc3()) { 390 /* 391 * For Xscale3, 392 * - shared device is TEXCB=00101 393 * - nonshared device is TEXCB=01000 394 * - write combine device mem is TEXCB=00100 395 * (Inner/Outer Uncacheable in xsc3 parlance) 396 */ 397 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED; 398 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2); 399 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1); 400 } else { 401 /* 402 * For ARMv6 and ARMv7 without TEX remapping, 403 * - shared device is TEXCB=00001 404 * - nonshared device is TEXCB=01000 405 * - write combine device mem is TEXCB=00100 406 * (Uncached Normal in ARMv6 parlance). 407 */ 408 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED; 409 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2); 410 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1); 411 } 412 } else { 413 /* 414 * On others, write combining is "Uncached/Buffered" 415 */ 416 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE; 417 } 418 419 /* 420 * Now deal with the memory-type mappings 421 */ 422 cp = &cache_policies[cachepolicy]; 423 vecs_pgprot = kern_pgprot = user_pgprot = cp->pte; 424 425 /* 426 * Enable CPU-specific coherency if supported. 427 * (Only available on XSC3 at the moment.) 428 */ 429 if (arch_is_coherent() && cpu_is_xsc3()) { 430 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; 431 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; 432 mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED; 433 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; 434 mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; 435 } 436 /* 437 * ARMv6 and above have extended page tables. 438 */ 439 if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { 440 #ifndef CONFIG_ARM_LPAE 441 /* 442 * Mark cache clean areas and XIP ROM read only 443 * from SVC mode and no access from userspace. 444 */ 445 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 446 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 447 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 448 #endif 449 450 if (is_smp()) { 451 /* 452 * Mark memory with the "shared" attribute 453 * for SMP systems 454 */ 455 user_pgprot |= L_PTE_SHARED; 456 kern_pgprot |= L_PTE_SHARED; 457 vecs_pgprot |= L_PTE_SHARED; 458 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S; 459 mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED; 460 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S; 461 mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED; 462 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; 463 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; 464 mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED; 465 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; 466 mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; 467 } 468 } 469 470 /* 471 * Non-cacheable Normal - intended for memory areas that must 472 * not cause dirty cache line writebacks when used 473 */ 474 if (cpu_arch >= CPU_ARCH_ARMv6) { 475 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) { 476 /* Non-cacheable Normal is XCB = 001 */ 477 mem_types[MT_MEMORY_NONCACHED].prot_sect |= 478 PMD_SECT_BUFFERED; 479 } else { 480 /* For both ARMv6 and non-TEX-remapping ARMv7 */ 481 mem_types[MT_MEMORY_NONCACHED].prot_sect |= 482 PMD_SECT_TEX(1); 483 } 484 } else { 485 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE; 486 } 487 488 #ifdef CONFIG_ARM_LPAE 489 /* 490 * Do not generate access flag faults for the kernel mappings. 491 */ 492 for (i = 0; i < ARRAY_SIZE(mem_types); i++) { 493 mem_types[i].prot_pte |= PTE_EXT_AF; 494 if (mem_types[i].prot_sect) 495 mem_types[i].prot_sect |= PMD_SECT_AF; 496 } 497 kern_pgprot |= PTE_EXT_AF; 498 vecs_pgprot |= PTE_EXT_AF; 499 #endif 500 501 for (i = 0; i < 16; i++) { 502 unsigned long v = pgprot_val(protection_map[i]); 503 protection_map[i] = __pgprot(v | user_pgprot); 504 } 505 506 mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot; 507 mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot; 508 509 pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot); 510 pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | 511 L_PTE_DIRTY | kern_pgprot); 512 513 mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; 514 mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; 515 mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd; 516 mem_types[MT_MEMORY].prot_pte |= kern_pgprot; 517 mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot; 518 mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask; 519 mem_types[MT_ROM].prot_sect |= cp->pmd; 520 521 switch (cp->pmd) { 522 case PMD_SECT_WT: 523 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT; 524 break; 525 case PMD_SECT_WB: 526 case PMD_SECT_WBWA: 527 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB; 528 break; 529 } 530 printk("Memory policy: ECC %sabled, Data cache %s\n", 531 ecc_mask ? "en" : "dis", cp->policy); 532 533 for (i = 0; i < ARRAY_SIZE(mem_types); i++) { 534 struct mem_type *t = &mem_types[i]; 535 if (t->prot_l1) 536 t->prot_l1 |= PMD_DOMAIN(t->domain); 537 if (t->prot_sect) 538 t->prot_sect |= PMD_DOMAIN(t->domain); 539 } 540 } 541 542 #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE 543 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 544 unsigned long size, pgprot_t vma_prot) 545 { 546 if (!pfn_valid(pfn)) 547 return pgprot_noncached(vma_prot); 548 else if (file->f_flags & O_SYNC) 549 return pgprot_writecombine(vma_prot); 550 return vma_prot; 551 } 552 EXPORT_SYMBOL(phys_mem_access_prot); 553 #endif 554 555 #define vectors_base() (vectors_high() ? 0xffff0000 : 0) 556 557 static void __init *early_alloc_aligned(unsigned long sz, unsigned long align) 558 { 559 void *ptr = __va(memblock_alloc(sz, align)); 560 memset(ptr, 0, sz); 561 return ptr; 562 } 563 564 static void __init *early_alloc(unsigned long sz) 565 { 566 return early_alloc_aligned(sz, sz); 567 } 568 569 static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot) 570 { 571 if (pmd_none(*pmd)) { 572 pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE); 573 __pmd_populate(pmd, __pa(pte), prot); 574 } 575 BUG_ON(pmd_bad(*pmd)); 576 return pte_offset_kernel(pmd, addr); 577 } 578 579 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, 580 unsigned long end, unsigned long pfn, 581 const struct mem_type *type) 582 { 583 pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1); 584 do { 585 set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0); 586 pfn++; 587 } while (pte++, addr += PAGE_SIZE, addr != end); 588 } 589 590 static void __init alloc_init_section(pud_t *pud, unsigned long addr, 591 unsigned long end, phys_addr_t phys, 592 const struct mem_type *type) 593 { 594 pmd_t *pmd = pmd_offset(pud, addr); 595 596 /* 597 * Try a section mapping - end, addr and phys must all be aligned 598 * to a section boundary. Note that PMDs refer to the individual 599 * L1 entries, whereas PGDs refer to a group of L1 entries making 600 * up one logical pointer to an L2 table. 601 */ 602 if (type->prot_sect && ((addr | end | phys) & ~SECTION_MASK) == 0) { 603 pmd_t *p = pmd; 604 605 #ifndef CONFIG_ARM_LPAE 606 if (addr & SECTION_SIZE) 607 pmd++; 608 #endif 609 610 do { 611 *pmd = __pmd(phys | type->prot_sect); 612 phys += SECTION_SIZE; 613 } while (pmd++, addr += SECTION_SIZE, addr != end); 614 615 flush_pmd_entry(p); 616 } else { 617 /* 618 * No need to loop; pte's aren't interested in the 619 * individual L1 entries. 620 */ 621 alloc_init_pte(pmd, addr, end, __phys_to_pfn(phys), type); 622 } 623 } 624 625 static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, 626 unsigned long end, unsigned long phys, const struct mem_type *type) 627 { 628 pud_t *pud = pud_offset(pgd, addr); 629 unsigned long next; 630 631 do { 632 next = pud_addr_end(addr, end); 633 alloc_init_section(pud, addr, next, phys, type); 634 phys += next - addr; 635 } while (pud++, addr = next, addr != end); 636 } 637 638 #ifndef CONFIG_ARM_LPAE 639 static void __init create_36bit_mapping(struct map_desc *md, 640 const struct mem_type *type) 641 { 642 unsigned long addr, length, end; 643 phys_addr_t phys; 644 pgd_t *pgd; 645 646 addr = md->virtual; 647 phys = __pfn_to_phys(md->pfn); 648 length = PAGE_ALIGN(md->length); 649 650 if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) { 651 printk(KERN_ERR "MM: CPU does not support supersection " 652 "mapping for 0x%08llx at 0x%08lx\n", 653 (long long)__pfn_to_phys((u64)md->pfn), addr); 654 return; 655 } 656 657 /* N.B. ARMv6 supersections are only defined to work with domain 0. 658 * Since domain assignments can in fact be arbitrary, the 659 * 'domain == 0' check below is required to insure that ARMv6 660 * supersections are only allocated for domain 0 regardless 661 * of the actual domain assignments in use. 662 */ 663 if (type->domain) { 664 printk(KERN_ERR "MM: invalid domain in supersection " 665 "mapping for 0x%08llx at 0x%08lx\n", 666 (long long)__pfn_to_phys((u64)md->pfn), addr); 667 return; 668 } 669 670 if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) { 671 printk(KERN_ERR "MM: cannot create mapping for 0x%08llx" 672 " at 0x%08lx invalid alignment\n", 673 (long long)__pfn_to_phys((u64)md->pfn), addr); 674 return; 675 } 676 677 /* 678 * Shift bits [35:32] of address into bits [23:20] of PMD 679 * (See ARMv6 spec). 680 */ 681 phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20); 682 683 pgd = pgd_offset_k(addr); 684 end = addr + length; 685 do { 686 pud_t *pud = pud_offset(pgd, addr); 687 pmd_t *pmd = pmd_offset(pud, addr); 688 int i; 689 690 for (i = 0; i < 16; i++) 691 *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER); 692 693 addr += SUPERSECTION_SIZE; 694 phys += SUPERSECTION_SIZE; 695 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT; 696 } while (addr != end); 697 } 698 #endif /* !CONFIG_ARM_LPAE */ 699 700 /* 701 * Create the page directory entries and any necessary 702 * page tables for the mapping specified by `md'. We 703 * are able to cope here with varying sizes and address 704 * offsets, and we take full advantage of sections and 705 * supersections. 706 */ 707 static void __init create_mapping(struct map_desc *md) 708 { 709 unsigned long addr, length, end; 710 phys_addr_t phys; 711 const struct mem_type *type; 712 pgd_t *pgd; 713 714 if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) { 715 printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx" 716 " at 0x%08lx in user region\n", 717 (long long)__pfn_to_phys((u64)md->pfn), md->virtual); 718 return; 719 } 720 721 if ((md->type == MT_DEVICE || md->type == MT_ROM) && 722 md->virtual >= PAGE_OFFSET && 723 (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) { 724 printk(KERN_WARNING "BUG: mapping for 0x%08llx" 725 " at 0x%08lx out of vmalloc space\n", 726 (long long)__pfn_to_phys((u64)md->pfn), md->virtual); 727 } 728 729 type = &mem_types[md->type]; 730 731 #ifndef CONFIG_ARM_LPAE 732 /* 733 * Catch 36-bit addresses 734 */ 735 if (md->pfn >= 0x100000) { 736 create_36bit_mapping(md, type); 737 return; 738 } 739 #endif 740 741 addr = md->virtual & PAGE_MASK; 742 phys = __pfn_to_phys(md->pfn); 743 length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK)); 744 745 if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) { 746 printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not " 747 "be mapped using pages, ignoring.\n", 748 (long long)__pfn_to_phys(md->pfn), addr); 749 return; 750 } 751 752 pgd = pgd_offset_k(addr); 753 end = addr + length; 754 do { 755 unsigned long next = pgd_addr_end(addr, end); 756 757 alloc_init_pud(pgd, addr, next, phys, type); 758 759 phys += next - addr; 760 addr = next; 761 } while (pgd++, addr != end); 762 } 763 764 /* 765 * Create the architecture specific mappings 766 */ 767 void __init iotable_init(struct map_desc *io_desc, int nr) 768 { 769 struct map_desc *md; 770 struct vm_struct *vm; 771 772 if (!nr) 773 return; 774 775 vm = early_alloc_aligned(sizeof(*vm) * nr, __alignof__(*vm)); 776 777 for (md = io_desc; nr; md++, nr--) { 778 create_mapping(md); 779 vm->addr = (void *)(md->virtual & PAGE_MASK); 780 vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK)); 781 vm->phys_addr = __pfn_to_phys(md->pfn); 782 vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING; 783 vm->flags |= VM_ARM_MTYPE(md->type); 784 vm->caller = iotable_init; 785 vm_area_add_early(vm++); 786 } 787 } 788 789 void __init vm_reserve_area_early(unsigned long addr, unsigned long size, 790 void *caller) 791 { 792 struct vm_struct *vm; 793 794 vm = early_alloc_aligned(sizeof(*vm), __alignof__(*vm)); 795 vm->addr = (void *)addr; 796 vm->size = size; 797 vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING; 798 vm->caller = caller; 799 vm_area_add_early(vm); 800 } 801 802 #ifndef CONFIG_ARM_LPAE 803 804 /* 805 * The Linux PMD is made of two consecutive section entries covering 2MB 806 * (see definition in include/asm/pgtable-2level.h). However a call to 807 * create_mapping() may optimize static mappings by using individual 808 * 1MB section mappings. This leaves the actual PMD potentially half 809 * initialized if the top or bottom section entry isn't used, leaving it 810 * open to problems if a subsequent ioremap() or vmalloc() tries to use 811 * the virtual space left free by that unused section entry. 812 * 813 * Let's avoid the issue by inserting dummy vm entries covering the unused 814 * PMD halves once the static mappings are in place. 815 */ 816 817 static void __init pmd_empty_section_gap(unsigned long addr) 818 { 819 vm_reserve_area_early(addr, SECTION_SIZE, pmd_empty_section_gap); 820 } 821 822 static void __init fill_pmd_gaps(void) 823 { 824 struct vm_struct *vm; 825 unsigned long addr, next = 0; 826 pmd_t *pmd; 827 828 /* we're still single threaded hence no lock needed here */ 829 for (vm = vmlist; vm; vm = vm->next) { 830 if (!(vm->flags & (VM_ARM_STATIC_MAPPING | VM_ARM_EMPTY_MAPPING))) 831 continue; 832 addr = (unsigned long)vm->addr; 833 if (addr < next) 834 continue; 835 836 /* 837 * Check if this vm starts on an odd section boundary. 838 * If so and the first section entry for this PMD is free 839 * then we block the corresponding virtual address. 840 */ 841 if ((addr & ~PMD_MASK) == SECTION_SIZE) { 842 pmd = pmd_off_k(addr); 843 if (pmd_none(*pmd)) 844 pmd_empty_section_gap(addr & PMD_MASK); 845 } 846 847 /* 848 * Then check if this vm ends on an odd section boundary. 849 * If so and the second section entry for this PMD is empty 850 * then we block the corresponding virtual address. 851 */ 852 addr += vm->size; 853 if ((addr & ~PMD_MASK) == SECTION_SIZE) { 854 pmd = pmd_off_k(addr) + 1; 855 if (pmd_none(*pmd)) 856 pmd_empty_section_gap(addr); 857 } 858 859 /* no need to look at any vm entry until we hit the next PMD */ 860 next = (addr + PMD_SIZE - 1) & PMD_MASK; 861 } 862 } 863 864 #else 865 #define fill_pmd_gaps() do { } while (0) 866 #endif 867 868 #if defined(CONFIG_PCI) && !defined(CONFIG_NEED_MACH_IO_H) 869 static void __init pci_reserve_io(void) 870 { 871 struct vm_struct *vm; 872 unsigned long addr; 873 874 /* we're still single threaded hence no lock needed here */ 875 for (vm = vmlist; vm; vm = vm->next) { 876 if (!(vm->flags & VM_ARM_STATIC_MAPPING)) 877 continue; 878 addr = (unsigned long)vm->addr; 879 addr &= ~(SZ_2M - 1); 880 if (addr == PCI_IO_VIRT_BASE) 881 return; 882 883 } 884 vm_reserve_area_early(PCI_IO_VIRT_BASE, SZ_2M, pci_reserve_io); 885 } 886 #else 887 #define pci_reserve_io() do { } while (0) 888 #endif 889 890 static void * __initdata vmalloc_min = 891 (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET); 892 893 /* 894 * vmalloc=size forces the vmalloc area to be exactly 'size' 895 * bytes. This can be used to increase (or decrease) the vmalloc 896 * area - the default is 240m. 897 */ 898 static int __init early_vmalloc(char *arg) 899 { 900 unsigned long vmalloc_reserve = memparse(arg, NULL); 901 902 if (vmalloc_reserve < SZ_16M) { 903 vmalloc_reserve = SZ_16M; 904 printk(KERN_WARNING 905 "vmalloc area too small, limiting to %luMB\n", 906 vmalloc_reserve >> 20); 907 } 908 909 if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) { 910 vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M); 911 printk(KERN_WARNING 912 "vmalloc area is too big, limiting to %luMB\n", 913 vmalloc_reserve >> 20); 914 } 915 916 vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve); 917 return 0; 918 } 919 early_param("vmalloc", early_vmalloc); 920 921 phys_addr_t arm_lowmem_limit __initdata = 0; 922 923 void __init sanity_check_meminfo(void) 924 { 925 int i, j, highmem = 0; 926 927 for (i = 0, j = 0; i < meminfo.nr_banks; i++) { 928 struct membank *bank = &meminfo.bank[j]; 929 *bank = meminfo.bank[i]; 930 931 if (bank->start > ULONG_MAX) 932 highmem = 1; 933 934 #ifdef CONFIG_HIGHMEM 935 if (__va(bank->start) >= vmalloc_min || 936 __va(bank->start) < (void *)PAGE_OFFSET) 937 highmem = 1; 938 939 bank->highmem = highmem; 940 941 /* 942 * Split those memory banks which are partially overlapping 943 * the vmalloc area greatly simplifying things later. 944 */ 945 if (!highmem && __va(bank->start) < vmalloc_min && 946 bank->size > vmalloc_min - __va(bank->start)) { 947 if (meminfo.nr_banks >= NR_BANKS) { 948 printk(KERN_CRIT "NR_BANKS too low, " 949 "ignoring high memory\n"); 950 } else { 951 memmove(bank + 1, bank, 952 (meminfo.nr_banks - i) * sizeof(*bank)); 953 meminfo.nr_banks++; 954 i++; 955 bank[1].size -= vmalloc_min - __va(bank->start); 956 bank[1].start = __pa(vmalloc_min - 1) + 1; 957 bank[1].highmem = highmem = 1; 958 j++; 959 } 960 bank->size = vmalloc_min - __va(bank->start); 961 } 962 #else 963 bank->highmem = highmem; 964 965 /* 966 * Highmem banks not allowed with !CONFIG_HIGHMEM. 967 */ 968 if (highmem) { 969 printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx " 970 "(!CONFIG_HIGHMEM).\n", 971 (unsigned long long)bank->start, 972 (unsigned long long)bank->start + bank->size - 1); 973 continue; 974 } 975 976 /* 977 * Check whether this memory bank would entirely overlap 978 * the vmalloc area. 979 */ 980 if (__va(bank->start) >= vmalloc_min || 981 __va(bank->start) < (void *)PAGE_OFFSET) { 982 printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx " 983 "(vmalloc region overlap).\n", 984 (unsigned long long)bank->start, 985 (unsigned long long)bank->start + bank->size - 1); 986 continue; 987 } 988 989 /* 990 * Check whether this memory bank would partially overlap 991 * the vmalloc area. 992 */ 993 if (__va(bank->start + bank->size - 1) >= vmalloc_min || 994 __va(bank->start + bank->size - 1) <= __va(bank->start)) { 995 unsigned long newsize = vmalloc_min - __va(bank->start); 996 printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx " 997 "to -%.8llx (vmalloc region overlap).\n", 998 (unsigned long long)bank->start, 999 (unsigned long long)bank->start + bank->size - 1, 1000 (unsigned long long)bank->start + newsize - 1); 1001 bank->size = newsize; 1002 } 1003 #endif 1004 if (!bank->highmem && bank->start + bank->size > arm_lowmem_limit) 1005 arm_lowmem_limit = bank->start + bank->size; 1006 1007 j++; 1008 } 1009 #ifdef CONFIG_HIGHMEM 1010 if (highmem) { 1011 const char *reason = NULL; 1012 1013 if (cache_is_vipt_aliasing()) { 1014 /* 1015 * Interactions between kmap and other mappings 1016 * make highmem support with aliasing VIPT caches 1017 * rather difficult. 1018 */ 1019 reason = "with VIPT aliasing cache"; 1020 } 1021 if (reason) { 1022 printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n", 1023 reason); 1024 while (j > 0 && meminfo.bank[j - 1].highmem) 1025 j--; 1026 } 1027 } 1028 #endif 1029 meminfo.nr_banks = j; 1030 high_memory = __va(arm_lowmem_limit - 1) + 1; 1031 memblock_set_current_limit(arm_lowmem_limit); 1032 } 1033 1034 static inline void prepare_page_table(void) 1035 { 1036 unsigned long addr; 1037 phys_addr_t end; 1038 1039 /* 1040 * Clear out all the mappings below the kernel image. 1041 */ 1042 for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE) 1043 pmd_clear(pmd_off_k(addr)); 1044 1045 #ifdef CONFIG_XIP_KERNEL 1046 /* The XIP kernel is mapped in the module area -- skip over it */ 1047 addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK; 1048 #endif 1049 for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE) 1050 pmd_clear(pmd_off_k(addr)); 1051 1052 /* 1053 * Find the end of the first block of lowmem. 1054 */ 1055 end = memblock.memory.regions[0].base + memblock.memory.regions[0].size; 1056 if (end >= arm_lowmem_limit) 1057 end = arm_lowmem_limit; 1058 1059 /* 1060 * Clear out all the kernel space mappings, except for the first 1061 * memory bank, up to the vmalloc region. 1062 */ 1063 for (addr = __phys_to_virt(end); 1064 addr < VMALLOC_START; addr += PMD_SIZE) 1065 pmd_clear(pmd_off_k(addr)); 1066 } 1067 1068 #ifdef CONFIG_ARM_LPAE 1069 /* the first page is reserved for pgd */ 1070 #define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \ 1071 PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t)) 1072 #else 1073 #define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) 1074 #endif 1075 1076 /* 1077 * Reserve the special regions of memory 1078 */ 1079 void __init arm_mm_memblock_reserve(void) 1080 { 1081 /* 1082 * Reserve the page tables. These are already in use, 1083 * and can only be in node 0. 1084 */ 1085 memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE); 1086 1087 #ifdef CONFIG_SA1111 1088 /* 1089 * Because of the SA1111 DMA bug, we want to preserve our 1090 * precious DMA-able memory... 1091 */ 1092 memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET); 1093 #endif 1094 } 1095 1096 /* 1097 * Set up the device mappings. Since we clear out the page tables for all 1098 * mappings above VMALLOC_START, we will remove any debug device mappings. 1099 * This means you have to be careful how you debug this function, or any 1100 * called function. This means you can't use any function or debugging 1101 * method which may touch any device, otherwise the kernel _will_ crash. 1102 */ 1103 static void __init devicemaps_init(struct machine_desc *mdesc) 1104 { 1105 struct map_desc map; 1106 unsigned long addr; 1107 void *vectors; 1108 1109 /* 1110 * Allocate the vector page early. 1111 */ 1112 vectors = early_alloc(PAGE_SIZE); 1113 1114 early_trap_init(vectors); 1115 1116 for (addr = VMALLOC_START; addr; addr += PMD_SIZE) 1117 pmd_clear(pmd_off_k(addr)); 1118 1119 /* 1120 * Map the kernel if it is XIP. 1121 * It is always first in the modulearea. 1122 */ 1123 #ifdef CONFIG_XIP_KERNEL 1124 map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK); 1125 map.virtual = MODULES_VADDR; 1126 map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK; 1127 map.type = MT_ROM; 1128 create_mapping(&map); 1129 #endif 1130 1131 /* 1132 * Map the cache flushing regions. 1133 */ 1134 #ifdef FLUSH_BASE 1135 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS); 1136 map.virtual = FLUSH_BASE; 1137 map.length = SZ_1M; 1138 map.type = MT_CACHECLEAN; 1139 create_mapping(&map); 1140 #endif 1141 #ifdef FLUSH_BASE_MINICACHE 1142 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M); 1143 map.virtual = FLUSH_BASE_MINICACHE; 1144 map.length = SZ_1M; 1145 map.type = MT_MINICLEAN; 1146 create_mapping(&map); 1147 #endif 1148 1149 /* 1150 * Create a mapping for the machine vectors at the high-vectors 1151 * location (0xffff0000). If we aren't using high-vectors, also 1152 * create a mapping at the low-vectors virtual address. 1153 */ 1154 map.pfn = __phys_to_pfn(virt_to_phys(vectors)); 1155 map.virtual = 0xffff0000; 1156 map.length = PAGE_SIZE; 1157 map.type = MT_HIGH_VECTORS; 1158 create_mapping(&map); 1159 1160 if (!vectors_high()) { 1161 map.virtual = 0; 1162 map.type = MT_LOW_VECTORS; 1163 create_mapping(&map); 1164 } 1165 1166 /* 1167 * Ask the machine support to map in the statically mapped devices. 1168 */ 1169 if (mdesc->map_io) 1170 mdesc->map_io(); 1171 fill_pmd_gaps(); 1172 1173 /* Reserve fixed i/o space in VMALLOC region */ 1174 pci_reserve_io(); 1175 1176 /* 1177 * Finally flush the caches and tlb to ensure that we're in a 1178 * consistent state wrt the writebuffer. This also ensures that 1179 * any write-allocated cache lines in the vector page are written 1180 * back. After this point, we can start to touch devices again. 1181 */ 1182 local_flush_tlb_all(); 1183 flush_cache_all(); 1184 } 1185 1186 static void __init kmap_init(void) 1187 { 1188 #ifdef CONFIG_HIGHMEM 1189 pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE), 1190 PKMAP_BASE, _PAGE_KERNEL_TABLE); 1191 #endif 1192 } 1193 1194 static void __init map_lowmem(void) 1195 { 1196 struct memblock_region *reg; 1197 1198 /* Map all the lowmem memory banks. */ 1199 for_each_memblock(memory, reg) { 1200 phys_addr_t start = reg->base; 1201 phys_addr_t end = start + reg->size; 1202 struct map_desc map; 1203 1204 if (end > arm_lowmem_limit) 1205 end = arm_lowmem_limit; 1206 if (start >= end) 1207 break; 1208 1209 map.pfn = __phys_to_pfn(start); 1210 map.virtual = __phys_to_virt(start); 1211 map.length = end - start; 1212 map.type = MT_MEMORY; 1213 1214 create_mapping(&map); 1215 } 1216 } 1217 1218 /* 1219 * paging_init() sets up the page tables, initialises the zone memory 1220 * maps, and sets up the zero page, bad page and bad page tables. 1221 */ 1222 void __init paging_init(struct machine_desc *mdesc) 1223 { 1224 void *zero_page; 1225 1226 memblock_set_current_limit(arm_lowmem_limit); 1227 1228 build_mem_type_table(); 1229 prepare_page_table(); 1230 map_lowmem(); 1231 dma_contiguous_remap(); 1232 devicemaps_init(mdesc); 1233 kmap_init(); 1234 1235 top_pmd = pmd_off_k(0xffff0000); 1236 1237 /* allocate the zero page. */ 1238 zero_page = early_alloc(PAGE_SIZE); 1239 1240 bootmem_init(); 1241 1242 empty_zero_page = virt_to_page(zero_page); 1243 __flush_dcache_page(NULL, empty_zero_page); 1244 } 1245