1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5 */ 6 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/memblock.h> 10 #include <linux/initrd.h> 11 #include <linux/swap.h> 12 #include <linux/sizes.h> 13 #include <linux/of_fdt.h> 14 #include <linux/libfdt.h> 15 #include <linux/set_memory.h> 16 #include <linux/dma-map-ops.h> 17 18 #include <asm/fixmap.h> 19 #include <asm/tlbflush.h> 20 #include <asm/sections.h> 21 #include <asm/soc.h> 22 #include <asm/io.h> 23 #include <asm/ptdump.h> 24 #include <asm/numa.h> 25 26 #include "../kernel/head.h" 27 28 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 29 __page_aligned_bss; 30 EXPORT_SYMBOL(empty_zero_page); 31 32 extern char _start[]; 33 #define DTB_EARLY_BASE_VA PGDIR_SIZE 34 void *dtb_early_va __initdata; 35 uintptr_t dtb_early_pa __initdata; 36 37 struct pt_alloc_ops { 38 pte_t *(*get_pte_virt)(phys_addr_t pa); 39 phys_addr_t (*alloc_pte)(uintptr_t va); 40 #ifndef __PAGETABLE_PMD_FOLDED 41 pmd_t *(*get_pmd_virt)(phys_addr_t pa); 42 phys_addr_t (*alloc_pmd)(uintptr_t va); 43 #endif 44 }; 45 46 static phys_addr_t dma32_phys_limit __ro_after_init; 47 48 static void __init zone_sizes_init(void) 49 { 50 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 51 52 #ifdef CONFIG_ZONE_DMA32 53 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); 54 #endif 55 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 56 57 free_area_init(max_zone_pfns); 58 } 59 60 static void setup_zero_page(void) 61 { 62 memset((void *)empty_zero_page, 0, PAGE_SIZE); 63 } 64 65 #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) 66 static inline void print_mlk(char *name, unsigned long b, unsigned long t) 67 { 68 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, 69 (((t) - (b)) >> 10)); 70 } 71 72 static inline void print_mlm(char *name, unsigned long b, unsigned long t) 73 { 74 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, 75 (((t) - (b)) >> 20)); 76 } 77 78 static void print_vm_layout(void) 79 { 80 pr_notice("Virtual kernel memory layout:\n"); 81 print_mlk("fixmap", (unsigned long)FIXADDR_START, 82 (unsigned long)FIXADDR_TOP); 83 print_mlm("pci io", (unsigned long)PCI_IO_START, 84 (unsigned long)PCI_IO_END); 85 print_mlm("vmemmap", (unsigned long)VMEMMAP_START, 86 (unsigned long)VMEMMAP_END); 87 print_mlm("vmalloc", (unsigned long)VMALLOC_START, 88 (unsigned long)VMALLOC_END); 89 print_mlm("lowmem", (unsigned long)PAGE_OFFSET, 90 (unsigned long)high_memory); 91 } 92 #else 93 static void print_vm_layout(void) { } 94 #endif /* CONFIG_DEBUG_VM */ 95 96 void __init mem_init(void) 97 { 98 #ifdef CONFIG_FLATMEM 99 BUG_ON(!mem_map); 100 #endif /* CONFIG_FLATMEM */ 101 102 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 103 memblock_free_all(); 104 105 mem_init_print_info(NULL); 106 print_vm_layout(); 107 } 108 109 void __init setup_bootmem(void) 110 { 111 phys_addr_t mem_start = 0; 112 phys_addr_t start, end = 0; 113 phys_addr_t vmlinux_end = __pa_symbol(&_end); 114 phys_addr_t vmlinux_start = __pa_symbol(&_start); 115 u64 i; 116 117 /* Find the memory region containing the kernel */ 118 for_each_mem_range(i, &start, &end) { 119 phys_addr_t size = end - start; 120 if (!mem_start) 121 mem_start = start; 122 if (start <= vmlinux_start && vmlinux_end <= end) 123 BUG_ON(size == 0); 124 } 125 126 /* 127 * The maximal physical memory size is -PAGE_OFFSET. 128 * Make sure that any memory beyond mem_start + (-PAGE_OFFSET) is removed 129 * as it is unusable by kernel. 130 */ 131 memblock_enforce_memory_limit(-PAGE_OFFSET); 132 133 /* Reserve from the start of the kernel to the end of the kernel */ 134 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 135 136 max_pfn = PFN_DOWN(memblock_end_of_DRAM()); 137 max_low_pfn = max_pfn; 138 dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); 139 set_max_mapnr(max_low_pfn); 140 141 reserve_initrd_mem(); 142 /* 143 * If DTB is built in, no need to reserve its memblock. 144 * Otherwise, do reserve it but avoid using 145 * early_init_fdt_reserve_self() since __pa() does 146 * not work for DTB pointers that are fixmap addresses 147 */ 148 if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) 149 memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); 150 151 early_init_fdt_scan_reserved_mem(); 152 dma_contiguous_reserve(dma32_phys_limit); 153 memblock_allow_resize(); 154 } 155 156 #ifdef CONFIG_MMU 157 static struct pt_alloc_ops pt_ops; 158 159 unsigned long va_pa_offset; 160 EXPORT_SYMBOL(va_pa_offset); 161 unsigned long pfn_base; 162 EXPORT_SYMBOL(pfn_base); 163 164 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 165 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 166 pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 167 168 #define MAX_EARLY_MAPPING_SIZE SZ_128M 169 170 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 171 172 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 173 { 174 unsigned long addr = __fix_to_virt(idx); 175 pte_t *ptep; 176 177 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 178 179 ptep = &fixmap_pte[pte_index(addr)]; 180 181 if (pgprot_val(prot)) 182 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 183 else 184 pte_clear(&init_mm, addr, ptep); 185 local_flush_tlb_page(addr); 186 } 187 188 static inline pte_t *__init get_pte_virt_early(phys_addr_t pa) 189 { 190 return (pte_t *)((uintptr_t)pa); 191 } 192 193 static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) 194 { 195 clear_fixmap(FIX_PTE); 196 return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 197 } 198 199 static inline pte_t *get_pte_virt_late(phys_addr_t pa) 200 { 201 return (pte_t *) __va(pa); 202 } 203 204 static inline phys_addr_t __init alloc_pte_early(uintptr_t va) 205 { 206 /* 207 * We only create PMD or PGD early mappings so we 208 * should never reach here with MMU disabled. 209 */ 210 BUG(); 211 } 212 213 static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) 214 { 215 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 216 } 217 218 static phys_addr_t alloc_pte_late(uintptr_t va) 219 { 220 unsigned long vaddr; 221 222 vaddr = __get_free_page(GFP_KERNEL); 223 if (!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr))) 224 BUG(); 225 return __pa(vaddr); 226 } 227 228 static void __init create_pte_mapping(pte_t *ptep, 229 uintptr_t va, phys_addr_t pa, 230 phys_addr_t sz, pgprot_t prot) 231 { 232 uintptr_t pte_idx = pte_index(va); 233 234 BUG_ON(sz != PAGE_SIZE); 235 236 if (pte_none(ptep[pte_idx])) 237 ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); 238 } 239 240 #ifndef __PAGETABLE_PMD_FOLDED 241 242 pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 243 pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 244 245 #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE 246 #define NUM_EARLY_PMDS 1UL 247 #else 248 #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) 249 #endif 250 pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); 251 pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 252 253 static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) 254 { 255 /* Before MMU is enabled */ 256 return (pmd_t *)((uintptr_t)pa); 257 } 258 259 static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) 260 { 261 clear_fixmap(FIX_PMD); 262 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 263 } 264 265 static pmd_t *get_pmd_virt_late(phys_addr_t pa) 266 { 267 return (pmd_t *) __va(pa); 268 } 269 270 static phys_addr_t __init alloc_pmd_early(uintptr_t va) 271 { 272 uintptr_t pmd_num; 273 274 pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; 275 BUG_ON(pmd_num >= NUM_EARLY_PMDS); 276 return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; 277 } 278 279 static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) 280 { 281 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 282 } 283 284 static phys_addr_t alloc_pmd_late(uintptr_t va) 285 { 286 unsigned long vaddr; 287 288 vaddr = __get_free_page(GFP_KERNEL); 289 BUG_ON(!vaddr); 290 return __pa(vaddr); 291 } 292 293 static void __init create_pmd_mapping(pmd_t *pmdp, 294 uintptr_t va, phys_addr_t pa, 295 phys_addr_t sz, pgprot_t prot) 296 { 297 pte_t *ptep; 298 phys_addr_t pte_phys; 299 uintptr_t pmd_idx = pmd_index(va); 300 301 if (sz == PMD_SIZE) { 302 if (pmd_none(pmdp[pmd_idx])) 303 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); 304 return; 305 } 306 307 if (pmd_none(pmdp[pmd_idx])) { 308 pte_phys = pt_ops.alloc_pte(va); 309 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 310 ptep = pt_ops.get_pte_virt(pte_phys); 311 memset(ptep, 0, PAGE_SIZE); 312 } else { 313 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); 314 ptep = pt_ops.get_pte_virt(pte_phys); 315 } 316 317 create_pte_mapping(ptep, va, pa, sz, prot); 318 } 319 320 #define pgd_next_t pmd_t 321 #define alloc_pgd_next(__va) pt_ops.alloc_pmd(__va) 322 #define get_pgd_next_virt(__pa) pt_ops.get_pmd_virt(__pa) 323 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 324 create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) 325 #define fixmap_pgd_next fixmap_pmd 326 #else 327 #define pgd_next_t pte_t 328 #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) 329 #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) 330 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 331 create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 332 #define fixmap_pgd_next fixmap_pte 333 #endif 334 335 void __init create_pgd_mapping(pgd_t *pgdp, 336 uintptr_t va, phys_addr_t pa, 337 phys_addr_t sz, pgprot_t prot) 338 { 339 pgd_next_t *nextp; 340 phys_addr_t next_phys; 341 uintptr_t pgd_idx = pgd_index(va); 342 343 if (sz == PGDIR_SIZE) { 344 if (pgd_val(pgdp[pgd_idx]) == 0) 345 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); 346 return; 347 } 348 349 if (pgd_val(pgdp[pgd_idx]) == 0) { 350 next_phys = alloc_pgd_next(va); 351 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 352 nextp = get_pgd_next_virt(next_phys); 353 memset(nextp, 0, PAGE_SIZE); 354 } else { 355 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); 356 nextp = get_pgd_next_virt(next_phys); 357 } 358 359 create_pgd_next_mapping(nextp, va, pa, sz, prot); 360 } 361 362 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 363 { 364 /* Upgrade to PMD_SIZE mappings whenever possible */ 365 if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) 366 return PAGE_SIZE; 367 368 return PMD_SIZE; 369 } 370 371 /* 372 * setup_vm() is called from head.S with MMU-off. 373 * 374 * Following requirements should be honoured for setup_vm() to work 375 * correctly: 376 * 1) It should use PC-relative addressing for accessing kernel symbols. 377 * To achieve this we always use GCC cmodel=medany. 378 * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 379 * so disable compiler instrumentation when FTRACE is enabled. 380 * 381 * Currently, the above requirements are honoured by using custom CFLAGS 382 * for init.o in mm/Makefile. 383 */ 384 385 #ifndef __riscv_cmodel_medany 386 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." 387 #endif 388 389 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 390 { 391 uintptr_t va, pa, end_va; 392 uintptr_t load_pa = (uintptr_t)(&_start); 393 uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; 394 uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); 395 #ifndef __PAGETABLE_PMD_FOLDED 396 pmd_t fix_bmap_spmd, fix_bmap_epmd; 397 #endif 398 399 va_pa_offset = PAGE_OFFSET - load_pa; 400 pfn_base = PFN_DOWN(load_pa); 401 402 /* 403 * Enforce boot alignment requirements of RV32 and 404 * RV64 by only allowing PMD or PGD mappings. 405 */ 406 BUG_ON(map_size == PAGE_SIZE); 407 408 /* Sanity check alignment and size */ 409 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 410 BUG_ON((load_pa % map_size) != 0); 411 BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); 412 413 pt_ops.alloc_pte = alloc_pte_early; 414 pt_ops.get_pte_virt = get_pte_virt_early; 415 #ifndef __PAGETABLE_PMD_FOLDED 416 pt_ops.alloc_pmd = alloc_pmd_early; 417 pt_ops.get_pmd_virt = get_pmd_virt_early; 418 #endif 419 /* Setup early PGD for fixmap */ 420 create_pgd_mapping(early_pg_dir, FIXADDR_START, 421 (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 422 423 #ifndef __PAGETABLE_PMD_FOLDED 424 /* Setup fixmap PMD */ 425 create_pmd_mapping(fixmap_pmd, FIXADDR_START, 426 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 427 /* Setup trampoline PGD and PMD */ 428 create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, 429 (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); 430 create_pmd_mapping(trampoline_pmd, PAGE_OFFSET, 431 load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); 432 #else 433 /* Setup trampoline PGD */ 434 create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, 435 load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); 436 #endif 437 438 /* 439 * Setup early PGD covering entire kernel which will allows 440 * us to reach paging_init(). We map all memory banks later 441 * in setup_vm_final() below. 442 */ 443 end_va = PAGE_OFFSET + load_sz; 444 for (va = PAGE_OFFSET; va < end_va; va += map_size) 445 create_pgd_mapping(early_pg_dir, va, 446 load_pa + (va - PAGE_OFFSET), 447 map_size, PAGE_KERNEL_EXEC); 448 449 #ifndef __PAGETABLE_PMD_FOLDED 450 /* Setup early PMD for DTB */ 451 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 452 (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE); 453 #ifndef CONFIG_BUILTIN_DTB 454 /* Create two consecutive PMD mappings for FDT early scan */ 455 pa = dtb_pa & ~(PMD_SIZE - 1); 456 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, 457 pa, PMD_SIZE, PAGE_KERNEL); 458 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, 459 pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); 460 dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); 461 #else /* CONFIG_BUILTIN_DTB */ 462 dtb_early_va = __va(dtb_pa); 463 #endif /* CONFIG_BUILTIN_DTB */ 464 #else 465 #ifndef CONFIG_BUILTIN_DTB 466 /* Create two consecutive PGD mappings for FDT early scan */ 467 pa = dtb_pa & ~(PGDIR_SIZE - 1); 468 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 469 pa, PGDIR_SIZE, PAGE_KERNEL); 470 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE, 471 pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL); 472 dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1)); 473 #else /* CONFIG_BUILTIN_DTB */ 474 dtb_early_va = __va(dtb_pa); 475 #endif /* CONFIG_BUILTIN_DTB */ 476 #endif 477 dtb_early_pa = dtb_pa; 478 479 /* 480 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap 481 * range can not span multiple pmds. 482 */ 483 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 484 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 485 486 #ifndef __PAGETABLE_PMD_FOLDED 487 /* 488 * Early ioremap fixmap is already created as it lies within first 2MB 489 * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END 490 * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn 491 * the user if not. 492 */ 493 fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))]; 494 fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))]; 495 if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) { 496 WARN_ON(1); 497 pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n", 498 pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd)); 499 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 500 fix_to_virt(FIX_BTMAP_BEGIN)); 501 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 502 fix_to_virt(FIX_BTMAP_END)); 503 504 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 505 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 506 } 507 #endif 508 } 509 510 static void __init setup_vm_final(void) 511 { 512 uintptr_t va, map_size; 513 phys_addr_t pa, start, end; 514 u64 i; 515 516 /** 517 * MMU is enabled at this point. But page table setup is not complete yet. 518 * fixmap page table alloc functions should be used at this point 519 */ 520 pt_ops.alloc_pte = alloc_pte_fixmap; 521 pt_ops.get_pte_virt = get_pte_virt_fixmap; 522 #ifndef __PAGETABLE_PMD_FOLDED 523 pt_ops.alloc_pmd = alloc_pmd_fixmap; 524 pt_ops.get_pmd_virt = get_pmd_virt_fixmap; 525 #endif 526 /* Setup swapper PGD for fixmap */ 527 create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 528 __pa_symbol(fixmap_pgd_next), 529 PGDIR_SIZE, PAGE_TABLE); 530 531 /* Map all memory banks */ 532 for_each_mem_range(i, &start, &end) { 533 if (start >= end) 534 break; 535 if (start <= __pa(PAGE_OFFSET) && 536 __pa(PAGE_OFFSET) < end) 537 start = __pa(PAGE_OFFSET); 538 539 map_size = best_map_size(start, end - start); 540 for (pa = start; pa < end; pa += map_size) { 541 va = (uintptr_t)__va(pa); 542 create_pgd_mapping(swapper_pg_dir, va, pa, 543 map_size, PAGE_KERNEL_EXEC); 544 } 545 } 546 547 /* Clear fixmap PTE and PMD mappings */ 548 clear_fixmap(FIX_PTE); 549 clear_fixmap(FIX_PMD); 550 551 /* Move to swapper page table */ 552 csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); 553 local_flush_tlb_all(); 554 555 /* generic page allocation functions must be used to setup page table */ 556 pt_ops.alloc_pte = alloc_pte_late; 557 pt_ops.get_pte_virt = get_pte_virt_late; 558 #ifndef __PAGETABLE_PMD_FOLDED 559 pt_ops.alloc_pmd = alloc_pmd_late; 560 pt_ops.get_pmd_virt = get_pmd_virt_late; 561 #endif 562 } 563 #else 564 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 565 { 566 dtb_early_va = (void *)dtb_pa; 567 dtb_early_pa = dtb_pa; 568 } 569 570 static inline void setup_vm_final(void) 571 { 572 } 573 #endif /* CONFIG_MMU */ 574 575 #ifdef CONFIG_STRICT_KERNEL_RWX 576 void protect_kernel_text_data(void) 577 { 578 unsigned long text_start = (unsigned long)_start; 579 unsigned long init_text_start = (unsigned long)__init_text_begin; 580 unsigned long init_data_start = (unsigned long)__init_data_begin; 581 unsigned long rodata_start = (unsigned long)__start_rodata; 582 unsigned long data_start = (unsigned long)_data; 583 unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn))); 584 585 set_memory_ro(text_start, (init_text_start - text_start) >> PAGE_SHIFT); 586 set_memory_ro(init_text_start, (init_data_start - init_text_start) >> PAGE_SHIFT); 587 set_memory_nx(init_data_start, (rodata_start - init_data_start) >> PAGE_SHIFT); 588 /* rodata section is marked readonly in mark_rodata_ro */ 589 set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 590 set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); 591 } 592 593 void mark_rodata_ro(void) 594 { 595 unsigned long rodata_start = (unsigned long)__start_rodata; 596 unsigned long data_start = (unsigned long)_data; 597 598 set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 599 600 debug_checkwx(); 601 } 602 #endif 603 604 void __init paging_init(void) 605 { 606 setup_vm_final(); 607 setup_zero_page(); 608 } 609 610 void __init misc_mem_init(void) 611 { 612 arch_numa_init(); 613 sparse_init(); 614 zone_sizes_init(); 615 memblock_dump_all(); 616 } 617 618 #ifdef CONFIG_SPARSEMEM_VMEMMAP 619 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 620 struct vmem_altmap *altmap) 621 { 622 return vmemmap_populate_basepages(start, end, node, NULL); 623 } 624 #endif 625