1*caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c1cc1552SCatalin Marinas /* 3c1cc1552SCatalin Marinas * Based on arch/arm/mm/init.c 4c1cc1552SCatalin Marinas * 5c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 6c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 7c1cc1552SCatalin Marinas */ 8c1cc1552SCatalin Marinas 9c1cc1552SCatalin Marinas #include <linux/kernel.h> 10c1cc1552SCatalin Marinas #include <linux/export.h> 11c1cc1552SCatalin Marinas #include <linux/errno.h> 12c1cc1552SCatalin Marinas #include <linux/swap.h> 13c1cc1552SCatalin Marinas #include <linux/init.h> 145a9e3e15SJisheng Zhang #include <linux/cache.h> 15c1cc1552SCatalin Marinas #include <linux/mman.h> 16c1cc1552SCatalin Marinas #include <linux/nodemask.h> 17c1cc1552SCatalin Marinas #include <linux/initrd.h> 18c1cc1552SCatalin Marinas #include <linux/gfp.h> 19c1cc1552SCatalin Marinas #include <linux/memblock.h> 20c1cc1552SCatalin Marinas #include <linux/sort.h> 21764b51eaSAKASHI Takahiro #include <linux/of.h> 22c1cc1552SCatalin Marinas #include <linux/of_fdt.h> 2319e7640dSCatalin Marinas #include <linux/dma-mapping.h> 246ac2104dSLaura Abbott #include <linux/dma-contiguous.h> 2586c8b27aSLeif Lindholm #include <linux/efi.h> 26a1e50a82SCatalin Marinas #include <linux/swiotlb.h> 27dae8c235SKefeng Wang #include <linux/vmalloc.h> 282077be67SLaura Abbott #include <linux/mm.h> 29764b51eaSAKASHI Takahiro #include <linux/kexec.h> 30e62aaeacSAKASHI Takahiro #include <linux/crash_dump.h> 31c1cc1552SCatalin Marinas 32a7f8de16SArd Biesheuvel #include <asm/boot.h> 3308375198SCatalin Marinas #include <asm/fixmap.h> 34f9040773SArd Biesheuvel #include <asm/kasan.h> 35a7f8de16SArd Biesheuvel #include <asm/kernel-pgtable.h> 36aa03c428SMark Rutland #include <asm/memory.h> 371a2db300SGanapatrao Kulkarni #include <asm/numa.h> 38c1cc1552SCatalin Marinas #include <asm/sections.h> 39c1cc1552SCatalin Marinas #include <asm/setup.h> 4087dfb311SMasahiro Yamada #include <linux/sizes.h> 41c1cc1552SCatalin Marinas #include <asm/tlb.h> 42e039ee4eSAndre Przywara #include <asm/alternative.h> 43c1cc1552SCatalin Marinas 44a7f8de16SArd Biesheuvel /* 45a7f8de16SArd Biesheuvel * We need to be able to catch inadvertent references to memstart_addr 46a7f8de16SArd Biesheuvel * that occur (potentially in generic code) before arm64_memblock_init() 47a7f8de16SArd Biesheuvel * executes, which assigns it its actual value. So use a default value 48a7f8de16SArd Biesheuvel * that cannot be mistaken for a real physical address. 49a7f8de16SArd Biesheuvel */ 505a9e3e15SJisheng Zhang s64 memstart_addr __ro_after_init = -1; 5103ef055fSMark Rutland EXPORT_SYMBOL(memstart_addr); 5203ef055fSMark Rutland 535a9e3e15SJisheng Zhang phys_addr_t arm64_dma_phys_limit __ro_after_init; 54c1cc1552SCatalin Marinas 55764b51eaSAKASHI Takahiro #ifdef CONFIG_KEXEC_CORE 56764b51eaSAKASHI Takahiro /* 57764b51eaSAKASHI Takahiro * reserve_crashkernel() - reserves memory for crash kernel 58764b51eaSAKASHI Takahiro * 59764b51eaSAKASHI Takahiro * This function reserves memory area given in "crashkernel=" kernel command 60764b51eaSAKASHI Takahiro * line parameter. The memory reserved is used by dump capture kernel when 61764b51eaSAKASHI Takahiro * primary kernel is crashing. 62764b51eaSAKASHI Takahiro */ 63764b51eaSAKASHI Takahiro static void __init reserve_crashkernel(void) 64764b51eaSAKASHI Takahiro { 65764b51eaSAKASHI Takahiro unsigned long long crash_base, crash_size; 66764b51eaSAKASHI Takahiro int ret; 67764b51eaSAKASHI Takahiro 68764b51eaSAKASHI Takahiro ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), 69764b51eaSAKASHI Takahiro &crash_size, &crash_base); 70764b51eaSAKASHI Takahiro /* no crashkernel= or invalid value specified */ 71764b51eaSAKASHI Takahiro if (ret || !crash_size) 72764b51eaSAKASHI Takahiro return; 73764b51eaSAKASHI Takahiro 74764b51eaSAKASHI Takahiro crash_size = PAGE_ALIGN(crash_size); 75764b51eaSAKASHI Takahiro 76764b51eaSAKASHI Takahiro if (crash_base == 0) { 77764b51eaSAKASHI Takahiro /* Current arm64 boot protocol requires 2MB alignment */ 78764b51eaSAKASHI Takahiro crash_base = memblock_find_in_range(0, ARCH_LOW_ADDRESS_LIMIT, 79764b51eaSAKASHI Takahiro crash_size, SZ_2M); 80764b51eaSAKASHI Takahiro if (crash_base == 0) { 81764b51eaSAKASHI Takahiro pr_warn("cannot allocate crashkernel (size:0x%llx)\n", 82764b51eaSAKASHI Takahiro crash_size); 83764b51eaSAKASHI Takahiro return; 84764b51eaSAKASHI Takahiro } 85764b51eaSAKASHI Takahiro } else { 86764b51eaSAKASHI Takahiro /* User specifies base address explicitly. */ 87764b51eaSAKASHI Takahiro if (!memblock_is_region_memory(crash_base, crash_size)) { 88764b51eaSAKASHI Takahiro pr_warn("cannot reserve crashkernel: region is not memory\n"); 89764b51eaSAKASHI Takahiro return; 90764b51eaSAKASHI Takahiro } 91764b51eaSAKASHI Takahiro 92764b51eaSAKASHI Takahiro if (memblock_is_region_reserved(crash_base, crash_size)) { 93764b51eaSAKASHI Takahiro pr_warn("cannot reserve crashkernel: region overlaps reserved memory\n"); 94764b51eaSAKASHI Takahiro return; 95764b51eaSAKASHI Takahiro } 96764b51eaSAKASHI Takahiro 97764b51eaSAKASHI Takahiro if (!IS_ALIGNED(crash_base, SZ_2M)) { 98764b51eaSAKASHI Takahiro pr_warn("cannot reserve crashkernel: base address is not 2MB aligned\n"); 99764b51eaSAKASHI Takahiro return; 100764b51eaSAKASHI Takahiro } 101764b51eaSAKASHI Takahiro } 102764b51eaSAKASHI Takahiro memblock_reserve(crash_base, crash_size); 103764b51eaSAKASHI Takahiro 104764b51eaSAKASHI Takahiro pr_info("crashkernel reserved: 0x%016llx - 0x%016llx (%lld MB)\n", 105764b51eaSAKASHI Takahiro crash_base, crash_base + crash_size, crash_size >> 20); 106764b51eaSAKASHI Takahiro 107764b51eaSAKASHI Takahiro crashk_res.start = crash_base; 108764b51eaSAKASHI Takahiro crashk_res.end = crash_base + crash_size - 1; 109764b51eaSAKASHI Takahiro } 110764b51eaSAKASHI Takahiro #else 111764b51eaSAKASHI Takahiro static void __init reserve_crashkernel(void) 112764b51eaSAKASHI Takahiro { 113764b51eaSAKASHI Takahiro } 114764b51eaSAKASHI Takahiro #endif /* CONFIG_KEXEC_CORE */ 115764b51eaSAKASHI Takahiro 116e62aaeacSAKASHI Takahiro #ifdef CONFIG_CRASH_DUMP 117e62aaeacSAKASHI Takahiro static int __init early_init_dt_scan_elfcorehdr(unsigned long node, 118e62aaeacSAKASHI Takahiro const char *uname, int depth, void *data) 119e62aaeacSAKASHI Takahiro { 120e62aaeacSAKASHI Takahiro const __be32 *reg; 121e62aaeacSAKASHI Takahiro int len; 122e62aaeacSAKASHI Takahiro 123e62aaeacSAKASHI Takahiro if (depth != 1 || strcmp(uname, "chosen") != 0) 124e62aaeacSAKASHI Takahiro return 0; 125e62aaeacSAKASHI Takahiro 126e62aaeacSAKASHI Takahiro reg = of_get_flat_dt_prop(node, "linux,elfcorehdr", &len); 127e62aaeacSAKASHI Takahiro if (!reg || (len < (dt_root_addr_cells + dt_root_size_cells))) 128e62aaeacSAKASHI Takahiro return 1; 129e62aaeacSAKASHI Takahiro 130e62aaeacSAKASHI Takahiro elfcorehdr_addr = dt_mem_next_cell(dt_root_addr_cells, ®); 131e62aaeacSAKASHI Takahiro elfcorehdr_size = dt_mem_next_cell(dt_root_size_cells, ®); 132e62aaeacSAKASHI Takahiro 133e62aaeacSAKASHI Takahiro return 1; 134e62aaeacSAKASHI Takahiro } 135e62aaeacSAKASHI Takahiro 136e62aaeacSAKASHI Takahiro /* 137e62aaeacSAKASHI Takahiro * reserve_elfcorehdr() - reserves memory for elf core header 138e62aaeacSAKASHI Takahiro * 139e62aaeacSAKASHI Takahiro * This function reserves the memory occupied by an elf core header 140e62aaeacSAKASHI Takahiro * described in the device tree. This region contains all the 141e62aaeacSAKASHI Takahiro * information about primary kernel's core image and is used by a dump 142e62aaeacSAKASHI Takahiro * capture kernel to access the system memory on primary kernel. 143e62aaeacSAKASHI Takahiro */ 144e62aaeacSAKASHI Takahiro static void __init reserve_elfcorehdr(void) 145e62aaeacSAKASHI Takahiro { 146e62aaeacSAKASHI Takahiro of_scan_flat_dt(early_init_dt_scan_elfcorehdr, NULL); 147e62aaeacSAKASHI Takahiro 148e62aaeacSAKASHI Takahiro if (!elfcorehdr_size) 149e62aaeacSAKASHI Takahiro return; 150e62aaeacSAKASHI Takahiro 151e62aaeacSAKASHI Takahiro if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) { 152e62aaeacSAKASHI Takahiro pr_warn("elfcorehdr is overlapped\n"); 153e62aaeacSAKASHI Takahiro return; 154e62aaeacSAKASHI Takahiro } 155e62aaeacSAKASHI Takahiro 156e62aaeacSAKASHI Takahiro memblock_reserve(elfcorehdr_addr, elfcorehdr_size); 157e62aaeacSAKASHI Takahiro 158e62aaeacSAKASHI Takahiro pr_info("Reserving %lldKB of memory at 0x%llx for elfcorehdr\n", 159e62aaeacSAKASHI Takahiro elfcorehdr_size >> 10, elfcorehdr_addr); 160e62aaeacSAKASHI Takahiro } 161e62aaeacSAKASHI Takahiro #else 162e62aaeacSAKASHI Takahiro static void __init reserve_elfcorehdr(void) 163e62aaeacSAKASHI Takahiro { 164e62aaeacSAKASHI Takahiro } 165e62aaeacSAKASHI Takahiro #endif /* CONFIG_CRASH_DUMP */ 166d50314a6SCatalin Marinas /* 167ad67f5a6SChristoph Hellwig * Return the maximum physical address for ZONE_DMA32 (DMA_BIT_MASK(32)). It 168d50314a6SCatalin Marinas * currently assumes that for memory starting above 4G, 32-bit devices will 169d50314a6SCatalin Marinas * use a DMA offset. 170d50314a6SCatalin Marinas */ 171a7c61a34SJisheng Zhang static phys_addr_t __init max_zone_dma_phys(void) 172d50314a6SCatalin Marinas { 173d50314a6SCatalin Marinas phys_addr_t offset = memblock_start_of_DRAM() & GENMASK_ULL(63, 32); 174d50314a6SCatalin Marinas return min(offset + (1ULL << 32), memblock_end_of_DRAM()); 175d50314a6SCatalin Marinas } 176d50314a6SCatalin Marinas 1771a2db300SGanapatrao Kulkarni #ifdef CONFIG_NUMA 1781a2db300SGanapatrao Kulkarni 1791a2db300SGanapatrao Kulkarni static void __init zone_sizes_init(unsigned long min, unsigned long max) 1801a2db300SGanapatrao Kulkarni { 1811a2db300SGanapatrao Kulkarni unsigned long max_zone_pfns[MAX_NR_ZONES] = {0}; 1821a2db300SGanapatrao Kulkarni 183ad67f5a6SChristoph Hellwig if (IS_ENABLED(CONFIG_ZONE_DMA32)) 184ad67f5a6SChristoph Hellwig max_zone_pfns[ZONE_DMA32] = PFN_DOWN(max_zone_dma_phys()); 1851a2db300SGanapatrao Kulkarni max_zone_pfns[ZONE_NORMAL] = max; 1861a2db300SGanapatrao Kulkarni 1871a2db300SGanapatrao Kulkarni free_area_init_nodes(max_zone_pfns); 1881a2db300SGanapatrao Kulkarni } 1891a2db300SGanapatrao Kulkarni 1901a2db300SGanapatrao Kulkarni #else 1911a2db300SGanapatrao Kulkarni 192c1cc1552SCatalin Marinas static void __init zone_sizes_init(unsigned long min, unsigned long max) 193c1cc1552SCatalin Marinas { 194c1cc1552SCatalin Marinas struct memblock_region *reg; 195c1cc1552SCatalin Marinas unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; 19619e7640dSCatalin Marinas unsigned long max_dma = min; 197c1cc1552SCatalin Marinas 198c1cc1552SCatalin Marinas memset(zone_size, 0, sizeof(zone_size)); 199c1cc1552SCatalin Marinas 200c1cc1552SCatalin Marinas /* 4GB maximum for 32-bit only capable devices */ 201ad67f5a6SChristoph Hellwig #ifdef CONFIG_ZONE_DMA32 202a1e50a82SCatalin Marinas max_dma = PFN_DOWN(arm64_dma_phys_limit); 203ad67f5a6SChristoph Hellwig zone_size[ZONE_DMA32] = max_dma - min; 20486a5906eSRobin Murphy #endif 20519e7640dSCatalin Marinas zone_size[ZONE_NORMAL] = max - max_dma; 206c1cc1552SCatalin Marinas 207c1cc1552SCatalin Marinas memcpy(zhole_size, zone_size, sizeof(zhole_size)); 208c1cc1552SCatalin Marinas 209c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 210c1cc1552SCatalin Marinas unsigned long start = memblock_region_memory_base_pfn(reg); 211c1cc1552SCatalin Marinas unsigned long end = memblock_region_memory_end_pfn(reg); 212c1cc1552SCatalin Marinas 213c1cc1552SCatalin Marinas if (start >= max) 214c1cc1552SCatalin Marinas continue; 21519e7640dSCatalin Marinas 216ad67f5a6SChristoph Hellwig #ifdef CONFIG_ZONE_DMA32 21786a5906eSRobin Murphy if (start < max_dma) { 21819e7640dSCatalin Marinas unsigned long dma_end = min(end, max_dma); 219ad67f5a6SChristoph Hellwig zhole_size[ZONE_DMA32] -= dma_end - start; 220c1cc1552SCatalin Marinas } 22186a5906eSRobin Murphy #endif 22219e7640dSCatalin Marinas if (end > max_dma) { 223c1cc1552SCatalin Marinas unsigned long normal_end = min(end, max); 22419e7640dSCatalin Marinas unsigned long normal_start = max(start, max_dma); 225c1cc1552SCatalin Marinas zhole_size[ZONE_NORMAL] -= normal_end - normal_start; 226c1cc1552SCatalin Marinas } 227c1cc1552SCatalin Marinas } 228c1cc1552SCatalin Marinas 229c1cc1552SCatalin Marinas free_area_init_node(0, zone_size, min, zhole_size); 230c1cc1552SCatalin Marinas } 231c1cc1552SCatalin Marinas 2321a2db300SGanapatrao Kulkarni #endif /* CONFIG_NUMA */ 2331a2db300SGanapatrao Kulkarni 234c1cc1552SCatalin Marinas int pfn_valid(unsigned long pfn) 235c1cc1552SCatalin Marinas { 2365ad356eaSGreg Hackmann phys_addr_t addr = pfn << PAGE_SHIFT; 2375ad356eaSGreg Hackmann 2385ad356eaSGreg Hackmann if ((addr >> PAGE_SHIFT) != pfn) 2395ad356eaSGreg Hackmann return 0; 2404ab21506SRobin Murphy 2414ab21506SRobin Murphy #ifdef CONFIG_SPARSEMEM 2424ab21506SRobin Murphy if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) 2434ab21506SRobin Murphy return 0; 2444ab21506SRobin Murphy 2454ab21506SRobin Murphy if (!valid_section(__nr_to_section(pfn_to_section_nr(pfn)))) 2464ab21506SRobin Murphy return 0; 2474ab21506SRobin Murphy #endif 2485ad356eaSGreg Hackmann return memblock_is_map_memory(addr); 249c1cc1552SCatalin Marinas } 250c1cc1552SCatalin Marinas EXPORT_SYMBOL(pfn_valid); 251c1cc1552SCatalin Marinas 252d7dc899aSStefan Agner static phys_addr_t memory_limit = PHYS_ADDR_MAX; 2536083fe74SMark Rutland 2546083fe74SMark Rutland /* 2556083fe74SMark Rutland * Limit the memory size that was specified via FDT. 2566083fe74SMark Rutland */ 2576083fe74SMark Rutland static int __init early_mem(char *p) 2586083fe74SMark Rutland { 2596083fe74SMark Rutland if (!p) 2606083fe74SMark Rutland return 1; 2616083fe74SMark Rutland 2626083fe74SMark Rutland memory_limit = memparse(p, &p) & PAGE_MASK; 2636083fe74SMark Rutland pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); 2646083fe74SMark Rutland 2656083fe74SMark Rutland return 0; 2666083fe74SMark Rutland } 2676083fe74SMark Rutland early_param("mem", early_mem); 2686083fe74SMark Rutland 2698f579b1cSAKASHI Takahiro static int __init early_init_dt_scan_usablemem(unsigned long node, 2708f579b1cSAKASHI Takahiro const char *uname, int depth, void *data) 2718f579b1cSAKASHI Takahiro { 2728f579b1cSAKASHI Takahiro struct memblock_region *usablemem = data; 2738f579b1cSAKASHI Takahiro const __be32 *reg; 2748f579b1cSAKASHI Takahiro int len; 2758f579b1cSAKASHI Takahiro 2768f579b1cSAKASHI Takahiro if (depth != 1 || strcmp(uname, "chosen") != 0) 2778f579b1cSAKASHI Takahiro return 0; 2788f579b1cSAKASHI Takahiro 2798f579b1cSAKASHI Takahiro reg = of_get_flat_dt_prop(node, "linux,usable-memory-range", &len); 2808f579b1cSAKASHI Takahiro if (!reg || (len < (dt_root_addr_cells + dt_root_size_cells))) 2818f579b1cSAKASHI Takahiro return 1; 2828f579b1cSAKASHI Takahiro 2838f579b1cSAKASHI Takahiro usablemem->base = dt_mem_next_cell(dt_root_addr_cells, ®); 2848f579b1cSAKASHI Takahiro usablemem->size = dt_mem_next_cell(dt_root_size_cells, ®); 2858f579b1cSAKASHI Takahiro 2868f579b1cSAKASHI Takahiro return 1; 2878f579b1cSAKASHI Takahiro } 2888f579b1cSAKASHI Takahiro 2898f579b1cSAKASHI Takahiro static void __init fdt_enforce_memory_region(void) 2908f579b1cSAKASHI Takahiro { 2918f579b1cSAKASHI Takahiro struct memblock_region reg = { 2928f579b1cSAKASHI Takahiro .size = 0, 2938f579b1cSAKASHI Takahiro }; 2948f579b1cSAKASHI Takahiro 2958f579b1cSAKASHI Takahiro of_scan_flat_dt(early_init_dt_scan_usablemem, ®); 2968f579b1cSAKASHI Takahiro 2978f579b1cSAKASHI Takahiro if (reg.size) 2988f579b1cSAKASHI Takahiro memblock_cap_memory_range(reg.base, reg.size); 2998f579b1cSAKASHI Takahiro } 3008f579b1cSAKASHI Takahiro 301c1cc1552SCatalin Marinas void __init arm64_memblock_init(void) 302c1cc1552SCatalin Marinas { 303a7f8de16SArd Biesheuvel const s64 linear_region_size = -(s64)PAGE_OFFSET; 304a7f8de16SArd Biesheuvel 3058f579b1cSAKASHI Takahiro /* Handle linux,usable-memory-range property */ 3068f579b1cSAKASHI Takahiro fdt_enforce_memory_region(); 3078f579b1cSAKASHI Takahiro 308e9eaa805SKristina Martsenko /* Remove memory above our supported physical address size */ 309e9eaa805SKristina Martsenko memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX); 310e9eaa805SKristina Martsenko 311a7f8de16SArd Biesheuvel /* 3126d2aa549SArd Biesheuvel * Ensure that the linear region takes up exactly half of the kernel 3136d2aa549SArd Biesheuvel * virtual address space. This way, we can distinguish a linear address 3146d2aa549SArd Biesheuvel * from a kernel/module/vmalloc address by testing a single bit. 3156d2aa549SArd Biesheuvel */ 3166d2aa549SArd Biesheuvel BUILD_BUG_ON(linear_region_size != BIT(VA_BITS - 1)); 3176d2aa549SArd Biesheuvel 3186d2aa549SArd Biesheuvel /* 319a7f8de16SArd Biesheuvel * Select a suitable value for the base of physical memory. 320a7f8de16SArd Biesheuvel */ 321a7f8de16SArd Biesheuvel memstart_addr = round_down(memblock_start_of_DRAM(), 322a7f8de16SArd Biesheuvel ARM64_MEMSTART_ALIGN); 323a7f8de16SArd Biesheuvel 324a7f8de16SArd Biesheuvel /* 325a7f8de16SArd Biesheuvel * Remove the memory that we will not be able to cover with the 326a7f8de16SArd Biesheuvel * linear mapping. Take care not to clip the kernel which may be 327a7f8de16SArd Biesheuvel * high in memory. 328a7f8de16SArd Biesheuvel */ 3292077be67SLaura Abbott memblock_remove(max_t(u64, memstart_addr + linear_region_size, 3302077be67SLaura Abbott __pa_symbol(_end)), ULLONG_MAX); 3312958987fSArd Biesheuvel if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) { 3322958987fSArd Biesheuvel /* ensure that memstart_addr remains sufficiently aligned */ 3332958987fSArd Biesheuvel memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size, 3342958987fSArd Biesheuvel ARM64_MEMSTART_ALIGN); 3352958987fSArd Biesheuvel memblock_remove(0, memstart_addr); 3362958987fSArd Biesheuvel } 337a7f8de16SArd Biesheuvel 338a7f8de16SArd Biesheuvel /* 339a7f8de16SArd Biesheuvel * Apply the memory limit if it was set. Since the kernel may be loaded 340a7f8de16SArd Biesheuvel * high up in memory, add back the kernel region that must be accessible 341a7f8de16SArd Biesheuvel * via the linear mapping. 342a7f8de16SArd Biesheuvel */ 343d7dc899aSStefan Agner if (memory_limit != PHYS_ADDR_MAX) { 344cb0a6502SDennis Chen memblock_mem_limit_remove_map(memory_limit); 3452077be67SLaura Abbott memblock_add(__pa_symbol(_text), (u64)(_end - _text)); 346a7f8de16SArd Biesheuvel } 3476083fe74SMark Rutland 348c756c592SFlorian Fainelli if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { 349177e15f0SArd Biesheuvel /* 350177e15f0SArd Biesheuvel * Add back the memory we just removed if it results in the 351177e15f0SArd Biesheuvel * initrd to become inaccessible via the linear mapping. 352177e15f0SArd Biesheuvel * Otherwise, this is a no-op 353177e15f0SArd Biesheuvel */ 354c756c592SFlorian Fainelli u64 base = phys_initrd_start & PAGE_MASK; 355d4d18e3eSBjorn Andersson u64 size = PAGE_ALIGN(phys_initrd_start + phys_initrd_size) - base; 356177e15f0SArd Biesheuvel 357177e15f0SArd Biesheuvel /* 358177e15f0SArd Biesheuvel * We can only add back the initrd memory if we don't end up 359177e15f0SArd Biesheuvel * with more memory than we can address via the linear mapping. 360177e15f0SArd Biesheuvel * It is up to the bootloader to position the kernel and the 361177e15f0SArd Biesheuvel * initrd reasonably close to each other (i.e., within 32 GB of 362177e15f0SArd Biesheuvel * each other) so that all granule/#levels combinations can 363177e15f0SArd Biesheuvel * always access both. 364177e15f0SArd Biesheuvel */ 365177e15f0SArd Biesheuvel if (WARN(base < memblock_start_of_DRAM() || 366177e15f0SArd Biesheuvel base + size > memblock_start_of_DRAM() + 367177e15f0SArd Biesheuvel linear_region_size, 368177e15f0SArd Biesheuvel "initrd not fully accessible via the linear mapping -- please check your bootloader ...\n")) { 36970b3d237SWill Deacon phys_initrd_size = 0; 370177e15f0SArd Biesheuvel } else { 371177e15f0SArd Biesheuvel memblock_remove(base, size); /* clear MEMBLOCK_ flags */ 372177e15f0SArd Biesheuvel memblock_add(base, size); 373177e15f0SArd Biesheuvel memblock_reserve(base, size); 374177e15f0SArd Biesheuvel } 375177e15f0SArd Biesheuvel } 376177e15f0SArd Biesheuvel 377c031a421SArd Biesheuvel if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { 378c031a421SArd Biesheuvel extern u16 memstart_offset_seed; 379c031a421SArd Biesheuvel u64 range = linear_region_size - 380c031a421SArd Biesheuvel (memblock_end_of_DRAM() - memblock_start_of_DRAM()); 381c031a421SArd Biesheuvel 382c031a421SArd Biesheuvel /* 383c031a421SArd Biesheuvel * If the size of the linear region exceeds, by a sufficient 384c031a421SArd Biesheuvel * margin, the size of the region that the available physical 385c031a421SArd Biesheuvel * memory spans, randomize the linear region as well. 386c031a421SArd Biesheuvel */ 387c031a421SArd Biesheuvel if (memstart_offset_seed > 0 && range >= ARM64_MEMSTART_ALIGN) { 388c8a43c18SYueyi Li range /= ARM64_MEMSTART_ALIGN; 389c031a421SArd Biesheuvel memstart_addr -= ARM64_MEMSTART_ALIGN * 390c031a421SArd Biesheuvel ((range * memstart_offset_seed) >> 16); 391c031a421SArd Biesheuvel } 392c031a421SArd Biesheuvel } 3932d5a5612SCatalin Marinas 394bd00cd5fSMark Rutland /* 395bd00cd5fSMark Rutland * Register the kernel text, kernel data, initrd, and initial 396bd00cd5fSMark Rutland * pagetables with memblock. 397bd00cd5fSMark Rutland */ 3982077be67SLaura Abbott memblock_reserve(__pa_symbol(_text), _end - _text); 399c756c592SFlorian Fainelli if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { 400a89dea58SArd Biesheuvel /* the generic initrd code expects virtual addresses */ 401c756c592SFlorian Fainelli initrd_start = __phys_to_virt(phys_initrd_start); 402c756c592SFlorian Fainelli initrd_end = initrd_start + phys_initrd_size; 403a89dea58SArd Biesheuvel } 404c1cc1552SCatalin Marinas 4059bf14b7cSMarek Szyprowski early_init_fdt_scan_reserved_mem(); 4062d5a5612SCatalin Marinas 4072d5a5612SCatalin Marinas /* 4GB maximum for 32-bit only capable devices */ 408ad67f5a6SChristoph Hellwig if (IS_ENABLED(CONFIG_ZONE_DMA32)) 409a1e50a82SCatalin Marinas arm64_dma_phys_limit = max_zone_dma_phys(); 410a1e50a82SCatalin Marinas else 411a1e50a82SCatalin Marinas arm64_dma_phys_limit = PHYS_MASK + 1; 412764b51eaSAKASHI Takahiro 413764b51eaSAKASHI Takahiro reserve_crashkernel(); 414764b51eaSAKASHI Takahiro 415e62aaeacSAKASHI Takahiro reserve_elfcorehdr(); 416e62aaeacSAKASHI Takahiro 417f24e5834SSteve Capper high_memory = __va(memblock_end_of_DRAM() - 1) + 1; 418f24e5834SSteve Capper 419a1e50a82SCatalin Marinas dma_contiguous_reserve(arm64_dma_phys_limit); 420c1cc1552SCatalin Marinas } 421c1cc1552SCatalin Marinas 422c1cc1552SCatalin Marinas void __init bootmem_init(void) 423c1cc1552SCatalin Marinas { 424c1cc1552SCatalin Marinas unsigned long min, max; 425c1cc1552SCatalin Marinas 426c1cc1552SCatalin Marinas min = PFN_UP(memblock_start_of_DRAM()); 427c1cc1552SCatalin Marinas max = PFN_DOWN(memblock_end_of_DRAM()); 428c1cc1552SCatalin Marinas 42936dd9086SVladimir Murzin early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT); 43036dd9086SVladimir Murzin 4311a2db300SGanapatrao Kulkarni max_pfn = max_low_pfn = max; 43219d6242eSMiles Chen min_low_pfn = min; 4331a2db300SGanapatrao Kulkarni 4341a2db300SGanapatrao Kulkarni arm64_numa_init(); 435c1cc1552SCatalin Marinas /* 436c1cc1552SCatalin Marinas * Sparsemem tries to allocate bootmem in memory_present(), so must be 437c1cc1552SCatalin Marinas * done after the fixed reservations. 438c1cc1552SCatalin Marinas */ 439a2c801c5SLogan Gunthorpe memblocks_present(); 440c1cc1552SCatalin Marinas 441c1cc1552SCatalin Marinas sparse_init(); 442c1cc1552SCatalin Marinas zone_sizes_init(min, max); 443c1cc1552SCatalin Marinas 4441a2db300SGanapatrao Kulkarni memblock_dump_all(); 445c1cc1552SCatalin Marinas } 446c1cc1552SCatalin Marinas 447c1cc1552SCatalin Marinas #ifndef CONFIG_SPARSEMEM_VMEMMAP 448c1cc1552SCatalin Marinas static inline void free_memmap(unsigned long start_pfn, unsigned long end_pfn) 449c1cc1552SCatalin Marinas { 450c1cc1552SCatalin Marinas struct page *start_pg, *end_pg; 451c1cc1552SCatalin Marinas unsigned long pg, pgend; 452c1cc1552SCatalin Marinas 453c1cc1552SCatalin Marinas /* 454c1cc1552SCatalin Marinas * Convert start_pfn/end_pfn to a struct page pointer. 455c1cc1552SCatalin Marinas */ 456c1cc1552SCatalin Marinas start_pg = pfn_to_page(start_pfn - 1) + 1; 457c1cc1552SCatalin Marinas end_pg = pfn_to_page(end_pfn - 1) + 1; 458c1cc1552SCatalin Marinas 459c1cc1552SCatalin Marinas /* 460c1cc1552SCatalin Marinas * Convert to physical addresses, and round start upwards and end 461c1cc1552SCatalin Marinas * downwards. 462c1cc1552SCatalin Marinas */ 463c1cc1552SCatalin Marinas pg = (unsigned long)PAGE_ALIGN(__pa(start_pg)); 464c1cc1552SCatalin Marinas pgend = (unsigned long)__pa(end_pg) & PAGE_MASK; 465c1cc1552SCatalin Marinas 466c1cc1552SCatalin Marinas /* 467c1cc1552SCatalin Marinas * If there are free pages between these, free the section of the 468c1cc1552SCatalin Marinas * memmap array. 469c1cc1552SCatalin Marinas */ 470c1cc1552SCatalin Marinas if (pg < pgend) 4712013288fSMike Rapoport memblock_free(pg, pgend - pg); 472c1cc1552SCatalin Marinas } 473c1cc1552SCatalin Marinas 474c1cc1552SCatalin Marinas /* 475c1cc1552SCatalin Marinas * The mem_map array can get very big. Free the unused area of the memory map. 476c1cc1552SCatalin Marinas */ 477c1cc1552SCatalin Marinas static void __init free_unused_memmap(void) 478c1cc1552SCatalin Marinas { 479c1cc1552SCatalin Marinas unsigned long start, prev_end = 0; 480c1cc1552SCatalin Marinas struct memblock_region *reg; 481c1cc1552SCatalin Marinas 482c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 483c1cc1552SCatalin Marinas start = __phys_to_pfn(reg->base); 484c1cc1552SCatalin Marinas 485c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM 486c1cc1552SCatalin Marinas /* 487c1cc1552SCatalin Marinas * Take care not to free memmap entries that don't exist due 488c1cc1552SCatalin Marinas * to SPARSEMEM sections which aren't present. 489c1cc1552SCatalin Marinas */ 490c1cc1552SCatalin Marinas start = min(start, ALIGN(prev_end, PAGES_PER_SECTION)); 491c1cc1552SCatalin Marinas #endif 492c1cc1552SCatalin Marinas /* 493c1cc1552SCatalin Marinas * If we had a previous bank, and there is a space between the 494c1cc1552SCatalin Marinas * current bank and the previous, free it. 495c1cc1552SCatalin Marinas */ 496c1cc1552SCatalin Marinas if (prev_end && prev_end < start) 497c1cc1552SCatalin Marinas free_memmap(prev_end, start); 498c1cc1552SCatalin Marinas 499c1cc1552SCatalin Marinas /* 500c1cc1552SCatalin Marinas * Align up here since the VM subsystem insists that the 501c1cc1552SCatalin Marinas * memmap entries are valid from the bank end aligned to 502c1cc1552SCatalin Marinas * MAX_ORDER_NR_PAGES. 503c1cc1552SCatalin Marinas */ 504b9bcc919SDave P Martin prev_end = ALIGN(__phys_to_pfn(reg->base + reg->size), 505c1cc1552SCatalin Marinas MAX_ORDER_NR_PAGES); 506c1cc1552SCatalin Marinas } 507c1cc1552SCatalin Marinas 508c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM 509c1cc1552SCatalin Marinas if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) 510c1cc1552SCatalin Marinas free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); 511c1cc1552SCatalin Marinas #endif 512c1cc1552SCatalin Marinas } 513c1cc1552SCatalin Marinas #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 514c1cc1552SCatalin Marinas 515c1cc1552SCatalin Marinas /* 516c1cc1552SCatalin Marinas * mem_init() marks the free areas in the mem_map and tells us how much memory 517c1cc1552SCatalin Marinas * is free. This is done after various parts of the system have claimed their 518c1cc1552SCatalin Marinas * memory after the kernel image. 519c1cc1552SCatalin Marinas */ 520c1cc1552SCatalin Marinas void __init mem_init(void) 521c1cc1552SCatalin Marinas { 522ae7871beSGeert Uytterhoeven if (swiotlb_force == SWIOTLB_FORCE || 523ae7871beSGeert Uytterhoeven max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) 524a1e50a82SCatalin Marinas swiotlb_init(1); 525524dabe1SAlexander Graf else 526524dabe1SAlexander Graf swiotlb_force = SWIOTLB_NO_FORCE; 527a1e50a82SCatalin Marinas 528344bf332SMuchun Song set_max_mapnr(max_pfn - PHYS_PFN_OFFSET); 529c1cc1552SCatalin Marinas 530c1cc1552SCatalin Marinas #ifndef CONFIG_SPARSEMEM_VMEMMAP 531c1cc1552SCatalin Marinas free_unused_memmap(); 532c1cc1552SCatalin Marinas #endif 533bee4ebd1SJiang Liu /* this will put all unused low memory onto the freelists */ 534c6ffc5caSMike Rapoport memblock_free_all(); 535c1cc1552SCatalin Marinas 5366879ea83SJiang Liu mem_init_print_info(NULL); 537c1cc1552SCatalin Marinas 538c1cc1552SCatalin Marinas /* 539c1cc1552SCatalin Marinas * Check boundaries twice: Some fundamental inconsistencies can be 540c1cc1552SCatalin Marinas * detected at build time already. 541c1cc1552SCatalin Marinas */ 542c1cc1552SCatalin Marinas #ifdef CONFIG_COMPAT 543363524d2SSteve Capper BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64); 544c1cc1552SCatalin Marinas #endif 545c1cc1552SCatalin Marinas 546bee4ebd1SJiang Liu if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { 547c1cc1552SCatalin Marinas extern int sysctl_overcommit_memory; 548c1cc1552SCatalin Marinas /* 549c1cc1552SCatalin Marinas * On a machine this small we won't get anywhere without 550c1cc1552SCatalin Marinas * overcommit, so turn it on by default. 551c1cc1552SCatalin Marinas */ 552c1cc1552SCatalin Marinas sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; 553c1cc1552SCatalin Marinas } 554c1cc1552SCatalin Marinas } 555c1cc1552SCatalin Marinas 556c1cc1552SCatalin Marinas void free_initmem(void) 557c1cc1552SCatalin Marinas { 5582077be67SLaura Abbott free_reserved_area(lm_alias(__init_begin), 5592077be67SLaura Abbott lm_alias(__init_end), 560d386825cSArd Biesheuvel 0, "unused kernel"); 561dae8c235SKefeng Wang /* 562dae8c235SKefeng Wang * Unmap the __init region but leave the VM area in place. This 563dae8c235SKefeng Wang * prevents the region from being reused for kernel modules, which 564dae8c235SKefeng Wang * is not supported by kallsyms. 565dae8c235SKefeng Wang */ 566dae8c235SKefeng Wang unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin)); 567c1cc1552SCatalin Marinas } 568c1cc1552SCatalin Marinas 569c1cc1552SCatalin Marinas #ifdef CONFIG_BLK_DEV_INITRD 570662ba3dbSWang Long void __init free_initrd_mem(unsigned long start, unsigned long end) 571c1cc1552SCatalin Marinas { 5729af5b807SJiang Liu free_reserved_area((void *)start, (void *)end, 0, "initrd"); 57305c58752SCHANDAN VN memblock_free(__virt_to_phys(start), end - start); 57405c58752SCHANDAN VN } 575c1cc1552SCatalin Marinas #endif 576a7f8de16SArd Biesheuvel 577a7f8de16SArd Biesheuvel /* 578a7f8de16SArd Biesheuvel * Dump out memory limit information on panic. 579a7f8de16SArd Biesheuvel */ 580a7f8de16SArd Biesheuvel static int dump_mem_limit(struct notifier_block *self, unsigned long v, void *p) 581a7f8de16SArd Biesheuvel { 582d7dc899aSStefan Agner if (memory_limit != PHYS_ADDR_MAX) { 583a7f8de16SArd Biesheuvel pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20); 584a7f8de16SArd Biesheuvel } else { 585a7f8de16SArd Biesheuvel pr_emerg("Memory Limit: none\n"); 586a7f8de16SArd Biesheuvel } 587a7f8de16SArd Biesheuvel return 0; 588a7f8de16SArd Biesheuvel } 589a7f8de16SArd Biesheuvel 590a7f8de16SArd Biesheuvel static struct notifier_block mem_limit_notifier = { 591a7f8de16SArd Biesheuvel .notifier_call = dump_mem_limit, 592a7f8de16SArd Biesheuvel }; 593a7f8de16SArd Biesheuvel 594a7f8de16SArd Biesheuvel static int __init register_mem_limit_dumper(void) 595a7f8de16SArd Biesheuvel { 596a7f8de16SArd Biesheuvel atomic_notifier_chain_register(&panic_notifier_list, 597a7f8de16SArd Biesheuvel &mem_limit_notifier); 598a7f8de16SArd Biesheuvel return 0; 599a7f8de16SArd Biesheuvel } 600a7f8de16SArd Biesheuvel __initcall(register_mem_limit_dumper); 601