1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c1cc1552SCatalin Marinas /* 3c1cc1552SCatalin Marinas * Based on arch/arm/mm/init.c 4c1cc1552SCatalin Marinas * 5c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 6c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 7c1cc1552SCatalin Marinas */ 8c1cc1552SCatalin Marinas 9c1cc1552SCatalin Marinas #include <linux/kernel.h> 10c1cc1552SCatalin Marinas #include <linux/export.h> 11c1cc1552SCatalin Marinas #include <linux/errno.h> 12c1cc1552SCatalin Marinas #include <linux/swap.h> 13c1cc1552SCatalin Marinas #include <linux/init.h> 145a9e3e15SJisheng Zhang #include <linux/cache.h> 15c1cc1552SCatalin Marinas #include <linux/mman.h> 16c1cc1552SCatalin Marinas #include <linux/nodemask.h> 17c1cc1552SCatalin Marinas #include <linux/initrd.h> 18c1cc1552SCatalin Marinas #include <linux/gfp.h> 19c1cc1552SCatalin Marinas #include <linux/memblock.h> 20c1cc1552SCatalin Marinas #include <linux/sort.h> 21764b51eaSAKASHI Takahiro #include <linux/of.h> 22c1cc1552SCatalin Marinas #include <linux/of_fdt.h> 238b5369eaSNicolas Saenz Julienne #include <linux/dma-direct.h> 240b1abd1fSChristoph Hellwig #include <linux/dma-map-ops.h> 2586c8b27aSLeif Lindholm #include <linux/efi.h> 26a1e50a82SCatalin Marinas #include <linux/swiotlb.h> 27dae8c235SKefeng Wang #include <linux/vmalloc.h> 282077be67SLaura Abbott #include <linux/mm.h> 29764b51eaSAKASHI Takahiro #include <linux/kexec.h> 30e62aaeacSAKASHI Takahiro #include <linux/crash_dump.h> 31cf11e85fSRoman Gushchin #include <linux/hugetlb.h> 322b865293SArd Biesheuvel #include <linux/acpi_iort.h> 3385f58eb1SChen Wandun #include <linux/kmemleak.h> 34c1cc1552SCatalin Marinas 35a7f8de16SArd Biesheuvel #include <asm/boot.h> 3608375198SCatalin Marinas #include <asm/fixmap.h> 37f9040773SArd Biesheuvel #include <asm/kasan.h> 38a7f8de16SArd Biesheuvel #include <asm/kernel-pgtable.h> 39f320bc74SQuentin Perret #include <asm/kvm_host.h> 40aa03c428SMark Rutland #include <asm/memory.h> 411a2db300SGanapatrao Kulkarni #include <asm/numa.h> 42c1cc1552SCatalin Marinas #include <asm/sections.h> 43c1cc1552SCatalin Marinas #include <asm/setup.h> 4487dfb311SMasahiro Yamada #include <linux/sizes.h> 45c1cc1552SCatalin Marinas #include <asm/tlb.h> 46e039ee4eSAndre Przywara #include <asm/alternative.h> 47687842ecSChristoph Hellwig #include <asm/xen/swiotlb-xen.h> 48c1cc1552SCatalin Marinas 49a7f8de16SArd Biesheuvel /* 50a7f8de16SArd Biesheuvel * We need to be able to catch inadvertent references to memstart_addr 51a7f8de16SArd Biesheuvel * that occur (potentially in generic code) before arm64_memblock_init() 52a7f8de16SArd Biesheuvel * executes, which assigns it its actual value. So use a default value 53a7f8de16SArd Biesheuvel * that cannot be mistaken for a real physical address. 54a7f8de16SArd Biesheuvel */ 555a9e3e15SJisheng Zhang s64 memstart_addr __ro_after_init = -1; 5603ef055fSMark Rutland EXPORT_SYMBOL(memstart_addr); 5703ef055fSMark Rutland 581a8e1cefSNicolas Saenz Julienne /* 59d78050eeSCatalin Marinas * If the corresponding config options are enabled, we create both ZONE_DMA 60d78050eeSCatalin Marinas * and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory 61d78050eeSCatalin Marinas * unless restricted on specific platforms (e.g. 30-bit on Raspberry Pi 4). 62d78050eeSCatalin Marinas * In such case, ZONE_DMA32 covers the rest of the 32-bit addressable memory, 63d78050eeSCatalin Marinas * otherwise it is empty. 641a8e1cefSNicolas Saenz Julienne */ 6503149563SVijay Balakrishna phys_addr_t __ro_after_init arm64_dma_phys_limit; 66c1cc1552SCatalin Marinas 67944a45abSChen Zhou /* Current arm64 boot protocol requires 2MB alignment */ 68944a45abSChen Zhou #define CRASH_ALIGN SZ_2M 69944a45abSChen Zhou 70944a45abSChen Zhou #define CRASH_ADDR_LOW_MAX arm64_dma_phys_limit 71944a45abSChen Zhou #define CRASH_ADDR_HIGH_MAX (PHYS_MASK + 1) 72*6c4dcaddSBaoquan He #define CRASH_HIGH_SEARCH_BASE SZ_4G 73944a45abSChen Zhou 74a149cf00SZhen Lei #define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20) 75a149cf00SZhen Lei 76944a45abSChen Zhou static int __init reserve_crashkernel_low(unsigned long long low_size) 77944a45abSChen Zhou { 78944a45abSChen Zhou unsigned long long low_base; 79944a45abSChen Zhou 80944a45abSChen Zhou low_base = memblock_phys_alloc_range(low_size, CRASH_ALIGN, 0, CRASH_ADDR_LOW_MAX); 81944a45abSChen Zhou if (!low_base) { 82944a45abSChen Zhou pr_err("cannot allocate crashkernel low memory (size:0x%llx).\n", low_size); 83944a45abSChen Zhou return -ENOMEM; 84944a45abSChen Zhou } 85944a45abSChen Zhou 86944a45abSChen Zhou pr_info("crashkernel low memory reserved: 0x%08llx - 0x%08llx (%lld MB)\n", 87944a45abSChen Zhou low_base, low_base + low_size, low_size >> 20); 88944a45abSChen Zhou 89944a45abSChen Zhou crashk_low_res.start = low_base; 90944a45abSChen Zhou crashk_low_res.end = low_base + low_size - 1; 91944a45abSChen Zhou insert_resource(&iomem_resource, &crashk_low_res); 92944a45abSChen Zhou 93944a45abSChen Zhou return 0; 94944a45abSChen Zhou } 95944a45abSChen Zhou 96764b51eaSAKASHI Takahiro /* 97764b51eaSAKASHI Takahiro * reserve_crashkernel() - reserves memory for crash kernel 98764b51eaSAKASHI Takahiro * 99764b51eaSAKASHI Takahiro * This function reserves memory area given in "crashkernel=" kernel command 100764b51eaSAKASHI Takahiro * line parameter. The memory reserved is used by dump capture kernel when 101764b51eaSAKASHI Takahiro * primary kernel is crashing. 102764b51eaSAKASHI Takahiro */ 103764b51eaSAKASHI Takahiro static void __init reserve_crashkernel(void) 104764b51eaSAKASHI Takahiro { 105*6c4dcaddSBaoquan He unsigned long long crash_low_size = 0, search_base = 0; 106944a45abSChen Zhou unsigned long long crash_max = CRASH_ADDR_LOW_MAX; 107*6c4dcaddSBaoquan He unsigned long long crash_base, crash_size; 108944a45abSChen Zhou char *cmdline = boot_command_line; 109a9ae89dfSZhen Lei bool fixed_base = false; 110*6c4dcaddSBaoquan He bool high = false; 111*6c4dcaddSBaoquan He int ret; 112764b51eaSAKASHI Takahiro 113d339f158SJisheng Zhang if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 114d339f158SJisheng Zhang return; 115d339f158SJisheng Zhang 116944a45abSChen Zhou /* crashkernel=X[@offset] */ 117944a45abSChen Zhou ret = parse_crashkernel(cmdline, memblock_phys_mem_size(), 118764b51eaSAKASHI Takahiro &crash_size, &crash_base); 119944a45abSChen Zhou if (ret == -ENOENT) { 120944a45abSChen Zhou ret = parse_crashkernel_high(cmdline, 0, &crash_size, &crash_base); 121764b51eaSAKASHI Takahiro if (ret || !crash_size) 122764b51eaSAKASHI Takahiro return; 123764b51eaSAKASHI Takahiro 124944a45abSChen Zhou /* 125944a45abSChen Zhou * crashkernel=Y,low can be specified or not, but invalid value 126944a45abSChen Zhou * is not allowed. 127944a45abSChen Zhou */ 128944a45abSChen Zhou ret = parse_crashkernel_low(cmdline, 0, &crash_low_size, &crash_base); 129a149cf00SZhen Lei if (ret == -ENOENT) 130a149cf00SZhen Lei crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE; 131a149cf00SZhen Lei else if (ret) 132944a45abSChen Zhou return; 133944a45abSChen Zhou 134*6c4dcaddSBaoquan He search_base = CRASH_HIGH_SEARCH_BASE; 135944a45abSChen Zhou crash_max = CRASH_ADDR_HIGH_MAX; 136*6c4dcaddSBaoquan He high = true; 137944a45abSChen Zhou } else if (ret || !crash_size) { 138944a45abSChen Zhou /* The specified value is invalid */ 139944a45abSChen Zhou return; 140944a45abSChen Zhou } 141944a45abSChen Zhou 142764b51eaSAKASHI Takahiro crash_size = PAGE_ALIGN(crash_size); 143764b51eaSAKASHI Takahiro 144a7259df7SMike Rapoport /* User specifies base address explicitly. */ 145a9ae89dfSZhen Lei if (crash_base) { 146a9ae89dfSZhen Lei fixed_base = true; 147*6c4dcaddSBaoquan He search_base = crash_base; 148a7259df7SMike Rapoport crash_max = crash_base + crash_size; 149a9ae89dfSZhen Lei } 150a7259df7SMike Rapoport 151a9ae89dfSZhen Lei retry: 152944a45abSChen Zhou crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, 153*6c4dcaddSBaoquan He search_base, crash_max); 154a7259df7SMike Rapoport if (!crash_base) { 155a9ae89dfSZhen Lei /* 156*6c4dcaddSBaoquan He * For crashkernel=size[KMG]@offset[KMG], print out failure 157*6c4dcaddSBaoquan He * message if can't reserve the specified region. 158a9ae89dfSZhen Lei */ 159*6c4dcaddSBaoquan He if (fixed_base) { 160*6c4dcaddSBaoquan He pr_warn("crashkernel reservation failed - memory is in use.\n"); 161*6c4dcaddSBaoquan He return; 162*6c4dcaddSBaoquan He } 163*6c4dcaddSBaoquan He 164*6c4dcaddSBaoquan He /* 165*6c4dcaddSBaoquan He * For crashkernel=size[KMG], if the first attempt was for 166*6c4dcaddSBaoquan He * low memory, fall back to high memory, the minimum required 167*6c4dcaddSBaoquan He * low memory will be reserved later. 168*6c4dcaddSBaoquan He */ 169*6c4dcaddSBaoquan He if (!high && crash_max == CRASH_ADDR_LOW_MAX) { 170a9ae89dfSZhen Lei crash_max = CRASH_ADDR_HIGH_MAX; 171*6c4dcaddSBaoquan He search_base = CRASH_ADDR_LOW_MAX; 172a9ae89dfSZhen Lei crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE; 173a9ae89dfSZhen Lei goto retry; 174a9ae89dfSZhen Lei } 175a9ae89dfSZhen Lei 176*6c4dcaddSBaoquan He /* 177*6c4dcaddSBaoquan He * For crashkernel=size[KMG],high, if the first attempt was 178*6c4dcaddSBaoquan He * for high memory, fall back to low memory. 179*6c4dcaddSBaoquan He */ 180*6c4dcaddSBaoquan He if (high && crash_max == CRASH_ADDR_HIGH_MAX) { 181*6c4dcaddSBaoquan He crash_max = CRASH_ADDR_LOW_MAX; 182*6c4dcaddSBaoquan He search_base = 0; 183*6c4dcaddSBaoquan He goto retry; 184*6c4dcaddSBaoquan He } 185764b51eaSAKASHI Takahiro pr_warn("cannot allocate crashkernel (size:0x%llx)\n", 186764b51eaSAKASHI Takahiro crash_size); 187764b51eaSAKASHI Takahiro return; 188764b51eaSAKASHI Takahiro } 189764b51eaSAKASHI Takahiro 190*6c4dcaddSBaoquan He if ((crash_base >= CRASH_ADDR_LOW_MAX) && crash_low_size && 191*6c4dcaddSBaoquan He reserve_crashkernel_low(crash_low_size)) { 192944a45abSChen Zhou memblock_phys_free(crash_base, crash_size); 193944a45abSChen Zhou return; 194944a45abSChen Zhou } 195944a45abSChen Zhou 196764b51eaSAKASHI Takahiro pr_info("crashkernel reserved: 0x%016llx - 0x%016llx (%lld MB)\n", 197764b51eaSAKASHI Takahiro crash_base, crash_base + crash_size, crash_size >> 20); 198764b51eaSAKASHI Takahiro 19985f58eb1SChen Wandun /* 20085f58eb1SChen Wandun * The crashkernel memory will be removed from the kernel linear 20185f58eb1SChen Wandun * map. Inform kmemleak so that it won't try to access it. 20285f58eb1SChen Wandun */ 20385f58eb1SChen Wandun kmemleak_ignore_phys(crash_base); 204944a45abSChen Zhou if (crashk_low_res.end) 205944a45abSChen Zhou kmemleak_ignore_phys(crashk_low_res.start); 206944a45abSChen Zhou 207764b51eaSAKASHI Takahiro crashk_res.start = crash_base; 208764b51eaSAKASHI Takahiro crashk_res.end = crash_base + crash_size - 1; 209e6b39442SZhen Lei insert_resource(&iomem_resource, &crashk_res); 210764b51eaSAKASHI Takahiro } 211764b51eaSAKASHI Takahiro 212d50314a6SCatalin Marinas /* 213791ab8b2SCatalin Marinas * Return the maximum physical address for a zone accessible by the given bits 214791ab8b2SCatalin Marinas * limit. If DRAM starts above 32-bit, expand the zone to the maximum 215791ab8b2SCatalin Marinas * available memory, otherwise cap it at 32-bit. 216d50314a6SCatalin Marinas */ 2171a8e1cefSNicolas Saenz Julienne static phys_addr_t __init max_zone_phys(unsigned int zone_bits) 218d50314a6SCatalin Marinas { 219791ab8b2SCatalin Marinas phys_addr_t zone_mask = DMA_BIT_MASK(zone_bits); 220791ab8b2SCatalin Marinas phys_addr_t phys_start = memblock_start_of_DRAM(); 221791ab8b2SCatalin Marinas 222791ab8b2SCatalin Marinas if (phys_start > U32_MAX) 223791ab8b2SCatalin Marinas zone_mask = PHYS_ADDR_MAX; 224791ab8b2SCatalin Marinas else if (phys_start > zone_mask) 225791ab8b2SCatalin Marinas zone_mask = U32_MAX; 226791ab8b2SCatalin Marinas 227791ab8b2SCatalin Marinas return min(zone_mask, memblock_end_of_DRAM() - 1) + 1; 228d50314a6SCatalin Marinas } 229d50314a6SCatalin Marinas 230f41ef4c2SKefeng Wang static void __init zone_sizes_init(void) 2311a2db300SGanapatrao Kulkarni { 2321a2db300SGanapatrao Kulkarni unsigned long max_zone_pfns[MAX_NR_ZONES] = {0}; 2332b865293SArd Biesheuvel unsigned int __maybe_unused acpi_zone_dma_bits; 2348424ecddSNicolas Saenz Julienne unsigned int __maybe_unused dt_zone_dma_bits; 235d78050eeSCatalin Marinas phys_addr_t __maybe_unused dma32_phys_limit = max_zone_phys(32); 2361a2db300SGanapatrao Kulkarni 2371a8e1cefSNicolas Saenz Julienne #ifdef CONFIG_ZONE_DMA 2382b865293SArd Biesheuvel acpi_zone_dma_bits = fls64(acpi_iort_dma_get_max_cpu_address()); 2398424ecddSNicolas Saenz Julienne dt_zone_dma_bits = fls64(of_dma_get_max_cpu_address(NULL)); 2402b865293SArd Biesheuvel zone_dma_bits = min3(32U, dt_zone_dma_bits, acpi_zone_dma_bits); 2419804f8c6SNicolas Saenz Julienne arm64_dma_phys_limit = max_zone_phys(zone_dma_bits); 2421a8e1cefSNicolas Saenz Julienne max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit); 2431a8e1cefSNicolas Saenz Julienne #endif 2440c1f14edSMiles Chen #ifdef CONFIG_ZONE_DMA32 245d78050eeSCatalin Marinas max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); 246d78050eeSCatalin Marinas if (!arm64_dma_phys_limit) 247d78050eeSCatalin Marinas arm64_dma_phys_limit = dma32_phys_limit; 2480c1f14edSMiles Chen #endif 249504cae45SBaoquan He if (!arm64_dma_phys_limit) 250504cae45SBaoquan He arm64_dma_phys_limit = PHYS_MASK + 1; 251f41ef4c2SKefeng Wang max_zone_pfns[ZONE_NORMAL] = max_pfn; 2521a2db300SGanapatrao Kulkarni 2539691a071SMike Rapoport free_area_init(max_zone_pfns); 2541a2db300SGanapatrao Kulkarni } 2551a2db300SGanapatrao Kulkarni 256873ba463SMike Rapoport int pfn_is_map_memory(unsigned long pfn) 257c1cc1552SCatalin Marinas { 258093bbe21SAnshuman Khandual phys_addr_t addr = PFN_PHYS(pfn); 2595ad356eaSGreg Hackmann 260873ba463SMike Rapoport /* avoid false positives for bogus PFNs, see comment in pfn_valid() */ 261093bbe21SAnshuman Khandual if (PHYS_PFN(addr) != pfn) 2625ad356eaSGreg Hackmann return 0; 2634ab21506SRobin Murphy 2645ad356eaSGreg Hackmann return memblock_is_map_memory(addr); 265c1cc1552SCatalin Marinas } 266873ba463SMike Rapoport EXPORT_SYMBOL(pfn_is_map_memory); 267c1cc1552SCatalin Marinas 268bb425a75SPeng Fan static phys_addr_t memory_limit __ro_after_init = PHYS_ADDR_MAX; 2696083fe74SMark Rutland 2706083fe74SMark Rutland /* 2716083fe74SMark Rutland * Limit the memory size that was specified via FDT. 2726083fe74SMark Rutland */ 2736083fe74SMark Rutland static int __init early_mem(char *p) 2746083fe74SMark Rutland { 2756083fe74SMark Rutland if (!p) 2766083fe74SMark Rutland return 1; 2776083fe74SMark Rutland 2786083fe74SMark Rutland memory_limit = memparse(p, &p) & PAGE_MASK; 2796083fe74SMark Rutland pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); 2806083fe74SMark Rutland 2816083fe74SMark Rutland return 0; 2826083fe74SMark Rutland } 2836083fe74SMark Rutland early_param("mem", early_mem); 2846083fe74SMark Rutland 285c1cc1552SCatalin Marinas void __init arm64_memblock_init(void) 286c1cc1552SCatalin Marinas { 28788053ec8SArd Biesheuvel s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); 28888053ec8SArd Biesheuvel 28988053ec8SArd Biesheuvel /* 29088053ec8SArd Biesheuvel * Corner case: 52-bit VA capable systems running KVM in nVHE mode may 29188053ec8SArd Biesheuvel * be limited in their ability to support a linear map that exceeds 51 29288053ec8SArd Biesheuvel * bits of VA space, depending on the placement of the ID map. Given 29388053ec8SArd Biesheuvel * that the placement of the ID map may be randomized, let's simply 29488053ec8SArd Biesheuvel * limit the kernel's linear map to 51 bits as well if we detect this 29588053ec8SArd Biesheuvel * configuration. 29688053ec8SArd Biesheuvel */ 29788053ec8SArd Biesheuvel if (IS_ENABLED(CONFIG_KVM) && vabits_actual == 52 && 29888053ec8SArd Biesheuvel is_hyp_mode_available() && !is_kernel_in_hyp_mode()) { 29988053ec8SArd Biesheuvel pr_info("Capping linear region to 51 bits for KVM in nVHE mode on LVA capable hardware.\n"); 30088053ec8SArd Biesheuvel linear_region_size = min_t(u64, linear_region_size, BIT(51)); 30188053ec8SArd Biesheuvel } 302a7f8de16SArd Biesheuvel 303e9eaa805SKristina Martsenko /* Remove memory above our supported physical address size */ 304e9eaa805SKristina Martsenko memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX); 305e9eaa805SKristina Martsenko 306a7f8de16SArd Biesheuvel /* 307a7f8de16SArd Biesheuvel * Select a suitable value for the base of physical memory. 308a7f8de16SArd Biesheuvel */ 309a7f8de16SArd Biesheuvel memstart_addr = round_down(memblock_start_of_DRAM(), 310a7f8de16SArd Biesheuvel ARM64_MEMSTART_ALIGN); 311a7f8de16SArd Biesheuvel 31231f80a4eSMarc Zyngier if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size) 31331f80a4eSMarc Zyngier pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n"); 31431f80a4eSMarc Zyngier 315a7f8de16SArd Biesheuvel /* 316a7f8de16SArd Biesheuvel * Remove the memory that we will not be able to cover with the 317a7f8de16SArd Biesheuvel * linear mapping. Take care not to clip the kernel which may be 318a7f8de16SArd Biesheuvel * high in memory. 319a7f8de16SArd Biesheuvel */ 3202077be67SLaura Abbott memblock_remove(max_t(u64, memstart_addr + linear_region_size, 3212077be67SLaura Abbott __pa_symbol(_end)), ULLONG_MAX); 3222958987fSArd Biesheuvel if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) { 3232958987fSArd Biesheuvel /* ensure that memstart_addr remains sufficiently aligned */ 3242958987fSArd Biesheuvel memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size, 3252958987fSArd Biesheuvel ARM64_MEMSTART_ALIGN); 3262958987fSArd Biesheuvel memblock_remove(0, memstart_addr); 3272958987fSArd Biesheuvel } 328a7f8de16SArd Biesheuvel 329a7f8de16SArd Biesheuvel /* 3307bc1a0f9SArd Biesheuvel * If we are running with a 52-bit kernel VA config on a system that 3317bc1a0f9SArd Biesheuvel * does not support it, we have to place the available physical 3327bc1a0f9SArd Biesheuvel * memory in the 48-bit addressable part of the linear region, i.e., 3337bc1a0f9SArd Biesheuvel * we have to move it upward. Since memstart_addr represents the 3347bc1a0f9SArd Biesheuvel * physical address of PAGE_OFFSET, we have to *subtract* from it. 3357bc1a0f9SArd Biesheuvel */ 3367bc1a0f9SArd Biesheuvel if (IS_ENABLED(CONFIG_ARM64_VA_BITS_52) && (vabits_actual != 52)) 3377bc1a0f9SArd Biesheuvel memstart_addr -= _PAGE_OFFSET(48) - _PAGE_OFFSET(52); 3387bc1a0f9SArd Biesheuvel 3397bc1a0f9SArd Biesheuvel /* 340a7f8de16SArd Biesheuvel * Apply the memory limit if it was set. Since the kernel may be loaded 341a7f8de16SArd Biesheuvel * high up in memory, add back the kernel region that must be accessible 342a7f8de16SArd Biesheuvel * via the linear mapping. 343a7f8de16SArd Biesheuvel */ 344d7dc899aSStefan Agner if (memory_limit != PHYS_ADDR_MAX) { 345cb0a6502SDennis Chen memblock_mem_limit_remove_map(memory_limit); 3462077be67SLaura Abbott memblock_add(__pa_symbol(_text), (u64)(_end - _text)); 347a7f8de16SArd Biesheuvel } 3486083fe74SMark Rutland 349c756c592SFlorian Fainelli if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { 350177e15f0SArd Biesheuvel /* 351177e15f0SArd Biesheuvel * Add back the memory we just removed if it results in the 352177e15f0SArd Biesheuvel * initrd to become inaccessible via the linear mapping. 353177e15f0SArd Biesheuvel * Otherwise, this is a no-op 354177e15f0SArd Biesheuvel */ 355c756c592SFlorian Fainelli u64 base = phys_initrd_start & PAGE_MASK; 356d4d18e3eSBjorn Andersson u64 size = PAGE_ALIGN(phys_initrd_start + phys_initrd_size) - base; 357177e15f0SArd Biesheuvel 358177e15f0SArd Biesheuvel /* 359177e15f0SArd Biesheuvel * We can only add back the initrd memory if we don't end up 360177e15f0SArd Biesheuvel * with more memory than we can address via the linear mapping. 361177e15f0SArd Biesheuvel * It is up to the bootloader to position the kernel and the 362177e15f0SArd Biesheuvel * initrd reasonably close to each other (i.e., within 32 GB of 363177e15f0SArd Biesheuvel * each other) so that all granule/#levels combinations can 364177e15f0SArd Biesheuvel * always access both. 365177e15f0SArd Biesheuvel */ 366177e15f0SArd Biesheuvel if (WARN(base < memblock_start_of_DRAM() || 367177e15f0SArd Biesheuvel base + size > memblock_start_of_DRAM() + 368177e15f0SArd Biesheuvel linear_region_size, 369177e15f0SArd Biesheuvel "initrd not fully accessible via the linear mapping -- please check your bootloader ...\n")) { 37070b3d237SWill Deacon phys_initrd_size = 0; 371177e15f0SArd Biesheuvel } else { 372177e15f0SArd Biesheuvel memblock_add(base, size); 373c0b978feSMa Wupeng memblock_clear_nomap(base, size); 374177e15f0SArd Biesheuvel memblock_reserve(base, size); 375177e15f0SArd Biesheuvel } 376177e15f0SArd Biesheuvel } 377177e15f0SArd Biesheuvel 378c031a421SArd Biesheuvel if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { 379c031a421SArd Biesheuvel extern u16 memstart_offset_seed; 38097d6786eSArd Biesheuvel u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); 38197d6786eSArd Biesheuvel int parange = cpuid_feature_extract_unsigned_field( 3822d987e64SMark Brown mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 38397d6786eSArd Biesheuvel s64 range = linear_region_size - 38497d6786eSArd Biesheuvel BIT(id_aa64mmfr0_parange_to_phys_shift(parange)); 385c031a421SArd Biesheuvel 386c031a421SArd Biesheuvel /* 387c031a421SArd Biesheuvel * If the size of the linear region exceeds, by a sufficient 38897d6786eSArd Biesheuvel * margin, the size of the region that the physical memory can 38997d6786eSArd Biesheuvel * span, randomize the linear region as well. 390c031a421SArd Biesheuvel */ 39197d6786eSArd Biesheuvel if (memstart_offset_seed > 0 && range >= (s64)ARM64_MEMSTART_ALIGN) { 392c8a43c18SYueyi Li range /= ARM64_MEMSTART_ALIGN; 393c031a421SArd Biesheuvel memstart_addr -= ARM64_MEMSTART_ALIGN * 394c031a421SArd Biesheuvel ((range * memstart_offset_seed) >> 16); 395c031a421SArd Biesheuvel } 396c031a421SArd Biesheuvel } 3972d5a5612SCatalin Marinas 398bd00cd5fSMark Rutland /* 399bd00cd5fSMark Rutland * Register the kernel text, kernel data, initrd, and initial 400bd00cd5fSMark Rutland * pagetables with memblock. 401bd00cd5fSMark Rutland */ 402e2a073ddSArd Biesheuvel memblock_reserve(__pa_symbol(_stext), _end - _stext); 403c756c592SFlorian Fainelli if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { 404a89dea58SArd Biesheuvel /* the generic initrd code expects virtual addresses */ 405c756c592SFlorian Fainelli initrd_start = __phys_to_virt(phys_initrd_start); 406c756c592SFlorian Fainelli initrd_end = initrd_start + phys_initrd_size; 407a89dea58SArd Biesheuvel } 408c1cc1552SCatalin Marinas 4099bf14b7cSMarek Szyprowski early_init_fdt_scan_reserved_mem(); 4102d5a5612SCatalin Marinas 411f24e5834SSteve Capper high_memory = __va(memblock_end_of_DRAM() - 1) + 1; 412c1cc1552SCatalin Marinas } 413c1cc1552SCatalin Marinas 414c1cc1552SCatalin Marinas void __init bootmem_init(void) 415c1cc1552SCatalin Marinas { 416c1cc1552SCatalin Marinas unsigned long min, max; 417c1cc1552SCatalin Marinas 418c1cc1552SCatalin Marinas min = PFN_UP(memblock_start_of_DRAM()); 419c1cc1552SCatalin Marinas max = PFN_DOWN(memblock_end_of_DRAM()); 420c1cc1552SCatalin Marinas 42136dd9086SVladimir Murzin early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT); 42236dd9086SVladimir Murzin 4231a2db300SGanapatrao Kulkarni max_pfn = max_low_pfn = max; 42419d6242eSMiles Chen min_low_pfn = min; 4251a2db300SGanapatrao Kulkarni 426eb75541fSAtish Patra arch_numa_init(); 427618e0786SBarry Song 428618e0786SBarry Song /* 429eb75541fSAtish Patra * must be done after arch_numa_init() which calls numa_init() to 430618e0786SBarry Song * initialize node_online_map that gets used in hugetlb_cma_reserve() 431618e0786SBarry Song * while allocating required CMA size across online nodes. 432618e0786SBarry Song */ 433abb7962aSAnshuman Khandual #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA) 434abb7962aSAnshuman Khandual arm64_hugetlb_cma_reserve(); 435618e0786SBarry Song #endif 436618e0786SBarry Song 437c6303ab9SBarry Song dma_pernuma_cma_reserve(); 438c6303ab9SBarry Song 439f320bc74SQuentin Perret kvm_hyp_reserve(); 440f320bc74SQuentin Perret 441c1cc1552SCatalin Marinas /* 442c89ab04fSMike Rapoport * sparse_init() tries to allocate memory from memblock, so must be 443c89ab04fSMike Rapoport * done after the fixed reservations 444c1cc1552SCatalin Marinas */ 445c1cc1552SCatalin Marinas sparse_init(); 446f41ef4c2SKefeng Wang zone_sizes_init(); 447c1cc1552SCatalin Marinas 4480a30c535SNicolas Saenz Julienne /* 449d78050eeSCatalin Marinas * Reserve the CMA area after arm64_dma_phys_limit was initialised. 450d78050eeSCatalin Marinas */ 451d78050eeSCatalin Marinas dma_contiguous_reserve(arm64_dma_phys_limit); 452d78050eeSCatalin Marinas 453d78050eeSCatalin Marinas /* 4540a30c535SNicolas Saenz Julienne * request_standard_resources() depends on crashkernel's memory being 4550a30c535SNicolas Saenz Julienne * reserved, so do it here. 4560a30c535SNicolas Saenz Julienne */ 4570a30c535SNicolas Saenz Julienne reserve_crashkernel(); 4580a30c535SNicolas Saenz Julienne 4591a2db300SGanapatrao Kulkarni memblock_dump_all(); 460c1cc1552SCatalin Marinas } 461c1cc1552SCatalin Marinas 462c1cc1552SCatalin Marinas /* 463c1cc1552SCatalin Marinas * mem_init() marks the free areas in the mem_map and tells us how much memory 464c1cc1552SCatalin Marinas * is free. This is done after various parts of the system have claimed their 465c1cc1552SCatalin Marinas * memory after the kernel image. 466c1cc1552SCatalin Marinas */ 467c1cc1552SCatalin Marinas void __init mem_init(void) 468c1cc1552SCatalin Marinas { 469c6af2aa9SChristoph Hellwig swiotlb_init(max_pfn > PFN_DOWN(arm64_dma_phys_limit), SWIOTLB_VERBOSE); 470a1e50a82SCatalin Marinas 471bee4ebd1SJiang Liu /* this will put all unused low memory onto the freelists */ 472c6ffc5caSMike Rapoport memblock_free_all(); 473c1cc1552SCatalin Marinas 474c1cc1552SCatalin Marinas /* 475c1cc1552SCatalin Marinas * Check boundaries twice: Some fundamental inconsistencies can be 476c1cc1552SCatalin Marinas * detected at build time already. 477c1cc1552SCatalin Marinas */ 478c1cc1552SCatalin Marinas #ifdef CONFIG_COMPAT 479363524d2SSteve Capper BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64); 480c1cc1552SCatalin Marinas #endif 481c1cc1552SCatalin Marinas 4827e04cc91SAnshuman Khandual /* 4837e04cc91SAnshuman Khandual * Selected page table levels should match when derived from 4847e04cc91SAnshuman Khandual * scratch using the virtual address range and page size. 4857e04cc91SAnshuman Khandual */ 4867e04cc91SAnshuman Khandual BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != 4877e04cc91SAnshuman Khandual CONFIG_PGTABLE_LEVELS); 4887e04cc91SAnshuman Khandual 489bee4ebd1SJiang Liu if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { 490c1cc1552SCatalin Marinas extern int sysctl_overcommit_memory; 491c1cc1552SCatalin Marinas /* 492c1cc1552SCatalin Marinas * On a machine this small we won't get anywhere without 493c1cc1552SCatalin Marinas * overcommit, so turn it on by default. 494c1cc1552SCatalin Marinas */ 495c1cc1552SCatalin Marinas sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; 496c1cc1552SCatalin Marinas } 497c1cc1552SCatalin Marinas } 498c1cc1552SCatalin Marinas 499c1cc1552SCatalin Marinas void free_initmem(void) 500c1cc1552SCatalin Marinas { 5012077be67SLaura Abbott free_reserved_area(lm_alias(__init_begin), 5022077be67SLaura Abbott lm_alias(__init_end), 5036ec939f8SAnshuman Khandual POISON_FREE_INITMEM, "unused kernel"); 504dae8c235SKefeng Wang /* 505dae8c235SKefeng Wang * Unmap the __init region but leave the VM area in place. This 506dae8c235SKefeng Wang * prevents the region from being reused for kernel modules, which 507dae8c235SKefeng Wang * is not supported by kallsyms. 508dae8c235SKefeng Wang */ 5094ad0ae8cSNicholas Piggin vunmap_range((u64)__init_begin, (u64)__init_end); 510c1cc1552SCatalin Marinas } 511c1cc1552SCatalin Marinas 512638d5031SAnshuman Khandual void dump_mem_limit(void) 513a7f8de16SArd Biesheuvel { 514d7dc899aSStefan Agner if (memory_limit != PHYS_ADDR_MAX) { 515a7f8de16SArd Biesheuvel pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20); 516a7f8de16SArd Biesheuvel } else { 517a7f8de16SArd Biesheuvel pr_emerg("Memory Limit: none\n"); 518a7f8de16SArd Biesheuvel } 519a7f8de16SArd Biesheuvel } 520