1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Based on arch/arm/mm/init.c
4 *
5 * Copyright (C) 1995-2005 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9 #include <linux/kernel.h>
10 #include <linux/export.h>
11 #include <linux/errno.h>
12 #include <linux/swap.h>
13 #include <linux/init.h>
14 #include <linux/cc_platform.h>
15 #include <linux/cache.h>
16 #include <linux/mman.h>
17 #include <linux/nodemask.h>
18 #include <linux/initrd.h>
19 #include <linux/gfp.h>
20 #include <linux/math.h>
21 #include <linux/memblock.h>
22 #include <linux/sort.h>
23 #include <linux/of.h>
24 #include <linux/of_fdt.h>
25 #include <linux/dma-direct.h>
26 #include <linux/dma-map-ops.h>
27 #include <linux/efi.h>
28 #include <linux/swiotlb.h>
29 #include <linux/vmalloc.h>
30 #include <linux/mm.h>
31 #include <linux/kexec.h>
32 #include <linux/crash_dump.h>
33 #include <linux/hugetlb.h>
34 #include <linux/acpi_iort.h>
35 #include <linux/kmemleak.h>
36 #include <linux/execmem.h>
37 #include <linux/arm-rsi-cmds.h>
38
39
40 #include <asm/boot.h>
41 #include <asm/fixmap.h>
42 #include <asm/hypervisor.h>
43 #include <asm/kasan.h>
44 #include <asm/kernel-pgtable.h>
45 #include <asm/kvm_host.h>
46 #include <asm/memory.h>
47 #include <asm/numa.h>
48 #include <asm/sections.h>
49 #include <asm/setup.h>
50 #include <linux/sizes.h>
51 #include <asm/tlb.h>
52 #include <asm/alternative.h>
53 #include <asm/xen/swiotlb-xen.h>
54
55 /*
56 * We need to be able to catch inadvertent references to memstart_addr
57 * that occur (potentially in generic code) before arm64_memblock_init()
58 * executes, which assigns it its actual value. So use a default value
59 * that cannot be mistaken for a real physical address.
60 */
61 s64 memstart_addr __ro_after_init = -1;
62 EXPORT_SYMBOL(memstart_addr);
63
64 /*
65 * If the corresponding config options are enabled, we create both ZONE_DMA
66 * and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory
67 * unless restricted on specific platforms (e.g. 30-bit on Raspberry Pi 4).
68 * In such case, ZONE_DMA32 covers the rest of the 32-bit addressable memory,
69 * otherwise it is empty.
70 */
71 phys_addr_t __ro_after_init arm64_dma_phys_limit;
72
73 /*
74 * To make optimal use of block mappings when laying out the linear
75 * mapping, round down the base of physical memory to a size that can
76 * be mapped efficiently, i.e., either PUD_SIZE (4k granule) or PMD_SIZE
77 * (64k granule), or a multiple that can be mapped using contiguous bits
78 * in the page tables: 32 * PMD_SIZE (16k granule)
79 */
80 #if defined(CONFIG_ARM64_4K_PAGES)
81 #define ARM64_MEMSTART_SHIFT PUD_SHIFT
82 #elif defined(CONFIG_ARM64_16K_PAGES)
83 #define ARM64_MEMSTART_SHIFT CONT_PMD_SHIFT
84 #else
85 #define ARM64_MEMSTART_SHIFT PMD_SHIFT
86 #endif
87
88 /*
89 * sparsemem vmemmap imposes an additional requirement on the alignment of
90 * memstart_addr, due to the fact that the base of the vmemmap region
91 * has a direct correspondence, and needs to appear sufficiently aligned
92 * in the virtual address space.
93 */
94 #if ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS
95 #define ARM64_MEMSTART_ALIGN (1UL << SECTION_SIZE_BITS)
96 #else
97 #define ARM64_MEMSTART_ALIGN (1UL << ARM64_MEMSTART_SHIFT)
98 #endif
99
arch_reserve_crashkernel(void)100 static void __init arch_reserve_crashkernel(void)
101 {
102 unsigned long long crash_base, crash_size, cma_size = 0;
103 unsigned long long low_size = 0;
104 bool high = false;
105 int ret;
106
107 if (!IS_ENABLED(CONFIG_CRASH_RESERVE))
108 return;
109
110 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
111 &crash_size, &crash_base,
112 &low_size, &cma_size, &high);
113 if (ret)
114 return;
115
116 reserve_crashkernel_generic(crash_size, crash_base, low_size, high);
117 reserve_crashkernel_cma(cma_size);
118 }
119
max_zone_phys(phys_addr_t zone_limit)120 static phys_addr_t __init max_zone_phys(phys_addr_t zone_limit)
121 {
122 return min(zone_limit, memblock_end_of_DRAM() - 1) + 1;
123 }
124
arch_zone_limits_init(unsigned long * max_zone_pfns)125 void __init arch_zone_limits_init(unsigned long *max_zone_pfns)
126 {
127 phys_addr_t __maybe_unused dma32_phys_limit =
128 max_zone_phys(DMA_BIT_MASK(32));
129
130 #ifdef CONFIG_ZONE_DMA
131 max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_phys(zone_dma_limit));
132 #endif
133 #ifdef CONFIG_ZONE_DMA32
134 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit);
135 #endif
136 max_zone_pfns[ZONE_NORMAL] = max_pfn;
137 }
138
dma_limits_init(void)139 static void __init dma_limits_init(void)
140 {
141 phys_addr_t __maybe_unused acpi_zone_dma_limit;
142 phys_addr_t __maybe_unused dt_zone_dma_limit;
143 phys_addr_t __maybe_unused dma32_phys_limit =
144 max_zone_phys(DMA_BIT_MASK(32));
145
146 #ifdef CONFIG_ZONE_DMA
147 acpi_zone_dma_limit = acpi_iort_dma_get_max_cpu_address();
148 dt_zone_dma_limit = of_dma_get_max_cpu_address(NULL);
149 zone_dma_limit = min(dt_zone_dma_limit, acpi_zone_dma_limit);
150 /*
151 * Information we get from firmware (e.g. DT dma-ranges) describe DMA
152 * bus constraints. Devices using DMA might have their own limitations.
153 * Some of them rely on DMA zone in low 32-bit memory. Keep low RAM
154 * DMA zone on platforms that have RAM there.
155 */
156 if (memblock_start_of_DRAM() < U32_MAX)
157 zone_dma_limit = min(zone_dma_limit, U32_MAX);
158 arm64_dma_phys_limit = max_zone_phys(zone_dma_limit);
159 #endif
160 #ifdef CONFIG_ZONE_DMA32
161 if (!arm64_dma_phys_limit)
162 arm64_dma_phys_limit = dma32_phys_limit;
163 #endif
164 if (!arm64_dma_phys_limit)
165 arm64_dma_phys_limit = PHYS_MASK + 1;
166 }
167
pfn_is_map_memory(unsigned long pfn)168 int pfn_is_map_memory(unsigned long pfn)
169 {
170 phys_addr_t addr = PFN_PHYS(pfn);
171
172 /* avoid false positives for bogus PFNs, see comment in pfn_valid() */
173 if (PHYS_PFN(addr) != pfn)
174 return 0;
175
176 return memblock_is_map_memory(addr);
177 }
178 EXPORT_SYMBOL(pfn_is_map_memory);
179
180 static phys_addr_t memory_limit __ro_after_init = PHYS_ADDR_MAX;
181
182 /*
183 * Limit the memory size that was specified via FDT.
184 */
early_mem(char * p)185 static int __init early_mem(char *p)
186 {
187 if (!p)
188 return 1;
189
190 memory_limit = memparse(p, &p) & PAGE_MASK;
191 pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
192
193 return 0;
194 }
195 early_param("mem", early_mem);
196
arm64_memblock_init(void)197 void __init arm64_memblock_init(void)
198 {
199 s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual);
200
201 /*
202 * Corner case: 52-bit VA capable systems running KVM in nVHE mode may
203 * be limited in their ability to support a linear map that exceeds 51
204 * bits of VA space, depending on the placement of the ID map. Given
205 * that the placement of the ID map may be randomized, let's simply
206 * limit the kernel's linear map to 51 bits as well if we detect this
207 * configuration.
208 */
209 if (IS_ENABLED(CONFIG_KVM) && vabits_actual == 52 &&
210 is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
211 pr_info("Capping linear region to 51 bits for KVM in nVHE mode on LVA capable hardware.\n");
212 linear_region_size = min_t(u64, linear_region_size, BIT(51));
213 }
214
215 /* Remove memory above our supported physical address size */
216 memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX);
217
218 /*
219 * Select a suitable value for the base of physical memory.
220 */
221 memstart_addr = round_down(memblock_start_of_DRAM(),
222 ARM64_MEMSTART_ALIGN);
223
224 if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size)
225 pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n");
226
227 /*
228 * Remove the memory that we will not be able to cover with the
229 * linear mapping. Take care not to clip the kernel which may be
230 * high in memory.
231 */
232 memblock_remove(max_t(u64, memstart_addr + linear_region_size,
233 __pa_symbol(_end)), ULLONG_MAX);
234 if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) {
235 /* ensure that memstart_addr remains sufficiently aligned */
236 memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size,
237 ARM64_MEMSTART_ALIGN);
238 memblock_remove(0, memstart_addr);
239 }
240
241 /*
242 * If we are running with a 52-bit kernel VA config on a system that
243 * does not support it, we have to place the available physical
244 * memory in the 48-bit addressable part of the linear region, i.e.,
245 * we have to move it upward. Since memstart_addr represents the
246 * physical address of PAGE_OFFSET, we have to *subtract* from it.
247 */
248 if (IS_ENABLED(CONFIG_ARM64_VA_BITS_52) && (vabits_actual != 52))
249 memstart_addr -= _PAGE_OFFSET(vabits_actual) - _PAGE_OFFSET(52);
250
251 /*
252 * Apply the memory limit if it was set. Since the kernel may be loaded
253 * high up in memory, add back the kernel region that must be accessible
254 * via the linear mapping.
255 */
256 if (memory_limit != PHYS_ADDR_MAX) {
257 memblock_mem_limit_remove_map(memory_limit);
258 memblock_add(__pa_symbol(_text), (resource_size_t)(_end - _text));
259 }
260
261 if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) {
262 /*
263 * Add back the memory we just removed if it results in the
264 * initrd to become inaccessible via the linear mapping.
265 * Otherwise, this is a no-op
266 */
267 phys_addr_t base = phys_initrd_start & PAGE_MASK;
268 resource_size_t size = PAGE_ALIGN(phys_initrd_start + phys_initrd_size) - base;
269
270 /*
271 * We can only add back the initrd memory if we don't end up
272 * with more memory than we can address via the linear mapping.
273 * It is up to the bootloader to position the kernel and the
274 * initrd reasonably close to each other (i.e., within 32 GB of
275 * each other) so that all granule/#levels combinations can
276 * always access both.
277 */
278 if (WARN(base < memblock_start_of_DRAM() ||
279 base + size > memblock_start_of_DRAM() +
280 linear_region_size,
281 "initrd not fully accessible via the linear mapping -- please check your bootloader ...\n")) {
282 phys_initrd_size = 0;
283 } else {
284 memblock_add(base, size);
285 memblock_clear_nomap(base, size);
286 memblock_reserve(base, size);
287 }
288 }
289
290 /*
291 * Register the kernel text, kernel data, initrd, and initial
292 * pagetables with memblock.
293 */
294 memblock_reserve(__pa_symbol(_text), _end - _text);
295 if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) {
296 /* the generic initrd code expects virtual addresses */
297 initrd_start = __phys_to_virt(phys_initrd_start);
298 initrd_end = initrd_start + phys_initrd_size;
299 }
300
301 early_init_fdt_scan_reserved_mem();
302 }
303
bootmem_init(void)304 void __init bootmem_init(void)
305 {
306 unsigned long min, max;
307
308 min = PFN_UP(memblock_start_of_DRAM());
309 max = PFN_DOWN(memblock_end_of_DRAM());
310
311 early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT);
312
313 max_pfn = max_low_pfn = max;
314 min_low_pfn = min;
315
316 arch_numa_init();
317
318 kvm_hyp_reserve();
319 dma_limits_init();
320
321 /*
322 * Reserve the CMA area after arm64_dma_phys_limit was initialised.
323 */
324 dma_contiguous_reserve(arm64_dma_phys_limit);
325
326 /*
327 * request_standard_resources() depends on crashkernel's memory being
328 * reserved, so do it here.
329 */
330 arch_reserve_crashkernel();
331
332 memblock_dump_all();
333 }
334
arch_setup_zero_pages(void)335 void __init arch_setup_zero_pages(void)
336 {
337 __zero_page = phys_to_page(__pa_symbol(empty_zero_page));
338 }
339
arch_mm_preinit(void)340 void __init arch_mm_preinit(void)
341 {
342 unsigned int flags = SWIOTLB_VERBOSE;
343
344 if (max_pfn <= PFN_DOWN(arm64_dma_phys_limit)) {
345 /*
346 * If no bouncing needed for ZONE_DMA, reduce the swiotlb
347 * buffer for kmalloc() bouncing to 1MB per 1GB of RAM.
348 */
349 unsigned long size =
350 DIV_ROUND_UP(memblock_phys_mem_size(), 1024);
351
352 swiotlb_adjust_size(min(swiotlb_size_or_default(), size));
353 }
354
355 swiotlb_init(true, flags);
356
357 /*
358 * Check boundaries twice: Some fundamental inconsistencies can be
359 * detected at build time already.
360 */
361 #ifdef CONFIG_COMPAT
362 BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64);
363 #endif
364
365 /*
366 * Selected page table levels should match when derived from
367 * scratch using the virtual address range and page size.
368 */
369 BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) !=
370 CONFIG_PGTABLE_LEVELS);
371
372 if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
373 extern int sysctl_overcommit_memory;
374 /*
375 * On a machine this small we won't get anywhere without
376 * overcommit, so turn it on by default.
377 */
378 sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
379 }
380 }
381
382 bool page_alloc_available __ro_after_init;
383
mem_init(void)384 void __init mem_init(void)
385 {
386 page_alloc_available = true;
387 swiotlb_update_mem_attributes();
388 }
389
free_initmem(void)390 void free_initmem(void)
391 {
392 void *lm_init_begin = lm_alias(__init_begin);
393 void *lm_init_end = lm_alias(__init_end);
394
395 WARN_ON(!IS_ALIGNED((unsigned long)lm_init_begin, PAGE_SIZE));
396 WARN_ON(!IS_ALIGNED((unsigned long)lm_init_end, PAGE_SIZE));
397
398 free_reserved_area(lm_init_begin, lm_init_end,
399 POISON_FREE_INITMEM, "unused kernel");
400 /*
401 * Unmap the __init region but leave the VM area in place. This
402 * prevents the region from being reused for kernel modules, which
403 * is not supported by kallsyms.
404 */
405 vunmap_range((u64)__init_begin, (u64)__init_end);
406 }
407
dump_mem_limit(void)408 void dump_mem_limit(void)
409 {
410 if (memory_limit != PHYS_ADDR_MAX) {
411 pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20);
412 } else {
413 pr_emerg("Memory Limit: none\n");
414 }
415 }
416
cc_platform_has(enum cc_attr attr)417 bool cc_platform_has(enum cc_attr attr)
418 {
419 switch (attr) {
420 case CC_ATTR_MEM_ENCRYPT:
421 case CC_ATTR_GUEST_MEM_ENCRYPT:
422 return is_realm_world() || is_protected_kvm_guest();
423 default:
424 return false;
425 }
426 }
427 EXPORT_SYMBOL_GPL(cc_platform_has);
428
429 #ifdef CONFIG_EXECMEM
430 static u64 module_direct_base __ro_after_init = 0;
431 static u64 module_plt_base __ro_after_init = 0;
432
433 /*
434 * Choose a random page-aligned base address for a window of 'size' bytes which
435 * entirely contains the interval [start, end - 1].
436 */
random_bounding_box(u64 size,u64 start,u64 end)437 static u64 __init random_bounding_box(u64 size, u64 start, u64 end)
438 {
439 u64 max_pgoff, pgoff;
440
441 if ((end - start) >= size)
442 return 0;
443
444 max_pgoff = (size - (end - start)) / PAGE_SIZE;
445 pgoff = get_random_u32_inclusive(0, max_pgoff);
446
447 return start - pgoff * PAGE_SIZE;
448 }
449
450 /*
451 * Modules may directly reference data and text anywhere within the kernel
452 * image and other modules. References using PREL32 relocations have a +/-2G
453 * range, and so we need to ensure that the entire kernel image and all modules
454 * fall within a 2G window such that these are always within range.
455 *
456 * Modules may directly branch to functions and code within the kernel text,
457 * and to functions and code within other modules. These branches will use
458 * CALL26/JUMP26 relocations with a +/-128M range. Without PLTs, we must ensure
459 * that the entire kernel text and all module text falls within a 128M window
460 * such that these are always within range. With PLTs, we can expand this to a
461 * 2G window.
462 *
463 * We chose the 128M region to surround the entire kernel image (rather than
464 * just the text) as using the same bounds for the 128M and 2G regions ensures
465 * by construction that we never select a 128M region that is not a subset of
466 * the 2G region. For very large and unusual kernel configurations this means
467 * we may fall back to PLTs where they could have been avoided, but this keeps
468 * the logic significantly simpler.
469 */
module_init_limits(void)470 static int __init module_init_limits(void)
471 {
472 u64 kernel_end = (u64)_end;
473 u64 kernel_start = (u64)_text;
474 u64 kernel_size = kernel_end - kernel_start;
475
476 /*
477 * The default modules region is placed immediately below the kernel
478 * image, and is large enough to use the full 2G relocation range.
479 */
480 BUILD_BUG_ON(KIMAGE_VADDR != MODULES_END);
481 BUILD_BUG_ON(MODULES_VSIZE < SZ_2G);
482
483 if (!kaslr_enabled()) {
484 if (kernel_size < SZ_128M)
485 module_direct_base = kernel_end - SZ_128M;
486 if (kernel_size < SZ_2G)
487 module_plt_base = kernel_end - SZ_2G;
488 } else {
489 u64 min = kernel_start;
490 u64 max = kernel_end;
491
492 if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) {
493 pr_info("2G module region forced by RANDOMIZE_MODULE_REGION_FULL\n");
494 } else {
495 module_direct_base = random_bounding_box(SZ_128M, min, max);
496 if (module_direct_base) {
497 min = module_direct_base;
498 max = module_direct_base + SZ_128M;
499 }
500 }
501
502 module_plt_base = random_bounding_box(SZ_2G, min, max);
503 }
504
505 pr_info("%llu pages in range for non-PLT usage",
506 module_direct_base ? (SZ_128M - kernel_size) / PAGE_SIZE : 0);
507 pr_info("%llu pages in range for PLT usage",
508 module_plt_base ? (SZ_2G - kernel_size) / PAGE_SIZE : 0);
509
510 return 0;
511 }
512
513 static struct execmem_info execmem_info __ro_after_init;
514
execmem_arch_setup(void)515 struct execmem_info __init *execmem_arch_setup(void)
516 {
517 unsigned long fallback_start = 0, fallback_end = 0;
518 unsigned long start = 0, end = 0;
519
520 module_init_limits();
521
522 /*
523 * Where possible, prefer to allocate within direct branch range of the
524 * kernel such that no PLTs are necessary.
525 */
526 if (module_direct_base) {
527 start = module_direct_base;
528 end = module_direct_base + SZ_128M;
529
530 if (module_plt_base) {
531 fallback_start = module_plt_base;
532 fallback_end = module_plt_base + SZ_2G;
533 }
534 } else if (module_plt_base) {
535 start = module_plt_base;
536 end = module_plt_base + SZ_2G;
537 }
538
539 execmem_info = (struct execmem_info){
540 .ranges = {
541 [EXECMEM_DEFAULT] = {
542 .start = start,
543 .end = end,
544 .pgprot = PAGE_KERNEL,
545 .alignment = 1,
546 .fallback_start = fallback_start,
547 .fallback_end = fallback_end,
548 },
549 [EXECMEM_KPROBES] = {
550 .start = VMALLOC_START,
551 .end = VMALLOC_END,
552 .pgprot = PAGE_KERNEL_ROX,
553 .alignment = 1,
554 },
555 [EXECMEM_BPF] = {
556 .start = VMALLOC_START,
557 .end = VMALLOC_END,
558 .pgprot = PAGE_KERNEL,
559 .alignment = 1,
560 },
561 },
562 };
563
564 return &execmem_info;
565 }
566 #endif /* CONFIG_EXECMEM */
567