1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/init.c 4 * 5 * Copyright (C) 1995-2005 Russell King 6 */ 7 #include <linux/kernel.h> 8 #include <linux/errno.h> 9 #include <linux/swap.h> 10 #include <linux/init.h> 11 #include <linux/mman.h> 12 #include <linux/sched/signal.h> 13 #include <linux/sched/task.h> 14 #include <linux/export.h> 15 #include <linux/nodemask.h> 16 #include <linux/initrd.h> 17 #include <linux/of_fdt.h> 18 #include <linux/highmem.h> 19 #include <linux/gfp.h> 20 #include <linux/memblock.h> 21 #include <linux/dma-map-ops.h> 22 #include <linux/sizes.h> 23 #include <linux/stop_machine.h> 24 #include <linux/swiotlb.h> 25 #include <linux/execmem.h> 26 27 #include <asm/cp15.h> 28 #include <asm/mach-types.h> 29 #include <asm/memblock.h> 30 #include <asm/page.h> 31 #include <asm/prom.h> 32 #include <asm/sections.h> 33 #include <asm/setup.h> 34 #include <asm/set_memory.h> 35 #include <asm/system_info.h> 36 #include <asm/tlb.h> 37 #include <asm/fixmap.h> 38 #include <asm/ptdump.h> 39 40 #include <asm/mach/arch.h> 41 #include <asm/mach/map.h> 42 43 #include "mm.h" 44 45 #ifdef CONFIG_CPU_CP15_MMU 46 unsigned long __init __clear_cr(unsigned long mask) 47 { 48 cr_alignment = cr_alignment & ~mask; 49 return cr_alignment; 50 } 51 #endif 52 53 #ifdef CONFIG_BLK_DEV_INITRD 54 static int __init parse_tag_initrd(const struct tag *tag) 55 { 56 pr_warn("ATAG_INITRD is deprecated; " 57 "please update your bootloader.\n"); 58 phys_initrd_start = __virt_to_phys(tag->u.initrd.start); 59 phys_initrd_size = tag->u.initrd.size; 60 return 0; 61 } 62 63 __tagtable(ATAG_INITRD, parse_tag_initrd); 64 65 static int __init parse_tag_initrd2(const struct tag *tag) 66 { 67 phys_initrd_start = tag->u.initrd.start; 68 phys_initrd_size = tag->u.initrd.size; 69 return 0; 70 } 71 72 __tagtable(ATAG_INITRD2, parse_tag_initrd2); 73 #endif 74 75 static void __init find_limits(unsigned long *min, unsigned long *max_low, 76 unsigned long *max_high) 77 { 78 *max_low = PFN_DOWN(memblock_get_current_limit()); 79 *min = PFN_UP(memblock_start_of_DRAM()); 80 *max_high = PFN_DOWN(memblock_end_of_DRAM()); 81 } 82 83 #ifdef CONFIG_ZONE_DMA 84 85 phys_addr_t arm_dma_zone_size __read_mostly; 86 EXPORT_SYMBOL(arm_dma_zone_size); 87 88 /* 89 * The DMA mask corresponding to the maximum bus address allocatable 90 * using GFP_DMA. The default here places no restriction on DMA 91 * allocations. This must be the smallest DMA mask in the system, 92 * so a successful GFP_DMA allocation will always satisfy this. 93 */ 94 phys_addr_t arm_dma_limit; 95 unsigned long arm_dma_pfn_limit; 96 #endif 97 98 void __init setup_dma_zone(const struct machine_desc *mdesc) 99 { 100 #ifdef CONFIG_ZONE_DMA 101 if (mdesc->dma_zone_size) { 102 arm_dma_zone_size = mdesc->dma_zone_size; 103 arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1; 104 } else 105 arm_dma_limit = 0xffffffff; 106 arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT; 107 #endif 108 } 109 110 void __init arch_zone_limits_init(unsigned long *max_zone_pfn) 111 { 112 #ifdef CONFIG_ZONE_DMA 113 max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low_pfn); 114 #endif 115 max_zone_pfn[ZONE_NORMAL] = max_low_pfn; 116 #ifdef CONFIG_HIGHMEM 117 max_zone_pfn[ZONE_HIGHMEM] = max_pfn; 118 #endif 119 } 120 121 #ifdef CONFIG_HAVE_ARCH_PFN_VALID 122 int pfn_valid(unsigned long pfn) 123 { 124 phys_addr_t addr = __pfn_to_phys(pfn); 125 unsigned long pageblock_size = PAGE_SIZE * pageblock_nr_pages; 126 127 if (__phys_to_pfn(addr) != pfn) 128 return 0; 129 130 /* 131 * If address less than pageblock_size bytes away from a present 132 * memory chunk there still will be a memory map entry for it 133 * because we round freed memory map to the pageblock boundaries. 134 */ 135 if (memblock_overlaps_region(&memblock.memory, 136 ALIGN_DOWN(addr, pageblock_size), 137 pageblock_size)) 138 return 1; 139 140 return 0; 141 } 142 EXPORT_SYMBOL(pfn_valid); 143 #endif 144 145 static bool arm_memblock_steal_permitted = true; 146 147 phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) 148 { 149 phys_addr_t phys; 150 151 BUG_ON(!arm_memblock_steal_permitted); 152 153 phys = memblock_phys_alloc(size, align); 154 if (!phys) 155 panic("Failed to steal %pa bytes at %pS\n", 156 &size, (void *)_RET_IP_); 157 158 memblock_phys_free(phys, size); 159 memblock_remove(phys, size); 160 161 return phys; 162 } 163 164 #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND 165 void check_cpu_icache_size(int cpuid) 166 { 167 u32 size, ctr; 168 169 asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr)); 170 171 size = 1 << ((ctr & 0xf) + 2); 172 if (cpuid != 0 && icache_size != size) 173 pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n", 174 cpuid); 175 if (icache_size > size) 176 icache_size = size; 177 } 178 #endif 179 180 void __init arm_memblock_init(const struct machine_desc *mdesc) 181 { 182 /* Register the kernel text, kernel data and initrd with memblock. */ 183 memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); 184 185 reserve_initrd_mem(); 186 187 arm_mm_memblock_reserve(); 188 189 /* reserve any platform specific memblock areas */ 190 if (mdesc->reserve) 191 mdesc->reserve(); 192 193 early_init_fdt_scan_reserved_mem(); 194 195 /* reserve memory for DMA contiguous allocations */ 196 dma_contiguous_reserve(arm_dma_limit); 197 198 arm_memblock_steal_permitted = false; 199 memblock_dump_all(); 200 } 201 202 void __init bootmem_init(void) 203 { 204 memblock_allow_resize(); 205 206 find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); 207 208 early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT, 209 (phys_addr_t)max_low_pfn << PAGE_SHIFT); 210 } 211 212 /* 213 * Poison init memory with an undefined instruction (ARM) or a branch to an 214 * undefined instruction (Thumb). 215 */ 216 static inline void poison_init_mem(void *s, size_t count) 217 { 218 u32 *p = (u32 *)s; 219 for (; count != 0; count -= 4) 220 *p++ = 0xe7fddef0; 221 } 222 223 void __init arch_mm_preinit(void) 224 { 225 #ifdef CONFIG_ARM_LPAE 226 swiotlb_init(max_pfn > arm_dma_pfn_limit, SWIOTLB_VERBOSE); 227 #endif 228 229 #ifdef CONFIG_SA1111 230 /* now that our DMA memory is actually so designated, we can free it */ 231 memblock_phys_free(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET); 232 #endif 233 234 /* 235 * Check boundaries twice: Some fundamental inconsistencies can 236 * be detected at build time already. 237 */ 238 #ifdef CONFIG_MMU 239 BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 240 BUG_ON(TASK_SIZE > MODULES_VADDR); 241 #endif 242 243 #ifdef CONFIG_HIGHMEM 244 BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 245 BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 246 #endif 247 } 248 249 #ifdef CONFIG_STRICT_KERNEL_RWX 250 struct section_perm { 251 const char *name; 252 unsigned long start; 253 unsigned long end; 254 pmdval_t mask; 255 pmdval_t prot; 256 pmdval_t clear; 257 }; 258 259 /* First section-aligned location at or after __start_rodata. */ 260 extern char __start_rodata_section_aligned[]; 261 262 static struct section_perm nx_perms[] = { 263 /* Make pages tables, etc before _stext RW (set NX). */ 264 { 265 .name = "pre-text NX", 266 .start = PAGE_OFFSET, 267 .end = (unsigned long)_stext, 268 .mask = ~PMD_SECT_XN, 269 .prot = PMD_SECT_XN, 270 }, 271 /* Make init RW (set NX). */ 272 { 273 .name = "init NX", 274 .start = (unsigned long)__init_begin, 275 .end = (unsigned long)_sdata, 276 .mask = ~PMD_SECT_XN, 277 .prot = PMD_SECT_XN, 278 }, 279 /* Make rodata NX (set RO in ro_perms below). */ 280 { 281 .name = "rodata NX", 282 .start = (unsigned long)__start_rodata_section_aligned, 283 .end = (unsigned long)__init_begin, 284 .mask = ~PMD_SECT_XN, 285 .prot = PMD_SECT_XN, 286 }, 287 }; 288 289 static struct section_perm ro_perms[] = { 290 /* Make kernel code and rodata RX (set RO). */ 291 { 292 .name = "text/rodata RO", 293 .start = (unsigned long)_stext, 294 .end = (unsigned long)__init_begin, 295 #ifdef CONFIG_ARM_LPAE 296 .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2), 297 .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2, 298 #else 299 .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE), 300 .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE, 301 .clear = PMD_SECT_AP_WRITE, 302 #endif 303 }, 304 }; 305 306 /* 307 * Updates section permissions only for the current mm (sections are 308 * copied into each mm). During startup, this is the init_mm. Is only 309 * safe to be called with preemption disabled, as under stop_machine(). 310 */ 311 static inline void section_update(unsigned long addr, pmdval_t mask, 312 pmdval_t prot, struct mm_struct *mm) 313 { 314 pmd_t *pmd; 315 316 pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr); 317 318 #ifdef CONFIG_ARM_LPAE 319 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 320 #else 321 if (addr & SECTION_SIZE) 322 pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot); 323 else 324 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 325 #endif 326 flush_pmd_entry(pmd); 327 local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE); 328 } 329 330 /* Make sure extended page tables are in use. */ 331 static inline bool arch_has_strict_perms(void) 332 { 333 if (cpu_architecture() < CPU_ARCH_ARMv6) 334 return false; 335 336 return !!(get_cr() & CR_XP); 337 } 338 339 static void set_section_perms(struct section_perm *perms, int n, bool set, 340 struct mm_struct *mm) 341 { 342 size_t i; 343 unsigned long addr; 344 345 if (!arch_has_strict_perms()) 346 return; 347 348 for (i = 0; i < n; i++) { 349 if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) || 350 !IS_ALIGNED(perms[i].end, SECTION_SIZE)) { 351 pr_err("BUG: %s section %lx-%lx not aligned to %lx\n", 352 perms[i].name, perms[i].start, perms[i].end, 353 SECTION_SIZE); 354 continue; 355 } 356 357 for (addr = perms[i].start; 358 addr < perms[i].end; 359 addr += SECTION_SIZE) 360 section_update(addr, perms[i].mask, 361 set ? perms[i].prot : perms[i].clear, mm); 362 } 363 364 } 365 366 /* 367 * update_sections_early intended to be called only through stop_machine 368 * framework and executed by only one CPU while all other CPUs will spin and 369 * wait, so no locking is required in this function. 370 */ 371 static void update_sections_early(struct section_perm perms[], int n) 372 { 373 struct task_struct *t, *s; 374 375 for_each_process(t) { 376 if (t->flags & PF_KTHREAD) 377 continue; 378 for_each_thread(t, s) 379 if (s->mm) 380 set_section_perms(perms, n, true, s->mm); 381 } 382 set_section_perms(perms, n, true, current->active_mm); 383 set_section_perms(perms, n, true, &init_mm); 384 } 385 386 static int __fix_kernmem_perms(void *unused) 387 { 388 update_sections_early(nx_perms, ARRAY_SIZE(nx_perms)); 389 return 0; 390 } 391 392 static void fix_kernmem_perms(void) 393 { 394 stop_machine(__fix_kernmem_perms, NULL, NULL); 395 } 396 397 static int __mark_rodata_ro(void *unused) 398 { 399 update_sections_early(ro_perms, ARRAY_SIZE(ro_perms)); 400 return 0; 401 } 402 403 void mark_rodata_ro(void) 404 { 405 stop_machine(__mark_rodata_ro, NULL, NULL); 406 arm_debug_checkwx(); 407 } 408 409 #else 410 static inline void fix_kernmem_perms(void) { } 411 #endif /* CONFIG_STRICT_KERNEL_RWX */ 412 413 void free_initmem(void) 414 { 415 fix_kernmem_perms(); 416 417 poison_init_mem(__init_begin, __init_end - __init_begin); 418 if (!machine_is_integrator() && !machine_is_cintegrator()) 419 free_initmem_default(-1); 420 } 421 422 #ifdef CONFIG_BLK_DEV_INITRD 423 void free_initrd_mem(unsigned long start, unsigned long end) 424 { 425 if (start == initrd_start) 426 start = round_down(start, PAGE_SIZE); 427 if (end == initrd_end) 428 end = round_up(end, PAGE_SIZE); 429 430 poison_init_mem((void *)start, PAGE_ALIGN(end) - start); 431 free_reserved_area((void *)start, (void *)end, -1, "initrd"); 432 } 433 #endif 434 435 #ifdef CONFIG_EXECMEM 436 437 #ifdef CONFIG_XIP_KERNEL 438 /* 439 * The XIP kernel text is mapped in the module area for modules and 440 * some other stuff to work without any indirect relocations. 441 * MODULES_VADDR is redefined here and not in asm/memory.h to avoid 442 * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. 443 */ 444 #undef MODULES_VADDR 445 #define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) 446 #endif 447 448 #ifdef CONFIG_MMU 449 static struct execmem_info execmem_info __ro_after_init; 450 451 struct execmem_info __init *execmem_arch_setup(void) 452 { 453 unsigned long fallback_start = 0, fallback_end = 0; 454 455 if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) { 456 fallback_start = VMALLOC_START; 457 fallback_end = VMALLOC_END; 458 } 459 460 execmem_info = (struct execmem_info){ 461 .ranges = { 462 [EXECMEM_DEFAULT] = { 463 .start = MODULES_VADDR, 464 .end = MODULES_END, 465 .pgprot = PAGE_KERNEL_EXEC, 466 .alignment = 1, 467 .fallback_start = fallback_start, 468 .fallback_end = fallback_end, 469 }, 470 }, 471 }; 472 473 return &execmem_info; 474 } 475 #endif /* CONFIG_MMU */ 476 477 #endif /* CONFIG_EXECMEM */ 478