1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Based on arch/arm/mm/mmu.c 4 * 5 * Copyright (C) 1995-2005 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9 #include <linux/cache.h> 10 #include <linux/export.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/init.h> 14 #include <linux/ioport.h> 15 #include <linux/kexec.h> 16 #include <linux/libfdt.h> 17 #include <linux/mman.h> 18 #include <linux/nodemask.h> 19 #include <linux/memblock.h> 20 #include <linux/memory.h> 21 #include <linux/fs.h> 22 #include <linux/io.h> 23 #include <linux/mm.h> 24 #include <linux/vmalloc.h> 25 26 #include <asm/barrier.h> 27 #include <asm/cputype.h> 28 #include <asm/fixmap.h> 29 #include <asm/kasan.h> 30 #include <asm/kernel-pgtable.h> 31 #include <asm/sections.h> 32 #include <asm/setup.h> 33 #include <linux/sizes.h> 34 #include <asm/tlb.h> 35 #include <asm/mmu_context.h> 36 #include <asm/ptdump.h> 37 #include <asm/tlbflush.h> 38 #include <asm/pgalloc.h> 39 40 #define NO_BLOCK_MAPPINGS BIT(0) 41 #define NO_CONT_MAPPINGS BIT(1) 42 #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ 43 44 u64 idmap_t0sz = TCR_T0SZ(VA_BITS_MIN); 45 u64 idmap_ptrs_per_pgd = PTRS_PER_PGD; 46 47 u64 __section(".mmuoff.data.write") vabits_actual; 48 EXPORT_SYMBOL(vabits_actual); 49 50 u64 kimage_voffset __ro_after_init; 51 EXPORT_SYMBOL(kimage_voffset); 52 53 /* 54 * Empty_zero_page is a special page that is used for zero-initialized data 55 * and COW. 56 */ 57 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; 58 EXPORT_SYMBOL(empty_zero_page); 59 60 static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 61 static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; 62 static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; 63 64 static DEFINE_SPINLOCK(swapper_pgdir_lock); 65 66 void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) 67 { 68 pgd_t *fixmap_pgdp; 69 70 spin_lock(&swapper_pgdir_lock); 71 fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp)); 72 WRITE_ONCE(*fixmap_pgdp, pgd); 73 /* 74 * We need dsb(ishst) here to ensure the page-table-walker sees 75 * our new entry before set_p?d() returns. The fixmap's 76 * flush_tlb_kernel_range() via clear_fixmap() does this for us. 77 */ 78 pgd_clear_fixmap(); 79 spin_unlock(&swapper_pgdir_lock); 80 } 81 82 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 83 unsigned long size, pgprot_t vma_prot) 84 { 85 if (!pfn_valid(pfn)) 86 return pgprot_noncached(vma_prot); 87 else if (file->f_flags & O_SYNC) 88 return pgprot_writecombine(vma_prot); 89 return vma_prot; 90 } 91 EXPORT_SYMBOL(phys_mem_access_prot); 92 93 static phys_addr_t __init early_pgtable_alloc(int shift) 94 { 95 phys_addr_t phys; 96 void *ptr; 97 98 phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 99 if (!phys) 100 panic("Failed to allocate page table page\n"); 101 102 /* 103 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE 104 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise 105 * any level of table. 106 */ 107 ptr = pte_set_fixmap(phys); 108 109 memset(ptr, 0, PAGE_SIZE); 110 111 /* 112 * Implicit barriers also ensure the zeroed page is visible to the page 113 * table walker 114 */ 115 pte_clear_fixmap(); 116 117 return phys; 118 } 119 120 static bool pgattr_change_is_safe(u64 old, u64 new) 121 { 122 /* 123 * The following mapping attributes may be updated in live 124 * kernel mappings without the need for break-before-make. 125 */ 126 pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG; 127 128 /* creating or taking down mappings is always safe */ 129 if (old == 0 || new == 0) 130 return true; 131 132 /* live contiguous mappings may not be manipulated at all */ 133 if ((old | new) & PTE_CONT) 134 return false; 135 136 /* Transitioning from Non-Global to Global is unsafe */ 137 if (old & ~new & PTE_NG) 138 return false; 139 140 /* 141 * Changing the memory type between Normal and Normal-Tagged is safe 142 * since Tagged is considered a permission attribute from the 143 * mismatched attribute aliases perspective. 144 */ 145 if (((old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) || 146 (old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)) && 147 ((new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) || 148 (new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED))) 149 mask |= PTE_ATTRINDX_MASK; 150 151 return ((old ^ new) & ~mask) == 0; 152 } 153 154 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, 155 phys_addr_t phys, pgprot_t prot) 156 { 157 pte_t *ptep; 158 159 ptep = pte_set_fixmap_offset(pmdp, addr); 160 do { 161 pte_t old_pte = READ_ONCE(*ptep); 162 163 set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot)); 164 165 /* 166 * After the PTE entry has been populated once, we 167 * only allow updates to the permission attributes. 168 */ 169 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), 170 READ_ONCE(pte_val(*ptep)))); 171 172 phys += PAGE_SIZE; 173 } while (ptep++, addr += PAGE_SIZE, addr != end); 174 175 pte_clear_fixmap(); 176 } 177 178 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, 179 unsigned long end, phys_addr_t phys, 180 pgprot_t prot, 181 phys_addr_t (*pgtable_alloc)(int), 182 int flags) 183 { 184 unsigned long next; 185 pmd_t pmd = READ_ONCE(*pmdp); 186 187 BUG_ON(pmd_sect(pmd)); 188 if (pmd_none(pmd)) { 189 pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN; 190 phys_addr_t pte_phys; 191 192 if (flags & NO_EXEC_MAPPINGS) 193 pmdval |= PMD_TABLE_PXN; 194 BUG_ON(!pgtable_alloc); 195 pte_phys = pgtable_alloc(PAGE_SHIFT); 196 __pmd_populate(pmdp, pte_phys, pmdval); 197 pmd = READ_ONCE(*pmdp); 198 } 199 BUG_ON(pmd_bad(pmd)); 200 201 do { 202 pgprot_t __prot = prot; 203 204 next = pte_cont_addr_end(addr, end); 205 206 /* use a contiguous mapping if the range is suitably aligned */ 207 if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && 208 (flags & NO_CONT_MAPPINGS) == 0) 209 __prot = __pgprot(pgprot_val(prot) | PTE_CONT); 210 211 init_pte(pmdp, addr, next, phys, __prot); 212 213 phys += next - addr; 214 } while (addr = next, addr != end); 215 } 216 217 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, 218 phys_addr_t phys, pgprot_t prot, 219 phys_addr_t (*pgtable_alloc)(int), int flags) 220 { 221 unsigned long next; 222 pmd_t *pmdp; 223 224 pmdp = pmd_set_fixmap_offset(pudp, addr); 225 do { 226 pmd_t old_pmd = READ_ONCE(*pmdp); 227 228 next = pmd_addr_end(addr, end); 229 230 /* try section mapping first */ 231 if (((addr | next | phys) & ~SECTION_MASK) == 0 && 232 (flags & NO_BLOCK_MAPPINGS) == 0) { 233 pmd_set_huge(pmdp, phys, prot); 234 235 /* 236 * After the PMD entry has been populated once, we 237 * only allow updates to the permission attributes. 238 */ 239 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), 240 READ_ONCE(pmd_val(*pmdp)))); 241 } else { 242 alloc_init_cont_pte(pmdp, addr, next, phys, prot, 243 pgtable_alloc, flags); 244 245 BUG_ON(pmd_val(old_pmd) != 0 && 246 pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); 247 } 248 phys += next - addr; 249 } while (pmdp++, addr = next, addr != end); 250 251 pmd_clear_fixmap(); 252 } 253 254 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, 255 unsigned long end, phys_addr_t phys, 256 pgprot_t prot, 257 phys_addr_t (*pgtable_alloc)(int), int flags) 258 { 259 unsigned long next; 260 pud_t pud = READ_ONCE(*pudp); 261 262 /* 263 * Check for initial section mappings in the pgd/pud. 264 */ 265 BUG_ON(pud_sect(pud)); 266 if (pud_none(pud)) { 267 pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN; 268 phys_addr_t pmd_phys; 269 270 if (flags & NO_EXEC_MAPPINGS) 271 pudval |= PUD_TABLE_PXN; 272 BUG_ON(!pgtable_alloc); 273 pmd_phys = pgtable_alloc(PMD_SHIFT); 274 __pud_populate(pudp, pmd_phys, pudval); 275 pud = READ_ONCE(*pudp); 276 } 277 BUG_ON(pud_bad(pud)); 278 279 do { 280 pgprot_t __prot = prot; 281 282 next = pmd_cont_addr_end(addr, end); 283 284 /* use a contiguous mapping if the range is suitably aligned */ 285 if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && 286 (flags & NO_CONT_MAPPINGS) == 0) 287 __prot = __pgprot(pgprot_val(prot) | PTE_CONT); 288 289 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); 290 291 phys += next - addr; 292 } while (addr = next, addr != end); 293 } 294 295 static inline bool use_1G_block(unsigned long addr, unsigned long next, 296 unsigned long phys) 297 { 298 if (PAGE_SHIFT != 12) 299 return false; 300 301 if (((addr | next | phys) & ~PUD_MASK) != 0) 302 return false; 303 304 return true; 305 } 306 307 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, 308 phys_addr_t phys, pgprot_t prot, 309 phys_addr_t (*pgtable_alloc)(int), 310 int flags) 311 { 312 unsigned long next; 313 pud_t *pudp; 314 p4d_t *p4dp = p4d_offset(pgdp, addr); 315 p4d_t p4d = READ_ONCE(*p4dp); 316 317 if (p4d_none(p4d)) { 318 p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN; 319 phys_addr_t pud_phys; 320 321 if (flags & NO_EXEC_MAPPINGS) 322 p4dval |= P4D_TABLE_PXN; 323 BUG_ON(!pgtable_alloc); 324 pud_phys = pgtable_alloc(PUD_SHIFT); 325 __p4d_populate(p4dp, pud_phys, p4dval); 326 p4d = READ_ONCE(*p4dp); 327 } 328 BUG_ON(p4d_bad(p4d)); 329 330 pudp = pud_set_fixmap_offset(p4dp, addr); 331 do { 332 pud_t old_pud = READ_ONCE(*pudp); 333 334 next = pud_addr_end(addr, end); 335 336 /* 337 * For 4K granule only, attempt to put down a 1GB block 338 */ 339 if (use_1G_block(addr, next, phys) && 340 (flags & NO_BLOCK_MAPPINGS) == 0) { 341 pud_set_huge(pudp, phys, prot); 342 343 /* 344 * After the PUD entry has been populated once, we 345 * only allow updates to the permission attributes. 346 */ 347 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud), 348 READ_ONCE(pud_val(*pudp)))); 349 } else { 350 alloc_init_cont_pmd(pudp, addr, next, phys, prot, 351 pgtable_alloc, flags); 352 353 BUG_ON(pud_val(old_pud) != 0 && 354 pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); 355 } 356 phys += next - addr; 357 } while (pudp++, addr = next, addr != end); 358 359 pud_clear_fixmap(); 360 } 361 362 static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, 363 unsigned long virt, phys_addr_t size, 364 pgprot_t prot, 365 phys_addr_t (*pgtable_alloc)(int), 366 int flags) 367 { 368 unsigned long addr, end, next; 369 pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); 370 371 /* 372 * If the virtual and physical address don't have the same offset 373 * within a page, we cannot map the region as the caller expects. 374 */ 375 if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) 376 return; 377 378 phys &= PAGE_MASK; 379 addr = virt & PAGE_MASK; 380 end = PAGE_ALIGN(virt + size); 381 382 do { 383 next = pgd_addr_end(addr, end); 384 alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, 385 flags); 386 phys += next - addr; 387 } while (pgdp++, addr = next, addr != end); 388 } 389 390 static phys_addr_t __pgd_pgtable_alloc(int shift) 391 { 392 void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); 393 BUG_ON(!ptr); 394 395 /* Ensure the zeroed page is visible to the page table walker */ 396 dsb(ishst); 397 return __pa(ptr); 398 } 399 400 static phys_addr_t pgd_pgtable_alloc(int shift) 401 { 402 phys_addr_t pa = __pgd_pgtable_alloc(shift); 403 404 /* 405 * Call proper page table ctor in case later we need to 406 * call core mm functions like apply_to_page_range() on 407 * this pre-allocated page table. 408 * 409 * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is 410 * folded, and if so pgtable_pmd_page_ctor() becomes nop. 411 */ 412 if (shift == PAGE_SHIFT) 413 BUG_ON(!pgtable_pte_page_ctor(phys_to_page(pa))); 414 else if (shift == PMD_SHIFT) 415 BUG_ON(!pgtable_pmd_page_ctor(phys_to_page(pa))); 416 417 return pa; 418 } 419 420 /* 421 * This function can only be used to modify existing table entries, 422 * without allocating new levels of table. Note that this permits the 423 * creation of new section or page entries. 424 */ 425 static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, 426 phys_addr_t size, pgprot_t prot) 427 { 428 if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { 429 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 430 &phys, virt); 431 return; 432 } 433 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 434 NO_CONT_MAPPINGS); 435 } 436 437 void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 438 unsigned long virt, phys_addr_t size, 439 pgprot_t prot, bool page_mappings_only) 440 { 441 int flags = 0; 442 443 BUG_ON(mm == &init_mm); 444 445 if (page_mappings_only) 446 flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; 447 448 __create_pgd_mapping(mm->pgd, phys, virt, size, prot, 449 pgd_pgtable_alloc, flags); 450 } 451 452 static void update_mapping_prot(phys_addr_t phys, unsigned long virt, 453 phys_addr_t size, pgprot_t prot) 454 { 455 if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { 456 pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n", 457 &phys, virt); 458 return; 459 } 460 461 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 462 NO_CONT_MAPPINGS); 463 464 /* flush the TLBs after updating live kernel mappings */ 465 flush_tlb_kernel_range(virt, virt + size); 466 } 467 468 static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, 469 phys_addr_t end, pgprot_t prot, int flags) 470 { 471 __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, 472 prot, early_pgtable_alloc, flags); 473 } 474 475 void __init mark_linear_text_alias_ro(void) 476 { 477 /* 478 * Remove the write permissions from the linear alias of .text/.rodata 479 */ 480 update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), 481 (unsigned long)__init_begin - (unsigned long)_stext, 482 PAGE_KERNEL_RO); 483 } 484 485 static bool crash_mem_map __initdata; 486 487 static int __init enable_crash_mem_map(char *arg) 488 { 489 /* 490 * Proper parameter parsing is done by reserve_crashkernel(). We only 491 * need to know if the linear map has to avoid block mappings so that 492 * the crashkernel reservations can be unmapped later. 493 */ 494 crash_mem_map = true; 495 496 return 0; 497 } 498 early_param("crashkernel", enable_crash_mem_map); 499 500 static void __init map_mem(pgd_t *pgdp) 501 { 502 static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); 503 phys_addr_t kernel_start = __pa_symbol(_stext); 504 phys_addr_t kernel_end = __pa_symbol(__init_begin); 505 phys_addr_t start, end; 506 int flags = NO_EXEC_MAPPINGS; 507 u64 i; 508 509 /* 510 * Setting hierarchical PXNTable attributes on table entries covering 511 * the linear region is only possible if it is guaranteed that no table 512 * entries at any level are being shared between the linear region and 513 * the vmalloc region. Check whether this is true for the PGD level, in 514 * which case it is guaranteed to be true for all other levels as well. 515 */ 516 BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end)); 517 518 if (rodata_full || crash_mem_map || debug_pagealloc_enabled()) 519 flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; 520 521 /* 522 * Take care not to create a writable alias for the 523 * read-only text and rodata sections of the kernel image. 524 * So temporarily mark them as NOMAP to skip mappings in 525 * the following for-loop 526 */ 527 memblock_mark_nomap(kernel_start, kernel_end - kernel_start); 528 529 /* map all the memory banks */ 530 for_each_mem_range(i, &start, &end) { 531 if (start >= end) 532 break; 533 /* 534 * The linear map must allow allocation tags reading/writing 535 * if MTE is present. Otherwise, it has the same attributes as 536 * PAGE_KERNEL. 537 */ 538 __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), 539 flags); 540 } 541 542 /* 543 * Map the linear alias of the [_stext, __init_begin) interval 544 * as non-executable now, and remove the write permission in 545 * mark_linear_text_alias_ro() below (which will be called after 546 * alternative patching has completed). This makes the contents 547 * of the region accessible to subsystems such as hibernate, 548 * but protects it from inadvertent modification or execution. 549 * Note that contiguous mappings cannot be remapped in this way, 550 * so we should avoid them here. 551 */ 552 __map_memblock(pgdp, kernel_start, kernel_end, 553 PAGE_KERNEL, NO_CONT_MAPPINGS); 554 memblock_clear_nomap(kernel_start, kernel_end - kernel_start); 555 } 556 557 void mark_rodata_ro(void) 558 { 559 unsigned long section_size; 560 561 /* 562 * mark .rodata as read only. Use __init_begin rather than __end_rodata 563 * to cover NOTES and EXCEPTION_TABLE. 564 */ 565 section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; 566 update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, 567 section_size, PAGE_KERNEL_RO); 568 569 debug_checkwx(); 570 } 571 572 static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, 573 pgprot_t prot, struct vm_struct *vma, 574 int flags, unsigned long vm_flags) 575 { 576 phys_addr_t pa_start = __pa_symbol(va_start); 577 unsigned long size = va_end - va_start; 578 579 BUG_ON(!PAGE_ALIGNED(pa_start)); 580 BUG_ON(!PAGE_ALIGNED(size)); 581 582 __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, 583 early_pgtable_alloc, flags); 584 585 if (!(vm_flags & VM_NO_GUARD)) 586 size += PAGE_SIZE; 587 588 vma->addr = va_start; 589 vma->phys_addr = pa_start; 590 vma->size = size; 591 vma->flags = VM_MAP | vm_flags; 592 vma->caller = __builtin_return_address(0); 593 594 vm_area_add_early(vma); 595 } 596 597 static int __init parse_rodata(char *arg) 598 { 599 int ret = strtobool(arg, &rodata_enabled); 600 if (!ret) { 601 rodata_full = false; 602 return 0; 603 } 604 605 /* permit 'full' in addition to boolean options */ 606 if (strcmp(arg, "full")) 607 return -EINVAL; 608 609 rodata_enabled = true; 610 rodata_full = true; 611 return 0; 612 } 613 early_param("rodata", parse_rodata); 614 615 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 616 static int __init map_entry_trampoline(void) 617 { 618 pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; 619 phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); 620 621 /* The trampoline is always mapped and can therefore be global */ 622 pgprot_val(prot) &= ~PTE_NG; 623 624 /* Map only the text into the trampoline page table */ 625 memset(tramp_pg_dir, 0, PGD_SIZE); 626 __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE, 627 prot, __pgd_pgtable_alloc, 0); 628 629 /* Map both the text and data into the kernel page table */ 630 __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot); 631 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { 632 extern char __entry_tramp_data_start[]; 633 634 __set_fixmap(FIX_ENTRY_TRAMP_DATA, 635 __pa_symbol(__entry_tramp_data_start), 636 PAGE_KERNEL_RO); 637 } 638 639 return 0; 640 } 641 core_initcall(map_entry_trampoline); 642 #endif 643 644 /* 645 * Open coded check for BTI, only for use to determine configuration 646 * for early mappings for before the cpufeature code has run. 647 */ 648 static bool arm64_early_this_cpu_has_bti(void) 649 { 650 u64 pfr1; 651 652 if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) 653 return false; 654 655 pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1); 656 return cpuid_feature_extract_unsigned_field(pfr1, 657 ID_AA64PFR1_BT_SHIFT); 658 } 659 660 /* 661 * Create fine-grained mappings for the kernel. 662 */ 663 static void __init map_kernel(pgd_t *pgdp) 664 { 665 static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, 666 vmlinux_initdata, vmlinux_data; 667 668 /* 669 * External debuggers may need to write directly to the text 670 * mapping to install SW breakpoints. Allow this (only) when 671 * explicitly requested with rodata=off. 672 */ 673 pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; 674 675 /* 676 * If we have a CPU that supports BTI and a kernel built for 677 * BTI then mark the kernel executable text as guarded pages 678 * now so we don't have to rewrite the page tables later. 679 */ 680 if (arm64_early_this_cpu_has_bti()) 681 text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP); 682 683 /* 684 * Only rodata will be remapped with different permissions later on, 685 * all other segments are allowed to use contiguous mappings. 686 */ 687 map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0, 688 VM_NO_GUARD); 689 map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, 690 &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); 691 map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, 692 &vmlinux_inittext, 0, VM_NO_GUARD); 693 map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, 694 &vmlinux_initdata, 0, VM_NO_GUARD); 695 map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); 696 697 if (!READ_ONCE(pgd_val(*pgd_offset_pgd(pgdp, FIXADDR_START)))) { 698 /* 699 * The fixmap falls in a separate pgd to the kernel, and doesn't 700 * live in the carveout for the swapper_pg_dir. We can simply 701 * re-use the existing dir for the fixmap. 702 */ 703 set_pgd(pgd_offset_pgd(pgdp, FIXADDR_START), 704 READ_ONCE(*pgd_offset_k(FIXADDR_START))); 705 } else if (CONFIG_PGTABLE_LEVELS > 3) { 706 pgd_t *bm_pgdp; 707 p4d_t *bm_p4dp; 708 pud_t *bm_pudp; 709 /* 710 * The fixmap shares its top level pgd entry with the kernel 711 * mapping. This can really only occur when we are running 712 * with 16k/4 levels, so we can simply reuse the pud level 713 * entry instead. 714 */ 715 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 716 bm_pgdp = pgd_offset_pgd(pgdp, FIXADDR_START); 717 bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_START); 718 bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_START); 719 pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd)); 720 pud_clear_fixmap(); 721 } else { 722 BUG(); 723 } 724 725 kasan_copy_shadow(pgdp); 726 } 727 728 void __init paging_init(void) 729 { 730 pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir)); 731 732 map_kernel(pgdp); 733 map_mem(pgdp); 734 735 pgd_clear_fixmap(); 736 737 cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); 738 init_mm.pgd = swapper_pg_dir; 739 740 memblock_free(__pa_symbol(init_pg_dir), 741 __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); 742 743 memblock_allow_resize(); 744 } 745 746 /* 747 * Check whether a kernel address is valid (derived from arch/x86/). 748 */ 749 int kern_addr_valid(unsigned long addr) 750 { 751 pgd_t *pgdp; 752 p4d_t *p4dp; 753 pud_t *pudp, pud; 754 pmd_t *pmdp, pmd; 755 pte_t *ptep, pte; 756 757 addr = arch_kasan_reset_tag(addr); 758 if ((((long)addr) >> VA_BITS) != -1UL) 759 return 0; 760 761 pgdp = pgd_offset_k(addr); 762 if (pgd_none(READ_ONCE(*pgdp))) 763 return 0; 764 765 p4dp = p4d_offset(pgdp, addr); 766 if (p4d_none(READ_ONCE(*p4dp))) 767 return 0; 768 769 pudp = pud_offset(p4dp, addr); 770 pud = READ_ONCE(*pudp); 771 if (pud_none(pud)) 772 return 0; 773 774 if (pud_sect(pud)) 775 return pfn_valid(pud_pfn(pud)); 776 777 pmdp = pmd_offset(pudp, addr); 778 pmd = READ_ONCE(*pmdp); 779 if (pmd_none(pmd)) 780 return 0; 781 782 if (pmd_sect(pmd)) 783 return pfn_valid(pmd_pfn(pmd)); 784 785 ptep = pte_offset_kernel(pmdp, addr); 786 pte = READ_ONCE(*ptep); 787 if (pte_none(pte)) 788 return 0; 789 790 return pfn_valid(pte_pfn(pte)); 791 } 792 793 #ifdef CONFIG_MEMORY_HOTPLUG 794 static void free_hotplug_page_range(struct page *page, size_t size, 795 struct vmem_altmap *altmap) 796 { 797 if (altmap) { 798 vmem_altmap_free(altmap, size >> PAGE_SHIFT); 799 } else { 800 WARN_ON(PageReserved(page)); 801 free_pages((unsigned long)page_address(page), get_order(size)); 802 } 803 } 804 805 static void free_hotplug_pgtable_page(struct page *page) 806 { 807 free_hotplug_page_range(page, PAGE_SIZE, NULL); 808 } 809 810 static bool pgtable_range_aligned(unsigned long start, unsigned long end, 811 unsigned long floor, unsigned long ceiling, 812 unsigned long mask) 813 { 814 start &= mask; 815 if (start < floor) 816 return false; 817 818 if (ceiling) { 819 ceiling &= mask; 820 if (!ceiling) 821 return false; 822 } 823 824 if (end - 1 > ceiling - 1) 825 return false; 826 return true; 827 } 828 829 static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr, 830 unsigned long end, bool free_mapped, 831 struct vmem_altmap *altmap) 832 { 833 pte_t *ptep, pte; 834 835 do { 836 ptep = pte_offset_kernel(pmdp, addr); 837 pte = READ_ONCE(*ptep); 838 if (pte_none(pte)) 839 continue; 840 841 WARN_ON(!pte_present(pte)); 842 pte_clear(&init_mm, addr, ptep); 843 flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 844 if (free_mapped) 845 free_hotplug_page_range(pte_page(pte), 846 PAGE_SIZE, altmap); 847 } while (addr += PAGE_SIZE, addr < end); 848 } 849 850 static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, 851 unsigned long end, bool free_mapped, 852 struct vmem_altmap *altmap) 853 { 854 unsigned long next; 855 pmd_t *pmdp, pmd; 856 857 do { 858 next = pmd_addr_end(addr, end); 859 pmdp = pmd_offset(pudp, addr); 860 pmd = READ_ONCE(*pmdp); 861 if (pmd_none(pmd)) 862 continue; 863 864 WARN_ON(!pmd_present(pmd)); 865 if (pmd_sect(pmd)) { 866 pmd_clear(pmdp); 867 868 /* 869 * One TLBI should be sufficient here as the PMD_SIZE 870 * range is mapped with a single block entry. 871 */ 872 flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 873 if (free_mapped) 874 free_hotplug_page_range(pmd_page(pmd), 875 PMD_SIZE, altmap); 876 continue; 877 } 878 WARN_ON(!pmd_table(pmd)); 879 unmap_hotplug_pte_range(pmdp, addr, next, free_mapped, altmap); 880 } while (addr = next, addr < end); 881 } 882 883 static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, 884 unsigned long end, bool free_mapped, 885 struct vmem_altmap *altmap) 886 { 887 unsigned long next; 888 pud_t *pudp, pud; 889 890 do { 891 next = pud_addr_end(addr, end); 892 pudp = pud_offset(p4dp, addr); 893 pud = READ_ONCE(*pudp); 894 if (pud_none(pud)) 895 continue; 896 897 WARN_ON(!pud_present(pud)); 898 if (pud_sect(pud)) { 899 pud_clear(pudp); 900 901 /* 902 * One TLBI should be sufficient here as the PUD_SIZE 903 * range is mapped with a single block entry. 904 */ 905 flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 906 if (free_mapped) 907 free_hotplug_page_range(pud_page(pud), 908 PUD_SIZE, altmap); 909 continue; 910 } 911 WARN_ON(!pud_table(pud)); 912 unmap_hotplug_pmd_range(pudp, addr, next, free_mapped, altmap); 913 } while (addr = next, addr < end); 914 } 915 916 static void unmap_hotplug_p4d_range(pgd_t *pgdp, unsigned long addr, 917 unsigned long end, bool free_mapped, 918 struct vmem_altmap *altmap) 919 { 920 unsigned long next; 921 p4d_t *p4dp, p4d; 922 923 do { 924 next = p4d_addr_end(addr, end); 925 p4dp = p4d_offset(pgdp, addr); 926 p4d = READ_ONCE(*p4dp); 927 if (p4d_none(p4d)) 928 continue; 929 930 WARN_ON(!p4d_present(p4d)); 931 unmap_hotplug_pud_range(p4dp, addr, next, free_mapped, altmap); 932 } while (addr = next, addr < end); 933 } 934 935 static void unmap_hotplug_range(unsigned long addr, unsigned long end, 936 bool free_mapped, struct vmem_altmap *altmap) 937 { 938 unsigned long next; 939 pgd_t *pgdp, pgd; 940 941 /* 942 * altmap can only be used as vmemmap mapping backing memory. 943 * In case the backing memory itself is not being freed, then 944 * altmap is irrelevant. Warn about this inconsistency when 945 * encountered. 946 */ 947 WARN_ON(!free_mapped && altmap); 948 949 do { 950 next = pgd_addr_end(addr, end); 951 pgdp = pgd_offset_k(addr); 952 pgd = READ_ONCE(*pgdp); 953 if (pgd_none(pgd)) 954 continue; 955 956 WARN_ON(!pgd_present(pgd)); 957 unmap_hotplug_p4d_range(pgdp, addr, next, free_mapped, altmap); 958 } while (addr = next, addr < end); 959 } 960 961 static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr, 962 unsigned long end, unsigned long floor, 963 unsigned long ceiling) 964 { 965 pte_t *ptep, pte; 966 unsigned long i, start = addr; 967 968 do { 969 ptep = pte_offset_kernel(pmdp, addr); 970 pte = READ_ONCE(*ptep); 971 972 /* 973 * This is just a sanity check here which verifies that 974 * pte clearing has been done by earlier unmap loops. 975 */ 976 WARN_ON(!pte_none(pte)); 977 } while (addr += PAGE_SIZE, addr < end); 978 979 if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK)) 980 return; 981 982 /* 983 * Check whether we can free the pte page if the rest of the 984 * entries are empty. Overlap with other regions have been 985 * handled by the floor/ceiling check. 986 */ 987 ptep = pte_offset_kernel(pmdp, 0UL); 988 for (i = 0; i < PTRS_PER_PTE; i++) { 989 if (!pte_none(READ_ONCE(ptep[i]))) 990 return; 991 } 992 993 pmd_clear(pmdp); 994 __flush_tlb_kernel_pgtable(start); 995 free_hotplug_pgtable_page(virt_to_page(ptep)); 996 } 997 998 static void free_empty_pmd_table(pud_t *pudp, unsigned long addr, 999 unsigned long end, unsigned long floor, 1000 unsigned long ceiling) 1001 { 1002 pmd_t *pmdp, pmd; 1003 unsigned long i, next, start = addr; 1004 1005 do { 1006 next = pmd_addr_end(addr, end); 1007 pmdp = pmd_offset(pudp, addr); 1008 pmd = READ_ONCE(*pmdp); 1009 if (pmd_none(pmd)) 1010 continue; 1011 1012 WARN_ON(!pmd_present(pmd) || !pmd_table(pmd) || pmd_sect(pmd)); 1013 free_empty_pte_table(pmdp, addr, next, floor, ceiling); 1014 } while (addr = next, addr < end); 1015 1016 if (CONFIG_PGTABLE_LEVELS <= 2) 1017 return; 1018 1019 if (!pgtable_range_aligned(start, end, floor, ceiling, PUD_MASK)) 1020 return; 1021 1022 /* 1023 * Check whether we can free the pmd page if the rest of the 1024 * entries are empty. Overlap with other regions have been 1025 * handled by the floor/ceiling check. 1026 */ 1027 pmdp = pmd_offset(pudp, 0UL); 1028 for (i = 0; i < PTRS_PER_PMD; i++) { 1029 if (!pmd_none(READ_ONCE(pmdp[i]))) 1030 return; 1031 } 1032 1033 pud_clear(pudp); 1034 __flush_tlb_kernel_pgtable(start); 1035 free_hotplug_pgtable_page(virt_to_page(pmdp)); 1036 } 1037 1038 static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr, 1039 unsigned long end, unsigned long floor, 1040 unsigned long ceiling) 1041 { 1042 pud_t *pudp, pud; 1043 unsigned long i, next, start = addr; 1044 1045 do { 1046 next = pud_addr_end(addr, end); 1047 pudp = pud_offset(p4dp, addr); 1048 pud = READ_ONCE(*pudp); 1049 if (pud_none(pud)) 1050 continue; 1051 1052 WARN_ON(!pud_present(pud) || !pud_table(pud) || pud_sect(pud)); 1053 free_empty_pmd_table(pudp, addr, next, floor, ceiling); 1054 } while (addr = next, addr < end); 1055 1056 if (CONFIG_PGTABLE_LEVELS <= 3) 1057 return; 1058 1059 if (!pgtable_range_aligned(start, end, floor, ceiling, PGDIR_MASK)) 1060 return; 1061 1062 /* 1063 * Check whether we can free the pud page if the rest of the 1064 * entries are empty. Overlap with other regions have been 1065 * handled by the floor/ceiling check. 1066 */ 1067 pudp = pud_offset(p4dp, 0UL); 1068 for (i = 0; i < PTRS_PER_PUD; i++) { 1069 if (!pud_none(READ_ONCE(pudp[i]))) 1070 return; 1071 } 1072 1073 p4d_clear(p4dp); 1074 __flush_tlb_kernel_pgtable(start); 1075 free_hotplug_pgtable_page(virt_to_page(pudp)); 1076 } 1077 1078 static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr, 1079 unsigned long end, unsigned long floor, 1080 unsigned long ceiling) 1081 { 1082 unsigned long next; 1083 p4d_t *p4dp, p4d; 1084 1085 do { 1086 next = p4d_addr_end(addr, end); 1087 p4dp = p4d_offset(pgdp, addr); 1088 p4d = READ_ONCE(*p4dp); 1089 if (p4d_none(p4d)) 1090 continue; 1091 1092 WARN_ON(!p4d_present(p4d)); 1093 free_empty_pud_table(p4dp, addr, next, floor, ceiling); 1094 } while (addr = next, addr < end); 1095 } 1096 1097 static void free_empty_tables(unsigned long addr, unsigned long end, 1098 unsigned long floor, unsigned long ceiling) 1099 { 1100 unsigned long next; 1101 pgd_t *pgdp, pgd; 1102 1103 do { 1104 next = pgd_addr_end(addr, end); 1105 pgdp = pgd_offset_k(addr); 1106 pgd = READ_ONCE(*pgdp); 1107 if (pgd_none(pgd)) 1108 continue; 1109 1110 WARN_ON(!pgd_present(pgd)); 1111 free_empty_p4d_table(pgdp, addr, next, floor, ceiling); 1112 } while (addr = next, addr < end); 1113 } 1114 #endif 1115 1116 #if !ARM64_SWAPPER_USES_SECTION_MAPS 1117 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1118 struct vmem_altmap *altmap) 1119 { 1120 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); 1121 return vmemmap_populate_basepages(start, end, node, altmap); 1122 } 1123 #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 1124 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1125 struct vmem_altmap *altmap) 1126 { 1127 unsigned long addr = start; 1128 unsigned long next; 1129 pgd_t *pgdp; 1130 p4d_t *p4dp; 1131 pud_t *pudp; 1132 pmd_t *pmdp; 1133 1134 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); 1135 do { 1136 next = pmd_addr_end(addr, end); 1137 1138 pgdp = vmemmap_pgd_populate(addr, node); 1139 if (!pgdp) 1140 return -ENOMEM; 1141 1142 p4dp = vmemmap_p4d_populate(pgdp, addr, node); 1143 if (!p4dp) 1144 return -ENOMEM; 1145 1146 pudp = vmemmap_pud_populate(p4dp, addr, node); 1147 if (!pudp) 1148 return -ENOMEM; 1149 1150 pmdp = pmd_offset(pudp, addr); 1151 if (pmd_none(READ_ONCE(*pmdp))) { 1152 void *p = NULL; 1153 1154 p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap); 1155 if (!p) { 1156 if (vmemmap_populate_basepages(addr, next, node, altmap)) 1157 return -ENOMEM; 1158 continue; 1159 } 1160 1161 pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL)); 1162 } else 1163 vmemmap_verify((pte_t *)pmdp, node, addr, next); 1164 } while (addr = next, addr != end); 1165 1166 return 0; 1167 } 1168 #endif /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 1169 void vmemmap_free(unsigned long start, unsigned long end, 1170 struct vmem_altmap *altmap) 1171 { 1172 #ifdef CONFIG_MEMORY_HOTPLUG 1173 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); 1174 1175 unmap_hotplug_range(start, end, true, altmap); 1176 free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END); 1177 #endif 1178 } 1179 1180 static inline pud_t *fixmap_pud(unsigned long addr) 1181 { 1182 pgd_t *pgdp = pgd_offset_k(addr); 1183 p4d_t *p4dp = p4d_offset(pgdp, addr); 1184 p4d_t p4d = READ_ONCE(*p4dp); 1185 1186 BUG_ON(p4d_none(p4d) || p4d_bad(p4d)); 1187 1188 return pud_offset_kimg(p4dp, addr); 1189 } 1190 1191 static inline pmd_t *fixmap_pmd(unsigned long addr) 1192 { 1193 pud_t *pudp = fixmap_pud(addr); 1194 pud_t pud = READ_ONCE(*pudp); 1195 1196 BUG_ON(pud_none(pud) || pud_bad(pud)); 1197 1198 return pmd_offset_kimg(pudp, addr); 1199 } 1200 1201 static inline pte_t *fixmap_pte(unsigned long addr) 1202 { 1203 return &bm_pte[pte_index(addr)]; 1204 } 1205 1206 /* 1207 * The p*d_populate functions call virt_to_phys implicitly so they can't be used 1208 * directly on kernel symbols (bm_p*d). This function is called too early to use 1209 * lm_alias so __p*d_populate functions must be used to populate with the 1210 * physical address from __pa_symbol. 1211 */ 1212 void __init early_fixmap_init(void) 1213 { 1214 pgd_t *pgdp; 1215 p4d_t *p4dp, p4d; 1216 pud_t *pudp; 1217 pmd_t *pmdp; 1218 unsigned long addr = FIXADDR_START; 1219 1220 pgdp = pgd_offset_k(addr); 1221 p4dp = p4d_offset(pgdp, addr); 1222 p4d = READ_ONCE(*p4dp); 1223 if (CONFIG_PGTABLE_LEVELS > 3 && 1224 !(p4d_none(p4d) || p4d_page_paddr(p4d) == __pa_symbol(bm_pud))) { 1225 /* 1226 * We only end up here if the kernel mapping and the fixmap 1227 * share the top level pgd entry, which should only happen on 1228 * 16k/4 levels configurations. 1229 */ 1230 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 1231 pudp = pud_offset_kimg(p4dp, addr); 1232 } else { 1233 if (p4d_none(p4d)) 1234 __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); 1235 pudp = fixmap_pud(addr); 1236 } 1237 if (pud_none(READ_ONCE(*pudp))) 1238 __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); 1239 pmdp = fixmap_pmd(addr); 1240 __pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE); 1241 1242 /* 1243 * The boot-ioremap range spans multiple pmds, for which 1244 * we are not prepared: 1245 */ 1246 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 1247 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 1248 1249 if ((pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 1250 || pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 1251 WARN_ON(1); 1252 pr_warn("pmdp %p != %p, %p\n", 1253 pmdp, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 1254 fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 1255 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 1256 fix_to_virt(FIX_BTMAP_BEGIN)); 1257 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 1258 fix_to_virt(FIX_BTMAP_END)); 1259 1260 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 1261 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 1262 } 1263 } 1264 1265 /* 1266 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we 1267 * ever need to use IPIs for TLB broadcasting, then we're in trouble here. 1268 */ 1269 void __set_fixmap(enum fixed_addresses idx, 1270 phys_addr_t phys, pgprot_t flags) 1271 { 1272 unsigned long addr = __fix_to_virt(idx); 1273 pte_t *ptep; 1274 1275 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 1276 1277 ptep = fixmap_pte(addr); 1278 1279 if (pgprot_val(flags)) { 1280 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); 1281 } else { 1282 pte_clear(&init_mm, addr, ptep); 1283 flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 1284 } 1285 } 1286 1287 void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) 1288 { 1289 const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 1290 int offset; 1291 void *dt_virt; 1292 1293 /* 1294 * Check whether the physical FDT address is set and meets the minimum 1295 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 1296 * at least 8 bytes so that we can always access the magic and size 1297 * fields of the FDT header after mapping the first chunk, double check 1298 * here if that is indeed the case. 1299 */ 1300 BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 1301 if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 1302 return NULL; 1303 1304 /* 1305 * Make sure that the FDT region can be mapped without the need to 1306 * allocate additional translation table pages, so that it is safe 1307 * to call create_mapping_noalloc() this early. 1308 * 1309 * On 64k pages, the FDT will be mapped using PTEs, so we need to 1310 * be in the same PMD as the rest of the fixmap. 1311 * On 4k pages, we'll use section mappings for the FDT so we only 1312 * have to be in the same PUD. 1313 */ 1314 BUILD_BUG_ON(dt_virt_base % SZ_2M); 1315 1316 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 1317 __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 1318 1319 offset = dt_phys % SWAPPER_BLOCK_SIZE; 1320 dt_virt = (void *)dt_virt_base + offset; 1321 1322 /* map the first chunk so we can read the size from the header */ 1323 create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), 1324 dt_virt_base, SWAPPER_BLOCK_SIZE, prot); 1325 1326 if (fdt_magic(dt_virt) != FDT_MAGIC) 1327 return NULL; 1328 1329 *size = fdt_totalsize(dt_virt); 1330 if (*size > MAX_FDT_SIZE) 1331 return NULL; 1332 1333 if (offset + *size > SWAPPER_BLOCK_SIZE) 1334 create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 1335 round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); 1336 1337 return dt_virt; 1338 } 1339 1340 int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot) 1341 { 1342 pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); 1343 1344 /* Only allow permission changes for now */ 1345 if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)), 1346 pud_val(new_pud))) 1347 return 0; 1348 1349 VM_BUG_ON(phys & ~PUD_MASK); 1350 set_pud(pudp, new_pud); 1351 return 1; 1352 } 1353 1354 int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) 1355 { 1356 pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); 1357 1358 /* Only allow permission changes for now */ 1359 if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), 1360 pmd_val(new_pmd))) 1361 return 0; 1362 1363 VM_BUG_ON(phys & ~PMD_MASK); 1364 set_pmd(pmdp, new_pmd); 1365 return 1; 1366 } 1367 1368 int pud_clear_huge(pud_t *pudp) 1369 { 1370 if (!pud_sect(READ_ONCE(*pudp))) 1371 return 0; 1372 pud_clear(pudp); 1373 return 1; 1374 } 1375 1376 int pmd_clear_huge(pmd_t *pmdp) 1377 { 1378 if (!pmd_sect(READ_ONCE(*pmdp))) 1379 return 0; 1380 pmd_clear(pmdp); 1381 return 1; 1382 } 1383 1384 int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) 1385 { 1386 pte_t *table; 1387 pmd_t pmd; 1388 1389 pmd = READ_ONCE(*pmdp); 1390 1391 if (!pmd_table(pmd)) { 1392 VM_WARN_ON(1); 1393 return 1; 1394 } 1395 1396 table = pte_offset_kernel(pmdp, addr); 1397 pmd_clear(pmdp); 1398 __flush_tlb_kernel_pgtable(addr); 1399 pte_free_kernel(NULL, table); 1400 return 1; 1401 } 1402 1403 int pud_free_pmd_page(pud_t *pudp, unsigned long addr) 1404 { 1405 pmd_t *table; 1406 pmd_t *pmdp; 1407 pud_t pud; 1408 unsigned long next, end; 1409 1410 pud = READ_ONCE(*pudp); 1411 1412 if (!pud_table(pud)) { 1413 VM_WARN_ON(1); 1414 return 1; 1415 } 1416 1417 table = pmd_offset(pudp, addr); 1418 pmdp = table; 1419 next = addr; 1420 end = addr + PUD_SIZE; 1421 do { 1422 pmd_free_pte_page(pmdp, next); 1423 } while (pmdp++, next += PMD_SIZE, next != end); 1424 1425 pud_clear(pudp); 1426 __flush_tlb_kernel_pgtable(addr); 1427 pmd_free(NULL, table); 1428 return 1; 1429 } 1430 1431 #ifdef CONFIG_MEMORY_HOTPLUG 1432 static void __remove_pgd_mapping(pgd_t *pgdir, unsigned long start, u64 size) 1433 { 1434 unsigned long end = start + size; 1435 1436 WARN_ON(pgdir != init_mm.pgd); 1437 WARN_ON((start < PAGE_OFFSET) || (end > PAGE_END)); 1438 1439 unmap_hotplug_range(start, end, false, NULL); 1440 free_empty_tables(start, end, PAGE_OFFSET, PAGE_END); 1441 } 1442 1443 struct range arch_get_mappable_range(void) 1444 { 1445 struct range mhp_range; 1446 u64 start_linear_pa = __pa(_PAGE_OFFSET(vabits_actual)); 1447 u64 end_linear_pa = __pa(PAGE_END - 1); 1448 1449 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { 1450 /* 1451 * Check for a wrap, it is possible because of randomized linear 1452 * mapping the start physical address is actually bigger than 1453 * the end physical address. In this case set start to zero 1454 * because [0, end_linear_pa] range must still be able to cover 1455 * all addressable physical addresses. 1456 */ 1457 if (start_linear_pa > end_linear_pa) 1458 start_linear_pa = 0; 1459 } 1460 1461 WARN_ON(start_linear_pa > end_linear_pa); 1462 1463 /* 1464 * Linear mapping region is the range [PAGE_OFFSET..(PAGE_END - 1)] 1465 * accommodating both its ends but excluding PAGE_END. Max physical 1466 * range which can be mapped inside this linear mapping range, must 1467 * also be derived from its end points. 1468 */ 1469 mhp_range.start = start_linear_pa; 1470 mhp_range.end = end_linear_pa; 1471 1472 return mhp_range; 1473 } 1474 1475 int arch_add_memory(int nid, u64 start, u64 size, 1476 struct mhp_params *params) 1477 { 1478 int ret, flags = NO_EXEC_MAPPINGS; 1479 1480 VM_BUG_ON(!mhp_range_allowed(start, size, true)); 1481 1482 /* 1483 * KFENCE requires linear map to be mapped at page granularity, so that 1484 * it is possible to protect/unprotect single pages in the KFENCE pool. 1485 */ 1486 if (rodata_full || debug_pagealloc_enabled() || 1487 IS_ENABLED(CONFIG_KFENCE)) 1488 flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; 1489 1490 __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), 1491 size, params->pgprot, __pgd_pgtable_alloc, 1492 flags); 1493 1494 memblock_clear_nomap(start, size); 1495 1496 ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT, 1497 params); 1498 if (ret) 1499 __remove_pgd_mapping(swapper_pg_dir, 1500 __phys_to_virt(start), size); 1501 return ret; 1502 } 1503 1504 void arch_remove_memory(int nid, u64 start, u64 size, 1505 struct vmem_altmap *altmap) 1506 { 1507 unsigned long start_pfn = start >> PAGE_SHIFT; 1508 unsigned long nr_pages = size >> PAGE_SHIFT; 1509 1510 __remove_pages(start_pfn, nr_pages, altmap); 1511 __remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size); 1512 } 1513 1514 /* 1515 * This memory hotplug notifier helps prevent boot memory from being 1516 * inadvertently removed as it blocks pfn range offlining process in 1517 * __offline_pages(). Hence this prevents both offlining as well as 1518 * removal process for boot memory which is initially always online. 1519 * In future if and when boot memory could be removed, this notifier 1520 * should be dropped and free_hotplug_page_range() should handle any 1521 * reserved pages allocated during boot. 1522 */ 1523 static int prevent_bootmem_remove_notifier(struct notifier_block *nb, 1524 unsigned long action, void *data) 1525 { 1526 struct mem_section *ms; 1527 struct memory_notify *arg = data; 1528 unsigned long end_pfn = arg->start_pfn + arg->nr_pages; 1529 unsigned long pfn = arg->start_pfn; 1530 1531 if ((action != MEM_GOING_OFFLINE) && (action != MEM_OFFLINE)) 1532 return NOTIFY_OK; 1533 1534 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1535 unsigned long start = PFN_PHYS(pfn); 1536 unsigned long end = start + (1UL << PA_SECTION_SHIFT); 1537 1538 ms = __pfn_to_section(pfn); 1539 if (!early_section(ms)) 1540 continue; 1541 1542 if (action == MEM_GOING_OFFLINE) { 1543 /* 1544 * Boot memory removal is not supported. Prevent 1545 * it via blocking any attempted offline request 1546 * for the boot memory and just report it. 1547 */ 1548 pr_warn("Boot memory [%lx %lx] offlining attempted\n", start, end); 1549 return NOTIFY_BAD; 1550 } else if (action == MEM_OFFLINE) { 1551 /* 1552 * This should have never happened. Boot memory 1553 * offlining should have been prevented by this 1554 * very notifier. Probably some memory removal 1555 * procedure might have changed which would then 1556 * require further debug. 1557 */ 1558 pr_err("Boot memory [%lx %lx] offlined\n", start, end); 1559 1560 /* 1561 * Core memory hotplug does not process a return 1562 * code from the notifier for MEM_OFFLINE events. 1563 * The error condition has been reported. Return 1564 * from here as if ignored. 1565 */ 1566 return NOTIFY_DONE; 1567 } 1568 } 1569 return NOTIFY_OK; 1570 } 1571 1572 static struct notifier_block prevent_bootmem_remove_nb = { 1573 .notifier_call = prevent_bootmem_remove_notifier, 1574 }; 1575 1576 /* 1577 * This ensures that boot memory sections on the platform are online 1578 * from early boot. Memory sections could not be prevented from being 1579 * offlined, unless for some reason they are not online to begin with. 1580 * This helps validate the basic assumption on which the above memory 1581 * event notifier works to prevent boot memory section offlining and 1582 * its possible removal. 1583 */ 1584 static void validate_bootmem_online(void) 1585 { 1586 phys_addr_t start, end, addr; 1587 struct mem_section *ms; 1588 u64 i; 1589 1590 /* 1591 * Scanning across all memblock might be expensive 1592 * on some big memory systems. Hence enable this 1593 * validation only with DEBUG_VM. 1594 */ 1595 if (!IS_ENABLED(CONFIG_DEBUG_VM)) 1596 return; 1597 1598 for_each_mem_range(i, &start, &end) { 1599 for (addr = start; addr < end; addr += (1UL << PA_SECTION_SHIFT)) { 1600 ms = __pfn_to_section(PHYS_PFN(addr)); 1601 1602 /* 1603 * All memory ranges in the system at this point 1604 * should have been marked as early sections. 1605 */ 1606 WARN_ON(!early_section(ms)); 1607 1608 /* 1609 * Memory notifier mechanism here to prevent boot 1610 * memory offlining depends on the fact that each 1611 * early section memory on the system is initially 1612 * online. Otherwise a given memory section which 1613 * is already offline will be overlooked and can 1614 * be removed completely. Call out such sections. 1615 */ 1616 if (!online_section(ms)) 1617 pr_err("Boot memory [%llx %llx] is offline, can be removed\n", 1618 addr, addr + (1UL << PA_SECTION_SHIFT)); 1619 } 1620 } 1621 } 1622 1623 static int __init prevent_bootmem_remove_init(void) 1624 { 1625 int ret = 0; 1626 1627 if (!IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 1628 return ret; 1629 1630 validate_bootmem_online(); 1631 ret = register_memory_notifier(&prevent_bootmem_remove_nb); 1632 if (ret) 1633 pr_err("%s: Notifier registration failed %d\n", __func__, ret); 1634 1635 return ret; 1636 } 1637 early_initcall(prevent_bootmem_remove_init); 1638 #endif 1639