1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Based on arch/arm/mm/mmu.c 4 * 5 * Copyright (C) 1995-2005 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9 #include <linux/cache.h> 10 #include <linux/export.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/init.h> 14 #include <linux/ioport.h> 15 #include <linux/kexec.h> 16 #include <linux/libfdt.h> 17 #include <linux/mman.h> 18 #include <linux/nodemask.h> 19 #include <linux/memblock.h> 20 #include <linux/memory.h> 21 #include <linux/fs.h> 22 #include <linux/io.h> 23 #include <linux/mm.h> 24 #include <linux/vmalloc.h> 25 26 #include <asm/barrier.h> 27 #include <asm/cputype.h> 28 #include <asm/fixmap.h> 29 #include <asm/kasan.h> 30 #include <asm/kernel-pgtable.h> 31 #include <asm/sections.h> 32 #include <asm/setup.h> 33 #include <linux/sizes.h> 34 #include <asm/tlb.h> 35 #include <asm/mmu_context.h> 36 #include <asm/ptdump.h> 37 #include <asm/tlbflush.h> 38 #include <asm/pgalloc.h> 39 40 #define NO_BLOCK_MAPPINGS BIT(0) 41 #define NO_CONT_MAPPINGS BIT(1) 42 #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ 43 44 u64 idmap_t0sz = TCR_T0SZ(VA_BITS_MIN); 45 u64 idmap_ptrs_per_pgd = PTRS_PER_PGD; 46 47 u64 __section(".mmuoff.data.write") vabits_actual; 48 EXPORT_SYMBOL(vabits_actual); 49 50 u64 kimage_voffset __ro_after_init; 51 EXPORT_SYMBOL(kimage_voffset); 52 53 /* 54 * Empty_zero_page is a special page that is used for zero-initialized data 55 * and COW. 56 */ 57 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; 58 EXPORT_SYMBOL(empty_zero_page); 59 60 static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 61 static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; 62 static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; 63 64 static DEFINE_SPINLOCK(swapper_pgdir_lock); 65 66 void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) 67 { 68 pgd_t *fixmap_pgdp; 69 70 spin_lock(&swapper_pgdir_lock); 71 fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp)); 72 WRITE_ONCE(*fixmap_pgdp, pgd); 73 /* 74 * We need dsb(ishst) here to ensure the page-table-walker sees 75 * our new entry before set_p?d() returns. The fixmap's 76 * flush_tlb_kernel_range() via clear_fixmap() does this for us. 77 */ 78 pgd_clear_fixmap(); 79 spin_unlock(&swapper_pgdir_lock); 80 } 81 82 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 83 unsigned long size, pgprot_t vma_prot) 84 { 85 if (!pfn_valid(pfn)) 86 return pgprot_noncached(vma_prot); 87 else if (file->f_flags & O_SYNC) 88 return pgprot_writecombine(vma_prot); 89 return vma_prot; 90 } 91 EXPORT_SYMBOL(phys_mem_access_prot); 92 93 static phys_addr_t __init early_pgtable_alloc(int shift) 94 { 95 phys_addr_t phys; 96 void *ptr; 97 98 phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 99 if (!phys) 100 panic("Failed to allocate page table page\n"); 101 102 /* 103 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE 104 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise 105 * any level of table. 106 */ 107 ptr = pte_set_fixmap(phys); 108 109 memset(ptr, 0, PAGE_SIZE); 110 111 /* 112 * Implicit barriers also ensure the zeroed page is visible to the page 113 * table walker 114 */ 115 pte_clear_fixmap(); 116 117 return phys; 118 } 119 120 static bool pgattr_change_is_safe(u64 old, u64 new) 121 { 122 /* 123 * The following mapping attributes may be updated in live 124 * kernel mappings without the need for break-before-make. 125 */ 126 pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG; 127 128 /* creating or taking down mappings is always safe */ 129 if (old == 0 || new == 0) 130 return true; 131 132 /* live contiguous mappings may not be manipulated at all */ 133 if ((old | new) & PTE_CONT) 134 return false; 135 136 /* Transitioning from Non-Global to Global is unsafe */ 137 if (old & ~new & PTE_NG) 138 return false; 139 140 /* 141 * Changing the memory type between Normal and Normal-Tagged is safe 142 * since Tagged is considered a permission attribute from the 143 * mismatched attribute aliases perspective. 144 */ 145 if (((old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) || 146 (old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)) && 147 ((new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) || 148 (new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED))) 149 mask |= PTE_ATTRINDX_MASK; 150 151 return ((old ^ new) & ~mask) == 0; 152 } 153 154 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, 155 phys_addr_t phys, pgprot_t prot) 156 { 157 pte_t *ptep; 158 159 ptep = pte_set_fixmap_offset(pmdp, addr); 160 do { 161 pte_t old_pte = READ_ONCE(*ptep); 162 163 set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot)); 164 165 /* 166 * After the PTE entry has been populated once, we 167 * only allow updates to the permission attributes. 168 */ 169 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), 170 READ_ONCE(pte_val(*ptep)))); 171 172 phys += PAGE_SIZE; 173 } while (ptep++, addr += PAGE_SIZE, addr != end); 174 175 pte_clear_fixmap(); 176 } 177 178 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, 179 unsigned long end, phys_addr_t phys, 180 pgprot_t prot, 181 phys_addr_t (*pgtable_alloc)(int), 182 int flags) 183 { 184 unsigned long next; 185 pmd_t pmd = READ_ONCE(*pmdp); 186 187 BUG_ON(pmd_sect(pmd)); 188 if (pmd_none(pmd)) { 189 pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN; 190 phys_addr_t pte_phys; 191 192 if (flags & NO_EXEC_MAPPINGS) 193 pmdval |= PMD_TABLE_PXN; 194 BUG_ON(!pgtable_alloc); 195 pte_phys = pgtable_alloc(PAGE_SHIFT); 196 __pmd_populate(pmdp, pte_phys, pmdval); 197 pmd = READ_ONCE(*pmdp); 198 } 199 BUG_ON(pmd_bad(pmd)); 200 201 do { 202 pgprot_t __prot = prot; 203 204 next = pte_cont_addr_end(addr, end); 205 206 /* use a contiguous mapping if the range is suitably aligned */ 207 if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && 208 (flags & NO_CONT_MAPPINGS) == 0) 209 __prot = __pgprot(pgprot_val(prot) | PTE_CONT); 210 211 init_pte(pmdp, addr, next, phys, __prot); 212 213 phys += next - addr; 214 } while (addr = next, addr != end); 215 } 216 217 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, 218 phys_addr_t phys, pgprot_t prot, 219 phys_addr_t (*pgtable_alloc)(int), int flags) 220 { 221 unsigned long next; 222 pmd_t *pmdp; 223 224 pmdp = pmd_set_fixmap_offset(pudp, addr); 225 do { 226 pmd_t old_pmd = READ_ONCE(*pmdp); 227 228 next = pmd_addr_end(addr, end); 229 230 /* try section mapping first */ 231 if (((addr | next | phys) & ~SECTION_MASK) == 0 && 232 (flags & NO_BLOCK_MAPPINGS) == 0) { 233 pmd_set_huge(pmdp, phys, prot); 234 235 /* 236 * After the PMD entry has been populated once, we 237 * only allow updates to the permission attributes. 238 */ 239 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), 240 READ_ONCE(pmd_val(*pmdp)))); 241 } else { 242 alloc_init_cont_pte(pmdp, addr, next, phys, prot, 243 pgtable_alloc, flags); 244 245 BUG_ON(pmd_val(old_pmd) != 0 && 246 pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); 247 } 248 phys += next - addr; 249 } while (pmdp++, addr = next, addr != end); 250 251 pmd_clear_fixmap(); 252 } 253 254 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, 255 unsigned long end, phys_addr_t phys, 256 pgprot_t prot, 257 phys_addr_t (*pgtable_alloc)(int), int flags) 258 { 259 unsigned long next; 260 pud_t pud = READ_ONCE(*pudp); 261 262 /* 263 * Check for initial section mappings in the pgd/pud. 264 */ 265 BUG_ON(pud_sect(pud)); 266 if (pud_none(pud)) { 267 pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN; 268 phys_addr_t pmd_phys; 269 270 if (flags & NO_EXEC_MAPPINGS) 271 pudval |= PUD_TABLE_PXN; 272 BUG_ON(!pgtable_alloc); 273 pmd_phys = pgtable_alloc(PMD_SHIFT); 274 __pud_populate(pudp, pmd_phys, pudval); 275 pud = READ_ONCE(*pudp); 276 } 277 BUG_ON(pud_bad(pud)); 278 279 do { 280 pgprot_t __prot = prot; 281 282 next = pmd_cont_addr_end(addr, end); 283 284 /* use a contiguous mapping if the range is suitably aligned */ 285 if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && 286 (flags & NO_CONT_MAPPINGS) == 0) 287 __prot = __pgprot(pgprot_val(prot) | PTE_CONT); 288 289 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); 290 291 phys += next - addr; 292 } while (addr = next, addr != end); 293 } 294 295 static inline bool use_1G_block(unsigned long addr, unsigned long next, 296 unsigned long phys) 297 { 298 if (PAGE_SHIFT != 12) 299 return false; 300 301 if (((addr | next | phys) & ~PUD_MASK) != 0) 302 return false; 303 304 return true; 305 } 306 307 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, 308 phys_addr_t phys, pgprot_t prot, 309 phys_addr_t (*pgtable_alloc)(int), 310 int flags) 311 { 312 unsigned long next; 313 pud_t *pudp; 314 p4d_t *p4dp = p4d_offset(pgdp, addr); 315 p4d_t p4d = READ_ONCE(*p4dp); 316 317 if (p4d_none(p4d)) { 318 p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN; 319 phys_addr_t pud_phys; 320 321 if (flags & NO_EXEC_MAPPINGS) 322 p4dval |= P4D_TABLE_PXN; 323 BUG_ON(!pgtable_alloc); 324 pud_phys = pgtable_alloc(PUD_SHIFT); 325 __p4d_populate(p4dp, pud_phys, p4dval); 326 p4d = READ_ONCE(*p4dp); 327 } 328 BUG_ON(p4d_bad(p4d)); 329 330 pudp = pud_set_fixmap_offset(p4dp, addr); 331 do { 332 pud_t old_pud = READ_ONCE(*pudp); 333 334 next = pud_addr_end(addr, end); 335 336 /* 337 * For 4K granule only, attempt to put down a 1GB block 338 */ 339 if (use_1G_block(addr, next, phys) && 340 (flags & NO_BLOCK_MAPPINGS) == 0) { 341 pud_set_huge(pudp, phys, prot); 342 343 /* 344 * After the PUD entry has been populated once, we 345 * only allow updates to the permission attributes. 346 */ 347 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud), 348 READ_ONCE(pud_val(*pudp)))); 349 } else { 350 alloc_init_cont_pmd(pudp, addr, next, phys, prot, 351 pgtable_alloc, flags); 352 353 BUG_ON(pud_val(old_pud) != 0 && 354 pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); 355 } 356 phys += next - addr; 357 } while (pudp++, addr = next, addr != end); 358 359 pud_clear_fixmap(); 360 } 361 362 static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, 363 unsigned long virt, phys_addr_t size, 364 pgprot_t prot, 365 phys_addr_t (*pgtable_alloc)(int), 366 int flags) 367 { 368 unsigned long addr, end, next; 369 pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); 370 371 /* 372 * If the virtual and physical address don't have the same offset 373 * within a page, we cannot map the region as the caller expects. 374 */ 375 if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) 376 return; 377 378 phys &= PAGE_MASK; 379 addr = virt & PAGE_MASK; 380 end = PAGE_ALIGN(virt + size); 381 382 do { 383 next = pgd_addr_end(addr, end); 384 alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, 385 flags); 386 phys += next - addr; 387 } while (pgdp++, addr = next, addr != end); 388 } 389 390 static phys_addr_t __pgd_pgtable_alloc(int shift) 391 { 392 void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); 393 BUG_ON(!ptr); 394 395 /* Ensure the zeroed page is visible to the page table walker */ 396 dsb(ishst); 397 return __pa(ptr); 398 } 399 400 static phys_addr_t pgd_pgtable_alloc(int shift) 401 { 402 phys_addr_t pa = __pgd_pgtable_alloc(shift); 403 404 /* 405 * Call proper page table ctor in case later we need to 406 * call core mm functions like apply_to_page_range() on 407 * this pre-allocated page table. 408 * 409 * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is 410 * folded, and if so pgtable_pmd_page_ctor() becomes nop. 411 */ 412 if (shift == PAGE_SHIFT) 413 BUG_ON(!pgtable_pte_page_ctor(phys_to_page(pa))); 414 else if (shift == PMD_SHIFT) 415 BUG_ON(!pgtable_pmd_page_ctor(phys_to_page(pa))); 416 417 return pa; 418 } 419 420 /* 421 * This function can only be used to modify existing table entries, 422 * without allocating new levels of table. Note that this permits the 423 * creation of new section or page entries. 424 */ 425 static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, 426 phys_addr_t size, pgprot_t prot) 427 { 428 if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { 429 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 430 &phys, virt); 431 return; 432 } 433 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 434 NO_CONT_MAPPINGS); 435 } 436 437 void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 438 unsigned long virt, phys_addr_t size, 439 pgprot_t prot, bool page_mappings_only) 440 { 441 int flags = 0; 442 443 BUG_ON(mm == &init_mm); 444 445 if (page_mappings_only) 446 flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; 447 448 __create_pgd_mapping(mm->pgd, phys, virt, size, prot, 449 pgd_pgtable_alloc, flags); 450 } 451 452 static void update_mapping_prot(phys_addr_t phys, unsigned long virt, 453 phys_addr_t size, pgprot_t prot) 454 { 455 if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { 456 pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n", 457 &phys, virt); 458 return; 459 } 460 461 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 462 NO_CONT_MAPPINGS); 463 464 /* flush the TLBs after updating live kernel mappings */ 465 flush_tlb_kernel_range(virt, virt + size); 466 } 467 468 static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, 469 phys_addr_t end, pgprot_t prot, int flags) 470 { 471 __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, 472 prot, early_pgtable_alloc, flags); 473 } 474 475 void __init mark_linear_text_alias_ro(void) 476 { 477 /* 478 * Remove the write permissions from the linear alias of .text/.rodata 479 */ 480 update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), 481 (unsigned long)__init_begin - (unsigned long)_stext, 482 PAGE_KERNEL_RO); 483 } 484 485 static bool crash_mem_map __initdata; 486 487 static int __init enable_crash_mem_map(char *arg) 488 { 489 /* 490 * Proper parameter parsing is done by reserve_crashkernel(). We only 491 * need to know if the linear map has to avoid block mappings so that 492 * the crashkernel reservations can be unmapped later. 493 */ 494 crash_mem_map = true; 495 496 return 0; 497 } 498 early_param("crashkernel", enable_crash_mem_map); 499 500 static void __init map_mem(pgd_t *pgdp) 501 { 502 static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); 503 phys_addr_t kernel_start = __pa_symbol(_stext); 504 phys_addr_t kernel_end = __pa_symbol(__init_begin); 505 phys_addr_t start, end; 506 int flags = NO_EXEC_MAPPINGS; 507 u64 i; 508 509 /* 510 * Setting hierarchical PXNTable attributes on table entries covering 511 * the linear region is only possible if it is guaranteed that no table 512 * entries at any level are being shared between the linear region and 513 * the vmalloc region. Check whether this is true for the PGD level, in 514 * which case it is guaranteed to be true for all other levels as well. 515 */ 516 BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end)); 517 518 if (rodata_full || crash_mem_map || debug_pagealloc_enabled()) 519 flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; 520 521 /* 522 * Take care not to create a writable alias for the 523 * read-only text and rodata sections of the kernel image. 524 * So temporarily mark them as NOMAP to skip mappings in 525 * the following for-loop 526 */ 527 memblock_mark_nomap(kernel_start, kernel_end - kernel_start); 528 529 /* map all the memory banks */ 530 for_each_mem_range(i, &start, &end) { 531 if (start >= end) 532 break; 533 /* 534 * The linear map must allow allocation tags reading/writing 535 * if MTE is present. Otherwise, it has the same attributes as 536 * PAGE_KERNEL. 537 */ 538 __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), 539 flags); 540 } 541 542 /* 543 * Map the linear alias of the [_stext, __init_begin) interval 544 * as non-executable now, and remove the write permission in 545 * mark_linear_text_alias_ro() below (which will be called after 546 * alternative patching has completed). This makes the contents 547 * of the region accessible to subsystems such as hibernate, 548 * but protects it from inadvertent modification or execution. 549 * Note that contiguous mappings cannot be remapped in this way, 550 * so we should avoid them here. 551 */ 552 __map_memblock(pgdp, kernel_start, kernel_end, 553 PAGE_KERNEL, NO_CONT_MAPPINGS); 554 memblock_clear_nomap(kernel_start, kernel_end - kernel_start); 555 } 556 557 void mark_rodata_ro(void) 558 { 559 unsigned long section_size; 560 561 /* 562 * mark .rodata as read only. Use __init_begin rather than __end_rodata 563 * to cover NOTES and EXCEPTION_TABLE. 564 */ 565 section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; 566 update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, 567 section_size, PAGE_KERNEL_RO); 568 569 debug_checkwx(); 570 } 571 572 static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, 573 pgprot_t prot, struct vm_struct *vma, 574 int flags, unsigned long vm_flags) 575 { 576 phys_addr_t pa_start = __pa_symbol(va_start); 577 unsigned long size = va_end - va_start; 578 579 BUG_ON(!PAGE_ALIGNED(pa_start)); 580 BUG_ON(!PAGE_ALIGNED(size)); 581 582 __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, 583 early_pgtable_alloc, flags); 584 585 if (!(vm_flags & VM_NO_GUARD)) 586 size += PAGE_SIZE; 587 588 vma->addr = va_start; 589 vma->phys_addr = pa_start; 590 vma->size = size; 591 vma->flags = VM_MAP | vm_flags; 592 vma->caller = __builtin_return_address(0); 593 594 vm_area_add_early(vma); 595 } 596 597 static int __init parse_rodata(char *arg) 598 { 599 int ret = strtobool(arg, &rodata_enabled); 600 if (!ret) { 601 rodata_full = false; 602 return 0; 603 } 604 605 /* permit 'full' in addition to boolean options */ 606 if (strcmp(arg, "full")) 607 return -EINVAL; 608 609 rodata_enabled = true; 610 rodata_full = true; 611 return 0; 612 } 613 early_param("rodata", parse_rodata); 614 615 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 616 static int __init map_entry_trampoline(void) 617 { 618 pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; 619 phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); 620 621 /* The trampoline is always mapped and can therefore be global */ 622 pgprot_val(prot) &= ~PTE_NG; 623 624 /* Map only the text into the trampoline page table */ 625 memset(tramp_pg_dir, 0, PGD_SIZE); 626 __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE, 627 prot, __pgd_pgtable_alloc, 0); 628 629 /* Map both the text and data into the kernel page table */ 630 __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot); 631 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { 632 extern char __entry_tramp_data_start[]; 633 634 __set_fixmap(FIX_ENTRY_TRAMP_DATA, 635 __pa_symbol(__entry_tramp_data_start), 636 PAGE_KERNEL_RO); 637 } 638 639 return 0; 640 } 641 core_initcall(map_entry_trampoline); 642 #endif 643 644 /* 645 * Open coded check for BTI, only for use to determine configuration 646 * for early mappings for before the cpufeature code has run. 647 */ 648 static bool arm64_early_this_cpu_has_bti(void) 649 { 650 u64 pfr1; 651 652 if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) 653 return false; 654 655 pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1); 656 return cpuid_feature_extract_unsigned_field(pfr1, 657 ID_AA64PFR1_BT_SHIFT); 658 } 659 660 /* 661 * Create fine-grained mappings for the kernel. 662 */ 663 static void __init map_kernel(pgd_t *pgdp) 664 { 665 static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, 666 vmlinux_initdata, vmlinux_data; 667 668 /* 669 * External debuggers may need to write directly to the text 670 * mapping to install SW breakpoints. Allow this (only) when 671 * explicitly requested with rodata=off. 672 */ 673 pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; 674 675 /* 676 * If we have a CPU that supports BTI and a kernel built for 677 * BTI then mark the kernel executable text as guarded pages 678 * now so we don't have to rewrite the page tables later. 679 */ 680 if (arm64_early_this_cpu_has_bti()) 681 text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP); 682 683 /* 684 * Only rodata will be remapped with different permissions later on, 685 * all other segments are allowed to use contiguous mappings. 686 */ 687 map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0, 688 VM_NO_GUARD); 689 map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, 690 &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); 691 map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, 692 &vmlinux_inittext, 0, VM_NO_GUARD); 693 map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, 694 &vmlinux_initdata, 0, VM_NO_GUARD); 695 map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); 696 697 if (!READ_ONCE(pgd_val(*pgd_offset_pgd(pgdp, FIXADDR_START)))) { 698 /* 699 * The fixmap falls in a separate pgd to the kernel, and doesn't 700 * live in the carveout for the swapper_pg_dir. We can simply 701 * re-use the existing dir for the fixmap. 702 */ 703 set_pgd(pgd_offset_pgd(pgdp, FIXADDR_START), 704 READ_ONCE(*pgd_offset_k(FIXADDR_START))); 705 } else if (CONFIG_PGTABLE_LEVELS > 3) { 706 pgd_t *bm_pgdp; 707 p4d_t *bm_p4dp; 708 pud_t *bm_pudp; 709 /* 710 * The fixmap shares its top level pgd entry with the kernel 711 * mapping. This can really only occur when we are running 712 * with 16k/4 levels, so we can simply reuse the pud level 713 * entry instead. 714 */ 715 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 716 bm_pgdp = pgd_offset_pgd(pgdp, FIXADDR_START); 717 bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_START); 718 bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_START); 719 pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd)); 720 pud_clear_fixmap(); 721 } else { 722 BUG(); 723 } 724 725 kasan_copy_shadow(pgdp); 726 } 727 728 void __init paging_init(void) 729 { 730 pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir)); 731 732 map_kernel(pgdp); 733 map_mem(pgdp); 734 735 pgd_clear_fixmap(); 736 737 cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); 738 init_mm.pgd = swapper_pg_dir; 739 740 memblock_free(__pa_symbol(init_pg_dir), 741 __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); 742 743 memblock_allow_resize(); 744 } 745 746 /* 747 * Check whether a kernel address is valid (derived from arch/x86/). 748 */ 749 int kern_addr_valid(unsigned long addr) 750 { 751 pgd_t *pgdp; 752 p4d_t *p4dp; 753 pud_t *pudp, pud; 754 pmd_t *pmdp, pmd; 755 pte_t *ptep, pte; 756 757 addr = arch_kasan_reset_tag(addr); 758 if ((((long)addr) >> VA_BITS) != -1UL) 759 return 0; 760 761 pgdp = pgd_offset_k(addr); 762 if (pgd_none(READ_ONCE(*pgdp))) 763 return 0; 764 765 p4dp = p4d_offset(pgdp, addr); 766 if (p4d_none(READ_ONCE(*p4dp))) 767 return 0; 768 769 pudp = pud_offset(p4dp, addr); 770 pud = READ_ONCE(*pudp); 771 if (pud_none(pud)) 772 return 0; 773 774 if (pud_sect(pud)) 775 return pfn_valid(pud_pfn(pud)); 776 777 pmdp = pmd_offset(pudp, addr); 778 pmd = READ_ONCE(*pmdp); 779 if (pmd_none(pmd)) 780 return 0; 781 782 if (pmd_sect(pmd)) 783 return pfn_valid(pmd_pfn(pmd)); 784 785 ptep = pte_offset_kernel(pmdp, addr); 786 pte = READ_ONCE(*ptep); 787 if (pte_none(pte)) 788 return 0; 789 790 return pfn_valid(pte_pfn(pte)); 791 } 792 793 #ifdef CONFIG_MEMORY_HOTPLUG 794 static void free_hotplug_page_range(struct page *page, size_t size, 795 struct vmem_altmap *altmap) 796 { 797 if (altmap) { 798 vmem_altmap_free(altmap, size >> PAGE_SHIFT); 799 } else { 800 WARN_ON(PageReserved(page)); 801 free_pages((unsigned long)page_address(page), get_order(size)); 802 } 803 } 804 805 static void free_hotplug_pgtable_page(struct page *page) 806 { 807 free_hotplug_page_range(page, PAGE_SIZE, NULL); 808 } 809 810 static bool pgtable_range_aligned(unsigned long start, unsigned long end, 811 unsigned long floor, unsigned long ceiling, 812 unsigned long mask) 813 { 814 start &= mask; 815 if (start < floor) 816 return false; 817 818 if (ceiling) { 819 ceiling &= mask; 820 if (!ceiling) 821 return false; 822 } 823 824 if (end - 1 > ceiling - 1) 825 return false; 826 return true; 827 } 828 829 static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr, 830 unsigned long end, bool free_mapped, 831 struct vmem_altmap *altmap) 832 { 833 pte_t *ptep, pte; 834 835 do { 836 ptep = pte_offset_kernel(pmdp, addr); 837 pte = READ_ONCE(*ptep); 838 if (pte_none(pte)) 839 continue; 840 841 WARN_ON(!pte_present(pte)); 842 pte_clear(&init_mm, addr, ptep); 843 flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 844 if (free_mapped) 845 free_hotplug_page_range(pte_page(pte), 846 PAGE_SIZE, altmap); 847 } while (addr += PAGE_SIZE, addr < end); 848 } 849 850 static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, 851 unsigned long end, bool free_mapped, 852 struct vmem_altmap *altmap) 853 { 854 unsigned long next; 855 pmd_t *pmdp, pmd; 856 857 do { 858 next = pmd_addr_end(addr, end); 859 pmdp = pmd_offset(pudp, addr); 860 pmd = READ_ONCE(*pmdp); 861 if (pmd_none(pmd)) 862 continue; 863 864 WARN_ON(!pmd_present(pmd)); 865 if (pmd_sect(pmd)) { 866 pmd_clear(pmdp); 867 868 /* 869 * One TLBI should be sufficient here as the PMD_SIZE 870 * range is mapped with a single block entry. 871 */ 872 flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 873 if (free_mapped) 874 free_hotplug_page_range(pmd_page(pmd), 875 PMD_SIZE, altmap); 876 continue; 877 } 878 WARN_ON(!pmd_table(pmd)); 879 unmap_hotplug_pte_range(pmdp, addr, next, free_mapped, altmap); 880 } while (addr = next, addr < end); 881 } 882 883 static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, 884 unsigned long end, bool free_mapped, 885 struct vmem_altmap *altmap) 886 { 887 unsigned long next; 888 pud_t *pudp, pud; 889 890 do { 891 next = pud_addr_end(addr, end); 892 pudp = pud_offset(p4dp, addr); 893 pud = READ_ONCE(*pudp); 894 if (pud_none(pud)) 895 continue; 896 897 WARN_ON(!pud_present(pud)); 898 if (pud_sect(pud)) { 899 pud_clear(pudp); 900 901 /* 902 * One TLBI should be sufficient here as the PUD_SIZE 903 * range is mapped with a single block entry. 904 */ 905 flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 906 if (free_mapped) 907 free_hotplug_page_range(pud_page(pud), 908 PUD_SIZE, altmap); 909 continue; 910 } 911 WARN_ON(!pud_table(pud)); 912 unmap_hotplug_pmd_range(pudp, addr, next, free_mapped, altmap); 913 } while (addr = next, addr < end); 914 } 915 916 static void unmap_hotplug_p4d_range(pgd_t *pgdp, unsigned long addr, 917 unsigned long end, bool free_mapped, 918 struct vmem_altmap *altmap) 919 { 920 unsigned long next; 921 p4d_t *p4dp, p4d; 922 923 do { 924 next = p4d_addr_end(addr, end); 925 p4dp = p4d_offset(pgdp, addr); 926 p4d = READ_ONCE(*p4dp); 927 if (p4d_none(p4d)) 928 continue; 929 930 WARN_ON(!p4d_present(p4d)); 931 unmap_hotplug_pud_range(p4dp, addr, next, free_mapped, altmap); 932 } while (addr = next, addr < end); 933 } 934 935 static void unmap_hotplug_range(unsigned long addr, unsigned long end, 936 bool free_mapped, struct vmem_altmap *altmap) 937 { 938 unsigned long next; 939 pgd_t *pgdp, pgd; 940 941 /* 942 * altmap can only be used as vmemmap mapping backing memory. 943 * In case the backing memory itself is not being freed, then 944 * altmap is irrelevant. Warn about this inconsistency when 945 * encountered. 946 */ 947 WARN_ON(!free_mapped && altmap); 948 949 do { 950 next = pgd_addr_end(addr, end); 951 pgdp = pgd_offset_k(addr); 952 pgd = READ_ONCE(*pgdp); 953 if (pgd_none(pgd)) 954 continue; 955 956 WARN_ON(!pgd_present(pgd)); 957 unmap_hotplug_p4d_range(pgdp, addr, next, free_mapped, altmap); 958 } while (addr = next, addr < end); 959 } 960 961 static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr, 962 unsigned long end, unsigned long floor, 963 unsigned long ceiling) 964 { 965 pte_t *ptep, pte; 966 unsigned long i, start = addr; 967 968 do { 969 ptep = pte_offset_kernel(pmdp, addr); 970 pte = READ_ONCE(*ptep); 971 972 /* 973 * This is just a sanity check here which verifies that 974 * pte clearing has been done by earlier unmap loops. 975 */ 976 WARN_ON(!pte_none(pte)); 977 } while (addr += PAGE_SIZE, addr < end); 978 979 if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK)) 980 return; 981 982 /* 983 * Check whether we can free the pte page if the rest of the 984 * entries are empty. Overlap with other regions have been 985 * handled by the floor/ceiling check. 986 */ 987 ptep = pte_offset_kernel(pmdp, 0UL); 988 for (i = 0; i < PTRS_PER_PTE; i++) { 989 if (!pte_none(READ_ONCE(ptep[i]))) 990 return; 991 } 992 993 pmd_clear(pmdp); 994 __flush_tlb_kernel_pgtable(start); 995 free_hotplug_pgtable_page(virt_to_page(ptep)); 996 } 997 998 static void free_empty_pmd_table(pud_t *pudp, unsigned long addr, 999 unsigned long end, unsigned long floor, 1000 unsigned long ceiling) 1001 { 1002 pmd_t *pmdp, pmd; 1003 unsigned long i, next, start = addr; 1004 1005 do { 1006 next = pmd_addr_end(addr, end); 1007 pmdp = pmd_offset(pudp, addr); 1008 pmd = READ_ONCE(*pmdp); 1009 if (pmd_none(pmd)) 1010 continue; 1011 1012 WARN_ON(!pmd_present(pmd) || !pmd_table(pmd) || pmd_sect(pmd)); 1013 free_empty_pte_table(pmdp, addr, next, floor, ceiling); 1014 } while (addr = next, addr < end); 1015 1016 if (CONFIG_PGTABLE_LEVELS <= 2) 1017 return; 1018 1019 if (!pgtable_range_aligned(start, end, floor, ceiling, PUD_MASK)) 1020 return; 1021 1022 /* 1023 * Check whether we can free the pmd page if the rest of the 1024 * entries are empty. Overlap with other regions have been 1025 * handled by the floor/ceiling check. 1026 */ 1027 pmdp = pmd_offset(pudp, 0UL); 1028 for (i = 0; i < PTRS_PER_PMD; i++) { 1029 if (!pmd_none(READ_ONCE(pmdp[i]))) 1030 return; 1031 } 1032 1033 pud_clear(pudp); 1034 __flush_tlb_kernel_pgtable(start); 1035 free_hotplug_pgtable_page(virt_to_page(pmdp)); 1036 } 1037 1038 static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr, 1039 unsigned long end, unsigned long floor, 1040 unsigned long ceiling) 1041 { 1042 pud_t *pudp, pud; 1043 unsigned long i, next, start = addr; 1044 1045 do { 1046 next = pud_addr_end(addr, end); 1047 pudp = pud_offset(p4dp, addr); 1048 pud = READ_ONCE(*pudp); 1049 if (pud_none(pud)) 1050 continue; 1051 1052 WARN_ON(!pud_present(pud) || !pud_table(pud) || pud_sect(pud)); 1053 free_empty_pmd_table(pudp, addr, next, floor, ceiling); 1054 } while (addr = next, addr < end); 1055 1056 if (CONFIG_PGTABLE_LEVELS <= 3) 1057 return; 1058 1059 if (!pgtable_range_aligned(start, end, floor, ceiling, PGDIR_MASK)) 1060 return; 1061 1062 /* 1063 * Check whether we can free the pud page if the rest of the 1064 * entries are empty. Overlap with other regions have been 1065 * handled by the floor/ceiling check. 1066 */ 1067 pudp = pud_offset(p4dp, 0UL); 1068 for (i = 0; i < PTRS_PER_PUD; i++) { 1069 if (!pud_none(READ_ONCE(pudp[i]))) 1070 return; 1071 } 1072 1073 p4d_clear(p4dp); 1074 __flush_tlb_kernel_pgtable(start); 1075 free_hotplug_pgtable_page(virt_to_page(pudp)); 1076 } 1077 1078 static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr, 1079 unsigned long end, unsigned long floor, 1080 unsigned long ceiling) 1081 { 1082 unsigned long next; 1083 p4d_t *p4dp, p4d; 1084 1085 do { 1086 next = p4d_addr_end(addr, end); 1087 p4dp = p4d_offset(pgdp, addr); 1088 p4d = READ_ONCE(*p4dp); 1089 if (p4d_none(p4d)) 1090 continue; 1091 1092 WARN_ON(!p4d_present(p4d)); 1093 free_empty_pud_table(p4dp, addr, next, floor, ceiling); 1094 } while (addr = next, addr < end); 1095 } 1096 1097 static void free_empty_tables(unsigned long addr, unsigned long end, 1098 unsigned long floor, unsigned long ceiling) 1099 { 1100 unsigned long next; 1101 pgd_t *pgdp, pgd; 1102 1103 do { 1104 next = pgd_addr_end(addr, end); 1105 pgdp = pgd_offset_k(addr); 1106 pgd = READ_ONCE(*pgdp); 1107 if (pgd_none(pgd)) 1108 continue; 1109 1110 WARN_ON(!pgd_present(pgd)); 1111 free_empty_p4d_table(pgdp, addr, next, floor, ceiling); 1112 } while (addr = next, addr < end); 1113 } 1114 #endif 1115 1116 #ifdef CONFIG_SPARSEMEM_VMEMMAP 1117 #if !ARM64_SWAPPER_USES_SECTION_MAPS 1118 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1119 struct vmem_altmap *altmap) 1120 { 1121 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); 1122 return vmemmap_populate_basepages(start, end, node, altmap); 1123 } 1124 #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 1125 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1126 struct vmem_altmap *altmap) 1127 { 1128 unsigned long addr = start; 1129 unsigned long next; 1130 pgd_t *pgdp; 1131 p4d_t *p4dp; 1132 pud_t *pudp; 1133 pmd_t *pmdp; 1134 1135 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); 1136 do { 1137 next = pmd_addr_end(addr, end); 1138 1139 pgdp = vmemmap_pgd_populate(addr, node); 1140 if (!pgdp) 1141 return -ENOMEM; 1142 1143 p4dp = vmemmap_p4d_populate(pgdp, addr, node); 1144 if (!p4dp) 1145 return -ENOMEM; 1146 1147 pudp = vmemmap_pud_populate(p4dp, addr, node); 1148 if (!pudp) 1149 return -ENOMEM; 1150 1151 pmdp = pmd_offset(pudp, addr); 1152 if (pmd_none(READ_ONCE(*pmdp))) { 1153 void *p = NULL; 1154 1155 p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap); 1156 if (!p) { 1157 if (vmemmap_populate_basepages(addr, next, node, altmap)) 1158 return -ENOMEM; 1159 continue; 1160 } 1161 1162 pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL)); 1163 } else 1164 vmemmap_verify((pte_t *)pmdp, node, addr, next); 1165 } while (addr = next, addr != end); 1166 1167 return 0; 1168 } 1169 #endif /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 1170 void vmemmap_free(unsigned long start, unsigned long end, 1171 struct vmem_altmap *altmap) 1172 { 1173 #ifdef CONFIG_MEMORY_HOTPLUG 1174 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); 1175 1176 unmap_hotplug_range(start, end, true, altmap); 1177 free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END); 1178 #endif 1179 } 1180 #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 1181 1182 static inline pud_t *fixmap_pud(unsigned long addr) 1183 { 1184 pgd_t *pgdp = pgd_offset_k(addr); 1185 p4d_t *p4dp = p4d_offset(pgdp, addr); 1186 p4d_t p4d = READ_ONCE(*p4dp); 1187 1188 BUG_ON(p4d_none(p4d) || p4d_bad(p4d)); 1189 1190 return pud_offset_kimg(p4dp, addr); 1191 } 1192 1193 static inline pmd_t *fixmap_pmd(unsigned long addr) 1194 { 1195 pud_t *pudp = fixmap_pud(addr); 1196 pud_t pud = READ_ONCE(*pudp); 1197 1198 BUG_ON(pud_none(pud) || pud_bad(pud)); 1199 1200 return pmd_offset_kimg(pudp, addr); 1201 } 1202 1203 static inline pte_t *fixmap_pte(unsigned long addr) 1204 { 1205 return &bm_pte[pte_index(addr)]; 1206 } 1207 1208 /* 1209 * The p*d_populate functions call virt_to_phys implicitly so they can't be used 1210 * directly on kernel symbols (bm_p*d). This function is called too early to use 1211 * lm_alias so __p*d_populate functions must be used to populate with the 1212 * physical address from __pa_symbol. 1213 */ 1214 void __init early_fixmap_init(void) 1215 { 1216 pgd_t *pgdp; 1217 p4d_t *p4dp, p4d; 1218 pud_t *pudp; 1219 pmd_t *pmdp; 1220 unsigned long addr = FIXADDR_START; 1221 1222 pgdp = pgd_offset_k(addr); 1223 p4dp = p4d_offset(pgdp, addr); 1224 p4d = READ_ONCE(*p4dp); 1225 if (CONFIG_PGTABLE_LEVELS > 3 && 1226 !(p4d_none(p4d) || p4d_page_paddr(p4d) == __pa_symbol(bm_pud))) { 1227 /* 1228 * We only end up here if the kernel mapping and the fixmap 1229 * share the top level pgd entry, which should only happen on 1230 * 16k/4 levels configurations. 1231 */ 1232 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 1233 pudp = pud_offset_kimg(p4dp, addr); 1234 } else { 1235 if (p4d_none(p4d)) 1236 __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); 1237 pudp = fixmap_pud(addr); 1238 } 1239 if (pud_none(READ_ONCE(*pudp))) 1240 __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); 1241 pmdp = fixmap_pmd(addr); 1242 __pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE); 1243 1244 /* 1245 * The boot-ioremap range spans multiple pmds, for which 1246 * we are not prepared: 1247 */ 1248 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 1249 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 1250 1251 if ((pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 1252 || pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 1253 WARN_ON(1); 1254 pr_warn("pmdp %p != %p, %p\n", 1255 pmdp, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 1256 fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 1257 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 1258 fix_to_virt(FIX_BTMAP_BEGIN)); 1259 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 1260 fix_to_virt(FIX_BTMAP_END)); 1261 1262 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 1263 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 1264 } 1265 } 1266 1267 /* 1268 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we 1269 * ever need to use IPIs for TLB broadcasting, then we're in trouble here. 1270 */ 1271 void __set_fixmap(enum fixed_addresses idx, 1272 phys_addr_t phys, pgprot_t flags) 1273 { 1274 unsigned long addr = __fix_to_virt(idx); 1275 pte_t *ptep; 1276 1277 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 1278 1279 ptep = fixmap_pte(addr); 1280 1281 if (pgprot_val(flags)) { 1282 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); 1283 } else { 1284 pte_clear(&init_mm, addr, ptep); 1285 flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 1286 } 1287 } 1288 1289 void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) 1290 { 1291 const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 1292 int offset; 1293 void *dt_virt; 1294 1295 /* 1296 * Check whether the physical FDT address is set and meets the minimum 1297 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 1298 * at least 8 bytes so that we can always access the magic and size 1299 * fields of the FDT header after mapping the first chunk, double check 1300 * here if that is indeed the case. 1301 */ 1302 BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 1303 if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 1304 return NULL; 1305 1306 /* 1307 * Make sure that the FDT region can be mapped without the need to 1308 * allocate additional translation table pages, so that it is safe 1309 * to call create_mapping_noalloc() this early. 1310 * 1311 * On 64k pages, the FDT will be mapped using PTEs, so we need to 1312 * be in the same PMD as the rest of the fixmap. 1313 * On 4k pages, we'll use section mappings for the FDT so we only 1314 * have to be in the same PUD. 1315 */ 1316 BUILD_BUG_ON(dt_virt_base % SZ_2M); 1317 1318 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 1319 __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 1320 1321 offset = dt_phys % SWAPPER_BLOCK_SIZE; 1322 dt_virt = (void *)dt_virt_base + offset; 1323 1324 /* map the first chunk so we can read the size from the header */ 1325 create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), 1326 dt_virt_base, SWAPPER_BLOCK_SIZE, prot); 1327 1328 if (fdt_magic(dt_virt) != FDT_MAGIC) 1329 return NULL; 1330 1331 *size = fdt_totalsize(dt_virt); 1332 if (*size > MAX_FDT_SIZE) 1333 return NULL; 1334 1335 if (offset + *size > SWAPPER_BLOCK_SIZE) 1336 create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 1337 round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); 1338 1339 return dt_virt; 1340 } 1341 1342 int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot) 1343 { 1344 pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); 1345 1346 /* Only allow permission changes for now */ 1347 if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)), 1348 pud_val(new_pud))) 1349 return 0; 1350 1351 VM_BUG_ON(phys & ~PUD_MASK); 1352 set_pud(pudp, new_pud); 1353 return 1; 1354 } 1355 1356 int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) 1357 { 1358 pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); 1359 1360 /* Only allow permission changes for now */ 1361 if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), 1362 pmd_val(new_pmd))) 1363 return 0; 1364 1365 VM_BUG_ON(phys & ~PMD_MASK); 1366 set_pmd(pmdp, new_pmd); 1367 return 1; 1368 } 1369 1370 int pud_clear_huge(pud_t *pudp) 1371 { 1372 if (!pud_sect(READ_ONCE(*pudp))) 1373 return 0; 1374 pud_clear(pudp); 1375 return 1; 1376 } 1377 1378 int pmd_clear_huge(pmd_t *pmdp) 1379 { 1380 if (!pmd_sect(READ_ONCE(*pmdp))) 1381 return 0; 1382 pmd_clear(pmdp); 1383 return 1; 1384 } 1385 1386 int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) 1387 { 1388 pte_t *table; 1389 pmd_t pmd; 1390 1391 pmd = READ_ONCE(*pmdp); 1392 1393 if (!pmd_table(pmd)) { 1394 VM_WARN_ON(1); 1395 return 1; 1396 } 1397 1398 table = pte_offset_kernel(pmdp, addr); 1399 pmd_clear(pmdp); 1400 __flush_tlb_kernel_pgtable(addr); 1401 pte_free_kernel(NULL, table); 1402 return 1; 1403 } 1404 1405 int pud_free_pmd_page(pud_t *pudp, unsigned long addr) 1406 { 1407 pmd_t *table; 1408 pmd_t *pmdp; 1409 pud_t pud; 1410 unsigned long next, end; 1411 1412 pud = READ_ONCE(*pudp); 1413 1414 if (!pud_table(pud)) { 1415 VM_WARN_ON(1); 1416 return 1; 1417 } 1418 1419 table = pmd_offset(pudp, addr); 1420 pmdp = table; 1421 next = addr; 1422 end = addr + PUD_SIZE; 1423 do { 1424 pmd_free_pte_page(pmdp, next); 1425 } while (pmdp++, next += PMD_SIZE, next != end); 1426 1427 pud_clear(pudp); 1428 __flush_tlb_kernel_pgtable(addr); 1429 pmd_free(NULL, table); 1430 return 1; 1431 } 1432 1433 #ifdef CONFIG_MEMORY_HOTPLUG 1434 static void __remove_pgd_mapping(pgd_t *pgdir, unsigned long start, u64 size) 1435 { 1436 unsigned long end = start + size; 1437 1438 WARN_ON(pgdir != init_mm.pgd); 1439 WARN_ON((start < PAGE_OFFSET) || (end > PAGE_END)); 1440 1441 unmap_hotplug_range(start, end, false, NULL); 1442 free_empty_tables(start, end, PAGE_OFFSET, PAGE_END); 1443 } 1444 1445 struct range arch_get_mappable_range(void) 1446 { 1447 struct range mhp_range; 1448 u64 start_linear_pa = __pa(_PAGE_OFFSET(vabits_actual)); 1449 u64 end_linear_pa = __pa(PAGE_END - 1); 1450 1451 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { 1452 /* 1453 * Check for a wrap, it is possible because of randomized linear 1454 * mapping the start physical address is actually bigger than 1455 * the end physical address. In this case set start to zero 1456 * because [0, end_linear_pa] range must still be able to cover 1457 * all addressable physical addresses. 1458 */ 1459 if (start_linear_pa > end_linear_pa) 1460 start_linear_pa = 0; 1461 } 1462 1463 WARN_ON(start_linear_pa > end_linear_pa); 1464 1465 /* 1466 * Linear mapping region is the range [PAGE_OFFSET..(PAGE_END - 1)] 1467 * accommodating both its ends but excluding PAGE_END. Max physical 1468 * range which can be mapped inside this linear mapping range, must 1469 * also be derived from its end points. 1470 */ 1471 mhp_range.start = start_linear_pa; 1472 mhp_range.end = end_linear_pa; 1473 1474 return mhp_range; 1475 } 1476 1477 int arch_add_memory(int nid, u64 start, u64 size, 1478 struct mhp_params *params) 1479 { 1480 int ret, flags = NO_EXEC_MAPPINGS; 1481 1482 VM_BUG_ON(!mhp_range_allowed(start, size, true)); 1483 1484 /* 1485 * KFENCE requires linear map to be mapped at page granularity, so that 1486 * it is possible to protect/unprotect single pages in the KFENCE pool. 1487 */ 1488 if (rodata_full || debug_pagealloc_enabled() || 1489 IS_ENABLED(CONFIG_KFENCE)) 1490 flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; 1491 1492 __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), 1493 size, params->pgprot, __pgd_pgtable_alloc, 1494 flags); 1495 1496 memblock_clear_nomap(start, size); 1497 1498 ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT, 1499 params); 1500 if (ret) 1501 __remove_pgd_mapping(swapper_pg_dir, 1502 __phys_to_virt(start), size); 1503 return ret; 1504 } 1505 1506 void arch_remove_memory(int nid, u64 start, u64 size, 1507 struct vmem_altmap *altmap) 1508 { 1509 unsigned long start_pfn = start >> PAGE_SHIFT; 1510 unsigned long nr_pages = size >> PAGE_SHIFT; 1511 1512 __remove_pages(start_pfn, nr_pages, altmap); 1513 __remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size); 1514 } 1515 1516 /* 1517 * This memory hotplug notifier helps prevent boot memory from being 1518 * inadvertently removed as it blocks pfn range offlining process in 1519 * __offline_pages(). Hence this prevents both offlining as well as 1520 * removal process for boot memory which is initially always online. 1521 * In future if and when boot memory could be removed, this notifier 1522 * should be dropped and free_hotplug_page_range() should handle any 1523 * reserved pages allocated during boot. 1524 */ 1525 static int prevent_bootmem_remove_notifier(struct notifier_block *nb, 1526 unsigned long action, void *data) 1527 { 1528 struct mem_section *ms; 1529 struct memory_notify *arg = data; 1530 unsigned long end_pfn = arg->start_pfn + arg->nr_pages; 1531 unsigned long pfn = arg->start_pfn; 1532 1533 if ((action != MEM_GOING_OFFLINE) && (action != MEM_OFFLINE)) 1534 return NOTIFY_OK; 1535 1536 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1537 unsigned long start = PFN_PHYS(pfn); 1538 unsigned long end = start + (1UL << PA_SECTION_SHIFT); 1539 1540 ms = __pfn_to_section(pfn); 1541 if (!early_section(ms)) 1542 continue; 1543 1544 if (action == MEM_GOING_OFFLINE) { 1545 /* 1546 * Boot memory removal is not supported. Prevent 1547 * it via blocking any attempted offline request 1548 * for the boot memory and just report it. 1549 */ 1550 pr_warn("Boot memory [%lx %lx] offlining attempted\n", start, end); 1551 return NOTIFY_BAD; 1552 } else if (action == MEM_OFFLINE) { 1553 /* 1554 * This should have never happened. Boot memory 1555 * offlining should have been prevented by this 1556 * very notifier. Probably some memory removal 1557 * procedure might have changed which would then 1558 * require further debug. 1559 */ 1560 pr_err("Boot memory [%lx %lx] offlined\n", start, end); 1561 1562 /* 1563 * Core memory hotplug does not process a return 1564 * code from the notifier for MEM_OFFLINE events. 1565 * The error condition has been reported. Return 1566 * from here as if ignored. 1567 */ 1568 return NOTIFY_DONE; 1569 } 1570 } 1571 return NOTIFY_OK; 1572 } 1573 1574 static struct notifier_block prevent_bootmem_remove_nb = { 1575 .notifier_call = prevent_bootmem_remove_notifier, 1576 }; 1577 1578 /* 1579 * This ensures that boot memory sections on the platform are online 1580 * from early boot. Memory sections could not be prevented from being 1581 * offlined, unless for some reason they are not online to begin with. 1582 * This helps validate the basic assumption on which the above memory 1583 * event notifier works to prevent boot memory section offlining and 1584 * its possible removal. 1585 */ 1586 static void validate_bootmem_online(void) 1587 { 1588 phys_addr_t start, end, addr; 1589 struct mem_section *ms; 1590 u64 i; 1591 1592 /* 1593 * Scanning across all memblock might be expensive 1594 * on some big memory systems. Hence enable this 1595 * validation only with DEBUG_VM. 1596 */ 1597 if (!IS_ENABLED(CONFIG_DEBUG_VM)) 1598 return; 1599 1600 for_each_mem_range(i, &start, &end) { 1601 for (addr = start; addr < end; addr += (1UL << PA_SECTION_SHIFT)) { 1602 ms = __pfn_to_section(PHYS_PFN(addr)); 1603 1604 /* 1605 * All memory ranges in the system at this point 1606 * should have been marked as early sections. 1607 */ 1608 WARN_ON(!early_section(ms)); 1609 1610 /* 1611 * Memory notifier mechanism here to prevent boot 1612 * memory offlining depends on the fact that each 1613 * early section memory on the system is initially 1614 * online. Otherwise a given memory section which 1615 * is already offline will be overlooked and can 1616 * be removed completely. Call out such sections. 1617 */ 1618 if (!online_section(ms)) 1619 pr_err("Boot memory [%llx %llx] is offline, can be removed\n", 1620 addr, addr + (1UL << PA_SECTION_SHIFT)); 1621 } 1622 } 1623 } 1624 1625 static int __init prevent_bootmem_remove_init(void) 1626 { 1627 int ret = 0; 1628 1629 if (!IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 1630 return ret; 1631 1632 validate_bootmem_online(); 1633 ret = register_memory_notifier(&prevent_bootmem_remove_nb); 1634 if (ret) 1635 pr_err("%s: Notifier registration failed %d\n", __func__, ret); 1636 1637 return ret; 1638 } 1639 early_initcall(prevent_bootmem_remove_init); 1640 #endif 1641