1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/x86_64/mm/init.c 4 * 5 * Copyright (C) 1995 Linus Torvalds 6 * Copyright (C) 2000 Pavel Machek <pavel@ucw.cz> 7 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de> 8 */ 9 10 #include <linux/signal.h> 11 #include <linux/sched.h> 12 #include <linux/kernel.h> 13 #include <linux/errno.h> 14 #include <linux/string.h> 15 #include <linux/types.h> 16 #include <linux/ptrace.h> 17 #include <linux/mman.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/smp.h> 21 #include <linux/init.h> 22 #include <linux/initrd.h> 23 #include <linux/pagemap.h> 24 #include <linux/memblock.h> 25 #include <linux/proc_fs.h> 26 #include <linux/pci.h> 27 #include <linux/pfn.h> 28 #include <linux/poison.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/memory.h> 31 #include <linux/memory_hotplug.h> 32 #include <linux/memremap.h> 33 #include <linux/nmi.h> 34 #include <linux/gfp.h> 35 #include <linux/kcore.h> 36 #include <linux/bootmem_info.h> 37 #include <linux/execmem.h> 38 39 #include <asm/processor.h> 40 #include <asm/bios_ebda.h> 41 #include <linux/uaccess.h> 42 #include <asm/pgalloc.h> 43 #include <asm/dma.h> 44 #include <asm/fixmap.h> 45 #include <asm/e820/api.h> 46 #include <asm/apic.h> 47 #include <asm/tlb.h> 48 #include <asm/mmu_context.h> 49 #include <asm/proto.h> 50 #include <asm/smp.h> 51 #include <asm/sections.h> 52 #include <asm/kdebug.h> 53 #include <asm/numa.h> 54 #include <asm/set_memory.h> 55 #include <asm/init.h> 56 #include <asm/uv/uv.h> 57 #include <asm/setup.h> 58 #include <asm/ftrace.h> 59 60 #include "mm_internal.h" 61 62 #include "ident_map.c" 63 64 #define DEFINE_POPULATE(fname, type1, type2, init) \ 65 static inline void fname##_init(struct mm_struct *mm, \ 66 type1##_t *arg1, type2##_t *arg2, bool init) \ 67 { \ 68 if (init) \ 69 fname##_safe(mm, arg1, arg2); \ 70 else \ 71 fname(mm, arg1, arg2); \ 72 } 73 74 DEFINE_POPULATE(p4d_populate, p4d, pud, init) 75 DEFINE_POPULATE(pgd_populate, pgd, p4d, init) 76 DEFINE_POPULATE(pud_populate, pud, pmd, init) 77 DEFINE_POPULATE(pmd_populate_kernel, pmd, pte, init) 78 79 #define DEFINE_ENTRY(type1, type2, init) \ 80 static inline void set_##type1##_init(type1##_t *arg1, \ 81 type2##_t arg2, bool init) \ 82 { \ 83 if (init) \ 84 set_##type1##_safe(arg1, arg2); \ 85 else \ 86 set_##type1(arg1, arg2); \ 87 } 88 89 DEFINE_ENTRY(p4d, p4d, init) 90 DEFINE_ENTRY(pud, pud, init) 91 DEFINE_ENTRY(pmd, pmd, init) 92 DEFINE_ENTRY(pte, pte, init) 93 94 static inline pgprot_t prot_sethuge(pgprot_t prot) 95 { 96 WARN_ON_ONCE(pgprot_val(prot) & _PAGE_PAT); 97 98 return __pgprot(pgprot_val(prot) | _PAGE_PSE); 99 } 100 101 /* 102 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the 103 * physical space so we can cache the place of the first one and move 104 * around without checking the pgd every time. 105 */ 106 107 /* Bits supported by the hardware: */ 108 pteval_t __supported_pte_mask __read_mostly = ~0; 109 /* Bits allowed in normal kernel mappings: */ 110 pteval_t __default_kernel_pte_mask __read_mostly = ~0; 111 EXPORT_SYMBOL_GPL(__supported_pte_mask); 112 /* Used in PAGE_KERNEL_* macros which are reasonably used out-of-tree: */ 113 EXPORT_SYMBOL(__default_kernel_pte_mask); 114 115 int force_personality32; 116 117 /* 118 * noexec32=on|off 119 * Control non executable heap for 32bit processes. 120 * 121 * on PROT_READ does not imply PROT_EXEC for 32-bit processes (default) 122 * off PROT_READ implies PROT_EXEC 123 */ 124 static int __init nonx32_setup(char *str) 125 { 126 if (!strcmp(str, "on")) 127 force_personality32 &= ~READ_IMPLIES_EXEC; 128 else if (!strcmp(str, "off")) 129 force_personality32 |= READ_IMPLIES_EXEC; 130 return 1; 131 } 132 __setup("noexec32=", nonx32_setup); 133 134 static void sync_global_pgds_l5(unsigned long start, unsigned long end) 135 { 136 unsigned long addr; 137 138 for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { 139 const pgd_t *pgd_ref = pgd_offset_k(addr); 140 struct page *page; 141 142 /* Check for overflow */ 143 if (addr < start) 144 break; 145 146 if (pgd_none(*pgd_ref)) 147 continue; 148 149 spin_lock(&pgd_lock); 150 list_for_each_entry(page, &pgd_list, lru) { 151 pgd_t *pgd; 152 spinlock_t *pgt_lock; 153 154 pgd = (pgd_t *)page_address(page) + pgd_index(addr); 155 /* the pgt_lock only for Xen */ 156 pgt_lock = &pgd_page_get_mm(page)->page_table_lock; 157 spin_lock(pgt_lock); 158 159 if (!pgd_none(*pgd_ref) && !pgd_none(*pgd)) 160 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref)); 161 162 if (pgd_none(*pgd)) 163 set_pgd(pgd, *pgd_ref); 164 165 spin_unlock(pgt_lock); 166 } 167 spin_unlock(&pgd_lock); 168 } 169 } 170 171 static void sync_global_pgds_l4(unsigned long start, unsigned long end) 172 { 173 unsigned long addr; 174 175 for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { 176 pgd_t *pgd_ref = pgd_offset_k(addr); 177 const p4d_t *p4d_ref; 178 struct page *page; 179 180 /* 181 * With folded p4d, pgd_none() is always false, we need to 182 * handle synchronization on p4d level. 183 */ 184 MAYBE_BUILD_BUG_ON(pgd_none(*pgd_ref)); 185 p4d_ref = p4d_offset(pgd_ref, addr); 186 187 if (p4d_none(*p4d_ref)) 188 continue; 189 190 spin_lock(&pgd_lock); 191 list_for_each_entry(page, &pgd_list, lru) { 192 pgd_t *pgd; 193 p4d_t *p4d; 194 spinlock_t *pgt_lock; 195 196 pgd = (pgd_t *)page_address(page) + pgd_index(addr); 197 p4d = p4d_offset(pgd, addr); 198 /* the pgt_lock only for Xen */ 199 pgt_lock = &pgd_page_get_mm(page)->page_table_lock; 200 spin_lock(pgt_lock); 201 202 if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) 203 BUG_ON(p4d_pgtable(*p4d) 204 != p4d_pgtable(*p4d_ref)); 205 206 if (p4d_none(*p4d)) 207 set_p4d(p4d, *p4d_ref); 208 209 spin_unlock(pgt_lock); 210 } 211 spin_unlock(&pgd_lock); 212 } 213 } 214 215 /* 216 * When memory was added make sure all the processes MM have 217 * suitable PGD entries in the local PGD level page. 218 */ 219 static void sync_global_pgds(unsigned long start, unsigned long end) 220 { 221 if (pgtable_l5_enabled()) 222 sync_global_pgds_l5(start, end); 223 else 224 sync_global_pgds_l4(start, end); 225 } 226 227 /* 228 * NOTE: This function is marked __ref because it calls __init function 229 * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0. 230 */ 231 static __ref void *spp_getpage(void) 232 { 233 void *ptr; 234 235 if (after_bootmem) 236 ptr = (void *) get_zeroed_page(GFP_ATOMIC); 237 else 238 ptr = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 239 240 if (!ptr || ((unsigned long)ptr & ~PAGE_MASK)) { 241 panic("set_pte_phys: cannot allocate page data %s\n", 242 after_bootmem ? "after bootmem" : ""); 243 } 244 245 pr_debug("spp_getpage %p\n", ptr); 246 247 return ptr; 248 } 249 250 static p4d_t *fill_p4d(pgd_t *pgd, unsigned long vaddr) 251 { 252 if (pgd_none(*pgd)) { 253 p4d_t *p4d = (p4d_t *)spp_getpage(); 254 pgd_populate(&init_mm, pgd, p4d); 255 if (p4d != p4d_offset(pgd, 0)) 256 printk(KERN_ERR "PAGETABLE BUG #00! %p <-> %p\n", 257 p4d, p4d_offset(pgd, 0)); 258 } 259 return p4d_offset(pgd, vaddr); 260 } 261 262 static pud_t *fill_pud(p4d_t *p4d, unsigned long vaddr) 263 { 264 if (p4d_none(*p4d)) { 265 pud_t *pud = (pud_t *)spp_getpage(); 266 p4d_populate(&init_mm, p4d, pud); 267 if (pud != pud_offset(p4d, 0)) 268 printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n", 269 pud, pud_offset(p4d, 0)); 270 } 271 return pud_offset(p4d, vaddr); 272 } 273 274 static pmd_t *fill_pmd(pud_t *pud, unsigned long vaddr) 275 { 276 if (pud_none(*pud)) { 277 pmd_t *pmd = (pmd_t *) spp_getpage(); 278 pud_populate(&init_mm, pud, pmd); 279 if (pmd != pmd_offset(pud, 0)) 280 printk(KERN_ERR "PAGETABLE BUG #02! %p <-> %p\n", 281 pmd, pmd_offset(pud, 0)); 282 } 283 return pmd_offset(pud, vaddr); 284 } 285 286 static pte_t *fill_pte(pmd_t *pmd, unsigned long vaddr) 287 { 288 if (pmd_none(*pmd)) { 289 pte_t *pte = (pte_t *) spp_getpage(); 290 pmd_populate_kernel(&init_mm, pmd, pte); 291 if (pte != pte_offset_kernel(pmd, 0)) 292 printk(KERN_ERR "PAGETABLE BUG #03!\n"); 293 } 294 return pte_offset_kernel(pmd, vaddr); 295 } 296 297 static void __set_pte_vaddr(pud_t *pud, unsigned long vaddr, pte_t new_pte) 298 { 299 pmd_t *pmd = fill_pmd(pud, vaddr); 300 pte_t *pte = fill_pte(pmd, vaddr); 301 302 set_pte(pte, new_pte); 303 304 /* 305 * It's enough to flush this one mapping. 306 * (PGE mappings get flushed as well) 307 */ 308 flush_tlb_one_kernel(vaddr); 309 } 310 311 void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte) 312 { 313 p4d_t *p4d = p4d_page + p4d_index(vaddr); 314 pud_t *pud = fill_pud(p4d, vaddr); 315 316 __set_pte_vaddr(pud, vaddr, new_pte); 317 } 318 319 void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte) 320 { 321 pud_t *pud = pud_page + pud_index(vaddr); 322 323 __set_pte_vaddr(pud, vaddr, new_pte); 324 } 325 326 void set_pte_vaddr(unsigned long vaddr, pte_t pteval) 327 { 328 pgd_t *pgd; 329 p4d_t *p4d_page; 330 331 pr_debug("set_pte_vaddr %lx to %lx\n", vaddr, native_pte_val(pteval)); 332 333 pgd = pgd_offset_k(vaddr); 334 if (pgd_none(*pgd)) { 335 printk(KERN_ERR 336 "PGD FIXMAP MISSING, it should be setup in head.S!\n"); 337 return; 338 } 339 340 p4d_page = p4d_offset(pgd, 0); 341 set_pte_vaddr_p4d(p4d_page, vaddr, pteval); 342 } 343 344 pmd_t * __init populate_extra_pmd(unsigned long vaddr) 345 { 346 pgd_t *pgd; 347 p4d_t *p4d; 348 pud_t *pud; 349 350 pgd = pgd_offset_k(vaddr); 351 p4d = fill_p4d(pgd, vaddr); 352 pud = fill_pud(p4d, vaddr); 353 return fill_pmd(pud, vaddr); 354 } 355 356 pte_t * __init populate_extra_pte(unsigned long vaddr) 357 { 358 pmd_t *pmd; 359 360 pmd = populate_extra_pmd(vaddr); 361 return fill_pte(pmd, vaddr); 362 } 363 364 /* 365 * Create large page table mappings for a range of physical addresses. 366 */ 367 static void __init __init_extra_mapping(unsigned long phys, unsigned long size, 368 enum page_cache_mode cache) 369 { 370 pgd_t *pgd; 371 p4d_t *p4d; 372 pud_t *pud; 373 pmd_t *pmd; 374 pgprot_t prot; 375 376 pgprot_val(prot) = pgprot_val(PAGE_KERNEL_LARGE) | 377 protval_4k_2_large(cachemode2protval(cache)); 378 BUG_ON((phys & ~PMD_MASK) || (size & ~PMD_MASK)); 379 for (; size; phys += PMD_SIZE, size -= PMD_SIZE) { 380 pgd = pgd_offset_k((unsigned long)__va(phys)); 381 if (pgd_none(*pgd)) { 382 p4d = (p4d_t *) spp_getpage(); 383 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE | 384 _PAGE_USER)); 385 } 386 p4d = p4d_offset(pgd, (unsigned long)__va(phys)); 387 if (p4d_none(*p4d)) { 388 pud = (pud_t *) spp_getpage(); 389 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE | 390 _PAGE_USER)); 391 } 392 pud = pud_offset(p4d, (unsigned long)__va(phys)); 393 if (pud_none(*pud)) { 394 pmd = (pmd_t *) spp_getpage(); 395 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | 396 _PAGE_USER)); 397 } 398 pmd = pmd_offset(pud, phys); 399 BUG_ON(!pmd_none(*pmd)); 400 set_pmd(pmd, __pmd(phys | pgprot_val(prot))); 401 } 402 } 403 404 void __init init_extra_mapping_wb(unsigned long phys, unsigned long size) 405 { 406 __init_extra_mapping(phys, size, _PAGE_CACHE_MODE_WB); 407 } 408 409 void __init init_extra_mapping_uc(unsigned long phys, unsigned long size) 410 { 411 __init_extra_mapping(phys, size, _PAGE_CACHE_MODE_UC); 412 } 413 414 /* 415 * The head.S code sets up the kernel high mapping: 416 * 417 * from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text) 418 * 419 * phys_base holds the negative offset to the kernel, which is added 420 * to the compile time generated pmds. This results in invalid pmds up 421 * to the point where we hit the physaddr 0 mapping. 422 * 423 * We limit the mappings to the region from _text to _brk_end. _brk_end 424 * is rounded up to the 2MB boundary. This catches the invalid pmds as 425 * well, as they are located before _text: 426 */ 427 void __init cleanup_highmap(void) 428 { 429 unsigned long vaddr = __START_KERNEL_map; 430 unsigned long vaddr_end = __START_KERNEL_map + KERNEL_IMAGE_SIZE; 431 unsigned long end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1; 432 pmd_t *pmd = level2_kernel_pgt; 433 434 /* 435 * Native path, max_pfn_mapped is not set yet. 436 * Xen has valid max_pfn_mapped set in 437 * arch/x86/xen/mmu.c:xen_setup_kernel_pagetable(). 438 */ 439 if (max_pfn_mapped) 440 vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT); 441 442 for (; vaddr + PMD_SIZE - 1 < vaddr_end; pmd++, vaddr += PMD_SIZE) { 443 if (pmd_none(*pmd)) 444 continue; 445 if (vaddr < (unsigned long) _text || vaddr > end) 446 set_pmd(pmd, __pmd(0)); 447 } 448 } 449 450 /* 451 * Create PTE level page table mapping for physical addresses. 452 * It returns the last physical address mapped. 453 */ 454 static unsigned long __meminit 455 phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end, 456 pgprot_t prot, bool init) 457 { 458 unsigned long pages = 0, paddr_next; 459 unsigned long paddr_last = paddr_end; 460 pte_t *pte; 461 int i; 462 463 pte = pte_page + pte_index(paddr); 464 i = pte_index(paddr); 465 466 for (; i < PTRS_PER_PTE; i++, paddr = paddr_next, pte++) { 467 paddr_next = (paddr & PAGE_MASK) + PAGE_SIZE; 468 if (paddr >= paddr_end) { 469 if (!after_bootmem && 470 !e820__mapped_any(paddr & PAGE_MASK, paddr_next, 471 E820_TYPE_RAM) && 472 !e820__mapped_any(paddr & PAGE_MASK, paddr_next, 473 E820_TYPE_ACPI)) 474 set_pte_init(pte, __pte(0), init); 475 continue; 476 } 477 478 /* 479 * We will re-use the existing mapping. 480 * Xen for example has some special requirements, like mapping 481 * pagetable pages as RO. So assume someone who pre-setup 482 * these mappings are more intelligent. 483 */ 484 if (!pte_none(*pte)) { 485 if (!after_bootmem) 486 pages++; 487 continue; 488 } 489 490 if (0) 491 pr_info(" pte=%p addr=%lx pte=%016lx\n", pte, paddr, 492 pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL).pte); 493 pages++; 494 set_pte_init(pte, pfn_pte(paddr >> PAGE_SHIFT, prot), init); 495 paddr_last = (paddr & PAGE_MASK) + PAGE_SIZE; 496 } 497 498 update_page_count(PG_LEVEL_4K, pages); 499 500 return paddr_last; 501 } 502 503 /* 504 * Create PMD level page table mapping for physical addresses. The virtual 505 * and physical address have to be aligned at this level. 506 * It returns the last physical address mapped. 507 */ 508 static unsigned long __meminit 509 phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end, 510 unsigned long page_size_mask, pgprot_t prot, bool init) 511 { 512 unsigned long pages = 0, paddr_next; 513 unsigned long paddr_last = paddr_end; 514 515 int i = pmd_index(paddr); 516 517 for (; i < PTRS_PER_PMD; i++, paddr = paddr_next) { 518 pmd_t *pmd = pmd_page + pmd_index(paddr); 519 pte_t *pte; 520 pgprot_t new_prot = prot; 521 522 paddr_next = (paddr & PMD_MASK) + PMD_SIZE; 523 if (paddr >= paddr_end) { 524 if (!after_bootmem && 525 !e820__mapped_any(paddr & PMD_MASK, paddr_next, 526 E820_TYPE_RAM) && 527 !e820__mapped_any(paddr & PMD_MASK, paddr_next, 528 E820_TYPE_ACPI)) 529 set_pmd_init(pmd, __pmd(0), init); 530 continue; 531 } 532 533 if (!pmd_none(*pmd)) { 534 if (!pmd_leaf(*pmd)) { 535 spin_lock(&init_mm.page_table_lock); 536 pte = (pte_t *)pmd_page_vaddr(*pmd); 537 paddr_last = phys_pte_init(pte, paddr, 538 paddr_end, prot, 539 init); 540 spin_unlock(&init_mm.page_table_lock); 541 continue; 542 } 543 /* 544 * If we are ok with PG_LEVEL_2M mapping, then we will 545 * use the existing mapping, 546 * 547 * Otherwise, we will split the large page mapping but 548 * use the same existing protection bits except for 549 * large page, so that we don't violate Intel's TLB 550 * Application note (317080) which says, while changing 551 * the page sizes, new and old translations should 552 * not differ with respect to page frame and 553 * attributes. 554 */ 555 if (page_size_mask & (1 << PG_LEVEL_2M)) { 556 if (!after_bootmem) 557 pages++; 558 paddr_last = paddr_next; 559 continue; 560 } 561 new_prot = pte_pgprot(pte_clrhuge(*(pte_t *)pmd)); 562 } 563 564 if (page_size_mask & (1<<PG_LEVEL_2M)) { 565 pages++; 566 spin_lock(&init_mm.page_table_lock); 567 set_pmd_init(pmd, 568 pfn_pmd(paddr >> PAGE_SHIFT, prot_sethuge(prot)), 569 init); 570 spin_unlock(&init_mm.page_table_lock); 571 paddr_last = paddr_next; 572 continue; 573 } 574 575 pte = alloc_low_page(); 576 paddr_last = phys_pte_init(pte, paddr, paddr_end, new_prot, init); 577 578 spin_lock(&init_mm.page_table_lock); 579 pmd_populate_kernel_init(&init_mm, pmd, pte, init); 580 spin_unlock(&init_mm.page_table_lock); 581 } 582 update_page_count(PG_LEVEL_2M, pages); 583 return paddr_last; 584 } 585 586 /* 587 * Create PUD level page table mapping for physical addresses. The virtual 588 * and physical address do not have to be aligned at this level. KASLR can 589 * randomize virtual addresses up to this level. 590 * It returns the last physical address mapped. 591 */ 592 static unsigned long __meminit 593 phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end, 594 unsigned long page_size_mask, pgprot_t _prot, bool init) 595 { 596 unsigned long pages = 0, paddr_next; 597 unsigned long paddr_last = paddr_end; 598 unsigned long vaddr = (unsigned long)__va(paddr); 599 int i = pud_index(vaddr); 600 601 for (; i < PTRS_PER_PUD; i++, paddr = paddr_next) { 602 pud_t *pud; 603 pmd_t *pmd; 604 pgprot_t prot = _prot; 605 606 vaddr = (unsigned long)__va(paddr); 607 pud = pud_page + pud_index(vaddr); 608 paddr_next = (paddr & PUD_MASK) + PUD_SIZE; 609 610 if (paddr >= paddr_end) { 611 if (!after_bootmem && 612 !e820__mapped_any(paddr & PUD_MASK, paddr_next, 613 E820_TYPE_RAM) && 614 !e820__mapped_any(paddr & PUD_MASK, paddr_next, 615 E820_TYPE_ACPI)) 616 set_pud_init(pud, __pud(0), init); 617 continue; 618 } 619 620 if (!pud_none(*pud)) { 621 if (!pud_leaf(*pud)) { 622 pmd = pmd_offset(pud, 0); 623 paddr_last = phys_pmd_init(pmd, paddr, 624 paddr_end, 625 page_size_mask, 626 prot, init); 627 continue; 628 } 629 /* 630 * If we are ok with PG_LEVEL_1G mapping, then we will 631 * use the existing mapping. 632 * 633 * Otherwise, we will split the gbpage mapping but use 634 * the same existing protection bits except for large 635 * page, so that we don't violate Intel's TLB 636 * Application note (317080) which says, while changing 637 * the page sizes, new and old translations should 638 * not differ with respect to page frame and 639 * attributes. 640 */ 641 if (page_size_mask & (1 << PG_LEVEL_1G)) { 642 if (!after_bootmem) 643 pages++; 644 paddr_last = paddr_next; 645 continue; 646 } 647 prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud)); 648 } 649 650 if (page_size_mask & (1<<PG_LEVEL_1G)) { 651 pages++; 652 spin_lock(&init_mm.page_table_lock); 653 set_pud_init(pud, 654 pfn_pud(paddr >> PAGE_SHIFT, prot_sethuge(prot)), 655 init); 656 spin_unlock(&init_mm.page_table_lock); 657 paddr_last = paddr_next; 658 continue; 659 } 660 661 pmd = alloc_low_page(); 662 paddr_last = phys_pmd_init(pmd, paddr, paddr_end, 663 page_size_mask, prot, init); 664 665 spin_lock(&init_mm.page_table_lock); 666 pud_populate_init(&init_mm, pud, pmd, init); 667 spin_unlock(&init_mm.page_table_lock); 668 } 669 670 update_page_count(PG_LEVEL_1G, pages); 671 672 return paddr_last; 673 } 674 675 static unsigned long __meminit 676 phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end, 677 unsigned long page_size_mask, pgprot_t prot, bool init) 678 { 679 unsigned long vaddr, vaddr_end, vaddr_next, paddr_next, paddr_last; 680 681 paddr_last = paddr_end; 682 vaddr = (unsigned long)__va(paddr); 683 vaddr_end = (unsigned long)__va(paddr_end); 684 685 if (!pgtable_l5_enabled()) 686 return phys_pud_init((pud_t *) p4d_page, paddr, paddr_end, 687 page_size_mask, prot, init); 688 689 for (; vaddr < vaddr_end; vaddr = vaddr_next) { 690 p4d_t *p4d = p4d_page + p4d_index(vaddr); 691 pud_t *pud; 692 693 vaddr_next = (vaddr & P4D_MASK) + P4D_SIZE; 694 paddr = __pa(vaddr); 695 696 if (paddr >= paddr_end) { 697 paddr_next = __pa(vaddr_next); 698 if (!after_bootmem && 699 !e820__mapped_any(paddr & P4D_MASK, paddr_next, 700 E820_TYPE_RAM) && 701 !e820__mapped_any(paddr & P4D_MASK, paddr_next, 702 E820_TYPE_ACPI)) 703 set_p4d_init(p4d, __p4d(0), init); 704 continue; 705 } 706 707 if (!p4d_none(*p4d)) { 708 pud = pud_offset(p4d, 0); 709 paddr_last = phys_pud_init(pud, paddr, __pa(vaddr_end), 710 page_size_mask, prot, init); 711 continue; 712 } 713 714 pud = alloc_low_page(); 715 paddr_last = phys_pud_init(pud, paddr, __pa(vaddr_end), 716 page_size_mask, prot, init); 717 718 spin_lock(&init_mm.page_table_lock); 719 p4d_populate_init(&init_mm, p4d, pud, init); 720 spin_unlock(&init_mm.page_table_lock); 721 } 722 723 return paddr_last; 724 } 725 726 static unsigned long __meminit 727 __kernel_physical_mapping_init(unsigned long paddr_start, 728 unsigned long paddr_end, 729 unsigned long page_size_mask, 730 pgprot_t prot, bool init) 731 { 732 bool pgd_changed = false; 733 unsigned long vaddr, vaddr_start, vaddr_end, vaddr_next, paddr_last; 734 735 paddr_last = paddr_end; 736 vaddr = (unsigned long)__va(paddr_start); 737 vaddr_end = (unsigned long)__va(paddr_end); 738 vaddr_start = vaddr; 739 740 for (; vaddr < vaddr_end; vaddr = vaddr_next) { 741 pgd_t *pgd = pgd_offset_k(vaddr); 742 p4d_t *p4d; 743 744 vaddr_next = (vaddr & PGDIR_MASK) + PGDIR_SIZE; 745 746 if (pgd_val(*pgd)) { 747 p4d = (p4d_t *)pgd_page_vaddr(*pgd); 748 paddr_last = phys_p4d_init(p4d, __pa(vaddr), 749 __pa(vaddr_end), 750 page_size_mask, 751 prot, init); 752 continue; 753 } 754 755 p4d = alloc_low_page(); 756 paddr_last = phys_p4d_init(p4d, __pa(vaddr), __pa(vaddr_end), 757 page_size_mask, prot, init); 758 759 spin_lock(&init_mm.page_table_lock); 760 if (pgtable_l5_enabled()) 761 pgd_populate_init(&init_mm, pgd, p4d, init); 762 else 763 p4d_populate_init(&init_mm, p4d_offset(pgd, vaddr), 764 (pud_t *) p4d, init); 765 766 spin_unlock(&init_mm.page_table_lock); 767 pgd_changed = true; 768 } 769 770 if (pgd_changed) 771 sync_global_pgds(vaddr_start, vaddr_end - 1); 772 773 return paddr_last; 774 } 775 776 777 /* 778 * Create page table mapping for the physical memory for specific physical 779 * addresses. Note that it can only be used to populate non-present entries. 780 * The virtual and physical addresses have to be aligned on PMD level 781 * down. It returns the last physical address mapped. 782 */ 783 unsigned long __meminit 784 kernel_physical_mapping_init(unsigned long paddr_start, 785 unsigned long paddr_end, 786 unsigned long page_size_mask, pgprot_t prot) 787 { 788 return __kernel_physical_mapping_init(paddr_start, paddr_end, 789 page_size_mask, prot, true); 790 } 791 792 /* 793 * This function is similar to kernel_physical_mapping_init() above with the 794 * exception that it uses set_{pud,pmd}() instead of the set_{pud,pte}_safe() 795 * when updating the mapping. The caller is responsible to flush the TLBs after 796 * the function returns. 797 */ 798 unsigned long __meminit 799 kernel_physical_mapping_change(unsigned long paddr_start, 800 unsigned long paddr_end, 801 unsigned long page_size_mask) 802 { 803 return __kernel_physical_mapping_init(paddr_start, paddr_end, 804 page_size_mask, PAGE_KERNEL, 805 false); 806 } 807 808 #ifndef CONFIG_NUMA 809 void __init initmem_init(void) 810 { 811 memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0); 812 } 813 #endif 814 815 void __init paging_init(void) 816 { 817 sparse_init(); 818 819 /* 820 * clear the default setting with node 0 821 * note: don't use nodes_clear here, that is really clearing when 822 * numa support is not compiled in, and later node_set_state 823 * will not set it back. 824 */ 825 node_clear_state(0, N_MEMORY); 826 node_clear_state(0, N_NORMAL_MEMORY); 827 828 zone_sizes_init(); 829 } 830 831 #ifdef CONFIG_SPARSEMEM_VMEMMAP 832 #define PAGE_UNUSED 0xFD 833 834 /* 835 * The unused vmemmap range, which was not yet memset(PAGE_UNUSED), ranges 836 * from unused_pmd_start to next PMD_SIZE boundary. 837 */ 838 static unsigned long unused_pmd_start __meminitdata; 839 840 static void __meminit vmemmap_flush_unused_pmd(void) 841 { 842 if (!unused_pmd_start) 843 return; 844 /* 845 * Clears (unused_pmd_start, PMD_END] 846 */ 847 memset((void *)unused_pmd_start, PAGE_UNUSED, 848 ALIGN(unused_pmd_start, PMD_SIZE) - unused_pmd_start); 849 unused_pmd_start = 0; 850 } 851 852 #ifdef CONFIG_MEMORY_HOTPLUG 853 /* Returns true if the PMD is completely unused and thus it can be freed */ 854 static bool __meminit vmemmap_pmd_is_unused(unsigned long addr, unsigned long end) 855 { 856 unsigned long start = ALIGN_DOWN(addr, PMD_SIZE); 857 858 /* 859 * Flush the unused range cache to ensure that memchr_inv() will work 860 * for the whole range. 861 */ 862 vmemmap_flush_unused_pmd(); 863 memset((void *)addr, PAGE_UNUSED, end - addr); 864 865 return !memchr_inv((void *)start, PAGE_UNUSED, PMD_SIZE); 866 } 867 #endif 868 869 static void __meminit __vmemmap_use_sub_pmd(unsigned long start) 870 { 871 /* 872 * As we expect to add in the same granularity as we remove, it's 873 * sufficient to mark only some piece used to block the memmap page from 874 * getting removed when removing some other adjacent memmap (just in 875 * case the first memmap never gets initialized e.g., because the memory 876 * block never gets onlined). 877 */ 878 memset((void *)start, 0, sizeof(struct page)); 879 } 880 881 static void __meminit vmemmap_use_sub_pmd(unsigned long start, unsigned long end) 882 { 883 /* 884 * We only optimize if the new used range directly follows the 885 * previously unused range (esp., when populating consecutive sections). 886 */ 887 if (unused_pmd_start == start) { 888 if (likely(IS_ALIGNED(end, PMD_SIZE))) 889 unused_pmd_start = 0; 890 else 891 unused_pmd_start = end; 892 return; 893 } 894 895 /* 896 * If the range does not contiguously follows previous one, make sure 897 * to mark the unused range of the previous one so it can be removed. 898 */ 899 vmemmap_flush_unused_pmd(); 900 __vmemmap_use_sub_pmd(start); 901 } 902 903 904 static void __meminit vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) 905 { 906 const unsigned long page = ALIGN_DOWN(start, PMD_SIZE); 907 908 vmemmap_flush_unused_pmd(); 909 910 /* 911 * Could be our memmap page is filled with PAGE_UNUSED already from a 912 * previous remove. Make sure to reset it. 913 */ 914 __vmemmap_use_sub_pmd(start); 915 916 /* 917 * Mark with PAGE_UNUSED the unused parts of the new memmap range 918 */ 919 if (!IS_ALIGNED(start, PMD_SIZE)) 920 memset((void *)page, PAGE_UNUSED, start - page); 921 922 /* 923 * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of 924 * consecutive sections. Remember for the last added PMD where the 925 * unused range begins. 926 */ 927 if (!IS_ALIGNED(end, PMD_SIZE)) 928 unused_pmd_start = end; 929 } 930 #endif 931 932 /* 933 * Memory hotplug specific functions 934 */ 935 #ifdef CONFIG_MEMORY_HOTPLUG 936 /* 937 * After memory hotplug the variables max_pfn, max_low_pfn and high_memory need 938 * updating. 939 */ 940 static void update_end_of_memory_vars(u64 start, u64 size) 941 { 942 unsigned long end_pfn = PFN_UP(start + size); 943 944 if (end_pfn > max_pfn) { 945 max_pfn = end_pfn; 946 max_low_pfn = end_pfn; 947 high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1; 948 } 949 } 950 951 int add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages, 952 struct mhp_params *params) 953 { 954 unsigned long end = ((start_pfn + nr_pages) << PAGE_SHIFT) - 1; 955 int ret; 956 957 if (WARN_ON_ONCE(end > DIRECT_MAP_PHYSMEM_END)) 958 return -ERANGE; 959 960 ret = __add_pages(nid, start_pfn, nr_pages, params); 961 WARN_ON_ONCE(ret); 962 963 /* 964 * Special case: add_pages() is called by memremap_pages() for adding device 965 * private pages. Do not bump up max_pfn in the device private path, 966 * because max_pfn changes affect dma_addressing_limited(). 967 * 968 * dma_addressing_limited() returning true when max_pfn is the device's 969 * addressable memory can force device drivers to use bounce buffers 970 * and impact their performance negatively: 971 */ 972 if (!params->pgmap) 973 /* update max_pfn, max_low_pfn and high_memory */ 974 update_end_of_memory_vars(start_pfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT); 975 976 return ret; 977 } 978 979 int arch_add_memory(int nid, u64 start, u64 size, 980 struct mhp_params *params) 981 { 982 unsigned long start_pfn = start >> PAGE_SHIFT; 983 unsigned long nr_pages = size >> PAGE_SHIFT; 984 985 init_memory_mapping(start, start + size, params->pgprot); 986 987 return add_pages(nid, start_pfn, nr_pages, params); 988 } 989 990 static void free_reserved_pages(struct page *page, unsigned long nr_pages) 991 { 992 while (nr_pages--) 993 free_reserved_page(page++); 994 } 995 996 static void __meminit free_pagetable(struct page *page, int order) 997 { 998 /* bootmem page has reserved flag */ 999 if (PageReserved(page)) { 1000 unsigned long nr_pages = 1 << order; 1001 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 1002 enum bootmem_type type = bootmem_type(page); 1003 1004 if (type == SECTION_INFO || type == MIX_SECTION_INFO) { 1005 while (nr_pages--) 1006 put_page_bootmem(page++); 1007 } else { 1008 free_reserved_pages(page, nr_pages); 1009 } 1010 #else 1011 free_reserved_pages(page, nr_pages); 1012 #endif 1013 } else { 1014 free_pages((unsigned long)page_address(page), order); 1015 } 1016 } 1017 1018 static void __meminit free_hugepage_table(struct page *page, 1019 struct vmem_altmap *altmap) 1020 { 1021 if (altmap) 1022 vmem_altmap_free(altmap, PMD_SIZE / PAGE_SIZE); 1023 else 1024 free_pagetable(page, get_order(PMD_SIZE)); 1025 } 1026 1027 static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) 1028 { 1029 pte_t *pte; 1030 int i; 1031 1032 for (i = 0; i < PTRS_PER_PTE; i++) { 1033 pte = pte_start + i; 1034 if (!pte_none(*pte)) 1035 return; 1036 } 1037 1038 /* free a pte table */ 1039 free_pagetable(pmd_page(*pmd), 0); 1040 spin_lock(&init_mm.page_table_lock); 1041 pmd_clear(pmd); 1042 spin_unlock(&init_mm.page_table_lock); 1043 } 1044 1045 static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) 1046 { 1047 pmd_t *pmd; 1048 int i; 1049 1050 for (i = 0; i < PTRS_PER_PMD; i++) { 1051 pmd = pmd_start + i; 1052 if (!pmd_none(*pmd)) 1053 return; 1054 } 1055 1056 /* free a pmd table */ 1057 free_pagetable(pud_page(*pud), 0); 1058 spin_lock(&init_mm.page_table_lock); 1059 pud_clear(pud); 1060 spin_unlock(&init_mm.page_table_lock); 1061 } 1062 1063 static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) 1064 { 1065 pud_t *pud; 1066 int i; 1067 1068 for (i = 0; i < PTRS_PER_PUD; i++) { 1069 pud = pud_start + i; 1070 if (!pud_none(*pud)) 1071 return; 1072 } 1073 1074 /* free a pud table */ 1075 free_pagetable(p4d_page(*p4d), 0); 1076 spin_lock(&init_mm.page_table_lock); 1077 p4d_clear(p4d); 1078 spin_unlock(&init_mm.page_table_lock); 1079 } 1080 1081 static void __meminit 1082 remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end, 1083 bool direct) 1084 { 1085 unsigned long next, pages = 0; 1086 pte_t *pte; 1087 phys_addr_t phys_addr; 1088 1089 pte = pte_start + pte_index(addr); 1090 for (; addr < end; addr = next, pte++) { 1091 next = (addr + PAGE_SIZE) & PAGE_MASK; 1092 if (next > end) 1093 next = end; 1094 1095 if (!pte_present(*pte)) 1096 continue; 1097 1098 /* 1099 * We mapped [0,1G) memory as identity mapping when 1100 * initializing, in arch/x86/kernel/head_64.S. These 1101 * pagetables cannot be removed. 1102 */ 1103 phys_addr = pte_val(*pte) + (addr & PAGE_MASK); 1104 if (phys_addr < (phys_addr_t)0x40000000) 1105 return; 1106 1107 if (!direct) 1108 free_pagetable(pte_page(*pte), 0); 1109 1110 spin_lock(&init_mm.page_table_lock); 1111 pte_clear(&init_mm, addr, pte); 1112 spin_unlock(&init_mm.page_table_lock); 1113 1114 /* For non-direct mapping, pages means nothing. */ 1115 pages++; 1116 } 1117 1118 /* Call free_pte_table() in remove_pmd_table(). */ 1119 flush_tlb_all(); 1120 if (direct) 1121 update_page_count(PG_LEVEL_4K, -pages); 1122 } 1123 1124 static void __meminit 1125 remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end, 1126 bool direct, struct vmem_altmap *altmap) 1127 { 1128 unsigned long next, pages = 0; 1129 pte_t *pte_base; 1130 pmd_t *pmd; 1131 1132 pmd = pmd_start + pmd_index(addr); 1133 for (; addr < end; addr = next, pmd++) { 1134 next = pmd_addr_end(addr, end); 1135 1136 if (!pmd_present(*pmd)) 1137 continue; 1138 1139 if (pmd_leaf(*pmd)) { 1140 if (IS_ALIGNED(addr, PMD_SIZE) && 1141 IS_ALIGNED(next, PMD_SIZE)) { 1142 if (!direct) 1143 free_hugepage_table(pmd_page(*pmd), 1144 altmap); 1145 1146 spin_lock(&init_mm.page_table_lock); 1147 pmd_clear(pmd); 1148 spin_unlock(&init_mm.page_table_lock); 1149 pages++; 1150 } 1151 #ifdef CONFIG_SPARSEMEM_VMEMMAP 1152 else if (vmemmap_pmd_is_unused(addr, next)) { 1153 free_hugepage_table(pmd_page(*pmd), 1154 altmap); 1155 spin_lock(&init_mm.page_table_lock); 1156 pmd_clear(pmd); 1157 spin_unlock(&init_mm.page_table_lock); 1158 } 1159 #endif 1160 continue; 1161 } 1162 1163 pte_base = (pte_t *)pmd_page_vaddr(*pmd); 1164 remove_pte_table(pte_base, addr, next, direct); 1165 free_pte_table(pte_base, pmd); 1166 } 1167 1168 /* Call free_pmd_table() in remove_pud_table(). */ 1169 if (direct) 1170 update_page_count(PG_LEVEL_2M, -pages); 1171 } 1172 1173 static void __meminit 1174 remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end, 1175 struct vmem_altmap *altmap, bool direct) 1176 { 1177 unsigned long next, pages = 0; 1178 pmd_t *pmd_base; 1179 pud_t *pud; 1180 1181 pud = pud_start + pud_index(addr); 1182 for (; addr < end; addr = next, pud++) { 1183 next = pud_addr_end(addr, end); 1184 1185 if (!pud_present(*pud)) 1186 continue; 1187 1188 if (pud_leaf(*pud) && 1189 IS_ALIGNED(addr, PUD_SIZE) && 1190 IS_ALIGNED(next, PUD_SIZE)) { 1191 spin_lock(&init_mm.page_table_lock); 1192 pud_clear(pud); 1193 spin_unlock(&init_mm.page_table_lock); 1194 pages++; 1195 continue; 1196 } 1197 1198 pmd_base = pmd_offset(pud, 0); 1199 remove_pmd_table(pmd_base, addr, next, direct, altmap); 1200 free_pmd_table(pmd_base, pud); 1201 } 1202 1203 if (direct) 1204 update_page_count(PG_LEVEL_1G, -pages); 1205 } 1206 1207 static void __meminit 1208 remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end, 1209 struct vmem_altmap *altmap, bool direct) 1210 { 1211 unsigned long next, pages = 0; 1212 pud_t *pud_base; 1213 p4d_t *p4d; 1214 1215 p4d = p4d_start + p4d_index(addr); 1216 for (; addr < end; addr = next, p4d++) { 1217 next = p4d_addr_end(addr, end); 1218 1219 if (!p4d_present(*p4d)) 1220 continue; 1221 1222 BUILD_BUG_ON(p4d_leaf(*p4d)); 1223 1224 pud_base = pud_offset(p4d, 0); 1225 remove_pud_table(pud_base, addr, next, altmap, direct); 1226 /* 1227 * For 4-level page tables we do not want to free PUDs, but in the 1228 * 5-level case we should free them. This code will have to change 1229 * to adapt for boot-time switching between 4 and 5 level page tables. 1230 */ 1231 if (pgtable_l5_enabled()) 1232 free_pud_table(pud_base, p4d); 1233 } 1234 1235 if (direct) 1236 update_page_count(PG_LEVEL_512G, -pages); 1237 } 1238 1239 /* start and end are both virtual address. */ 1240 static void __meminit 1241 remove_pagetable(unsigned long start, unsigned long end, bool direct, 1242 struct vmem_altmap *altmap) 1243 { 1244 unsigned long next; 1245 unsigned long addr; 1246 pgd_t *pgd; 1247 p4d_t *p4d; 1248 1249 for (addr = start; addr < end; addr = next) { 1250 next = pgd_addr_end(addr, end); 1251 1252 pgd = pgd_offset_k(addr); 1253 if (!pgd_present(*pgd)) 1254 continue; 1255 1256 p4d = p4d_offset(pgd, 0); 1257 remove_p4d_table(p4d, addr, next, altmap, direct); 1258 } 1259 1260 flush_tlb_all(); 1261 } 1262 1263 void __ref vmemmap_free(unsigned long start, unsigned long end, 1264 struct vmem_altmap *altmap) 1265 { 1266 VM_BUG_ON(!PAGE_ALIGNED(start)); 1267 VM_BUG_ON(!PAGE_ALIGNED(end)); 1268 1269 remove_pagetable(start, end, false, altmap); 1270 } 1271 1272 static void __meminit 1273 kernel_physical_mapping_remove(unsigned long start, unsigned long end) 1274 { 1275 start = (unsigned long)__va(start); 1276 end = (unsigned long)__va(end); 1277 1278 remove_pagetable(start, end, true, NULL); 1279 } 1280 1281 void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) 1282 { 1283 unsigned long start_pfn = start >> PAGE_SHIFT; 1284 unsigned long nr_pages = size >> PAGE_SHIFT; 1285 1286 __remove_pages(start_pfn, nr_pages, altmap); 1287 kernel_physical_mapping_remove(start, start + size); 1288 } 1289 #endif /* CONFIG_MEMORY_HOTPLUG */ 1290 1291 static struct kcore_list kcore_vsyscall; 1292 1293 static void __init register_page_bootmem_info(void) 1294 { 1295 #if defined(CONFIG_NUMA) || defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) 1296 int i; 1297 1298 for_each_online_node(i) 1299 register_page_bootmem_info_node(NODE_DATA(i)); 1300 #endif 1301 } 1302 1303 /* 1304 * Pre-allocates page-table pages for the vmalloc area in the kernel page-table. 1305 * Only the level which needs to be synchronized between all page-tables is 1306 * allocated because the synchronization can be expensive. 1307 */ 1308 static void __init preallocate_vmalloc_pages(void) 1309 { 1310 unsigned long addr; 1311 const char *lvl; 1312 1313 for (addr = VMALLOC_START; addr <= VMEMORY_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) { 1314 pgd_t *pgd = pgd_offset_k(addr); 1315 p4d_t *p4d; 1316 pud_t *pud; 1317 1318 lvl = "p4d"; 1319 p4d = p4d_alloc(&init_mm, pgd, addr); 1320 if (!p4d) 1321 goto failed; 1322 1323 if (pgtable_l5_enabled()) 1324 continue; 1325 1326 /* 1327 * The goal here is to allocate all possibly required 1328 * hardware page tables pointed to by the top hardware 1329 * level. 1330 * 1331 * On 4-level systems, the P4D layer is folded away and 1332 * the above code does no preallocation. Below, go down 1333 * to the pud _software_ level to ensure the second 1334 * hardware level is allocated on 4-level systems too. 1335 */ 1336 lvl = "pud"; 1337 pud = pud_alloc(&init_mm, p4d, addr); 1338 if (!pud) 1339 goto failed; 1340 } 1341 1342 return; 1343 1344 failed: 1345 1346 /* 1347 * The pages have to be there now or they will be missing in 1348 * process page-tables later. 1349 */ 1350 panic("Failed to pre-allocate %s pages for vmalloc area\n", lvl); 1351 } 1352 1353 void __init arch_mm_preinit(void) 1354 { 1355 pci_iommu_alloc(); 1356 } 1357 1358 void __init mem_init(void) 1359 { 1360 /* clear_bss() already clear the empty_zero_page */ 1361 1362 after_bootmem = 1; 1363 x86_init.hyper.init_after_bootmem(); 1364 1365 /* 1366 * Must be done after boot memory is put on freelist, because here we 1367 * might set fields in deferred struct pages that have not yet been 1368 * initialized, and memblock_free_all() initializes all the reserved 1369 * deferred pages for us. 1370 */ 1371 register_page_bootmem_info(); 1372 1373 /* Register memory areas for /proc/kcore */ 1374 if (get_gate_vma(&init_mm)) 1375 kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); 1376 1377 preallocate_vmalloc_pages(); 1378 } 1379 1380 int kernel_set_to_readonly; 1381 1382 void mark_rodata_ro(void) 1383 { 1384 unsigned long start = PFN_ALIGN(_text); 1385 unsigned long rodata_start = PFN_ALIGN(__start_rodata); 1386 unsigned long end = (unsigned long)__end_rodata_hpage_align; 1387 unsigned long text_end = PFN_ALIGN(_etext); 1388 unsigned long rodata_end = PFN_ALIGN(__end_rodata); 1389 unsigned long all_end; 1390 1391 printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", 1392 (end - start) >> 10); 1393 set_memory_ro(start, (end - start) >> PAGE_SHIFT); 1394 1395 execmem_cache_make_ro(); 1396 1397 kernel_set_to_readonly = 1; 1398 1399 /* 1400 * The rodata/data/bss/brk section (but not the kernel text!) 1401 * should also be not-executable. 1402 * 1403 * We align all_end to PMD_SIZE because the existing mapping 1404 * is a full PMD. If we would align _brk_end to PAGE_SIZE we 1405 * split the PMD and the reminder between _brk_end and the end 1406 * of the PMD will remain mapped executable. 1407 * 1408 * Any PMD which was setup after the one which covers _brk_end 1409 * has been zapped already via cleanup_highmem(). 1410 */ 1411 all_end = roundup((unsigned long)_brk_end, PMD_SIZE); 1412 set_memory_nx(text_end, (all_end - text_end) >> PAGE_SHIFT); 1413 1414 set_ftrace_ops_ro(); 1415 1416 #ifdef CONFIG_CPA_DEBUG 1417 printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, end); 1418 set_memory_rw(start, (end-start) >> PAGE_SHIFT); 1419 1420 printk(KERN_INFO "Testing CPA: again\n"); 1421 set_memory_ro(start, (end-start) >> PAGE_SHIFT); 1422 #endif 1423 1424 free_kernel_image_pages("unused kernel image (text/rodata gap)", 1425 (void *)text_end, (void *)rodata_start); 1426 free_kernel_image_pages("unused kernel image (rodata/data gap)", 1427 (void *)rodata_end, (void *)_sdata); 1428 } 1429 1430 /* 1431 * Block size is the minimum amount of memory which can be hotplugged or 1432 * hotremoved. It must be power of two and must be equal or larger than 1433 * MIN_MEMORY_BLOCK_SIZE. 1434 */ 1435 #define MAX_BLOCK_SIZE (2UL << 30) 1436 1437 /* Amount of ram needed to start using large blocks */ 1438 #define MEM_SIZE_FOR_LARGE_BLOCK (64UL << 30) 1439 1440 /* Adjustable memory block size */ 1441 static unsigned long set_memory_block_size; 1442 int __init set_memory_block_size_order(unsigned int order) 1443 { 1444 unsigned long size = 1UL << order; 1445 1446 if (size > MEM_SIZE_FOR_LARGE_BLOCK || size < MIN_MEMORY_BLOCK_SIZE) 1447 return -EINVAL; 1448 1449 set_memory_block_size = size; 1450 return 0; 1451 } 1452 1453 static unsigned long probe_memory_block_size(void) 1454 { 1455 unsigned long boot_mem_end = max_pfn << PAGE_SHIFT; 1456 unsigned long bz; 1457 1458 /* If memory block size has been set, then use it */ 1459 bz = set_memory_block_size; 1460 if (bz) 1461 goto done; 1462 1463 /* Use regular block if RAM is smaller than MEM_SIZE_FOR_LARGE_BLOCK */ 1464 if (boot_mem_end < MEM_SIZE_FOR_LARGE_BLOCK) { 1465 bz = MIN_MEMORY_BLOCK_SIZE; 1466 goto done; 1467 } 1468 1469 /* 1470 * Use max block size to minimize overhead on bare metal, where 1471 * alignment for memory hotplug isn't a concern. 1472 */ 1473 if (!boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1474 bz = MAX_BLOCK_SIZE; 1475 goto done; 1476 } 1477 1478 /* Find the largest allowed block size that aligns to memory end */ 1479 for (bz = MAX_BLOCK_SIZE; bz > MIN_MEMORY_BLOCK_SIZE; bz >>= 1) { 1480 if (IS_ALIGNED(boot_mem_end, bz)) 1481 break; 1482 } 1483 done: 1484 pr_info("x86/mm: Memory block size: %ldMB\n", bz >> 20); 1485 1486 return bz; 1487 } 1488 1489 static unsigned long memory_block_size_probed; 1490 unsigned long memory_block_size_bytes(void) 1491 { 1492 if (!memory_block_size_probed) 1493 memory_block_size_probed = probe_memory_block_size(); 1494 1495 return memory_block_size_probed; 1496 } 1497 1498 #ifdef CONFIG_SPARSEMEM_VMEMMAP 1499 /* 1500 * Initialise the sparsemem vmemmap using huge-pages at the PMD level. 1501 */ 1502 static long __meminitdata addr_start, addr_end; 1503 static void __meminitdata *p_start, *p_end; 1504 static int __meminitdata node_start; 1505 1506 void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node, 1507 unsigned long addr, unsigned long next) 1508 { 1509 pte_t entry; 1510 1511 entry = pfn_pte(__pa(p) >> PAGE_SHIFT, 1512 PAGE_KERNEL_LARGE); 1513 set_pmd(pmd, __pmd(pte_val(entry))); 1514 1515 /* check to see if we have contiguous blocks */ 1516 if (p_end != p || node_start != node) { 1517 if (p_start) 1518 pr_debug(" [%lx-%lx] PMD -> [%p-%p] on node %d\n", 1519 addr_start, addr_end-1, p_start, p_end-1, node_start); 1520 addr_start = addr; 1521 node_start = node; 1522 p_start = p; 1523 } 1524 1525 addr_end = addr + PMD_SIZE; 1526 p_end = p + PMD_SIZE; 1527 1528 if (!IS_ALIGNED(addr, PMD_SIZE) || 1529 !IS_ALIGNED(next, PMD_SIZE)) 1530 vmemmap_use_new_sub_pmd(addr, next); 1531 } 1532 1533 int __meminit vmemmap_check_pmd(pmd_t *pmd, int node, 1534 unsigned long addr, unsigned long next) 1535 { 1536 int large = pmd_leaf(*pmd); 1537 1538 if (pmd_leaf(*pmd)) { 1539 vmemmap_verify((pte_t *)pmd, node, addr, next); 1540 vmemmap_use_sub_pmd(addr, next); 1541 } 1542 1543 return large; 1544 } 1545 1546 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1547 struct vmem_altmap *altmap) 1548 { 1549 int err; 1550 1551 VM_BUG_ON(!PAGE_ALIGNED(start)); 1552 VM_BUG_ON(!PAGE_ALIGNED(end)); 1553 1554 if (end - start < PAGES_PER_SECTION * sizeof(struct page)) 1555 err = vmemmap_populate_basepages(start, end, node, NULL); 1556 else if (boot_cpu_has(X86_FEATURE_PSE)) 1557 err = vmemmap_populate_hugepages(start, end, node, altmap); 1558 else if (altmap) { 1559 pr_err_once("%s: no cpu support for altmap allocations\n", 1560 __func__); 1561 err = -ENOMEM; 1562 } else 1563 err = vmemmap_populate_basepages(start, end, node, NULL); 1564 if (!err) 1565 sync_global_pgds(start, end - 1); 1566 return err; 1567 } 1568 1569 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 1570 void register_page_bootmem_memmap(unsigned long section_nr, 1571 struct page *start_page, unsigned long nr_pages) 1572 { 1573 unsigned long addr = (unsigned long)start_page; 1574 unsigned long end = (unsigned long)(start_page + nr_pages); 1575 unsigned long next; 1576 pgd_t *pgd; 1577 p4d_t *p4d; 1578 pud_t *pud; 1579 pmd_t *pmd; 1580 unsigned int nr_pmd_pages; 1581 struct page *page; 1582 1583 for (; addr < end; addr = next) { 1584 pte_t *pte = NULL; 1585 1586 pgd = pgd_offset_k(addr); 1587 if (pgd_none(*pgd)) { 1588 next = (addr + PAGE_SIZE) & PAGE_MASK; 1589 continue; 1590 } 1591 get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO); 1592 1593 p4d = p4d_offset(pgd, addr); 1594 if (p4d_none(*p4d)) { 1595 next = (addr + PAGE_SIZE) & PAGE_MASK; 1596 continue; 1597 } 1598 get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO); 1599 1600 pud = pud_offset(p4d, addr); 1601 if (pud_none(*pud)) { 1602 next = (addr + PAGE_SIZE) & PAGE_MASK; 1603 continue; 1604 } 1605 get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO); 1606 1607 pmd = pmd_offset(pud, addr); 1608 if (pmd_none(*pmd)) { 1609 next = (addr + PAGE_SIZE) & PAGE_MASK; 1610 continue; 1611 } 1612 1613 if (!boot_cpu_has(X86_FEATURE_PSE) || !pmd_leaf(*pmd)) { 1614 next = (addr + PAGE_SIZE) & PAGE_MASK; 1615 get_page_bootmem(section_nr, pmd_page(*pmd), 1616 MIX_SECTION_INFO); 1617 1618 pte = pte_offset_kernel(pmd, addr); 1619 if (pte_none(*pte)) 1620 continue; 1621 get_page_bootmem(section_nr, pte_page(*pte), 1622 SECTION_INFO); 1623 } else { 1624 next = pmd_addr_end(addr, end); 1625 nr_pmd_pages = (next - addr) >> PAGE_SHIFT; 1626 page = pmd_page(*pmd); 1627 while (nr_pmd_pages--) 1628 get_page_bootmem(section_nr, page++, 1629 SECTION_INFO); 1630 } 1631 } 1632 } 1633 #endif 1634 1635 void __meminit vmemmap_populate_print_last(void) 1636 { 1637 if (p_start) { 1638 pr_debug(" [%lx-%lx] PMD -> [%p-%p] on node %d\n", 1639 addr_start, addr_end-1, p_start, p_end-1, node_start); 1640 p_start = NULL; 1641 p_end = NULL; 1642 node_start = 0; 1643 } 1644 } 1645 #endif 1646