1 /* 2 * linux/arch/x86_64/mm/init.c 3 * 4 * Copyright (C) 1995 Linus Torvalds 5 * Copyright (C) 2000 Pavel Machek <pavel@ucw.cz> 6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de> 7 */ 8 9 #include <linux/signal.h> 10 #include <linux/sched.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/string.h> 14 #include <linux/types.h> 15 #include <linux/ptrace.h> 16 #include <linux/mman.h> 17 #include <linux/mm.h> 18 #include <linux/swap.h> 19 #include <linux/smp.h> 20 #include <linux/init.h> 21 #include <linux/initrd.h> 22 #include <linux/pagemap.h> 23 #include <linux/memblock.h> 24 #include <linux/proc_fs.h> 25 #include <linux/pci.h> 26 #include <linux/pfn.h> 27 #include <linux/poison.h> 28 #include <linux/dma-mapping.h> 29 #include <linux/memory.h> 30 #include <linux/memory_hotplug.h> 31 #include <linux/memremap.h> 32 #include <linux/nmi.h> 33 #include <linux/gfp.h> 34 #include <linux/kcore.h> 35 36 #include <asm/processor.h> 37 #include <asm/bios_ebda.h> 38 #include <linux/uaccess.h> 39 #include <asm/pgtable.h> 40 #include <asm/pgalloc.h> 41 #include <asm/dma.h> 42 #include <asm/fixmap.h> 43 #include <asm/e820/api.h> 44 #include <asm/apic.h> 45 #include <asm/tlb.h> 46 #include <asm/mmu_context.h> 47 #include <asm/proto.h> 48 #include <asm/smp.h> 49 #include <asm/sections.h> 50 #include <asm/kdebug.h> 51 #include <asm/numa.h> 52 #include <asm/set_memory.h> 53 #include <asm/init.h> 54 #include <asm/uv/uv.h> 55 #include <asm/setup.h> 56 57 #include "mm_internal.h" 58 59 #include "ident_map.c" 60 61 /* 62 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the 63 * physical space so we can cache the place of the first one and move 64 * around without checking the pgd every time. 65 */ 66 67 /* Bits supported by the hardware: */ 68 pteval_t __supported_pte_mask __read_mostly = ~0; 69 /* Bits allowed in normal kernel mappings: */ 70 pteval_t __default_kernel_pte_mask __read_mostly = ~0; 71 EXPORT_SYMBOL_GPL(__supported_pte_mask); 72 /* Used in PAGE_KERNEL_* macros which are reasonably used out-of-tree: */ 73 EXPORT_SYMBOL(__default_kernel_pte_mask); 74 75 int force_personality32; 76 77 /* 78 * noexec32=on|off 79 * Control non executable heap for 32bit processes. 80 * To control the stack too use noexec=off 81 * 82 * on PROT_READ does not imply PROT_EXEC for 32-bit processes (default) 83 * off PROT_READ implies PROT_EXEC 84 */ 85 static int __init nonx32_setup(char *str) 86 { 87 if (!strcmp(str, "on")) 88 force_personality32 &= ~READ_IMPLIES_EXEC; 89 else if (!strcmp(str, "off")) 90 force_personality32 |= READ_IMPLIES_EXEC; 91 return 1; 92 } 93 __setup("noexec32=", nonx32_setup); 94 95 static void sync_global_pgds_l5(unsigned long start, unsigned long end) 96 { 97 unsigned long addr; 98 99 for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { 100 const pgd_t *pgd_ref = pgd_offset_k(addr); 101 struct page *page; 102 103 /* Check for overflow */ 104 if (addr < start) 105 break; 106 107 if (pgd_none(*pgd_ref)) 108 continue; 109 110 spin_lock(&pgd_lock); 111 list_for_each_entry(page, &pgd_list, lru) { 112 pgd_t *pgd; 113 spinlock_t *pgt_lock; 114 115 pgd = (pgd_t *)page_address(page) + pgd_index(addr); 116 /* the pgt_lock only for Xen */ 117 pgt_lock = &pgd_page_get_mm(page)->page_table_lock; 118 spin_lock(pgt_lock); 119 120 if (!pgd_none(*pgd_ref) && !pgd_none(*pgd)) 121 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref)); 122 123 if (pgd_none(*pgd)) 124 set_pgd(pgd, *pgd_ref); 125 126 spin_unlock(pgt_lock); 127 } 128 spin_unlock(&pgd_lock); 129 } 130 } 131 132 static void sync_global_pgds_l4(unsigned long start, unsigned long end) 133 { 134 unsigned long addr; 135 136 for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { 137 pgd_t *pgd_ref = pgd_offset_k(addr); 138 const p4d_t *p4d_ref; 139 struct page *page; 140 141 /* 142 * With folded p4d, pgd_none() is always false, we need to 143 * handle synchonization on p4d level. 144 */ 145 MAYBE_BUILD_BUG_ON(pgd_none(*pgd_ref)); 146 p4d_ref = p4d_offset(pgd_ref, addr); 147 148 if (p4d_none(*p4d_ref)) 149 continue; 150 151 spin_lock(&pgd_lock); 152 list_for_each_entry(page, &pgd_list, lru) { 153 pgd_t *pgd; 154 p4d_t *p4d; 155 spinlock_t *pgt_lock; 156 157 pgd = (pgd_t *)page_address(page) + pgd_index(addr); 158 p4d = p4d_offset(pgd, addr); 159 /* the pgt_lock only for Xen */ 160 pgt_lock = &pgd_page_get_mm(page)->page_table_lock; 161 spin_lock(pgt_lock); 162 163 if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) 164 BUG_ON(p4d_page_vaddr(*p4d) 165 != p4d_page_vaddr(*p4d_ref)); 166 167 if (p4d_none(*p4d)) 168 set_p4d(p4d, *p4d_ref); 169 170 spin_unlock(pgt_lock); 171 } 172 spin_unlock(&pgd_lock); 173 } 174 } 175 176 /* 177 * When memory was added make sure all the processes MM have 178 * suitable PGD entries in the local PGD level page. 179 */ 180 void sync_global_pgds(unsigned long start, unsigned long end) 181 { 182 if (pgtable_l5_enabled()) 183 sync_global_pgds_l5(start, end); 184 else 185 sync_global_pgds_l4(start, end); 186 } 187 188 /* 189 * NOTE: This function is marked __ref because it calls __init function 190 * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0. 191 */ 192 static __ref void *spp_getpage(void) 193 { 194 void *ptr; 195 196 if (after_bootmem) 197 ptr = (void *) get_zeroed_page(GFP_ATOMIC); 198 else 199 ptr = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 200 201 if (!ptr || ((unsigned long)ptr & ~PAGE_MASK)) { 202 panic("set_pte_phys: cannot allocate page data %s\n", 203 after_bootmem ? "after bootmem" : ""); 204 } 205 206 pr_debug("spp_getpage %p\n", ptr); 207 208 return ptr; 209 } 210 211 static p4d_t *fill_p4d(pgd_t *pgd, unsigned long vaddr) 212 { 213 if (pgd_none(*pgd)) { 214 p4d_t *p4d = (p4d_t *)spp_getpage(); 215 pgd_populate(&init_mm, pgd, p4d); 216 if (p4d != p4d_offset(pgd, 0)) 217 printk(KERN_ERR "PAGETABLE BUG #00! %p <-> %p\n", 218 p4d, p4d_offset(pgd, 0)); 219 } 220 return p4d_offset(pgd, vaddr); 221 } 222 223 static pud_t *fill_pud(p4d_t *p4d, unsigned long vaddr) 224 { 225 if (p4d_none(*p4d)) { 226 pud_t *pud = (pud_t *)spp_getpage(); 227 p4d_populate(&init_mm, p4d, pud); 228 if (pud != pud_offset(p4d, 0)) 229 printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n", 230 pud, pud_offset(p4d, 0)); 231 } 232 return pud_offset(p4d, vaddr); 233 } 234 235 static pmd_t *fill_pmd(pud_t *pud, unsigned long vaddr) 236 { 237 if (pud_none(*pud)) { 238 pmd_t *pmd = (pmd_t *) spp_getpage(); 239 pud_populate(&init_mm, pud, pmd); 240 if (pmd != pmd_offset(pud, 0)) 241 printk(KERN_ERR "PAGETABLE BUG #02! %p <-> %p\n", 242 pmd, pmd_offset(pud, 0)); 243 } 244 return pmd_offset(pud, vaddr); 245 } 246 247 static pte_t *fill_pte(pmd_t *pmd, unsigned long vaddr) 248 { 249 if (pmd_none(*pmd)) { 250 pte_t *pte = (pte_t *) spp_getpage(); 251 pmd_populate_kernel(&init_mm, pmd, pte); 252 if (pte != pte_offset_kernel(pmd, 0)) 253 printk(KERN_ERR "PAGETABLE BUG #03!\n"); 254 } 255 return pte_offset_kernel(pmd, vaddr); 256 } 257 258 static void __set_pte_vaddr(pud_t *pud, unsigned long vaddr, pte_t new_pte) 259 { 260 pmd_t *pmd = fill_pmd(pud, vaddr); 261 pte_t *pte = fill_pte(pmd, vaddr); 262 263 set_pte(pte, new_pte); 264 265 /* 266 * It's enough to flush this one mapping. 267 * (PGE mappings get flushed as well) 268 */ 269 __flush_tlb_one_kernel(vaddr); 270 } 271 272 void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte) 273 { 274 p4d_t *p4d = p4d_page + p4d_index(vaddr); 275 pud_t *pud = fill_pud(p4d, vaddr); 276 277 __set_pte_vaddr(pud, vaddr, new_pte); 278 } 279 280 void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte) 281 { 282 pud_t *pud = pud_page + pud_index(vaddr); 283 284 __set_pte_vaddr(pud, vaddr, new_pte); 285 } 286 287 void set_pte_vaddr(unsigned long vaddr, pte_t pteval) 288 { 289 pgd_t *pgd; 290 p4d_t *p4d_page; 291 292 pr_debug("set_pte_vaddr %lx to %lx\n", vaddr, native_pte_val(pteval)); 293 294 pgd = pgd_offset_k(vaddr); 295 if (pgd_none(*pgd)) { 296 printk(KERN_ERR 297 "PGD FIXMAP MISSING, it should be setup in head.S!\n"); 298 return; 299 } 300 301 p4d_page = p4d_offset(pgd, 0); 302 set_pte_vaddr_p4d(p4d_page, vaddr, pteval); 303 } 304 305 pmd_t * __init populate_extra_pmd(unsigned long vaddr) 306 { 307 pgd_t *pgd; 308 p4d_t *p4d; 309 pud_t *pud; 310 311 pgd = pgd_offset_k(vaddr); 312 p4d = fill_p4d(pgd, vaddr); 313 pud = fill_pud(p4d, vaddr); 314 return fill_pmd(pud, vaddr); 315 } 316 317 pte_t * __init populate_extra_pte(unsigned long vaddr) 318 { 319 pmd_t *pmd; 320 321 pmd = populate_extra_pmd(vaddr); 322 return fill_pte(pmd, vaddr); 323 } 324 325 /* 326 * Create large page table mappings for a range of physical addresses. 327 */ 328 static void __init __init_extra_mapping(unsigned long phys, unsigned long size, 329 enum page_cache_mode cache) 330 { 331 pgd_t *pgd; 332 p4d_t *p4d; 333 pud_t *pud; 334 pmd_t *pmd; 335 pgprot_t prot; 336 337 pgprot_val(prot) = pgprot_val(PAGE_KERNEL_LARGE) | 338 pgprot_val(pgprot_4k_2_large(cachemode2pgprot(cache))); 339 BUG_ON((phys & ~PMD_MASK) || (size & ~PMD_MASK)); 340 for (; size; phys += PMD_SIZE, size -= PMD_SIZE) { 341 pgd = pgd_offset_k((unsigned long)__va(phys)); 342 if (pgd_none(*pgd)) { 343 p4d = (p4d_t *) spp_getpage(); 344 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE | 345 _PAGE_USER)); 346 } 347 p4d = p4d_offset(pgd, (unsigned long)__va(phys)); 348 if (p4d_none(*p4d)) { 349 pud = (pud_t *) spp_getpage(); 350 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE | 351 _PAGE_USER)); 352 } 353 pud = pud_offset(p4d, (unsigned long)__va(phys)); 354 if (pud_none(*pud)) { 355 pmd = (pmd_t *) spp_getpage(); 356 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | 357 _PAGE_USER)); 358 } 359 pmd = pmd_offset(pud, phys); 360 BUG_ON(!pmd_none(*pmd)); 361 set_pmd(pmd, __pmd(phys | pgprot_val(prot))); 362 } 363 } 364 365 void __init init_extra_mapping_wb(unsigned long phys, unsigned long size) 366 { 367 __init_extra_mapping(phys, size, _PAGE_CACHE_MODE_WB); 368 } 369 370 void __init init_extra_mapping_uc(unsigned long phys, unsigned long size) 371 { 372 __init_extra_mapping(phys, size, _PAGE_CACHE_MODE_UC); 373 } 374 375 /* 376 * The head.S code sets up the kernel high mapping: 377 * 378 * from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text) 379 * 380 * phys_base holds the negative offset to the kernel, which is added 381 * to the compile time generated pmds. This results in invalid pmds up 382 * to the point where we hit the physaddr 0 mapping. 383 * 384 * We limit the mappings to the region from _text to _brk_end. _brk_end 385 * is rounded up to the 2MB boundary. This catches the invalid pmds as 386 * well, as they are located before _text: 387 */ 388 void __init cleanup_highmap(void) 389 { 390 unsigned long vaddr = __START_KERNEL_map; 391 unsigned long vaddr_end = __START_KERNEL_map + KERNEL_IMAGE_SIZE; 392 unsigned long end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1; 393 pmd_t *pmd = level2_kernel_pgt; 394 395 /* 396 * Native path, max_pfn_mapped is not set yet. 397 * Xen has valid max_pfn_mapped set in 398 * arch/x86/xen/mmu.c:xen_setup_kernel_pagetable(). 399 */ 400 if (max_pfn_mapped) 401 vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT); 402 403 for (; vaddr + PMD_SIZE - 1 < vaddr_end; pmd++, vaddr += PMD_SIZE) { 404 if (pmd_none(*pmd)) 405 continue; 406 if (vaddr < (unsigned long) _text || vaddr > end) 407 set_pmd(pmd, __pmd(0)); 408 } 409 } 410 411 /* 412 * Create PTE level page table mapping for physical addresses. 413 * It returns the last physical address mapped. 414 */ 415 static unsigned long __meminit 416 phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end, 417 pgprot_t prot) 418 { 419 unsigned long pages = 0, paddr_next; 420 unsigned long paddr_last = paddr_end; 421 pte_t *pte; 422 int i; 423 424 pte = pte_page + pte_index(paddr); 425 i = pte_index(paddr); 426 427 for (; i < PTRS_PER_PTE; i++, paddr = paddr_next, pte++) { 428 paddr_next = (paddr & PAGE_MASK) + PAGE_SIZE; 429 if (paddr >= paddr_end) { 430 if (!after_bootmem && 431 !e820__mapped_any(paddr & PAGE_MASK, paddr_next, 432 E820_TYPE_RAM) && 433 !e820__mapped_any(paddr & PAGE_MASK, paddr_next, 434 E820_TYPE_RESERVED_KERN)) 435 set_pte_safe(pte, __pte(0)); 436 continue; 437 } 438 439 /* 440 * We will re-use the existing mapping. 441 * Xen for example has some special requirements, like mapping 442 * pagetable pages as RO. So assume someone who pre-setup 443 * these mappings are more intelligent. 444 */ 445 if (!pte_none(*pte)) { 446 if (!after_bootmem) 447 pages++; 448 continue; 449 } 450 451 if (0) 452 pr_info(" pte=%p addr=%lx pte=%016lx\n", pte, paddr, 453 pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL).pte); 454 pages++; 455 set_pte_safe(pte, pfn_pte(paddr >> PAGE_SHIFT, prot)); 456 paddr_last = (paddr & PAGE_MASK) + PAGE_SIZE; 457 } 458 459 update_page_count(PG_LEVEL_4K, pages); 460 461 return paddr_last; 462 } 463 464 /* 465 * Create PMD level page table mapping for physical addresses. The virtual 466 * and physical address have to be aligned at this level. 467 * It returns the last physical address mapped. 468 */ 469 static unsigned long __meminit 470 phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end, 471 unsigned long page_size_mask, pgprot_t prot) 472 { 473 unsigned long pages = 0, paddr_next; 474 unsigned long paddr_last = paddr_end; 475 476 int i = pmd_index(paddr); 477 478 for (; i < PTRS_PER_PMD; i++, paddr = paddr_next) { 479 pmd_t *pmd = pmd_page + pmd_index(paddr); 480 pte_t *pte; 481 pgprot_t new_prot = prot; 482 483 paddr_next = (paddr & PMD_MASK) + PMD_SIZE; 484 if (paddr >= paddr_end) { 485 if (!after_bootmem && 486 !e820__mapped_any(paddr & PMD_MASK, paddr_next, 487 E820_TYPE_RAM) && 488 !e820__mapped_any(paddr & PMD_MASK, paddr_next, 489 E820_TYPE_RESERVED_KERN)) 490 set_pmd_safe(pmd, __pmd(0)); 491 continue; 492 } 493 494 if (!pmd_none(*pmd)) { 495 if (!pmd_large(*pmd)) { 496 spin_lock(&init_mm.page_table_lock); 497 pte = (pte_t *)pmd_page_vaddr(*pmd); 498 paddr_last = phys_pte_init(pte, paddr, 499 paddr_end, prot); 500 spin_unlock(&init_mm.page_table_lock); 501 continue; 502 } 503 /* 504 * If we are ok with PG_LEVEL_2M mapping, then we will 505 * use the existing mapping, 506 * 507 * Otherwise, we will split the large page mapping but 508 * use the same existing protection bits except for 509 * large page, so that we don't violate Intel's TLB 510 * Application note (317080) which says, while changing 511 * the page sizes, new and old translations should 512 * not differ with respect to page frame and 513 * attributes. 514 */ 515 if (page_size_mask & (1 << PG_LEVEL_2M)) { 516 if (!after_bootmem) 517 pages++; 518 paddr_last = paddr_next; 519 continue; 520 } 521 new_prot = pte_pgprot(pte_clrhuge(*(pte_t *)pmd)); 522 } 523 524 if (page_size_mask & (1<<PG_LEVEL_2M)) { 525 pages++; 526 spin_lock(&init_mm.page_table_lock); 527 set_pte_safe((pte_t *)pmd, 528 pfn_pte((paddr & PMD_MASK) >> PAGE_SHIFT, 529 __pgprot(pgprot_val(prot) | _PAGE_PSE))); 530 spin_unlock(&init_mm.page_table_lock); 531 paddr_last = paddr_next; 532 continue; 533 } 534 535 pte = alloc_low_page(); 536 paddr_last = phys_pte_init(pte, paddr, paddr_end, new_prot); 537 538 spin_lock(&init_mm.page_table_lock); 539 pmd_populate_kernel_safe(&init_mm, pmd, pte); 540 spin_unlock(&init_mm.page_table_lock); 541 } 542 update_page_count(PG_LEVEL_2M, pages); 543 return paddr_last; 544 } 545 546 /* 547 * Create PUD level page table mapping for physical addresses. The virtual 548 * and physical address do not have to be aligned at this level. KASLR can 549 * randomize virtual addresses up to this level. 550 * It returns the last physical address mapped. 551 */ 552 static unsigned long __meminit 553 phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end, 554 unsigned long page_size_mask) 555 { 556 unsigned long pages = 0, paddr_next; 557 unsigned long paddr_last = paddr_end; 558 unsigned long vaddr = (unsigned long)__va(paddr); 559 int i = pud_index(vaddr); 560 561 for (; i < PTRS_PER_PUD; i++, paddr = paddr_next) { 562 pud_t *pud; 563 pmd_t *pmd; 564 pgprot_t prot = PAGE_KERNEL; 565 566 vaddr = (unsigned long)__va(paddr); 567 pud = pud_page + pud_index(vaddr); 568 paddr_next = (paddr & PUD_MASK) + PUD_SIZE; 569 570 if (paddr >= paddr_end) { 571 if (!after_bootmem && 572 !e820__mapped_any(paddr & PUD_MASK, paddr_next, 573 E820_TYPE_RAM) && 574 !e820__mapped_any(paddr & PUD_MASK, paddr_next, 575 E820_TYPE_RESERVED_KERN)) 576 set_pud_safe(pud, __pud(0)); 577 continue; 578 } 579 580 if (!pud_none(*pud)) { 581 if (!pud_large(*pud)) { 582 pmd = pmd_offset(pud, 0); 583 paddr_last = phys_pmd_init(pmd, paddr, 584 paddr_end, 585 page_size_mask, 586 prot); 587 continue; 588 } 589 /* 590 * If we are ok with PG_LEVEL_1G mapping, then we will 591 * use the existing mapping. 592 * 593 * Otherwise, we will split the gbpage mapping but use 594 * the same existing protection bits except for large 595 * page, so that we don't violate Intel's TLB 596 * Application note (317080) which says, while changing 597 * the page sizes, new and old translations should 598 * not differ with respect to page frame and 599 * attributes. 600 */ 601 if (page_size_mask & (1 << PG_LEVEL_1G)) { 602 if (!after_bootmem) 603 pages++; 604 paddr_last = paddr_next; 605 continue; 606 } 607 prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud)); 608 } 609 610 if (page_size_mask & (1<<PG_LEVEL_1G)) { 611 pages++; 612 spin_lock(&init_mm.page_table_lock); 613 set_pte_safe((pte_t *)pud, 614 pfn_pte((paddr & PUD_MASK) >> PAGE_SHIFT, 615 PAGE_KERNEL_LARGE)); 616 spin_unlock(&init_mm.page_table_lock); 617 paddr_last = paddr_next; 618 continue; 619 } 620 621 pmd = alloc_low_page(); 622 paddr_last = phys_pmd_init(pmd, paddr, paddr_end, 623 page_size_mask, prot); 624 625 spin_lock(&init_mm.page_table_lock); 626 pud_populate_safe(&init_mm, pud, pmd); 627 spin_unlock(&init_mm.page_table_lock); 628 } 629 630 update_page_count(PG_LEVEL_1G, pages); 631 632 return paddr_last; 633 } 634 635 static unsigned long __meminit 636 phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end, 637 unsigned long page_size_mask) 638 { 639 unsigned long paddr_next, paddr_last = paddr_end; 640 unsigned long vaddr = (unsigned long)__va(paddr); 641 int i = p4d_index(vaddr); 642 643 if (!pgtable_l5_enabled()) 644 return phys_pud_init((pud_t *) p4d_page, paddr, paddr_end, page_size_mask); 645 646 for (; i < PTRS_PER_P4D; i++, paddr = paddr_next) { 647 p4d_t *p4d; 648 pud_t *pud; 649 650 vaddr = (unsigned long)__va(paddr); 651 p4d = p4d_page + p4d_index(vaddr); 652 paddr_next = (paddr & P4D_MASK) + P4D_SIZE; 653 654 if (paddr >= paddr_end) { 655 if (!after_bootmem && 656 !e820__mapped_any(paddr & P4D_MASK, paddr_next, 657 E820_TYPE_RAM) && 658 !e820__mapped_any(paddr & P4D_MASK, paddr_next, 659 E820_TYPE_RESERVED_KERN)) 660 set_p4d_safe(p4d, __p4d(0)); 661 continue; 662 } 663 664 if (!p4d_none(*p4d)) { 665 pud = pud_offset(p4d, 0); 666 paddr_last = phys_pud_init(pud, paddr, 667 paddr_end, 668 page_size_mask); 669 continue; 670 } 671 672 pud = alloc_low_page(); 673 paddr_last = phys_pud_init(pud, paddr, paddr_end, 674 page_size_mask); 675 676 spin_lock(&init_mm.page_table_lock); 677 p4d_populate_safe(&init_mm, p4d, pud); 678 spin_unlock(&init_mm.page_table_lock); 679 } 680 681 return paddr_last; 682 } 683 684 /* 685 * Create page table mapping for the physical memory for specific physical 686 * addresses. The virtual and physical addresses have to be aligned on PMD level 687 * down. It returns the last physical address mapped. 688 */ 689 unsigned long __meminit 690 kernel_physical_mapping_init(unsigned long paddr_start, 691 unsigned long paddr_end, 692 unsigned long page_size_mask) 693 { 694 bool pgd_changed = false; 695 unsigned long vaddr, vaddr_start, vaddr_end, vaddr_next, paddr_last; 696 697 paddr_last = paddr_end; 698 vaddr = (unsigned long)__va(paddr_start); 699 vaddr_end = (unsigned long)__va(paddr_end); 700 vaddr_start = vaddr; 701 702 for (; vaddr < vaddr_end; vaddr = vaddr_next) { 703 pgd_t *pgd = pgd_offset_k(vaddr); 704 p4d_t *p4d; 705 706 vaddr_next = (vaddr & PGDIR_MASK) + PGDIR_SIZE; 707 708 if (pgd_val(*pgd)) { 709 p4d = (p4d_t *)pgd_page_vaddr(*pgd); 710 paddr_last = phys_p4d_init(p4d, __pa(vaddr), 711 __pa(vaddr_end), 712 page_size_mask); 713 continue; 714 } 715 716 p4d = alloc_low_page(); 717 paddr_last = phys_p4d_init(p4d, __pa(vaddr), __pa(vaddr_end), 718 page_size_mask); 719 720 spin_lock(&init_mm.page_table_lock); 721 if (pgtable_l5_enabled()) 722 pgd_populate_safe(&init_mm, pgd, p4d); 723 else 724 p4d_populate_safe(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d); 725 spin_unlock(&init_mm.page_table_lock); 726 pgd_changed = true; 727 } 728 729 if (pgd_changed) 730 sync_global_pgds(vaddr_start, vaddr_end - 1); 731 732 return paddr_last; 733 } 734 735 #ifndef CONFIG_NUMA 736 void __init initmem_init(void) 737 { 738 memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0); 739 } 740 #endif 741 742 void __init paging_init(void) 743 { 744 sparse_memory_present_with_active_regions(MAX_NUMNODES); 745 sparse_init(); 746 747 /* 748 * clear the default setting with node 0 749 * note: don't use nodes_clear here, that is really clearing when 750 * numa support is not compiled in, and later node_set_state 751 * will not set it back. 752 */ 753 node_clear_state(0, N_MEMORY); 754 if (N_MEMORY != N_NORMAL_MEMORY) 755 node_clear_state(0, N_NORMAL_MEMORY); 756 757 zone_sizes_init(); 758 } 759 760 /* 761 * Memory hotplug specific functions 762 */ 763 #ifdef CONFIG_MEMORY_HOTPLUG 764 /* 765 * After memory hotplug the variables max_pfn, max_low_pfn and high_memory need 766 * updating. 767 */ 768 static void update_end_of_memory_vars(u64 start, u64 size) 769 { 770 unsigned long end_pfn = PFN_UP(start + size); 771 772 if (end_pfn > max_pfn) { 773 max_pfn = end_pfn; 774 max_low_pfn = end_pfn; 775 high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1; 776 } 777 } 778 779 int add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages, 780 struct vmem_altmap *altmap, bool want_memblock) 781 { 782 int ret; 783 784 ret = __add_pages(nid, start_pfn, nr_pages, altmap, want_memblock); 785 WARN_ON_ONCE(ret); 786 787 /* update max_pfn, max_low_pfn and high_memory */ 788 update_end_of_memory_vars(start_pfn << PAGE_SHIFT, 789 nr_pages << PAGE_SHIFT); 790 791 return ret; 792 } 793 794 int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap, 795 bool want_memblock) 796 { 797 unsigned long start_pfn = start >> PAGE_SHIFT; 798 unsigned long nr_pages = size >> PAGE_SHIFT; 799 800 init_memory_mapping(start, start + size); 801 802 return add_pages(nid, start_pfn, nr_pages, altmap, want_memblock); 803 } 804 805 #define PAGE_INUSE 0xFD 806 807 static void __meminit free_pagetable(struct page *page, int order) 808 { 809 unsigned long magic; 810 unsigned int nr_pages = 1 << order; 811 812 /* bootmem page has reserved flag */ 813 if (PageReserved(page)) { 814 __ClearPageReserved(page); 815 816 magic = (unsigned long)page->freelist; 817 if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) { 818 while (nr_pages--) 819 put_page_bootmem(page++); 820 } else 821 while (nr_pages--) 822 free_reserved_page(page++); 823 } else 824 free_pages((unsigned long)page_address(page), order); 825 } 826 827 static void __meminit free_hugepage_table(struct page *page, 828 struct vmem_altmap *altmap) 829 { 830 if (altmap) 831 vmem_altmap_free(altmap, PMD_SIZE / PAGE_SIZE); 832 else 833 free_pagetable(page, get_order(PMD_SIZE)); 834 } 835 836 static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) 837 { 838 pte_t *pte; 839 int i; 840 841 for (i = 0; i < PTRS_PER_PTE; i++) { 842 pte = pte_start + i; 843 if (!pte_none(*pte)) 844 return; 845 } 846 847 /* free a pte talbe */ 848 free_pagetable(pmd_page(*pmd), 0); 849 spin_lock(&init_mm.page_table_lock); 850 pmd_clear(pmd); 851 spin_unlock(&init_mm.page_table_lock); 852 } 853 854 static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) 855 { 856 pmd_t *pmd; 857 int i; 858 859 for (i = 0; i < PTRS_PER_PMD; i++) { 860 pmd = pmd_start + i; 861 if (!pmd_none(*pmd)) 862 return; 863 } 864 865 /* free a pmd talbe */ 866 free_pagetable(pud_page(*pud), 0); 867 spin_lock(&init_mm.page_table_lock); 868 pud_clear(pud); 869 spin_unlock(&init_mm.page_table_lock); 870 } 871 872 static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) 873 { 874 pud_t *pud; 875 int i; 876 877 for (i = 0; i < PTRS_PER_PUD; i++) { 878 pud = pud_start + i; 879 if (!pud_none(*pud)) 880 return; 881 } 882 883 /* free a pud talbe */ 884 free_pagetable(p4d_page(*p4d), 0); 885 spin_lock(&init_mm.page_table_lock); 886 p4d_clear(p4d); 887 spin_unlock(&init_mm.page_table_lock); 888 } 889 890 static void __meminit 891 remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end, 892 bool direct) 893 { 894 unsigned long next, pages = 0; 895 pte_t *pte; 896 void *page_addr; 897 phys_addr_t phys_addr; 898 899 pte = pte_start + pte_index(addr); 900 for (; addr < end; addr = next, pte++) { 901 next = (addr + PAGE_SIZE) & PAGE_MASK; 902 if (next > end) 903 next = end; 904 905 if (!pte_present(*pte)) 906 continue; 907 908 /* 909 * We mapped [0,1G) memory as identity mapping when 910 * initializing, in arch/x86/kernel/head_64.S. These 911 * pagetables cannot be removed. 912 */ 913 phys_addr = pte_val(*pte) + (addr & PAGE_MASK); 914 if (phys_addr < (phys_addr_t)0x40000000) 915 return; 916 917 if (PAGE_ALIGNED(addr) && PAGE_ALIGNED(next)) { 918 /* 919 * Do not free direct mapping pages since they were 920 * freed when offlining, or simplely not in use. 921 */ 922 if (!direct) 923 free_pagetable(pte_page(*pte), 0); 924 925 spin_lock(&init_mm.page_table_lock); 926 pte_clear(&init_mm, addr, pte); 927 spin_unlock(&init_mm.page_table_lock); 928 929 /* For non-direct mapping, pages means nothing. */ 930 pages++; 931 } else { 932 /* 933 * If we are here, we are freeing vmemmap pages since 934 * direct mapped memory ranges to be freed are aligned. 935 * 936 * If we are not removing the whole page, it means 937 * other page structs in this page are being used and 938 * we canot remove them. So fill the unused page_structs 939 * with 0xFD, and remove the page when it is wholly 940 * filled with 0xFD. 941 */ 942 memset((void *)addr, PAGE_INUSE, next - addr); 943 944 page_addr = page_address(pte_page(*pte)); 945 if (!memchr_inv(page_addr, PAGE_INUSE, PAGE_SIZE)) { 946 free_pagetable(pte_page(*pte), 0); 947 948 spin_lock(&init_mm.page_table_lock); 949 pte_clear(&init_mm, addr, pte); 950 spin_unlock(&init_mm.page_table_lock); 951 } 952 } 953 } 954 955 /* Call free_pte_table() in remove_pmd_table(). */ 956 flush_tlb_all(); 957 if (direct) 958 update_page_count(PG_LEVEL_4K, -pages); 959 } 960 961 static void __meminit 962 remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end, 963 bool direct, struct vmem_altmap *altmap) 964 { 965 unsigned long next, pages = 0; 966 pte_t *pte_base; 967 pmd_t *pmd; 968 void *page_addr; 969 970 pmd = pmd_start + pmd_index(addr); 971 for (; addr < end; addr = next, pmd++) { 972 next = pmd_addr_end(addr, end); 973 974 if (!pmd_present(*pmd)) 975 continue; 976 977 if (pmd_large(*pmd)) { 978 if (IS_ALIGNED(addr, PMD_SIZE) && 979 IS_ALIGNED(next, PMD_SIZE)) { 980 if (!direct) 981 free_hugepage_table(pmd_page(*pmd), 982 altmap); 983 984 spin_lock(&init_mm.page_table_lock); 985 pmd_clear(pmd); 986 spin_unlock(&init_mm.page_table_lock); 987 pages++; 988 } else { 989 /* If here, we are freeing vmemmap pages. */ 990 memset((void *)addr, PAGE_INUSE, next - addr); 991 992 page_addr = page_address(pmd_page(*pmd)); 993 if (!memchr_inv(page_addr, PAGE_INUSE, 994 PMD_SIZE)) { 995 free_hugepage_table(pmd_page(*pmd), 996 altmap); 997 998 spin_lock(&init_mm.page_table_lock); 999 pmd_clear(pmd); 1000 spin_unlock(&init_mm.page_table_lock); 1001 } 1002 } 1003 1004 continue; 1005 } 1006 1007 pte_base = (pte_t *)pmd_page_vaddr(*pmd); 1008 remove_pte_table(pte_base, addr, next, direct); 1009 free_pte_table(pte_base, pmd); 1010 } 1011 1012 /* Call free_pmd_table() in remove_pud_table(). */ 1013 if (direct) 1014 update_page_count(PG_LEVEL_2M, -pages); 1015 } 1016 1017 static void __meminit 1018 remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end, 1019 struct vmem_altmap *altmap, bool direct) 1020 { 1021 unsigned long next, pages = 0; 1022 pmd_t *pmd_base; 1023 pud_t *pud; 1024 void *page_addr; 1025 1026 pud = pud_start + pud_index(addr); 1027 for (; addr < end; addr = next, pud++) { 1028 next = pud_addr_end(addr, end); 1029 1030 if (!pud_present(*pud)) 1031 continue; 1032 1033 if (pud_large(*pud)) { 1034 if (IS_ALIGNED(addr, PUD_SIZE) && 1035 IS_ALIGNED(next, PUD_SIZE)) { 1036 if (!direct) 1037 free_pagetable(pud_page(*pud), 1038 get_order(PUD_SIZE)); 1039 1040 spin_lock(&init_mm.page_table_lock); 1041 pud_clear(pud); 1042 spin_unlock(&init_mm.page_table_lock); 1043 pages++; 1044 } else { 1045 /* If here, we are freeing vmemmap pages. */ 1046 memset((void *)addr, PAGE_INUSE, next - addr); 1047 1048 page_addr = page_address(pud_page(*pud)); 1049 if (!memchr_inv(page_addr, PAGE_INUSE, 1050 PUD_SIZE)) { 1051 free_pagetable(pud_page(*pud), 1052 get_order(PUD_SIZE)); 1053 1054 spin_lock(&init_mm.page_table_lock); 1055 pud_clear(pud); 1056 spin_unlock(&init_mm.page_table_lock); 1057 } 1058 } 1059 1060 continue; 1061 } 1062 1063 pmd_base = pmd_offset(pud, 0); 1064 remove_pmd_table(pmd_base, addr, next, direct, altmap); 1065 free_pmd_table(pmd_base, pud); 1066 } 1067 1068 if (direct) 1069 update_page_count(PG_LEVEL_1G, -pages); 1070 } 1071 1072 static void __meminit 1073 remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end, 1074 struct vmem_altmap *altmap, bool direct) 1075 { 1076 unsigned long next, pages = 0; 1077 pud_t *pud_base; 1078 p4d_t *p4d; 1079 1080 p4d = p4d_start + p4d_index(addr); 1081 for (; addr < end; addr = next, p4d++) { 1082 next = p4d_addr_end(addr, end); 1083 1084 if (!p4d_present(*p4d)) 1085 continue; 1086 1087 BUILD_BUG_ON(p4d_large(*p4d)); 1088 1089 pud_base = pud_offset(p4d, 0); 1090 remove_pud_table(pud_base, addr, next, altmap, direct); 1091 /* 1092 * For 4-level page tables we do not want to free PUDs, but in the 1093 * 5-level case we should free them. This code will have to change 1094 * to adapt for boot-time switching between 4 and 5 level page tables. 1095 */ 1096 if (pgtable_l5_enabled()) 1097 free_pud_table(pud_base, p4d); 1098 } 1099 1100 if (direct) 1101 update_page_count(PG_LEVEL_512G, -pages); 1102 } 1103 1104 /* start and end are both virtual address. */ 1105 static void __meminit 1106 remove_pagetable(unsigned long start, unsigned long end, bool direct, 1107 struct vmem_altmap *altmap) 1108 { 1109 unsigned long next; 1110 unsigned long addr; 1111 pgd_t *pgd; 1112 p4d_t *p4d; 1113 1114 for (addr = start; addr < end; addr = next) { 1115 next = pgd_addr_end(addr, end); 1116 1117 pgd = pgd_offset_k(addr); 1118 if (!pgd_present(*pgd)) 1119 continue; 1120 1121 p4d = p4d_offset(pgd, 0); 1122 remove_p4d_table(p4d, addr, next, altmap, direct); 1123 } 1124 1125 flush_tlb_all(); 1126 } 1127 1128 void __ref vmemmap_free(unsigned long start, unsigned long end, 1129 struct vmem_altmap *altmap) 1130 { 1131 remove_pagetable(start, end, false, altmap); 1132 } 1133 1134 #ifdef CONFIG_MEMORY_HOTREMOVE 1135 static void __meminit 1136 kernel_physical_mapping_remove(unsigned long start, unsigned long end) 1137 { 1138 start = (unsigned long)__va(start); 1139 end = (unsigned long)__va(end); 1140 1141 remove_pagetable(start, end, true, NULL); 1142 } 1143 1144 int __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) 1145 { 1146 unsigned long start_pfn = start >> PAGE_SHIFT; 1147 unsigned long nr_pages = size >> PAGE_SHIFT; 1148 struct page *page = pfn_to_page(start_pfn); 1149 struct zone *zone; 1150 int ret; 1151 1152 /* With altmap the first mapped page is offset from @start */ 1153 if (altmap) 1154 page += vmem_altmap_offset(altmap); 1155 zone = page_zone(page); 1156 ret = __remove_pages(zone, start_pfn, nr_pages, altmap); 1157 WARN_ON_ONCE(ret); 1158 kernel_physical_mapping_remove(start, start + size); 1159 1160 return ret; 1161 } 1162 #endif 1163 #endif /* CONFIG_MEMORY_HOTPLUG */ 1164 1165 static struct kcore_list kcore_vsyscall; 1166 1167 static void __init register_page_bootmem_info(void) 1168 { 1169 #ifdef CONFIG_NUMA 1170 int i; 1171 1172 for_each_online_node(i) 1173 register_page_bootmem_info_node(NODE_DATA(i)); 1174 #endif 1175 } 1176 1177 void __init mem_init(void) 1178 { 1179 pci_iommu_alloc(); 1180 1181 /* clear_bss() already clear the empty_zero_page */ 1182 1183 /* this will put all memory onto the freelists */ 1184 memblock_free_all(); 1185 after_bootmem = 1; 1186 x86_init.hyper.init_after_bootmem(); 1187 1188 /* 1189 * Must be done after boot memory is put on freelist, because here we 1190 * might set fields in deferred struct pages that have not yet been 1191 * initialized, and memblock_free_all() initializes all the reserved 1192 * deferred pages for us. 1193 */ 1194 register_page_bootmem_info(); 1195 1196 /* Register memory areas for /proc/kcore */ 1197 if (get_gate_vma(&init_mm)) 1198 kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); 1199 1200 mem_init_print_info(NULL); 1201 } 1202 1203 int kernel_set_to_readonly; 1204 1205 void set_kernel_text_rw(void) 1206 { 1207 unsigned long start = PFN_ALIGN(_text); 1208 unsigned long end = PFN_ALIGN(__stop___ex_table); 1209 1210 if (!kernel_set_to_readonly) 1211 return; 1212 1213 pr_debug("Set kernel text: %lx - %lx for read write\n", 1214 start, end); 1215 1216 /* 1217 * Make the kernel identity mapping for text RW. Kernel text 1218 * mapping will always be RO. Refer to the comment in 1219 * static_protections() in pageattr.c 1220 */ 1221 set_memory_rw(start, (end - start) >> PAGE_SHIFT); 1222 } 1223 1224 void set_kernel_text_ro(void) 1225 { 1226 unsigned long start = PFN_ALIGN(_text); 1227 unsigned long end = PFN_ALIGN(__stop___ex_table); 1228 1229 if (!kernel_set_to_readonly) 1230 return; 1231 1232 pr_debug("Set kernel text: %lx - %lx for read only\n", 1233 start, end); 1234 1235 /* 1236 * Set the kernel identity mapping for text RO. 1237 */ 1238 set_memory_ro(start, (end - start) >> PAGE_SHIFT); 1239 } 1240 1241 void mark_rodata_ro(void) 1242 { 1243 unsigned long start = PFN_ALIGN(_text); 1244 unsigned long rodata_start = PFN_ALIGN(__start_rodata); 1245 unsigned long end = (unsigned long) &__end_rodata_hpage_align; 1246 unsigned long text_end = PFN_ALIGN(&__stop___ex_table); 1247 unsigned long rodata_end = PFN_ALIGN(&__end_rodata); 1248 unsigned long all_end; 1249 1250 printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", 1251 (end - start) >> 10); 1252 set_memory_ro(start, (end - start) >> PAGE_SHIFT); 1253 1254 kernel_set_to_readonly = 1; 1255 1256 /* 1257 * The rodata/data/bss/brk section (but not the kernel text!) 1258 * should also be not-executable. 1259 * 1260 * We align all_end to PMD_SIZE because the existing mapping 1261 * is a full PMD. If we would align _brk_end to PAGE_SIZE we 1262 * split the PMD and the reminder between _brk_end and the end 1263 * of the PMD will remain mapped executable. 1264 * 1265 * Any PMD which was setup after the one which covers _brk_end 1266 * has been zapped already via cleanup_highmem(). 1267 */ 1268 all_end = roundup((unsigned long)_brk_end, PMD_SIZE); 1269 set_memory_nx(text_end, (all_end - text_end) >> PAGE_SHIFT); 1270 1271 #ifdef CONFIG_CPA_DEBUG 1272 printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, end); 1273 set_memory_rw(start, (end-start) >> PAGE_SHIFT); 1274 1275 printk(KERN_INFO "Testing CPA: again\n"); 1276 set_memory_ro(start, (end-start) >> PAGE_SHIFT); 1277 #endif 1278 1279 free_kernel_image_pages((void *)text_end, (void *)rodata_start); 1280 free_kernel_image_pages((void *)rodata_end, (void *)_sdata); 1281 1282 debug_checkwx(); 1283 } 1284 1285 int kern_addr_valid(unsigned long addr) 1286 { 1287 unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT; 1288 pgd_t *pgd; 1289 p4d_t *p4d; 1290 pud_t *pud; 1291 pmd_t *pmd; 1292 pte_t *pte; 1293 1294 if (above != 0 && above != -1UL) 1295 return 0; 1296 1297 pgd = pgd_offset_k(addr); 1298 if (pgd_none(*pgd)) 1299 return 0; 1300 1301 p4d = p4d_offset(pgd, addr); 1302 if (p4d_none(*p4d)) 1303 return 0; 1304 1305 pud = pud_offset(p4d, addr); 1306 if (pud_none(*pud)) 1307 return 0; 1308 1309 if (pud_large(*pud)) 1310 return pfn_valid(pud_pfn(*pud)); 1311 1312 pmd = pmd_offset(pud, addr); 1313 if (pmd_none(*pmd)) 1314 return 0; 1315 1316 if (pmd_large(*pmd)) 1317 return pfn_valid(pmd_pfn(*pmd)); 1318 1319 pte = pte_offset_kernel(pmd, addr); 1320 if (pte_none(*pte)) 1321 return 0; 1322 1323 return pfn_valid(pte_pfn(*pte)); 1324 } 1325 1326 /* 1327 * Block size is the minimum amount of memory which can be hotplugged or 1328 * hotremoved. It must be power of two and must be equal or larger than 1329 * MIN_MEMORY_BLOCK_SIZE. 1330 */ 1331 #define MAX_BLOCK_SIZE (2UL << 30) 1332 1333 /* Amount of ram needed to start using large blocks */ 1334 #define MEM_SIZE_FOR_LARGE_BLOCK (64UL << 30) 1335 1336 /* Adjustable memory block size */ 1337 static unsigned long set_memory_block_size; 1338 int __init set_memory_block_size_order(unsigned int order) 1339 { 1340 unsigned long size = 1UL << order; 1341 1342 if (size > MEM_SIZE_FOR_LARGE_BLOCK || size < MIN_MEMORY_BLOCK_SIZE) 1343 return -EINVAL; 1344 1345 set_memory_block_size = size; 1346 return 0; 1347 } 1348 1349 static unsigned long probe_memory_block_size(void) 1350 { 1351 unsigned long boot_mem_end = max_pfn << PAGE_SHIFT; 1352 unsigned long bz; 1353 1354 /* If memory block size has been set, then use it */ 1355 bz = set_memory_block_size; 1356 if (bz) 1357 goto done; 1358 1359 /* Use regular block if RAM is smaller than MEM_SIZE_FOR_LARGE_BLOCK */ 1360 if (boot_mem_end < MEM_SIZE_FOR_LARGE_BLOCK) { 1361 bz = MIN_MEMORY_BLOCK_SIZE; 1362 goto done; 1363 } 1364 1365 /* Find the largest allowed block size that aligns to memory end */ 1366 for (bz = MAX_BLOCK_SIZE; bz > MIN_MEMORY_BLOCK_SIZE; bz >>= 1) { 1367 if (IS_ALIGNED(boot_mem_end, bz)) 1368 break; 1369 } 1370 done: 1371 pr_info("x86/mm: Memory block size: %ldMB\n", bz >> 20); 1372 1373 return bz; 1374 } 1375 1376 static unsigned long memory_block_size_probed; 1377 unsigned long memory_block_size_bytes(void) 1378 { 1379 if (!memory_block_size_probed) 1380 memory_block_size_probed = probe_memory_block_size(); 1381 1382 return memory_block_size_probed; 1383 } 1384 1385 #ifdef CONFIG_SPARSEMEM_VMEMMAP 1386 /* 1387 * Initialise the sparsemem vmemmap using huge-pages at the PMD level. 1388 */ 1389 static long __meminitdata addr_start, addr_end; 1390 static void __meminitdata *p_start, *p_end; 1391 static int __meminitdata node_start; 1392 1393 static int __meminit vmemmap_populate_hugepages(unsigned long start, 1394 unsigned long end, int node, struct vmem_altmap *altmap) 1395 { 1396 unsigned long addr; 1397 unsigned long next; 1398 pgd_t *pgd; 1399 p4d_t *p4d; 1400 pud_t *pud; 1401 pmd_t *pmd; 1402 1403 for (addr = start; addr < end; addr = next) { 1404 next = pmd_addr_end(addr, end); 1405 1406 pgd = vmemmap_pgd_populate(addr, node); 1407 if (!pgd) 1408 return -ENOMEM; 1409 1410 p4d = vmemmap_p4d_populate(pgd, addr, node); 1411 if (!p4d) 1412 return -ENOMEM; 1413 1414 pud = vmemmap_pud_populate(p4d, addr, node); 1415 if (!pud) 1416 return -ENOMEM; 1417 1418 pmd = pmd_offset(pud, addr); 1419 if (pmd_none(*pmd)) { 1420 void *p; 1421 1422 if (altmap) 1423 p = altmap_alloc_block_buf(PMD_SIZE, altmap); 1424 else 1425 p = vmemmap_alloc_block_buf(PMD_SIZE, node); 1426 if (p) { 1427 pte_t entry; 1428 1429 entry = pfn_pte(__pa(p) >> PAGE_SHIFT, 1430 PAGE_KERNEL_LARGE); 1431 set_pmd(pmd, __pmd(pte_val(entry))); 1432 1433 /* check to see if we have contiguous blocks */ 1434 if (p_end != p || node_start != node) { 1435 if (p_start) 1436 pr_debug(" [%lx-%lx] PMD -> [%p-%p] on node %d\n", 1437 addr_start, addr_end-1, p_start, p_end-1, node_start); 1438 addr_start = addr; 1439 node_start = node; 1440 p_start = p; 1441 } 1442 1443 addr_end = addr + PMD_SIZE; 1444 p_end = p + PMD_SIZE; 1445 continue; 1446 } else if (altmap) 1447 return -ENOMEM; /* no fallback */ 1448 } else if (pmd_large(*pmd)) { 1449 vmemmap_verify((pte_t *)pmd, node, addr, next); 1450 continue; 1451 } 1452 if (vmemmap_populate_basepages(addr, next, node)) 1453 return -ENOMEM; 1454 } 1455 return 0; 1456 } 1457 1458 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1459 struct vmem_altmap *altmap) 1460 { 1461 int err; 1462 1463 if (boot_cpu_has(X86_FEATURE_PSE)) 1464 err = vmemmap_populate_hugepages(start, end, node, altmap); 1465 else if (altmap) { 1466 pr_err_once("%s: no cpu support for altmap allocations\n", 1467 __func__); 1468 err = -ENOMEM; 1469 } else 1470 err = vmemmap_populate_basepages(start, end, node); 1471 if (!err) 1472 sync_global_pgds(start, end - 1); 1473 return err; 1474 } 1475 1476 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HAVE_BOOTMEM_INFO_NODE) 1477 void register_page_bootmem_memmap(unsigned long section_nr, 1478 struct page *start_page, unsigned long nr_pages) 1479 { 1480 unsigned long addr = (unsigned long)start_page; 1481 unsigned long end = (unsigned long)(start_page + nr_pages); 1482 unsigned long next; 1483 pgd_t *pgd; 1484 p4d_t *p4d; 1485 pud_t *pud; 1486 pmd_t *pmd; 1487 unsigned int nr_pmd_pages; 1488 struct page *page; 1489 1490 for (; addr < end; addr = next) { 1491 pte_t *pte = NULL; 1492 1493 pgd = pgd_offset_k(addr); 1494 if (pgd_none(*pgd)) { 1495 next = (addr + PAGE_SIZE) & PAGE_MASK; 1496 continue; 1497 } 1498 get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO); 1499 1500 p4d = p4d_offset(pgd, addr); 1501 if (p4d_none(*p4d)) { 1502 next = (addr + PAGE_SIZE) & PAGE_MASK; 1503 continue; 1504 } 1505 get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO); 1506 1507 pud = pud_offset(p4d, addr); 1508 if (pud_none(*pud)) { 1509 next = (addr + PAGE_SIZE) & PAGE_MASK; 1510 continue; 1511 } 1512 get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO); 1513 1514 if (!boot_cpu_has(X86_FEATURE_PSE)) { 1515 next = (addr + PAGE_SIZE) & PAGE_MASK; 1516 pmd = pmd_offset(pud, addr); 1517 if (pmd_none(*pmd)) 1518 continue; 1519 get_page_bootmem(section_nr, pmd_page(*pmd), 1520 MIX_SECTION_INFO); 1521 1522 pte = pte_offset_kernel(pmd, addr); 1523 if (pte_none(*pte)) 1524 continue; 1525 get_page_bootmem(section_nr, pte_page(*pte), 1526 SECTION_INFO); 1527 } else { 1528 next = pmd_addr_end(addr, end); 1529 1530 pmd = pmd_offset(pud, addr); 1531 if (pmd_none(*pmd)) 1532 continue; 1533 1534 nr_pmd_pages = 1 << get_order(PMD_SIZE); 1535 page = pmd_page(*pmd); 1536 while (nr_pmd_pages--) 1537 get_page_bootmem(section_nr, page++, 1538 SECTION_INFO); 1539 } 1540 } 1541 } 1542 #endif 1543 1544 void __meminit vmemmap_populate_print_last(void) 1545 { 1546 if (p_start) { 1547 pr_debug(" [%lx-%lx] PMD -> [%p-%p] on node %d\n", 1548 addr_start, addr_end-1, p_start, p_end-1, node_start); 1549 p_start = NULL; 1550 p_end = NULL; 1551 node_start = 0; 1552 } 1553 } 1554 #endif 1555