1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2006 4 */ 5 6 #include <linux/memory_hotplug.h> 7 #include <linux/cpufeature.h> 8 #include <linux/memblock.h> 9 #include <linux/pfn.h> 10 #include <linux/mm.h> 11 #include <linux/init.h> 12 #include <linux/list.h> 13 #include <linux/hugetlb.h> 14 #include <linux/slab.h> 15 #include <linux/sort.h> 16 #include <asm/page-states.h> 17 #include <asm/abs_lowcore.h> 18 #include <asm/cacheflush.h> 19 #include <asm/maccess.h> 20 #include <asm/nospec-branch.h> 21 #include <asm/ctlreg.h> 22 #include <asm/pgalloc.h> 23 #include <asm/setup.h> 24 #include <asm/tlbflush.h> 25 #include <asm/sections.h> 26 #include <asm/set_memory.h> 27 #include <asm/physmem_info.h> 28 29 static DEFINE_MUTEX(vmem_mutex); 30 31 static void __ref *vmem_alloc_pages(unsigned int order) 32 { 33 unsigned long size = PAGE_SIZE << order; 34 35 if (slab_is_available()) 36 return (void *)__get_free_pages(GFP_KERNEL, order); 37 return memblock_alloc(size, size); 38 } 39 40 static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap) 41 { 42 struct page *page; 43 44 if (altmap) { 45 vmem_altmap_free(altmap, 1 << order); 46 return; 47 } 48 page = virt_to_page((void *)addr); 49 if (PageReserved(page)) { 50 /* allocated from memblock */ 51 free_reserved_pages(page, order); 52 } else { 53 free_pages(addr, order); 54 } 55 } 56 57 void *vmem_crst_alloc(unsigned long val) 58 { 59 unsigned long *table; 60 61 table = vmem_alloc_pages(CRST_ALLOC_ORDER); 62 if (!table) 63 return NULL; 64 crst_table_init(table, val); 65 __arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER); 66 return table; 67 } 68 69 pte_t __ref *vmem_pte_alloc(void) 70 { 71 pte_t *pte; 72 73 if (slab_is_available()) 74 pte = (pte_t *)page_table_alloc(&init_mm); 75 else 76 pte = (pte_t *)memblock_alloc(PAGE_SIZE, PAGE_SIZE); 77 if (!pte) 78 return NULL; 79 memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); 80 __arch_set_page_dat(pte, 1); 81 return pte; 82 } 83 84 static void vmem_pte_free(unsigned long *table) 85 { 86 page_table_free(&init_mm, table); 87 } 88 89 #define PAGE_UNUSED 0xFD 90 91 /* 92 * The unused vmemmap range, which was not yet memset(PAGE_UNUSED) ranges 93 * from unused_sub_pmd_start to next PMD_SIZE boundary. 94 */ 95 static unsigned long unused_sub_pmd_start; 96 97 static void vmemmap_flush_unused_sub_pmd(void) 98 { 99 if (!unused_sub_pmd_start) 100 return; 101 memset((void *)unused_sub_pmd_start, PAGE_UNUSED, 102 ALIGN(unused_sub_pmd_start, PMD_SIZE) - unused_sub_pmd_start); 103 unused_sub_pmd_start = 0; 104 } 105 106 static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end) 107 { 108 /* 109 * As we expect to add in the same granularity as we remove, it's 110 * sufficient to mark only some piece used to block the memmap page from 111 * getting removed (just in case the memmap never gets initialized, 112 * e.g., because the memory block never gets onlined). 113 */ 114 memset((void *)start, 0, sizeof(struct page)); 115 } 116 117 static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) 118 { 119 /* 120 * We only optimize if the new used range directly follows the 121 * previously unused range (esp., when populating consecutive sections). 122 */ 123 if (unused_sub_pmd_start == start) { 124 unused_sub_pmd_start = end; 125 if (likely(IS_ALIGNED(unused_sub_pmd_start, PMD_SIZE))) 126 unused_sub_pmd_start = 0; 127 return; 128 } 129 vmemmap_flush_unused_sub_pmd(); 130 vmemmap_mark_sub_pmd_used(start, end); 131 } 132 133 static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) 134 { 135 unsigned long page = ALIGN_DOWN(start, PMD_SIZE); 136 137 vmemmap_flush_unused_sub_pmd(); 138 139 /* Could be our memmap page is filled with PAGE_UNUSED already ... */ 140 vmemmap_mark_sub_pmd_used(start, end); 141 142 /* Mark the unused parts of the new memmap page PAGE_UNUSED. */ 143 if (!IS_ALIGNED(start, PMD_SIZE)) 144 memset((void *)page, PAGE_UNUSED, start - page); 145 /* 146 * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of 147 * consecutive sections. Remember for the last added PMD the last 148 * unused range in the populated PMD. 149 */ 150 if (!IS_ALIGNED(end, PMD_SIZE)) 151 unused_sub_pmd_start = end; 152 } 153 154 /* Returns true if the PMD is completely unused and can be freed. */ 155 static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end) 156 { 157 unsigned long page = ALIGN_DOWN(start, PMD_SIZE); 158 159 vmemmap_flush_unused_sub_pmd(); 160 memset((void *)start, PAGE_UNUSED, end - start); 161 return !memchr_inv((void *)page, PAGE_UNUSED, PMD_SIZE); 162 } 163 164 /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */ 165 static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr, 166 unsigned long end, bool add, bool direct, 167 struct vmem_altmap *altmap) 168 { 169 unsigned long prot, pages = 0; 170 int ret = -ENOMEM; 171 pte_t *pte, entry; 172 173 prot = pgprot_val(PAGE_KERNEL); 174 pte = pte_offset_kernel(pmd, addr); 175 for (; addr < end; addr += PAGE_SIZE, pte++) { 176 entry = ptep_get(pte); 177 if (!add) { 178 if (pte_none(entry)) 179 continue; 180 if (!direct) 181 vmem_free_pages((unsigned long)pfn_to_virt(pte_pfn(entry)), get_order(PAGE_SIZE), altmap); 182 pte_clear(&init_mm, addr, pte); 183 } else if (pte_none(entry)) { 184 if (!direct) { 185 void *new_page = vmemmap_alloc_block_buf(PAGE_SIZE, NUMA_NO_NODE, altmap); 186 187 if (!new_page) 188 goto out; 189 set_pte(pte, __pte(__pa(new_page) | prot)); 190 } else { 191 set_pte(pte, __pte(__pa(addr) | prot)); 192 } 193 } else { 194 continue; 195 } 196 pages++; 197 } 198 ret = 0; 199 out: 200 if (direct) 201 update_page_count(PG_DIRECT_MAP_4K, add ? pages : -pages); 202 return ret; 203 } 204 205 static void try_free_pte_table(pmd_t *pmd, unsigned long start) 206 { 207 pte_t *pte; 208 int i; 209 210 /* We can safely assume this is fully in 1:1 mapping & vmemmap area */ 211 pte = pte_offset_kernel(pmd, start); 212 for (i = 0; i < PTRS_PER_PTE; i++, pte++) { 213 if (!pte_none(ptep_get(pte))) 214 return; 215 } 216 vmem_pte_free((unsigned long *)pmd_deref(pmdp_get(pmd))); 217 pmd_clear(pmd); 218 } 219 220 /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */ 221 static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, 222 unsigned long end, bool add, bool direct, 223 struct vmem_altmap *altmap) 224 { 225 unsigned long next, prot, pages = 0; 226 int ret = -ENOMEM; 227 pmd_t entry; 228 pmd_t *pmd; 229 pte_t *pte; 230 231 prot = pgprot_val(SEGMENT_KERNEL); 232 pmd = pmd_offset(pud, addr); 233 for (; addr < end; addr = next, pmd++) { 234 next = pmd_addr_end(addr, end); 235 entry = pmdp_get(pmd); 236 if (!add) { 237 if (pmd_none(entry)) 238 continue; 239 if (pmd_leaf(entry)) { 240 if (IS_ALIGNED(addr, PMD_SIZE) && 241 IS_ALIGNED(next, PMD_SIZE)) { 242 if (!direct) 243 vmem_free_pages(pmd_deref(entry), get_order(PMD_SIZE), altmap); 244 pmd_clear(pmd); 245 pages++; 246 } else if (!direct && vmemmap_unuse_sub_pmd(addr, next)) { 247 vmem_free_pages(pmd_deref(entry), get_order(PMD_SIZE), altmap); 248 pmd_clear(pmd); 249 } 250 continue; 251 } 252 } else if (pmd_none(entry)) { 253 if (IS_ALIGNED(addr, PMD_SIZE) && 254 IS_ALIGNED(next, PMD_SIZE) && 255 cpu_has_edat1() && direct && 256 !debug_pagealloc_enabled()) { 257 set_pmd(pmd, __pmd(__pa(addr) | prot)); 258 pages++; 259 continue; 260 } else if (!direct && cpu_has_edat1()) { 261 void *new_page; 262 263 /* 264 * Use 1MB frames for vmemmap if available. We 265 * always use large frames even if they are only 266 * partially used. Otherwise we would have also 267 * page tables since vmemmap_populate gets 268 * called for each section separately. 269 */ 270 new_page = vmemmap_alloc_block_buf(PMD_SIZE, NUMA_NO_NODE, altmap); 271 if (new_page) { 272 set_pmd(pmd, __pmd(__pa(new_page) | prot)); 273 if (!IS_ALIGNED(addr, PMD_SIZE) || 274 !IS_ALIGNED(next, PMD_SIZE)) { 275 vmemmap_use_new_sub_pmd(addr, next); 276 } 277 continue; 278 } 279 } 280 pte = vmem_pte_alloc(); 281 if (!pte) 282 goto out; 283 pmd_populate(&init_mm, pmd, pte); 284 } else if (pmd_leaf(entry)) { 285 if (!direct) 286 vmemmap_use_sub_pmd(addr, next); 287 continue; 288 } 289 ret = modify_pte_table(pmd, addr, next, add, direct, altmap); 290 if (ret) 291 goto out; 292 if (!add) 293 try_free_pte_table(pmd, addr & PMD_MASK); 294 } 295 ret = 0; 296 out: 297 if (direct) 298 update_page_count(PG_DIRECT_MAP_1M, add ? pages : -pages); 299 return ret; 300 } 301 302 static void try_free_pmd_table(pud_t *pud, unsigned long start) 303 { 304 pmd_t *pmd; 305 int i; 306 307 pmd = pmd_offset(pud, start); 308 for (i = 0; i < PTRS_PER_PMD; i++, pmd++) 309 if (!pmd_none(pmdp_get(pmd))) 310 return; 311 vmem_free_pages(pud_deref(pudp_get(pud)), CRST_ALLOC_ORDER, NULL); 312 pud_clear(pud); 313 } 314 315 static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, 316 bool add, bool direct, struct vmem_altmap *altmap) 317 { 318 unsigned long next, prot, pages = 0; 319 int ret = -ENOMEM; 320 pud_t *pud, entry; 321 pmd_t *pmd; 322 323 prot = pgprot_val(REGION3_KERNEL); 324 pud = pud_offset(p4d, addr); 325 for (; addr < end; addr = next, pud++) { 326 next = pud_addr_end(addr, end); 327 entry = pudp_get(pud); 328 if (!add) { 329 if (pud_none(entry)) 330 continue; 331 if (pud_leaf(entry)) { 332 if (IS_ALIGNED(addr, PUD_SIZE) && 333 IS_ALIGNED(next, PUD_SIZE)) { 334 if (!direct) 335 vmem_free_pages(pud_deref(entry), get_order(PUD_SIZE), altmap); 336 pud_clear(pud); 337 pages++; 338 continue; 339 } else { 340 split_pud_page(pud, addr & PUD_MASK); 341 } 342 } 343 } else if (pud_none(entry)) { 344 if (IS_ALIGNED(addr, PUD_SIZE) && 345 IS_ALIGNED(next, PUD_SIZE) && 346 cpu_has_edat2() && direct && 347 !debug_pagealloc_enabled()) { 348 set_pud(pud, __pud(__pa(addr) | prot)); 349 pages++; 350 continue; 351 } 352 pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); 353 if (!pmd) 354 goto out; 355 pud_populate(&init_mm, pud, pmd); 356 } else if (pud_leaf(entry)) { 357 continue; 358 } 359 ret = modify_pmd_table(pud, addr, next, add, direct, altmap); 360 if (ret) 361 goto out; 362 if (!add) 363 try_free_pmd_table(pud, addr & PUD_MASK); 364 } 365 ret = 0; 366 out: 367 if (direct) 368 update_page_count(PG_DIRECT_MAP_2G, add ? pages : -pages); 369 return ret; 370 } 371 372 static void try_free_pud_table(p4d_t *p4d, unsigned long start) 373 { 374 pud_t *pud; 375 int i; 376 377 pud = pud_offset(p4d, start); 378 for (i = 0; i < PTRS_PER_PUD; i++, pud++) { 379 if (!pud_none(pudp_get(pud))) 380 return; 381 } 382 vmem_free_pages(p4d_deref(p4dp_get(p4d)), CRST_ALLOC_ORDER, NULL); 383 p4d_clear(p4d); 384 } 385 386 static int modify_p4d_table(pgd_t *pgd, unsigned long addr, unsigned long end, 387 bool add, bool direct, struct vmem_altmap *altmap) 388 { 389 unsigned long next; 390 int ret = -ENOMEM; 391 p4d_t *p4d, entry; 392 pud_t *pud; 393 394 p4d = p4d_offset(pgd, addr); 395 for (; addr < end; addr = next, p4d++) { 396 next = p4d_addr_end(addr, end); 397 entry = p4dp_get(p4d); 398 if (!add) { 399 if (p4d_none(entry)) 400 continue; 401 } else if (p4d_none(entry)) { 402 pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); 403 if (!pud) 404 goto out; 405 p4d_populate(&init_mm, p4d, pud); 406 } 407 ret = modify_pud_table(p4d, addr, next, add, direct, altmap); 408 if (ret) 409 goto out; 410 if (!add) 411 try_free_pud_table(p4d, addr & P4D_MASK); 412 } 413 ret = 0; 414 out: 415 return ret; 416 } 417 418 static void try_free_p4d_table(pgd_t *pgd, unsigned long start) 419 { 420 p4d_t *p4d; 421 int i; 422 423 p4d = p4d_offset(pgd, start); 424 for (i = 0; i < PTRS_PER_P4D; i++, p4d++) { 425 if (!p4d_none(p4dp_get(p4d))) 426 return; 427 } 428 vmem_free_pages(pgd_deref(pgdp_get(pgd)), CRST_ALLOC_ORDER, NULL); 429 pgd_clear(pgd); 430 } 431 432 static int modify_pagetable(unsigned long start, unsigned long end, bool add, 433 bool direct, struct vmem_altmap *altmap) 434 { 435 unsigned long addr, next; 436 int ret = -ENOMEM; 437 pgd_t *pgd, entry; 438 p4d_t *p4d; 439 440 if (WARN_ON_ONCE(!PAGE_ALIGNED(start | end))) 441 return -EINVAL; 442 /* Don't mess with any tables not fully in 1:1 mapping, vmemmap & kasan area */ 443 #ifdef CONFIG_KASAN 444 if (WARN_ON_ONCE(!(start >= KASAN_SHADOW_START && end <= KASAN_SHADOW_END) && 445 end > __abs_lowcore)) 446 return -EINVAL; 447 #else 448 if (WARN_ON_ONCE(end > __abs_lowcore)) 449 return -EINVAL; 450 #endif 451 for (addr = start; addr < end; addr = next) { 452 next = pgd_addr_end(addr, end); 453 pgd = pgd_offset_k(addr); 454 entry = pgdp_get(pgd); 455 456 if (!add) { 457 if (pgd_none(entry)) 458 continue; 459 } else if (pgd_none(entry)) { 460 p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); 461 if (!p4d) 462 goto out; 463 pgd_populate(&init_mm, pgd, p4d); 464 } 465 ret = modify_p4d_table(pgd, addr, next, add, direct, altmap); 466 if (ret) 467 goto out; 468 if (!add) 469 try_free_p4d_table(pgd, addr & PGDIR_MASK); 470 } 471 ret = 0; 472 out: 473 if (!add) 474 flush_tlb_kernel_range(start, end); 475 return ret; 476 } 477 478 static int add_pagetable(unsigned long start, unsigned long end, bool direct, 479 struct vmem_altmap *altmap) 480 { 481 return modify_pagetable(start, end, true, direct, altmap); 482 } 483 484 static int remove_pagetable(unsigned long start, unsigned long end, bool direct, 485 struct vmem_altmap *altmap) 486 { 487 return modify_pagetable(start, end, false, direct, altmap); 488 } 489 490 /* 491 * Add a physical memory range to the 1:1 mapping. 492 */ 493 static int vmem_add_range(unsigned long start, unsigned long size) 494 { 495 start = (unsigned long)__va(start); 496 return add_pagetable(start, start + size, true, NULL); 497 } 498 499 /* 500 * Remove a physical memory range from the 1:1 mapping. 501 */ 502 static void vmem_remove_range(unsigned long start, unsigned long size) 503 { 504 start = (unsigned long)__va(start); 505 remove_pagetable(start, start + size, true, NULL); 506 } 507 508 /* 509 * Add a backed mem_map array to the virtual mem_map array. 510 */ 511 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 512 struct vmem_altmap *altmap) 513 { 514 int ret; 515 516 mutex_lock(&vmem_mutex); 517 /* We don't care about the node, just use NUMA_NO_NODE on allocations */ 518 ret = add_pagetable(start, end, false, altmap); 519 if (ret) 520 remove_pagetable(start, end, false, altmap); 521 mutex_unlock(&vmem_mutex); 522 return ret; 523 } 524 525 #ifdef CONFIG_MEMORY_HOTPLUG 526 527 void vmemmap_free(unsigned long start, unsigned long end, 528 struct vmem_altmap *altmap) 529 { 530 mutex_lock(&vmem_mutex); 531 remove_pagetable(start, end, false, altmap); 532 mutex_unlock(&vmem_mutex); 533 } 534 535 #endif 536 537 void vmem_remove_mapping(unsigned long start, unsigned long size) 538 { 539 mutex_lock(&vmem_mutex); 540 vmem_remove_range(start, size); 541 mutex_unlock(&vmem_mutex); 542 } 543 544 struct range arch_get_mappable_range(void) 545 { 546 struct range mhp_range; 547 548 mhp_range.start = 0; 549 mhp_range.end = max_mappable - 1; 550 return mhp_range; 551 } 552 553 int vmem_add_mapping(unsigned long start, unsigned long size) 554 { 555 struct range range = arch_get_mappable_range(); 556 int ret; 557 558 if (start < range.start || 559 start + size > range.end + 1 || 560 start + size < start) 561 return -ERANGE; 562 563 mutex_lock(&vmem_mutex); 564 ret = vmem_add_range(start, size); 565 if (ret) 566 vmem_remove_range(start, size); 567 mutex_unlock(&vmem_mutex); 568 return ret; 569 } 570 571 /* 572 * Allocate new or return existing page-table entry, but do not map it 573 * to any physical address. If missing, allocate segment- and region- 574 * table entries along. Meeting a large segment- or region-table entry 575 * while traversing is an error, since the function is expected to be 576 * called against virtual regions reserved for 4KB mappings only. 577 */ 578 pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc) 579 { 580 pte_t *ptep = NULL; 581 pud_t pud_entry; 582 pmd_t pmd_entry; 583 pgd_t *pgd; 584 p4d_t *p4d; 585 pud_t *pud; 586 pmd_t *pmd; 587 pte_t *pte; 588 589 pgd = pgd_offset_k(addr); 590 if (pgd_none(pgdp_get(pgd))) { 591 if (!alloc) 592 goto out; 593 p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); 594 if (!p4d) 595 goto out; 596 pgd_populate(&init_mm, pgd, p4d); 597 } 598 p4d = p4d_offset(pgd, addr); 599 if (p4d_none(p4dp_get(p4d))) { 600 if (!alloc) 601 goto out; 602 pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); 603 if (!pud) 604 goto out; 605 p4d_populate(&init_mm, p4d, pud); 606 } 607 pud = pud_offset(p4d, addr); 608 pud_entry = pudp_get(pud); 609 if (pud_none(pud_entry)) { 610 if (!alloc) 611 goto out; 612 pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); 613 if (!pmd) 614 goto out; 615 pud_populate(&init_mm, pud, pmd); 616 } else if (WARN_ON_ONCE(pud_leaf(pud_entry))) { 617 goto out; 618 } 619 pmd = pmd_offset(pud, addr); 620 pmd_entry = pmdp_get(pmd); 621 if (pmd_none(pmd_entry)) { 622 if (!alloc) 623 goto out; 624 pte = vmem_pte_alloc(); 625 if (!pte) 626 goto out; 627 pmd_populate(&init_mm, pmd, pte); 628 } else if (WARN_ON_ONCE(pmd_leaf(pmd_entry))) { 629 goto out; 630 } 631 ptep = pte_offset_kernel(pmd, addr); 632 out: 633 return ptep; 634 } 635 636 int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc) 637 { 638 pte_t *ptep, pte; 639 640 if (!IS_ALIGNED(addr, PAGE_SIZE)) 641 return -EINVAL; 642 ptep = vmem_get_alloc_pte(addr, alloc); 643 if (!ptep) 644 return -ENOMEM; 645 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); 646 pte = mk_pte_phys(phys, prot); 647 set_pte(ptep, pte); 648 return 0; 649 } 650 651 int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot) 652 { 653 int rc; 654 655 mutex_lock(&vmem_mutex); 656 rc = __vmem_map_4k_page(addr, phys, prot, true); 657 mutex_unlock(&vmem_mutex); 658 return rc; 659 } 660 661 void vmem_unmap_4k_page(unsigned long addr) 662 { 663 pte_t *ptep; 664 665 mutex_lock(&vmem_mutex); 666 ptep = virt_to_kpte(addr); 667 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); 668 pte_clear(&init_mm, addr, ptep); 669 mutex_unlock(&vmem_mutex); 670 } 671 672 void __init vmem_map_init(void) 673 { 674 __set_memory_rox(_stext, _etext); 675 __set_memory_ro(_etext, __end_rodata); 676 __set_memory_rox(__stext_amode31, __etext_amode31); 677 /* 678 * If the BEAR-enhancement facility is not installed the first 679 * prefix page is used to return to the previous context with 680 * an LPSWE instruction and therefore must be executable. 681 */ 682 if (!cpu_has_bear()) 683 set_memory_x(0, 1); 684 if (debug_pagealloc_enabled()) 685 __set_memory_4k(__va(0), absolute_pointer(__va(0)) + ident_map_size); 686 pr_info("Write protected kernel read-only data: %luk\n", 687 (unsigned long)(__end_rodata - _stext) >> 10); 688 } 689