1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Procedures for maintaining information about logical memory blocks. 4 * 5 * Peter Bergner, IBM Corp. June 2001. 6 * Copyright (C) 2001 Peter Bergner. 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/slab.h> 11 #include <linux/init.h> 12 #include <linux/bitops.h> 13 #include <linux/poison.h> 14 #include <linux/pfn.h> 15 #include <linux/debugfs.h> 16 #include <linux/kmemleak.h> 17 #include <linux/seq_file.h> 18 #include <linux/memblock.h> 19 #include <linux/mutex.h> 20 #include <linux/string_helpers.h> 21 22 #ifdef CONFIG_KEXEC_HANDOVER 23 #include <linux/libfdt.h> 24 #include <linux/kexec_handover.h> 25 #include <linux/kho/abi/memblock.h> 26 #endif /* CONFIG_KEXEC_HANDOVER */ 27 28 #include <asm/sections.h> 29 #include <linux/io.h> 30 31 #include "internal.h" 32 #include "mm_init.h" 33 34 #define INIT_MEMBLOCK_REGIONS 128 35 #define INIT_PHYSMEM_REGIONS 4 36 37 #ifndef INIT_MEMBLOCK_RESERVED_REGIONS 38 # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS 39 #endif 40 41 #ifndef INIT_MEMBLOCK_MEMORY_REGIONS 42 #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS 43 #endif 44 45 /** 46 * DOC: memblock overview 47 * 48 * Memblock is a method of managing memory regions during the early 49 * boot period when the usual kernel memory allocators are not up and 50 * running. 51 * 52 * Memblock views the system memory as collections of contiguous 53 * regions. There are several types of these collections: 54 * 55 * * ``memory`` - describes the physical memory available to the 56 * kernel; this may differ from the actual physical memory installed 57 * in the system, for instance when the memory is restricted with 58 * ``mem=`` command line parameter 59 * * ``reserved`` - describes the regions that were allocated 60 * * ``physmem`` - describes the actual physical memory available during 61 * boot regardless of the possible restrictions and memory hot(un)plug; 62 * the ``physmem`` type is only available on some architectures. 63 * 64 * Each region is represented by struct memblock_region that 65 * defines the region extents, its attributes and NUMA node id on NUMA 66 * systems. Every memory type is described by the struct memblock_type 67 * which contains an array of memory regions along with 68 * the allocator metadata. The "memory" and "reserved" types are nicely 69 * wrapped with struct memblock. This structure is statically 70 * initialized at build time. The region arrays are initially sized to 71 * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and 72 * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array 73 * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS. 74 * The memblock_allow_resize() enables automatic resizing of the region 75 * arrays during addition of new regions. This feature should be used 76 * with care so that memory allocated for the region array will not 77 * overlap with areas that should be reserved, for example initrd. 78 * 79 * The early architecture setup should tell memblock what the physical 80 * memory layout is by using memblock_add() or memblock_add_node() 81 * functions. The first function does not assign the region to a NUMA 82 * node and it is appropriate for UMA systems. Yet, it is possible to 83 * use it on NUMA systems as well and assign the region to a NUMA node 84 * later in the setup process using memblock_set_node(). The 85 * memblock_add_node() performs such an assignment directly. 86 * 87 * Once memblock is setup the memory can be allocated using one of the 88 * API variants: 89 * 90 * * memblock_phys_alloc*() - these functions return the **physical** 91 * address of the allocated memory 92 * * memblock_alloc*() - these functions return the **virtual** address 93 * of the allocated memory. 94 * 95 * Note, that both API variants use implicit assumptions about allowed 96 * memory ranges and the fallback methods. Consult the documentation 97 * of memblock_alloc_internal() and memblock_alloc_range_nid() 98 * functions for more elaborate description. 99 * 100 * As the system boot progresses, the architecture specific mem_init() 101 * function frees all the memory to the buddy page allocator. 102 * 103 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the 104 * memblock data structures (except "physmem") will be discarded after the 105 * system initialization completes. 106 */ 107 108 #ifndef CONFIG_NUMA 109 struct pglist_data __refdata contig_page_data; 110 EXPORT_SYMBOL(contig_page_data); 111 #endif 112 113 unsigned long max_low_pfn; 114 unsigned long min_low_pfn; 115 unsigned long max_pfn; 116 unsigned long long max_possible_pfn; 117 118 #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 119 /* When set to true, only allocate from MEMBLOCK_KHO_SCRATCH ranges */ 120 static bool kho_scratch_only; 121 #else 122 #define kho_scratch_only false 123 #endif 124 125 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock; 126 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock; 127 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 128 static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS]; 129 #endif 130 131 struct memblock memblock __initdata_memblock = { 132 .memory.regions = memblock_memory_init_regions, 133 .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS, 134 .memory.name = "memory", 135 136 .reserved.regions = memblock_reserved_init_regions, 137 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS, 138 .reserved.name = "reserved", 139 140 .bottom_up = false, 141 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 142 }; 143 144 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 145 struct memblock_type physmem = { 146 .regions = memblock_physmem_init_regions, 147 .max = INIT_PHYSMEM_REGIONS, 148 .name = "physmem", 149 }; 150 #endif 151 152 /* 153 * keep a pointer to &memblock.memory in the text section to use it in 154 * __next_mem_range() and its helpers. 155 * For architectures that do not keep memblock data after init, this 156 * pointer will be reset to NULL at memblock_discard() 157 */ 158 static __refdata struct memblock_type *memblock_memory = &memblock.memory; 159 160 #define for_each_memblock_type(i, memblock_type, rgn) \ 161 for (i = 0, rgn = &memblock_type->regions[0]; \ 162 i < memblock_type->cnt; \ 163 i++, rgn = &memblock_type->regions[i]) 164 165 #define memblock_dbg(fmt, ...) \ 166 do { \ 167 if (memblock_debug) \ 168 pr_info(fmt, ##__VA_ARGS__); \ 169 } while (0) 170 171 static int memblock_debug __initdata_memblock; 172 static bool system_has_some_mirror __initdata_memblock; 173 static int memblock_can_resize __initdata_memblock; 174 static int memblock_memory_in_slab __initdata_memblock; 175 static int memblock_reserved_in_slab __initdata_memblock; 176 177 bool __init_memblock memblock_has_mirror(void) 178 { 179 return system_has_some_mirror; 180 } 181 182 static enum memblock_flags __init_memblock choose_memblock_flags(void) 183 { 184 /* skip non-scratch memory for kho early boot allocations */ 185 if (kho_scratch_only) 186 return MEMBLOCK_KHO_SCRATCH; 187 188 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; 189 } 190 191 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 192 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 193 { 194 return *size = min(*size, PHYS_ADDR_MAX - base); 195 } 196 197 /* 198 * Address comparison utilities 199 */ 200 unsigned long __init_memblock 201 memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, phys_addr_t base2, 202 phys_addr_t size2) 203 { 204 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 205 } 206 207 bool __init_memblock memblock_overlaps_region(struct memblock_type *type, 208 phys_addr_t base, phys_addr_t size) 209 { 210 unsigned long i; 211 212 memblock_cap_size(base, &size); 213 214 for (i = 0; i < type->cnt; i++) 215 if (memblock_addrs_overlap(base, size, type->regions[i].base, 216 type->regions[i].size)) 217 return true; 218 return false; 219 } 220 221 /** 222 * __memblock_find_range_bottom_up - find free area utility in bottom-up 223 * @start: start of candidate range 224 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 225 * %MEMBLOCK_ALLOC_ACCESSIBLE 226 * @size: size of free area to find 227 * @align: alignment of free area to find 228 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 229 * @flags: pick from blocks based on memory attributes 230 * 231 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 232 * 233 * Return: 234 * Found address on success, 0 on failure. 235 */ 236 static phys_addr_t __init_memblock 237 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 238 phys_addr_t size, phys_addr_t align, int nid, 239 enum memblock_flags flags) 240 { 241 phys_addr_t this_start, this_end, cand; 242 u64 i; 243 244 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) { 245 this_start = clamp(this_start, start, end); 246 this_end = clamp(this_end, start, end); 247 248 cand = round_up(this_start, align); 249 if (cand < this_end && this_end - cand >= size) 250 return cand; 251 } 252 253 return 0; 254 } 255 256 /** 257 * __memblock_find_range_top_down - find free area utility, in top-down 258 * @start: start of candidate range 259 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 260 * %MEMBLOCK_ALLOC_ACCESSIBLE 261 * @size: size of free area to find 262 * @align: alignment of free area to find 263 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 264 * @flags: pick from blocks based on memory attributes 265 * 266 * Utility called from memblock_find_in_range_node(), find free area top-down. 267 * 268 * Return: 269 * Found address on success, 0 on failure. 270 */ 271 static phys_addr_t __init_memblock 272 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 273 phys_addr_t size, phys_addr_t align, int nid, 274 enum memblock_flags flags) 275 { 276 phys_addr_t this_start, this_end, cand; 277 u64 i; 278 279 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end, 280 NULL) { 281 this_start = clamp(this_start, start, end); 282 this_end = clamp(this_end, start, end); 283 284 if (this_end < size) 285 continue; 286 287 cand = round_down(this_end - size, align); 288 if (cand >= this_start) 289 return cand; 290 } 291 292 return 0; 293 } 294 295 /** 296 * memblock_find_in_range_node - find free area in given range and node 297 * @size: size of free area to find 298 * @align: alignment of free area to find 299 * @start: start of candidate range 300 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 301 * %MEMBLOCK_ALLOC_ACCESSIBLE 302 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 303 * @flags: pick from blocks based on memory attributes 304 * 305 * Find @size free area aligned to @align in the specified range and node. 306 * 307 * Return: 308 * Found address on success, 0 on failure. 309 */ 310 static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 311 phys_addr_t align, phys_addr_t start, 312 phys_addr_t end, int nid, 313 enum memblock_flags flags) 314 { 315 /* pump up @end */ 316 if (end == MEMBLOCK_ALLOC_ACCESSIBLE || 317 end == MEMBLOCK_ALLOC_NOLEAKTRACE) 318 end = memblock.current_limit; 319 320 /* avoid allocating the first page */ 321 start = max_t(phys_addr_t, start, PAGE_SIZE); 322 end = max(start, end); 323 324 if (memblock_bottom_up()) 325 return __memblock_find_range_bottom_up(start, end, size, align, 326 nid, flags); 327 else 328 return __memblock_find_range_top_down(start, end, size, align, 329 nid, flags); 330 } 331 332 /** 333 * memblock_find_in_range - find free area in given range 334 * @start: start of candidate range 335 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 336 * %MEMBLOCK_ALLOC_ACCESSIBLE 337 * @size: size of free area to find 338 * @align: alignment of free area to find 339 * 340 * Find @size free area aligned to @align in the specified range. 341 * 342 * Return: 343 * Found address on success, 0 on failure. 344 */ 345 static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 346 phys_addr_t end, phys_addr_t size, 347 phys_addr_t align) 348 { 349 phys_addr_t ret; 350 enum memblock_flags flags = choose_memblock_flags(); 351 352 again: 353 ret = memblock_find_in_range_node(size, align, start, end, 354 NUMA_NO_NODE, flags); 355 356 if (!ret && (flags & MEMBLOCK_MIRROR)) { 357 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 358 &size); 359 flags &= ~MEMBLOCK_MIRROR; 360 goto again; 361 } 362 363 return ret; 364 } 365 366 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 367 { 368 type->total_size -= type->regions[r].size; 369 memmove(&type->regions[r], &type->regions[r + 1], 370 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 371 type->cnt--; 372 373 /* Special case for empty arrays */ 374 if (type->cnt == 0) { 375 WARN_ON(type->total_size != 0); 376 type->regions[0].base = 0; 377 type->regions[0].size = 0; 378 type->regions[0].flags = 0; 379 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 380 } 381 } 382 383 #ifndef CONFIG_ARCH_KEEP_MEMBLOCK 384 /** 385 * memblock_discard - discard memory and reserved arrays if they were allocated 386 */ 387 void __init memblock_discard(void) 388 { 389 phys_addr_t size; 390 void *addr; 391 392 if (memblock.reserved.regions != memblock_reserved_init_regions) { 393 addr = memblock.reserved.regions; 394 size = PAGE_ALIGN(sizeof(struct memblock_region) * 395 memblock.reserved.max); 396 if (memblock_reserved_in_slab) 397 kfree(addr); 398 else 399 memblock_free(addr, size); 400 } 401 402 if (memblock.memory.regions != memblock_memory_init_regions) { 403 addr = memblock.memory.regions; 404 size = PAGE_ALIGN(sizeof(struct memblock_region) * 405 memblock.memory.max); 406 if (memblock_memory_in_slab) 407 kfree(addr); 408 else 409 memblock_free(addr, size); 410 } 411 412 memblock_memory = NULL; 413 } 414 #endif 415 416 /** 417 * memblock_double_array - double the size of the memblock regions array 418 * @type: memblock type of the regions array being doubled 419 * @new_area_start: starting address of memory range to avoid overlap with 420 * @new_area_size: size of memory range to avoid overlap with 421 * 422 * Double the size of the @type regions array. If memblock is being used to 423 * allocate memory for a new reserved regions array and there is a previously 424 * allocated memory range [@new_area_start, @new_area_start + @new_area_size] 425 * waiting to be reserved, ensure the memory used by the new array does 426 * not overlap. 427 * 428 * Return: 429 * 0 on success, -1 on failure. 430 */ 431 static int __init_memblock memblock_double_array(struct memblock_type *type, 432 phys_addr_t new_area_start, 433 phys_addr_t new_area_size) 434 { 435 struct memblock_region *new_array, *old_array; 436 phys_addr_t old_alloc_size, new_alloc_size; 437 phys_addr_t old_size, new_size, addr, new_end; 438 int use_slab = slab_is_available(); 439 int *in_slab; 440 441 /* We don't allow resizing until we know about the reserved regions 442 * of memory that aren't suitable for allocation 443 */ 444 if (!memblock_can_resize) 445 panic("memblock: cannot resize %s array\n", type->name); 446 447 /* Calculate new doubled size */ 448 old_size = type->max * sizeof(struct memblock_region); 449 new_size = old_size << 1; 450 /* 451 * We need to allocated new one align to PAGE_SIZE, 452 * so we can free them completely later. 453 */ 454 old_alloc_size = PAGE_ALIGN(old_size); 455 new_alloc_size = PAGE_ALIGN(new_size); 456 457 /* Retrieve the slab flag */ 458 if (type == &memblock.memory) 459 in_slab = &memblock_memory_in_slab; 460 else 461 in_slab = &memblock_reserved_in_slab; 462 463 /* Try to find some space for it */ 464 if (use_slab) { 465 new_array = kmalloc(new_size, GFP_KERNEL); 466 addr = new_array ? __pa(new_array) : 0; 467 } else { 468 /* only exclude range when trying to double reserved.regions */ 469 if (type != &memblock.reserved) 470 new_area_start = new_area_size = 0; 471 472 addr = memblock_find_in_range(new_area_start + new_area_size, 473 memblock.current_limit, 474 new_alloc_size, PAGE_SIZE); 475 if (!addr && new_area_size) 476 addr = memblock_find_in_range(0, 477 min(new_area_start, memblock.current_limit), 478 new_alloc_size, PAGE_SIZE); 479 480 if (addr) { 481 /* The memory may not have been accepted, yet. */ 482 accept_memory(addr, new_alloc_size); 483 484 new_array = __va(addr); 485 } else { 486 new_array = NULL; 487 } 488 } 489 if (!addr) { 490 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 491 type->name, type->max, type->max * 2); 492 return -1; 493 } 494 495 new_end = addr + new_size - 1; 496 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]", 497 type->name, type->max * 2, &addr, &new_end); 498 499 /* 500 * Found space, we now need to move the array over before we add the 501 * reserved region since it may be our reserved array itself that is 502 * full. 503 */ 504 memcpy(new_array, type->regions, old_size); 505 memset(new_array + type->max, 0, old_size); 506 old_array = type->regions; 507 type->regions = new_array; 508 type->max <<= 1; 509 510 /* Free old array. We needn't free it if the array is the static one */ 511 if (*in_slab) 512 kfree(old_array); 513 else if (old_array != memblock_memory_init_regions && 514 old_array != memblock_reserved_init_regions) 515 memblock_free(old_array, old_alloc_size); 516 517 /* 518 * Reserve the new array if that comes from the memblock. Otherwise, we 519 * needn't do it 520 */ 521 if (!use_slab) 522 BUG_ON(memblock_reserve_kern(addr, new_alloc_size)); 523 524 /* Update slab flag */ 525 *in_slab = use_slab; 526 527 return 0; 528 } 529 530 /** 531 * memblock_merge_regions - merge neighboring compatible regions 532 * @type: memblock type to scan 533 * @start_rgn: start scanning from (@start_rgn - 1) 534 * @end_rgn: end scanning at (@end_rgn - 1) 535 * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn) 536 */ 537 static void __init_memblock memblock_merge_regions(struct memblock_type *type, 538 unsigned long start_rgn, 539 unsigned long end_rgn) 540 { 541 int i = 0; 542 if (start_rgn) 543 i = start_rgn - 1; 544 end_rgn = min(end_rgn, type->cnt - 1); 545 while (i < end_rgn) { 546 struct memblock_region *this = &type->regions[i]; 547 struct memblock_region *next = &type->regions[i + 1]; 548 549 if (this->base + this->size != next->base || 550 memblock_get_region_node(this) != 551 memblock_get_region_node(next) || 552 this->flags != next->flags) { 553 BUG_ON(this->base + this->size > next->base); 554 i++; 555 continue; 556 } 557 558 this->size += next->size; 559 /* move forward from next + 1, index of which is i + 2 */ 560 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 561 type->cnt--; 562 end_rgn--; 563 } 564 } 565 566 /** 567 * memblock_insert_region - insert new memblock region 568 * @type: memblock type to insert into 569 * @idx: index for the insertion point 570 * @base: base address of the new region 571 * @size: size of the new region 572 * @nid: node id of the new region 573 * @flags: flags of the new region 574 * 575 * Insert new memblock region [@base, @base + @size) into @type at @idx. 576 * @type must already have extra room to accommodate the new region. 577 */ 578 static void __init_memblock memblock_insert_region(struct memblock_type *type, 579 int idx, phys_addr_t base, 580 phys_addr_t size, 581 int nid, 582 enum memblock_flags flags) 583 { 584 struct memblock_region *rgn = &type->regions[idx]; 585 586 BUG_ON(type->cnt >= type->max); 587 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 588 rgn->base = base; 589 rgn->size = size; 590 rgn->flags = flags; 591 memblock_set_region_node(rgn, nid); 592 type->cnt++; 593 type->total_size += size; 594 } 595 596 /** 597 * memblock_add_range - add new memblock region 598 * @type: memblock type to add new region into 599 * @base: base address of the new region 600 * @size: size of the new region 601 * @nid: nid of the new region 602 * @flags: flags of the new region 603 * 604 * Add new memblock region [@base, @base + @size) into @type. The new region 605 * is allowed to overlap with existing ones - overlaps don't affect already 606 * existing regions. @type is guaranteed to be minimal (all neighbouring 607 * compatible regions are merged) after the addition. 608 * 609 * Return: 610 * 0 on success, -errno on failure. 611 */ 612 static int __init_memblock memblock_add_range(struct memblock_type *type, 613 phys_addr_t base, phys_addr_t size, 614 int nid, enum memblock_flags flags) 615 { 616 bool insert = false; 617 phys_addr_t obase = base; 618 phys_addr_t end = base + memblock_cap_size(base, &size); 619 int idx, nr_new, start_rgn = -1, end_rgn; 620 struct memblock_region *rgn; 621 622 if (!size) 623 return 0; 624 625 /* special case for empty array */ 626 if (type->regions[0].size == 0) { 627 WARN_ON(type->cnt != 0 || type->total_size); 628 type->regions[0].base = base; 629 type->regions[0].size = size; 630 type->regions[0].flags = flags; 631 memblock_set_region_node(&type->regions[0], nid); 632 type->total_size = size; 633 type->cnt = 1; 634 return 0; 635 } 636 637 /* 638 * The worst case is when new range overlaps all existing regions, 639 * then we'll need type->cnt + 1 empty regions in @type. So if 640 * type->cnt * 2 + 1 is less than or equal to type->max, we know 641 * that there is enough empty regions in @type, and we can insert 642 * regions directly. 643 */ 644 if (type->cnt * 2 + 1 <= type->max) 645 insert = true; 646 647 repeat: 648 /* 649 * The following is executed twice. Once with %false @insert and 650 * then with %true. The first counts the number of regions needed 651 * to accommodate the new area. The second actually inserts them. 652 */ 653 base = obase; 654 nr_new = 0; 655 656 for_each_memblock_type(idx, type, rgn) { 657 phys_addr_t rbase = rgn->base; 658 phys_addr_t rend = rbase + rgn->size; 659 660 if (rbase >= end) 661 break; 662 if (rend <= base) 663 continue; 664 /* 665 * @rgn overlaps. If it separates the lower part of new 666 * area, insert that portion. 667 */ 668 if (rbase > base) { 669 #ifdef CONFIG_NUMA 670 WARN_ON(nid != memblock_get_region_node(rgn)); 671 #endif 672 WARN_ON(flags != MEMBLOCK_NONE && flags != rgn->flags); 673 nr_new++; 674 if (insert) { 675 if (start_rgn == -1) 676 start_rgn = idx; 677 end_rgn = idx + 1; 678 memblock_insert_region(type, idx++, base, 679 rbase - base, nid, 680 flags); 681 } 682 } 683 /* area below @rend is dealt with, forget about it */ 684 base = min(rend, end); 685 } 686 687 /* insert the remaining portion */ 688 if (base < end) { 689 nr_new++; 690 if (insert) { 691 if (start_rgn == -1) 692 start_rgn = idx; 693 end_rgn = idx + 1; 694 memblock_insert_region(type, idx, base, end - base, 695 nid, flags); 696 } 697 } 698 699 if (!nr_new) 700 return 0; 701 702 /* 703 * If this was the first round, resize array and repeat for actual 704 * insertions; otherwise, merge and return. 705 */ 706 if (!insert) { 707 while (type->cnt + nr_new > type->max) 708 if (memblock_double_array(type, obase, size) < 0) 709 return -ENOMEM; 710 insert = true; 711 goto repeat; 712 } else { 713 memblock_merge_regions(type, start_rgn, end_rgn); 714 return 0; 715 } 716 } 717 718 /** 719 * memblock_add_node - add new memblock region within a NUMA node 720 * @base: base address of the new region 721 * @size: size of the new region 722 * @nid: nid of the new region 723 * @flags: flags of the new region 724 * 725 * Add new memblock region [@base, @base + @size) to the "memory" 726 * type. See memblock_add_range() description for mode details 727 * 728 * Return: 729 * 0 on success, -errno on failure. 730 */ 731 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 732 int nid, enum memblock_flags flags) 733 { 734 phys_addr_t end = base + size - 1; 735 736 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 737 &base, &end, nid, flags, (void *)_RET_IP_); 738 739 return memblock_add_range(&memblock.memory, base, size, nid, flags); 740 } 741 742 /** 743 * memblock_add - add new memblock region 744 * @base: base address of the new region 745 * @size: size of the new region 746 * 747 * Add new memblock region [@base, @base + @size) to the "memory" 748 * type. See memblock_add_range() description for mode details 749 * 750 * Return: 751 * 0 on success, -errno on failure. 752 */ 753 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 754 { 755 phys_addr_t end = base + size - 1; 756 757 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 758 &base, &end, (void *)_RET_IP_); 759 760 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0); 761 } 762 763 /** 764 * memblock_validate_numa_coverage - check if amount of memory with 765 * no node ID assigned is less than a threshold 766 * @threshold_bytes: maximal memory size that can have unassigned node 767 * ID (in bytes). 768 * 769 * A buggy firmware may report memory that does not belong to any node. 770 * Check if amount of such memory is below @threshold_bytes. 771 * 772 * Return: true on success, false on failure. 773 */ 774 bool __init_memblock memblock_validate_numa_coverage(unsigned long threshold_bytes) 775 { 776 unsigned long nr_pages = 0; 777 unsigned long start_pfn, end_pfn, mem_size_mb; 778 int nid, i; 779 780 /* calculate lost page */ 781 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 782 if (!numa_valid_node(nid)) 783 nr_pages += end_pfn - start_pfn; 784 } 785 786 if ((nr_pages << PAGE_SHIFT) > threshold_bytes) { 787 mem_size_mb = memblock_phys_mem_size() / SZ_1M; 788 pr_err("NUMA: no nodes coverage for %luMB of %luMB RAM\n", 789 (nr_pages << PAGE_SHIFT) / SZ_1M, mem_size_mb); 790 return false; 791 } 792 793 return true; 794 } 795 796 797 /** 798 * memblock_isolate_range - isolate given range into disjoint memblocks 799 * @type: memblock type to isolate range for 800 * @base: base of range to isolate 801 * @size: size of range to isolate 802 * @start_rgn: out parameter for the start of isolated region 803 * @end_rgn: out parameter for the end of isolated region 804 * 805 * Walk @type and ensure that regions don't cross the boundaries defined by 806 * [@base, @base + @size). Crossing regions are split at the boundaries, 807 * which may create at most two more regions. The index of the first 808 * region inside the range is returned in *@start_rgn and the index of the 809 * first region after the range is returned in *@end_rgn. 810 * 811 * Return: 812 * 0 on success, -errno on failure. 813 */ 814 static int __init_memblock memblock_isolate_range(struct memblock_type *type, 815 phys_addr_t base, phys_addr_t size, 816 int *start_rgn, int *end_rgn) 817 { 818 phys_addr_t end = base + memblock_cap_size(base, &size); 819 int idx; 820 struct memblock_region *rgn; 821 822 *start_rgn = *end_rgn = 0; 823 824 if (!size) 825 return 0; 826 827 /* we'll create at most two more regions */ 828 while (type->cnt + 2 > type->max) 829 if (memblock_double_array(type, base, size) < 0) 830 return -ENOMEM; 831 832 for_each_memblock_type(idx, type, rgn) { 833 phys_addr_t rbase = rgn->base; 834 phys_addr_t rend = rbase + rgn->size; 835 836 if (rbase >= end) 837 break; 838 if (rend <= base) 839 continue; 840 841 if (rbase < base) { 842 /* 843 * @rgn intersects from below. Split and continue 844 * to process the next region - the new top half. 845 */ 846 rgn->base = base; 847 rgn->size -= base - rbase; 848 type->total_size -= base - rbase; 849 memblock_insert_region(type, idx, rbase, base - rbase, 850 memblock_get_region_node(rgn), 851 rgn->flags); 852 } else if (rend > end) { 853 /* 854 * @rgn intersects from above. Split and redo the 855 * current region - the new bottom half. 856 */ 857 rgn->base = end; 858 rgn->size -= end - rbase; 859 type->total_size -= end - rbase; 860 memblock_insert_region(type, idx--, rbase, end - rbase, 861 memblock_get_region_node(rgn), 862 rgn->flags); 863 } else { 864 /* @rgn is fully contained, record it */ 865 if (!*end_rgn) 866 *start_rgn = idx; 867 *end_rgn = idx + 1; 868 } 869 } 870 871 return 0; 872 } 873 874 static int __init_memblock memblock_remove_range(struct memblock_type *type, 875 phys_addr_t base, phys_addr_t size) 876 { 877 int start_rgn, end_rgn; 878 int i, ret; 879 880 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 881 if (ret) 882 return ret; 883 884 for (i = end_rgn - 1; i >= start_rgn; i--) 885 memblock_remove_region(type, i); 886 return 0; 887 } 888 889 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 890 { 891 phys_addr_t end = base + size - 1; 892 893 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 894 &base, &end, (void *)_RET_IP_); 895 896 return memblock_remove_range(&memblock.memory, base, size); 897 } 898 899 static unsigned long __free_reserved_area(phys_addr_t start, phys_addr_t end, 900 int poison) 901 { 902 unsigned long pages = 0, pfn; 903 904 if (deferred_pages_enabled()) { 905 WARN(1, "Cannot free reserved memory because of deferred initialization of the memory map"); 906 return 0; 907 } 908 909 for_each_valid_pfn(pfn, PFN_UP(start), PFN_DOWN(end)) { 910 struct page *page = pfn_to_page(pfn); 911 void *direct_map_addr; 912 913 /* 914 * 'direct_map_addr' might be different from the kernel virtual 915 * address because some architectures use aliases. 916 * Going via physical address, pfn_to_page() and page_address() 917 * ensures that we get a _writeable_ alias for the memset(). 918 */ 919 direct_map_addr = page_address(page); 920 /* 921 * Perform a kasan-unchecked memset() since this memory 922 * has not been initialized. 923 */ 924 direct_map_addr = kasan_reset_tag(direct_map_addr); 925 if ((unsigned int)poison <= 0xFF) 926 memset(direct_map_addr, poison, PAGE_SIZE); 927 928 free_reserved_page(page); 929 pages++; 930 } 931 return pages; 932 } 933 934 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 935 { 936 phys_addr_t start_pa, end_pa; 937 unsigned long pages; 938 939 /* 940 * end is the first address past the region and it may be beyond what 941 * __pa() or __pa_symbol() can handle. 942 * Use the address included in the range for the conversion and add back 943 * 1 afterwards. 944 */ 945 if (__is_kernel((unsigned long)start)) { 946 start_pa = __pa_symbol(start); 947 end_pa = __pa_symbol(end - 1) + 1; 948 } else { 949 start_pa = __pa(start); 950 end_pa = __pa(end - 1) + 1; 951 } 952 953 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) { 954 if (start_pa < end_pa) 955 memblock_remove_range(&memblock.reserved, 956 start_pa, end_pa - start_pa); 957 } 958 959 pages = __free_reserved_area(start_pa, end_pa, poison); 960 if (pages && s) 961 pr_info("Freeing %s memory: %ldK\n", s, K(pages)); 962 963 return pages; 964 } 965 966 /** 967 * memblock_free - free boot memory allocation 968 * @ptr: starting address of the boot memory allocation 969 * @size: size of the boot memory block in bytes 970 * 971 * Free boot memory block previously allocated by memblock_alloc_xx() API. 972 * If called after the buddy allocator is available, the memory is released to 973 * the buddy allocator. 974 */ 975 void __init_memblock memblock_free(void *ptr, size_t size) 976 { 977 if (ptr) 978 memblock_phys_free(__pa(ptr), size); 979 } 980 981 /** 982 * memblock_phys_free - free boot memory block 983 * @base: phys starting address of the boot memory block 984 * @size: size of the boot memory block in bytes 985 * 986 * Free boot memory block previously allocated by memblock_phys_alloc_xx() API. 987 * If called after the buddy allocator is available, the memory is released to 988 * the buddy allocator. 989 */ 990 int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size) 991 { 992 phys_addr_t end = base + size - 1; 993 int ret = 0; 994 995 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 996 &base, &end, (void *)_RET_IP_); 997 998 kmemleak_free_part_phys(base, size); 999 1000 if (!slab_is_available() || IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) 1001 ret = memblock_remove_range(&memblock.reserved, base, size); 1002 1003 if (slab_is_available()) 1004 __free_reserved_area(base, base + size, -1); 1005 1006 return ret; 1007 } 1008 1009 int __init_memblock __memblock_reserve(phys_addr_t base, phys_addr_t size, 1010 int nid, enum memblock_flags flags) 1011 { 1012 phys_addr_t end = base + size - 1; 1013 1014 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 1015 &base, &end, nid, flags, (void *)_RET_IP_); 1016 1017 return memblock_add_range(&memblock.reserved, base, size, nid, flags); 1018 } 1019 1020 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 1021 int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) 1022 { 1023 phys_addr_t end = base + size - 1; 1024 1025 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 1026 &base, &end, (void *)_RET_IP_); 1027 1028 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0); 1029 } 1030 #endif 1031 1032 /** 1033 * memblock_setclr_flag - set or clear flag for a memory region 1034 * @type: memblock type to set/clear flag for 1035 * @base: base address of the region 1036 * @size: size of the region 1037 * @set: set or clear the flag 1038 * @flag: the flag to update 1039 * 1040 * This function isolates region [@base, @base + @size), and sets/clears flag 1041 * 1042 * Return: 0 on success, -errno on failure. 1043 */ 1044 static int __init_memblock memblock_setclr_flag(struct memblock_type *type, 1045 phys_addr_t base, phys_addr_t size, int set, int flag) 1046 { 1047 int i, ret, start_rgn, end_rgn; 1048 1049 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1050 if (ret) 1051 return ret; 1052 1053 for (i = start_rgn; i < end_rgn; i++) { 1054 struct memblock_region *r = &type->regions[i]; 1055 1056 if (set) 1057 r->flags |= flag; 1058 else 1059 r->flags &= ~flag; 1060 } 1061 1062 memblock_merge_regions(type, start_rgn, end_rgn); 1063 return 0; 1064 } 1065 1066 /** 1067 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 1068 * @base: the base phys addr of the region 1069 * @size: the size of the region 1070 * 1071 * Return: 0 on success, -errno on failure. 1072 */ 1073 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 1074 { 1075 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_HOTPLUG); 1076 } 1077 1078 /** 1079 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 1080 * @base: the base phys addr of the region 1081 * @size: the size of the region 1082 * 1083 * Return: 0 on success, -errno on failure. 1084 */ 1085 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 1086 { 1087 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_HOTPLUG); 1088 } 1089 1090 /** 1091 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR. 1092 * @base: the base phys addr of the region 1093 * @size: the size of the region 1094 * 1095 * Return: 0 on success, -errno on failure. 1096 */ 1097 int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size) 1098 { 1099 if (!mirrored_kernelcore) 1100 return 0; 1101 1102 system_has_some_mirror = true; 1103 1104 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_MIRROR); 1105 } 1106 1107 /** 1108 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP. 1109 * @base: the base phys addr of the region 1110 * @size: the size of the region 1111 * 1112 * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the 1113 * direct mapping of the physical memory. These regions will still be 1114 * covered by the memory map. The struct page representing NOMAP memory 1115 * frames in the memory map will be PageReserved() 1116 * 1117 * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from 1118 * memblock, the caller must inform kmemleak to ignore that memory 1119 * 1120 * Return: 0 on success, -errno on failure. 1121 */ 1122 int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size) 1123 { 1124 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_NOMAP); 1125 } 1126 1127 /** 1128 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region. 1129 * @base: the base phys addr of the region 1130 * @size: the size of the region 1131 * 1132 * Return: 0 on success, -errno on failure. 1133 */ 1134 int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size) 1135 { 1136 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_NOMAP); 1137 } 1138 1139 /** 1140 * memblock_reserved_mark_noinit - Mark a reserved memory region with flag 1141 * MEMBLOCK_RSRV_NOINIT 1142 * 1143 * @base: the base phys addr of the region 1144 * @size: the size of the region 1145 * 1146 * The struct pages for the reserved regions marked %MEMBLOCK_RSRV_NOINIT will 1147 * not be fully initialized to allow the caller optimize their initialization. 1148 * 1149 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled, setting this flag 1150 * completely bypasses the initialization of struct pages for such region. 1151 * 1152 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is disabled, struct pages in this 1153 * region will be initialized with default values but won't be marked as 1154 * reserved. 1155 * 1156 * Return: 0 on success, -errno on failure. 1157 */ 1158 int __init_memblock memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size) 1159 { 1160 return memblock_setclr_flag(&memblock.reserved, base, size, 1, 1161 MEMBLOCK_RSRV_NOINIT); 1162 } 1163 1164 /** 1165 * memblock_reserved_mark_kern - Mark a reserved memory region with flag 1166 * MEMBLOCK_RSRV_KERN 1167 * 1168 * @base: the base phys addr of the region 1169 * @size: the size of the region 1170 * 1171 * Return: 0 on success, -errno on failure. 1172 */ 1173 int __init_memblock memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t size) 1174 { 1175 return memblock_setclr_flag(&memblock.reserved, base, size, 1, 1176 MEMBLOCK_RSRV_KERN); 1177 } 1178 1179 /** 1180 * memblock_mark_kho_scratch - Mark a memory region as MEMBLOCK_KHO_SCRATCH. 1181 * @base: the base phys addr of the region 1182 * @size: the size of the region 1183 * 1184 * Only memory regions marked with %MEMBLOCK_KHO_SCRATCH will be considered 1185 * for allocations during early boot with kexec handover. 1186 * 1187 * Return: 0 on success, -errno on failure. 1188 */ 1189 __init int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size) 1190 { 1191 return memblock_setclr_flag(&memblock.memory, base, size, 1, 1192 MEMBLOCK_KHO_SCRATCH); 1193 } 1194 1195 /** 1196 * memblock_clear_kho_scratch - Clear MEMBLOCK_KHO_SCRATCH flag for a 1197 * specified region. 1198 * @base: the base phys addr of the region 1199 * @size: the size of the region 1200 * 1201 * Return: 0 on success, -errno on failure. 1202 */ 1203 __init int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size) 1204 { 1205 return memblock_setclr_flag(&memblock.memory, base, size, 0, 1206 MEMBLOCK_KHO_SCRATCH); 1207 } 1208 1209 static bool should_skip_region(struct memblock_type *type, 1210 struct memblock_region *m, 1211 int nid, int flags) 1212 { 1213 int m_nid = memblock_get_region_node(m); 1214 1215 /* we never skip regions when iterating memblock.reserved or physmem */ 1216 if (type != memblock_memory) 1217 return false; 1218 1219 /* only memory regions are associated with nodes, check it */ 1220 if (numa_valid_node(nid) && nid != m_nid) 1221 return true; 1222 1223 /* skip hotpluggable memory regions if needed */ 1224 if (movable_node_is_enabled() && memblock_is_hotpluggable(m) && 1225 !(flags & MEMBLOCK_HOTPLUG)) 1226 return true; 1227 1228 /* if we want mirror memory skip non-mirror memory regions */ 1229 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 1230 return true; 1231 1232 /* skip nomap memory unless we were asked for it explicitly */ 1233 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 1234 return true; 1235 1236 /* skip driver-managed memory unless we were asked for it explicitly */ 1237 if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m)) 1238 return true; 1239 1240 /* 1241 * In early alloc during kexec handover, we can only consider 1242 * MEMBLOCK_KHO_SCRATCH regions for the allocations 1243 */ 1244 if ((flags & MEMBLOCK_KHO_SCRATCH) && !memblock_is_kho_scratch(m)) 1245 return true; 1246 1247 return false; 1248 } 1249 1250 /** 1251 * __next_mem_range - next function for for_each_free_mem_range() etc. 1252 * @idx: pointer to u64 loop variable 1253 * @nid: node selector, %NUMA_NO_NODE for all nodes 1254 * @flags: pick from blocks based on memory attributes 1255 * @type_a: pointer to memblock_type from where the range is taken 1256 * @type_b: pointer to memblock_type which excludes memory from being taken 1257 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1258 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1259 * @out_nid: ptr to int for nid of the range, can be %NULL 1260 * 1261 * Find the first area from *@idx which matches @nid, fill the out 1262 * parameters, and update *@idx for the next iteration. The lower 32bit of 1263 * *@idx contains index into type_a and the upper 32bit indexes the 1264 * areas before each region in type_b. For example, if type_b regions 1265 * look like the following, 1266 * 1267 * 0:[0-16), 1:[32-48), 2:[128-130) 1268 * 1269 * The upper 32bit indexes the following regions. 1270 * 1271 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 1272 * 1273 * As both region arrays are sorted, the function advances the two indices 1274 * in lockstep and returns each intersection. 1275 */ 1276 void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags, 1277 struct memblock_type *type_a, 1278 struct memblock_type *type_b, phys_addr_t *out_start, 1279 phys_addr_t *out_end, int *out_nid) 1280 { 1281 int idx_a = *idx & 0xffffffff; 1282 int idx_b = *idx >> 32; 1283 1284 for (; idx_a < type_a->cnt; idx_a++) { 1285 struct memblock_region *m = &type_a->regions[idx_a]; 1286 1287 phys_addr_t m_start = m->base; 1288 phys_addr_t m_end = m->base + m->size; 1289 int m_nid = memblock_get_region_node(m); 1290 1291 if (should_skip_region(type_a, m, nid, flags)) 1292 continue; 1293 1294 if (!type_b) { 1295 if (out_start) 1296 *out_start = m_start; 1297 if (out_end) 1298 *out_end = m_end; 1299 if (out_nid) 1300 *out_nid = m_nid; 1301 idx_a++; 1302 *idx = (u32)idx_a | (u64)idx_b << 32; 1303 return; 1304 } 1305 1306 /* scan areas before each reservation */ 1307 for (; idx_b < type_b->cnt + 1; idx_b++) { 1308 struct memblock_region *r; 1309 phys_addr_t r_start; 1310 phys_addr_t r_end; 1311 1312 r = &type_b->regions[idx_b]; 1313 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1314 r_end = idx_b < type_b->cnt ? 1315 r->base : PHYS_ADDR_MAX; 1316 1317 /* 1318 * if idx_b advanced past idx_a, 1319 * break out to advance idx_a 1320 */ 1321 if (r_start >= m_end) 1322 break; 1323 /* if the two regions intersect, we're done */ 1324 if (m_start < r_end) { 1325 if (out_start) 1326 *out_start = 1327 max(m_start, r_start); 1328 if (out_end) 1329 *out_end = min(m_end, r_end); 1330 if (out_nid) 1331 *out_nid = m_nid; 1332 /* 1333 * The region which ends first is 1334 * advanced for the next iteration. 1335 */ 1336 if (m_end <= r_end) 1337 idx_a++; 1338 else 1339 idx_b++; 1340 *idx = (u32)idx_a | (u64)idx_b << 32; 1341 return; 1342 } 1343 } 1344 } 1345 1346 /* signal end of iteration */ 1347 *idx = ULLONG_MAX; 1348 } 1349 1350 /** 1351 * __next_mem_range_rev - generic next function for for_each_*_range_rev() 1352 * 1353 * @idx: pointer to u64 loop variable 1354 * @nid: node selector, %NUMA_NO_NODE for all nodes 1355 * @flags: pick from blocks based on memory attributes 1356 * @type_a: pointer to memblock_type from where the range is taken 1357 * @type_b: pointer to memblock_type which excludes memory from being taken 1358 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1359 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1360 * @out_nid: ptr to int for nid of the range, can be %NULL 1361 * 1362 * Finds the next range from type_a which is not marked as unsuitable 1363 * in type_b. 1364 * 1365 * Reverse of __next_mem_range(). 1366 */ 1367 void __init_memblock __next_mem_range_rev(u64 *idx, int nid, 1368 enum memblock_flags flags, 1369 struct memblock_type *type_a, 1370 struct memblock_type *type_b, 1371 phys_addr_t *out_start, 1372 phys_addr_t *out_end, int *out_nid) 1373 { 1374 int idx_a = *idx & 0xffffffff; 1375 int idx_b = *idx >> 32; 1376 1377 if (*idx == (u64)ULLONG_MAX) { 1378 idx_a = type_a->cnt - 1; 1379 if (type_b != NULL) 1380 idx_b = type_b->cnt; 1381 else 1382 idx_b = 0; 1383 } 1384 1385 for (; idx_a >= 0; idx_a--) { 1386 struct memblock_region *m = &type_a->regions[idx_a]; 1387 1388 phys_addr_t m_start = m->base; 1389 phys_addr_t m_end = m->base + m->size; 1390 int m_nid = memblock_get_region_node(m); 1391 1392 if (should_skip_region(type_a, m, nid, flags)) 1393 continue; 1394 1395 if (!type_b) { 1396 if (out_start) 1397 *out_start = m_start; 1398 if (out_end) 1399 *out_end = m_end; 1400 if (out_nid) 1401 *out_nid = m_nid; 1402 idx_a--; 1403 *idx = (u32)idx_a | (u64)idx_b << 32; 1404 return; 1405 } 1406 1407 /* scan areas before each reservation */ 1408 for (; idx_b >= 0; idx_b--) { 1409 struct memblock_region *r; 1410 phys_addr_t r_start; 1411 phys_addr_t r_end; 1412 1413 r = &type_b->regions[idx_b]; 1414 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1415 r_end = idx_b < type_b->cnt ? 1416 r->base : PHYS_ADDR_MAX; 1417 /* 1418 * if idx_b advanced past idx_a, 1419 * break out to advance idx_a 1420 */ 1421 1422 if (r_end <= m_start) 1423 break; 1424 /* if the two regions intersect, we're done */ 1425 if (m_end > r_start) { 1426 if (out_start) 1427 *out_start = max(m_start, r_start); 1428 if (out_end) 1429 *out_end = min(m_end, r_end); 1430 if (out_nid) 1431 *out_nid = m_nid; 1432 if (m_start >= r_start) 1433 idx_a--; 1434 else 1435 idx_b--; 1436 *idx = (u32)idx_a | (u64)idx_b << 32; 1437 return; 1438 } 1439 } 1440 } 1441 /* signal end of iteration */ 1442 *idx = ULLONG_MAX; 1443 } 1444 1445 /* 1446 * Common iterator interface used to define for_each_mem_pfn_range(). 1447 */ 1448 void __init_memblock __next_mem_pfn_range(int *idx, int nid, 1449 unsigned long *out_start_pfn, 1450 unsigned long *out_end_pfn, int *out_nid) 1451 { 1452 struct memblock_type *type = &memblock.memory; 1453 struct memblock_region *r; 1454 int r_nid; 1455 1456 while (++*idx < type->cnt) { 1457 r = &type->regions[*idx]; 1458 r_nid = memblock_get_region_node(r); 1459 1460 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 1461 continue; 1462 if (!numa_valid_node(nid) || nid == r_nid) 1463 break; 1464 } 1465 if (*idx >= type->cnt) { 1466 *idx = -1; 1467 return; 1468 } 1469 1470 if (out_start_pfn) 1471 *out_start_pfn = PFN_UP(r->base); 1472 if (out_end_pfn) 1473 *out_end_pfn = PFN_DOWN(r->base + r->size); 1474 if (out_nid) 1475 *out_nid = r_nid; 1476 } 1477 1478 /** 1479 * memblock_set_node - set node ID on memblock regions 1480 * @base: base of area to set node ID for 1481 * @size: size of area to set node ID for 1482 * @type: memblock type to set node ID for 1483 * @nid: node ID to set 1484 * 1485 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid. 1486 * Regions which cross the area boundaries are split as necessary. 1487 * 1488 * Return: 1489 * 0 on success, -errno on failure. 1490 */ 1491 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 1492 struct memblock_type *type, int nid) 1493 { 1494 #ifdef CONFIG_NUMA 1495 int start_rgn, end_rgn; 1496 int i, ret; 1497 1498 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1499 if (ret) 1500 return ret; 1501 1502 for (i = start_rgn; i < end_rgn; i++) 1503 memblock_set_region_node(&type->regions[i], nid); 1504 1505 memblock_merge_regions(type, start_rgn, end_rgn); 1506 #endif 1507 return 0; 1508 } 1509 1510 /** 1511 * memblock_alloc_range_nid - allocate boot memory block 1512 * @size: size of memory block to be allocated in bytes 1513 * @align: alignment of the region and block's size 1514 * @start: the lower bound of the memory region to allocate (phys address) 1515 * @end: the upper bound of the memory region to allocate (phys address) 1516 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1517 * @exact_nid: control the allocation fall back to other nodes 1518 * 1519 * The allocation is performed from memory region limited by 1520 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE. 1521 * 1522 * If the specified node can not hold the requested memory and @exact_nid 1523 * is false, the allocation falls back to any node in the system. 1524 * 1525 * For systems with memory mirroring, the allocation is attempted first 1526 * from the regions with mirroring enabled and then retried from any 1527 * memory region. 1528 * 1529 * In addition, function using kmemleak_alloc_phys for allocated boot 1530 * memory block, it is never reported as leaks. 1531 * 1532 * Return: 1533 * Physical address of allocated memory block on success, %0 on failure. 1534 */ 1535 phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, 1536 phys_addr_t align, phys_addr_t start, 1537 phys_addr_t end, int nid, 1538 bool exact_nid) 1539 { 1540 enum memblock_flags flags = choose_memblock_flags(); 1541 phys_addr_t found; 1542 1543 /* 1544 * Detect any accidental use of these APIs after slab is ready, as at 1545 * this moment memblock may be deinitialized already and its 1546 * internal data may be destroyed (after execution of memblock_free_all) 1547 */ 1548 if (WARN_ON_ONCE(slab_is_available())) { 1549 void *vaddr = kzalloc_node(size, GFP_NOWAIT, nid); 1550 1551 return vaddr ? virt_to_phys(vaddr) : 0; 1552 } 1553 1554 if (!align) { 1555 /* Can't use WARNs this early in boot on powerpc */ 1556 dump_stack(); 1557 align = SMP_CACHE_BYTES; 1558 } 1559 1560 again: 1561 found = memblock_find_in_range_node(size, align, start, end, nid, 1562 flags); 1563 if (found && !__memblock_reserve(found, size, nid, MEMBLOCK_RSRV_KERN)) 1564 goto done; 1565 1566 if (numa_valid_node(nid) && !exact_nid) { 1567 found = memblock_find_in_range_node(size, align, start, 1568 end, NUMA_NO_NODE, 1569 flags); 1570 if (found && !memblock_reserve_kern(found, size)) 1571 goto done; 1572 } 1573 1574 if (flags & MEMBLOCK_MIRROR) { 1575 flags &= ~MEMBLOCK_MIRROR; 1576 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 1577 &size); 1578 goto again; 1579 } 1580 1581 return 0; 1582 1583 done: 1584 /* 1585 * Skip kmemleak for those places like kasan_init() and 1586 * early_pgtable_alloc() due to high volume. 1587 */ 1588 if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) 1589 /* 1590 * Memblock allocated blocks are never reported as 1591 * leaks. This is because many of these blocks are 1592 * only referred via the physical address which is 1593 * not looked up by kmemleak. 1594 */ 1595 kmemleak_alloc_phys(found, size, 0); 1596 1597 /* 1598 * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, 1599 * require memory to be accepted before it can be used by the 1600 * guest. 1601 * 1602 * Accept the memory of the allocated buffer. 1603 */ 1604 accept_memory(found, size); 1605 1606 return found; 1607 } 1608 1609 /** 1610 * memblock_phys_alloc_range - allocate a memory block inside specified range 1611 * @size: size of memory block to be allocated in bytes 1612 * @align: alignment of the region and block's size 1613 * @start: the lower bound of the memory region to allocate (physical address) 1614 * @end: the upper bound of the memory region to allocate (physical address) 1615 * 1616 * Allocate @size bytes in the between @start and @end. 1617 * 1618 * Return: physical address of the allocated memory block on success, 1619 * %0 on failure. 1620 */ 1621 phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, 1622 phys_addr_t align, 1623 phys_addr_t start, 1624 phys_addr_t end) 1625 { 1626 memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n", 1627 __func__, (u64)size, (u64)align, &start, &end, 1628 (void *)_RET_IP_); 1629 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, 1630 false); 1631 } 1632 1633 /** 1634 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node 1635 * @size: size of memory block to be allocated in bytes 1636 * @align: alignment of the region and block's size 1637 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1638 * 1639 * Allocates memory block from the specified NUMA node. If the node 1640 * has no available memory, attempts to allocated from any node in the 1641 * system. 1642 * 1643 * Return: physical address of the allocated memory block on success, 1644 * %0 on failure. 1645 */ 1646 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1647 { 1648 return memblock_alloc_range_nid(size, align, 0, 1649 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false); 1650 } 1651 1652 /** 1653 * memblock_alloc_internal - allocate boot memory block 1654 * @size: size of memory block to be allocated in bytes 1655 * @align: alignment of the region and block's size 1656 * @min_addr: the lower bound of the memory region to allocate (phys address) 1657 * @max_addr: the upper bound of the memory region to allocate (phys address) 1658 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1659 * @exact_nid: control the allocation fall back to other nodes 1660 * 1661 * Allocates memory block using memblock_alloc_range_nid() and 1662 * converts the returned physical address to virtual. 1663 * 1664 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1665 * will fall back to memory below @min_addr. Other constraints, such 1666 * as node and mirrored memory will be handled again in 1667 * memblock_alloc_range_nid(). 1668 * 1669 * Return: 1670 * Virtual address of allocated memory block on success, NULL on failure. 1671 */ 1672 static void * __init memblock_alloc_internal( 1673 phys_addr_t size, phys_addr_t align, 1674 phys_addr_t min_addr, phys_addr_t max_addr, 1675 int nid, bool exact_nid) 1676 { 1677 phys_addr_t alloc; 1678 1679 1680 if (max_addr > memblock.current_limit) 1681 max_addr = memblock.current_limit; 1682 1683 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid, 1684 exact_nid); 1685 1686 /* retry allocation without lower limit */ 1687 if (!alloc && min_addr) 1688 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid, 1689 exact_nid); 1690 1691 if (!alloc) 1692 return NULL; 1693 1694 return phys_to_virt(alloc); 1695 } 1696 1697 /** 1698 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node 1699 * without zeroing memory 1700 * @size: size of memory block to be allocated in bytes 1701 * @align: alignment of the region and block's size 1702 * @min_addr: the lower bound of the memory region from where the allocation 1703 * is preferred (phys address) 1704 * @max_addr: the upper bound of the memory region from where the allocation 1705 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1706 * allocate only from memory limited by memblock.current_limit value 1707 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1708 * 1709 * Public function, provides additional debug information (including caller 1710 * info), if enabled. Does not zero allocated memory. 1711 * 1712 * Return: 1713 * Virtual address of allocated memory block on success, NULL on failure. 1714 */ 1715 void * __init memblock_alloc_exact_nid_raw( 1716 phys_addr_t size, phys_addr_t align, 1717 phys_addr_t min_addr, phys_addr_t max_addr, 1718 int nid) 1719 { 1720 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1721 __func__, (u64)size, (u64)align, nid, &min_addr, 1722 &max_addr, (void *)_RET_IP_); 1723 1724 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1725 true); 1726 } 1727 1728 /** 1729 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing 1730 * memory and without panicking 1731 * @size: size of memory block to be allocated in bytes 1732 * @align: alignment of the region and block's size 1733 * @min_addr: the lower bound of the memory region from where the allocation 1734 * is preferred (phys address) 1735 * @max_addr: the upper bound of the memory region from where the allocation 1736 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1737 * allocate only from memory limited by memblock.current_limit value 1738 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1739 * 1740 * Public function, provides additional debug information (including caller 1741 * info), if enabled. Does not zero allocated memory, does not panic if request 1742 * cannot be satisfied. 1743 * 1744 * Return: 1745 * Virtual address of allocated memory block on success, NULL on failure. 1746 */ 1747 void * __init memblock_alloc_try_nid_raw( 1748 phys_addr_t size, phys_addr_t align, 1749 phys_addr_t min_addr, phys_addr_t max_addr, 1750 int nid) 1751 { 1752 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1753 __func__, (u64)size, (u64)align, nid, &min_addr, 1754 &max_addr, (void *)_RET_IP_); 1755 1756 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1757 false); 1758 } 1759 1760 /** 1761 * memblock_alloc_try_nid - allocate boot memory block 1762 * @size: size of memory block to be allocated in bytes 1763 * @align: alignment of the region and block's size 1764 * @min_addr: the lower bound of the memory region from where the allocation 1765 * is preferred (phys address) 1766 * @max_addr: the upper bound of the memory region from where the allocation 1767 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1768 * allocate only from memory limited by memblock.current_limit value 1769 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1770 * 1771 * Public function, provides additional debug information (including caller 1772 * info), if enabled. This function zeroes the allocated memory. 1773 * 1774 * Return: 1775 * Virtual address of allocated memory block on success, NULL on failure. 1776 */ 1777 void * __init memblock_alloc_try_nid( 1778 phys_addr_t size, phys_addr_t align, 1779 phys_addr_t min_addr, phys_addr_t max_addr, 1780 int nid) 1781 { 1782 void *ptr; 1783 1784 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1785 __func__, (u64)size, (u64)align, nid, &min_addr, 1786 &max_addr, (void *)_RET_IP_); 1787 ptr = memblock_alloc_internal(size, align, 1788 min_addr, max_addr, nid, false); 1789 if (ptr) 1790 memset(ptr, 0, size); 1791 1792 return ptr; 1793 } 1794 1795 /** 1796 * __memblock_alloc_or_panic - Try to allocate memory and panic on failure 1797 * @size: size of memory block to be allocated in bytes 1798 * @align: alignment of the region and block's size 1799 * @func: caller func name 1800 * 1801 * This function attempts to allocate memory using memblock_alloc, 1802 * and in case of failure, it calls panic with the formatted message. 1803 * This function should not be used directly, please use the macro memblock_alloc_or_panic. 1804 */ 1805 void *__init __memblock_alloc_or_panic(phys_addr_t size, phys_addr_t align, 1806 const char *func) 1807 { 1808 void *addr = memblock_alloc(size, align); 1809 1810 if (unlikely(!addr)) 1811 panic("%s: Failed to allocate %pap bytes\n", func, &size); 1812 return addr; 1813 } 1814 1815 /* 1816 * Remaining API functions 1817 */ 1818 1819 phys_addr_t __init_memblock memblock_phys_mem_size(void) 1820 { 1821 return memblock.memory.total_size; 1822 } 1823 1824 phys_addr_t __init_memblock memblock_reserved_size(void) 1825 { 1826 return memblock.reserved.total_size; 1827 } 1828 1829 phys_addr_t __init_memblock memblock_reserved_kern_size(phys_addr_t limit, int nid) 1830 { 1831 struct memblock_region *r; 1832 phys_addr_t total = 0; 1833 1834 for_each_reserved_mem_region(r) { 1835 phys_addr_t size = r->size; 1836 1837 if (r->base > limit) 1838 break; 1839 1840 if (r->base + r->size > limit) 1841 size = limit - r->base; 1842 1843 if (nid == memblock_get_region_node(r) || !numa_valid_node(nid)) 1844 if (r->flags & MEMBLOCK_RSRV_KERN) 1845 total += size; 1846 } 1847 1848 return total; 1849 } 1850 1851 /** 1852 * memblock_estimated_nr_free_pages - return estimated number of free pages 1853 * from memblock point of view 1854 * 1855 * During bootup, subsystems might need a rough estimate of the number of free 1856 * pages in the whole system, before precise numbers are available from the 1857 * buddy. Especially with CONFIG_DEFERRED_STRUCT_PAGE_INIT, the numbers 1858 * obtained from the buddy might be very imprecise during bootup. 1859 * 1860 * Return: 1861 * An estimated number of free pages from memblock point of view. 1862 */ 1863 unsigned long __init memblock_estimated_nr_free_pages(void) 1864 { 1865 return PHYS_PFN(memblock_phys_mem_size() - 1866 memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, NUMA_NO_NODE)); 1867 } 1868 1869 /* lowest address */ 1870 phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1871 { 1872 return memblock.memory.regions[0].base; 1873 } 1874 1875 phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1876 { 1877 int idx = memblock.memory.cnt - 1; 1878 1879 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1880 } 1881 1882 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) 1883 { 1884 phys_addr_t max_addr = PHYS_ADDR_MAX; 1885 struct memblock_region *r; 1886 1887 /* 1888 * translate the memory @limit size into the max address within one of 1889 * the memory memblock regions, if the @limit exceeds the total size 1890 * of those regions, max_addr will keep original value PHYS_ADDR_MAX 1891 */ 1892 for_each_mem_region(r) { 1893 if (limit <= r->size) { 1894 max_addr = r->base + limit; 1895 break; 1896 } 1897 limit -= r->size; 1898 } 1899 1900 return max_addr; 1901 } 1902 1903 void __init memblock_enforce_memory_limit(phys_addr_t limit) 1904 { 1905 phys_addr_t max_addr; 1906 1907 if (!limit) 1908 return; 1909 1910 max_addr = __find_max_addr(limit); 1911 1912 /* @limit exceeds the total size of the memory, do nothing */ 1913 if (max_addr == PHYS_ADDR_MAX) 1914 return; 1915 1916 /* truncate both memory and reserved regions */ 1917 memblock_remove_range(&memblock.memory, max_addr, 1918 PHYS_ADDR_MAX); 1919 memblock_remove_range(&memblock.reserved, max_addr, 1920 PHYS_ADDR_MAX); 1921 } 1922 1923 void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size) 1924 { 1925 int start_rgn, end_rgn; 1926 int i, ret; 1927 1928 if (!size) 1929 return; 1930 1931 if (!memblock_memory->total_size) { 1932 pr_warn("%s: No memory registered yet\n", __func__); 1933 return; 1934 } 1935 1936 ret = memblock_isolate_range(&memblock.memory, base, size, 1937 &start_rgn, &end_rgn); 1938 if (ret) 1939 return; 1940 1941 /* remove all the MAP regions */ 1942 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--) 1943 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1944 memblock_remove_region(&memblock.memory, i); 1945 1946 for (i = start_rgn - 1; i >= 0; i--) 1947 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1948 memblock_remove_region(&memblock.memory, i); 1949 1950 /* truncate the reserved regions */ 1951 memblock_remove_range(&memblock.reserved, 0, base); 1952 memblock_remove_range(&memblock.reserved, 1953 base + size, PHYS_ADDR_MAX); 1954 } 1955 1956 void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1957 { 1958 phys_addr_t max_addr; 1959 1960 if (!limit) 1961 return; 1962 1963 max_addr = __find_max_addr(limit); 1964 1965 /* @limit exceeds the total size of the memory, do nothing */ 1966 if (max_addr == PHYS_ADDR_MAX) 1967 return; 1968 1969 memblock_cap_memory_range(0, max_addr); 1970 } 1971 1972 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1973 { 1974 unsigned int left = 0, right = type->cnt; 1975 1976 do { 1977 unsigned int mid = (right + left) / 2; 1978 1979 if (addr < type->regions[mid].base) 1980 right = mid; 1981 else if (addr >= (type->regions[mid].base + 1982 type->regions[mid].size)) 1983 left = mid + 1; 1984 else 1985 return mid; 1986 } while (left < right); 1987 return -1; 1988 } 1989 1990 bool __init_memblock memblock_is_reserved(phys_addr_t addr) 1991 { 1992 return memblock_search(&memblock.reserved, addr) != -1; 1993 } 1994 1995 bool __init_memblock memblock_is_memory(phys_addr_t addr) 1996 { 1997 return memblock_search(&memblock.memory, addr) != -1; 1998 } 1999 2000 bool __init_memblock memblock_is_map_memory(phys_addr_t addr) 2001 { 2002 int i = memblock_search(&memblock.memory, addr); 2003 2004 if (i == -1) 2005 return false; 2006 return !memblock_is_nomap(&memblock.memory.regions[i]); 2007 } 2008 2009 int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 2010 unsigned long *start_pfn, unsigned long *end_pfn) 2011 { 2012 struct memblock_type *type = &memblock.memory; 2013 int mid = memblock_search(type, PFN_PHYS(pfn)); 2014 2015 if (mid == -1) 2016 return NUMA_NO_NODE; 2017 2018 *start_pfn = PFN_DOWN(type->regions[mid].base); 2019 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size); 2020 2021 return memblock_get_region_node(&type->regions[mid]); 2022 } 2023 2024 /** 2025 * memblock_is_region_memory - check if a region is a subset of memory 2026 * @base: base of region to check 2027 * @size: size of region to check 2028 * 2029 * Check if the region [@base, @base + @size) is a subset of a memory block. 2030 * 2031 * Return: 2032 * 0 if false, non-zero if true 2033 */ 2034 bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 2035 { 2036 int idx = memblock_search(&memblock.memory, base); 2037 phys_addr_t end = base + memblock_cap_size(base, &size); 2038 2039 if (idx == -1) 2040 return false; 2041 return (memblock.memory.regions[idx].base + 2042 memblock.memory.regions[idx].size) >= end; 2043 } 2044 2045 /** 2046 * memblock_is_region_reserved - check if a region intersects reserved memory 2047 * @base: base of region to check 2048 * @size: size of region to check 2049 * 2050 * Check if the region [@base, @base + @size) intersects a reserved 2051 * memory block. 2052 * 2053 * Return: 2054 * True if they intersect, false if not. 2055 */ 2056 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 2057 { 2058 return memblock_overlaps_region(&memblock.reserved, base, size); 2059 } 2060 2061 void __init_memblock memblock_trim_memory(phys_addr_t align) 2062 { 2063 phys_addr_t start, end, orig_start, orig_end; 2064 struct memblock_region *r; 2065 2066 for_each_mem_region(r) { 2067 orig_start = r->base; 2068 orig_end = r->base + r->size; 2069 start = round_up(orig_start, align); 2070 end = round_down(orig_end, align); 2071 2072 if (start == orig_start && end == orig_end) 2073 continue; 2074 2075 if (start < end) { 2076 r->base = start; 2077 r->size = end - start; 2078 } else { 2079 memblock_remove_region(&memblock.memory, 2080 r - memblock.memory.regions); 2081 r--; 2082 } 2083 } 2084 } 2085 2086 void __init_memblock memblock_set_current_limit(phys_addr_t limit) 2087 { 2088 memblock.current_limit = limit; 2089 } 2090 2091 phys_addr_t __init_memblock memblock_get_current_limit(void) 2092 { 2093 return memblock.current_limit; 2094 } 2095 2096 static void __init_memblock memblock_dump(struct memblock_type *type) 2097 { 2098 phys_addr_t base, end, size; 2099 enum memblock_flags flags; 2100 int idx; 2101 struct memblock_region *rgn; 2102 2103 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt); 2104 2105 for_each_memblock_type(idx, type, rgn) { 2106 char nid_buf[32] = ""; 2107 2108 base = rgn->base; 2109 size = rgn->size; 2110 end = base + size - 1; 2111 flags = rgn->flags; 2112 #ifdef CONFIG_NUMA 2113 if (numa_valid_node(memblock_get_region_node(rgn))) 2114 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 2115 memblock_get_region_node(rgn)); 2116 #endif 2117 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n", 2118 type->name, idx, &base, &end, &size, nid_buf, flags); 2119 } 2120 } 2121 2122 static void __init_memblock __memblock_dump_all(void) 2123 { 2124 pr_info("MEMBLOCK configuration:\n"); 2125 pr_info(" memory size = %pa reserved size = %pa\n", 2126 &memblock.memory.total_size, 2127 &memblock.reserved.total_size); 2128 2129 memblock_dump(&memblock.memory); 2130 memblock_dump(&memblock.reserved); 2131 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2132 memblock_dump(&physmem); 2133 #endif 2134 } 2135 2136 void __init_memblock memblock_dump_all(void) 2137 { 2138 if (memblock_debug) 2139 __memblock_dump_all(); 2140 } 2141 2142 void __init memblock_allow_resize(void) 2143 { 2144 memblock_can_resize = 1; 2145 } 2146 2147 static int __init early_memblock(char *p) 2148 { 2149 if (p && strstr(p, "debug")) 2150 memblock_debug = 1; 2151 return 0; 2152 } 2153 early_param("memblock", early_memblock); 2154 2155 static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn) 2156 { 2157 struct page *start_pg, *end_pg; 2158 phys_addr_t pg, pgend; 2159 2160 /* 2161 * Convert start_pfn/end_pfn to a struct page pointer. 2162 */ 2163 start_pg = pfn_to_page(start_pfn - 1) + 1; 2164 end_pg = pfn_to_page(end_pfn - 1) + 1; 2165 2166 /* 2167 * Convert to physical addresses, and round start upwards and end 2168 * downwards. 2169 */ 2170 pg = PAGE_ALIGN(__pa(start_pg)); 2171 pgend = PAGE_ALIGN_DOWN(__pa(end_pg)); 2172 2173 /* 2174 * If there are free pages between these, free the section of the 2175 * memmap array. 2176 */ 2177 if (pg < pgend) 2178 memblock_phys_free(pg, pgend - pg); 2179 } 2180 2181 /* 2182 * The mem_map array can get very big. Free the unused area of the memory map. 2183 */ 2184 static void __init free_unused_memmap(void) 2185 { 2186 unsigned long start, end, prev_end = 0; 2187 int i; 2188 2189 if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) || 2190 IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 2191 return; 2192 2193 /* 2194 * This relies on each bank being in address order. 2195 * The banks are sorted previously in bootmem_init(). 2196 */ 2197 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { 2198 #ifdef CONFIG_SPARSEMEM 2199 /* 2200 * Take care not to free memmap entries that don't exist 2201 * due to SPARSEMEM sections which aren't present. 2202 */ 2203 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION)); 2204 #endif 2205 /* 2206 * Align down here since many operations in VM subsystem 2207 * presume that there are no holes in the memory map inside 2208 * a pageblock 2209 */ 2210 start = pageblock_start_pfn(start); 2211 2212 /* 2213 * If we had a previous bank, and there is a space 2214 * between the current bank and the previous, free it. 2215 */ 2216 if (prev_end && prev_end < start) 2217 free_memmap(prev_end, start); 2218 2219 /* 2220 * Align up here since many operations in VM subsystem 2221 * presume that there are no holes in the memory map inside 2222 * a pageblock 2223 */ 2224 prev_end = pageblock_align(end); 2225 } 2226 2227 #ifdef CONFIG_SPARSEMEM 2228 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) 2229 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); 2230 #endif 2231 } 2232 2233 static void __init __free_pages_memory(unsigned long start, unsigned long end) 2234 { 2235 int order; 2236 2237 while (start < end) { 2238 /* 2239 * Free the pages in the largest chunks alignment allows. 2240 * 2241 * __ffs() behaviour is undefined for 0. start == 0 is 2242 * MAX_PAGE_ORDER-aligned, set order to MAX_PAGE_ORDER for 2243 * the case. 2244 */ 2245 if (start) 2246 order = min_t(int, MAX_PAGE_ORDER, __ffs(start)); 2247 else 2248 order = MAX_PAGE_ORDER; 2249 2250 while (start + (1UL << order) > end) 2251 order--; 2252 2253 memblock_free_pages(start, order); 2254 2255 start += (1UL << order); 2256 } 2257 } 2258 2259 static unsigned long __init __free_memory_core(phys_addr_t start, 2260 phys_addr_t end) 2261 { 2262 unsigned long start_pfn = PFN_UP(start); 2263 unsigned long end_pfn = PFN_DOWN(end); 2264 2265 if (!IS_ENABLED(CONFIG_HIGHMEM) && end_pfn > max_low_pfn) 2266 end_pfn = max_low_pfn; 2267 2268 if (start_pfn >= end_pfn) 2269 return 0; 2270 2271 __free_pages_memory(start_pfn, end_pfn); 2272 2273 return end_pfn - start_pfn; 2274 } 2275 2276 /* 2277 * Initialised pages do not have PageReserved set. This function is called 2278 * for each reserved range and marks the pages PageReserved. 2279 * When deferred initialization of struct pages is enabled it also ensures 2280 * that struct pages are properly initialised. 2281 */ 2282 static void __init memmap_init_reserved_range(phys_addr_t start, 2283 phys_addr_t end, int nid) 2284 { 2285 unsigned long pfn; 2286 2287 for_each_valid_pfn(pfn, PFN_DOWN(start), PFN_UP(end)) { 2288 struct page *page = pfn_to_page(pfn); 2289 2290 init_deferred_page(pfn, nid); 2291 2292 /* 2293 * no need for atomic set_bit because the struct 2294 * page is not visible yet so nobody should 2295 * access it yet. 2296 */ 2297 __SetPageReserved(page); 2298 } 2299 } 2300 2301 static void __init memmap_init_reserved_pages(void) 2302 { 2303 struct memblock_region *region; 2304 phys_addr_t start, end; 2305 int nid; 2306 unsigned long max_reserved; 2307 2308 /* 2309 * set nid on all reserved pages and also treat struct 2310 * pages for the NOMAP regions as PageReserved 2311 */ 2312 repeat: 2313 max_reserved = memblock.reserved.max; 2314 for_each_mem_region(region) { 2315 nid = memblock_get_region_node(region); 2316 start = region->base; 2317 end = start + region->size; 2318 2319 if (memblock_is_nomap(region)) 2320 memmap_init_reserved_range(start, end, nid); 2321 2322 memblock_set_node(start, region->size, &memblock.reserved, nid); 2323 } 2324 /* 2325 * 'max' is changed means memblock.reserved has been doubled its 2326 * array, which may result a new reserved region before current 2327 * 'start'. Now we should repeat the procedure to set its node id. 2328 */ 2329 if (max_reserved != memblock.reserved.max) 2330 goto repeat; 2331 2332 /* 2333 * initialize struct pages for reserved regions that don't have 2334 * the MEMBLOCK_RSRV_NOINIT flag set 2335 */ 2336 for_each_reserved_mem_region(region) { 2337 if (!memblock_is_reserved_noinit(region)) { 2338 nid = memblock_get_region_node(region); 2339 start = region->base; 2340 end = start + region->size; 2341 2342 if (!numa_valid_node(nid)) 2343 nid = early_pfn_to_nid(PFN_DOWN(start)); 2344 2345 memmap_init_reserved_range(start, end, nid); 2346 } 2347 } 2348 } 2349 2350 static unsigned long __init free_low_memory_core_early(void) 2351 { 2352 unsigned long count = 0; 2353 phys_addr_t start, end; 2354 u64 i; 2355 2356 memblock_clear_hotplug(0, -1); 2357 2358 memmap_init_reserved_pages(); 2359 2360 /* 2361 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id 2362 * because in some case like Node0 doesn't have RAM installed 2363 * low ram will be on Node1 2364 */ 2365 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, 2366 NULL) 2367 count += __free_memory_core(start, end); 2368 2369 return count; 2370 } 2371 2372 static int reset_managed_pages_done __initdata; 2373 2374 static void __init reset_node_managed_pages(pg_data_t *pgdat) 2375 { 2376 struct zone *z; 2377 2378 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 2379 atomic_long_set(&z->managed_pages, 0); 2380 } 2381 2382 void __init reset_all_zones_managed_pages(void) 2383 { 2384 struct pglist_data *pgdat; 2385 2386 if (reset_managed_pages_done) 2387 return; 2388 2389 for_each_online_pgdat(pgdat) 2390 reset_node_managed_pages(pgdat); 2391 2392 reset_managed_pages_done = 1; 2393 } 2394 2395 /** 2396 * memblock_free_all - release free pages to the buddy allocator 2397 */ 2398 void __init memblock_free_all(void) 2399 { 2400 unsigned long pages; 2401 2402 free_unused_memmap(); 2403 reset_all_zones_managed_pages(); 2404 2405 memblock_clear_kho_scratch_only(); 2406 pages = free_low_memory_core_early(); 2407 totalram_pages_add(pages); 2408 } 2409 2410 /* Keep a table to reserve named memory */ 2411 #define RESERVE_MEM_MAX_ENTRIES 8 2412 #define RESERVE_MEM_NAME_SIZE 16 2413 struct reserve_mem_table { 2414 char name[RESERVE_MEM_NAME_SIZE]; 2415 phys_addr_t start; 2416 phys_addr_t size; 2417 }; 2418 static struct reserve_mem_table reserved_mem_table[RESERVE_MEM_MAX_ENTRIES]; 2419 static int reserved_mem_count; 2420 static DEFINE_MUTEX(reserve_mem_lock); 2421 2422 /* Add wildcard region with a lookup name */ 2423 static void __init reserved_mem_add(phys_addr_t start, phys_addr_t size, 2424 const char *name) 2425 { 2426 struct reserve_mem_table *map; 2427 2428 map = &reserved_mem_table[reserved_mem_count++]; 2429 map->start = start; 2430 map->size = size; 2431 strscpy(map->name, name); 2432 } 2433 2434 static struct reserve_mem_table *reserve_mem_find_by_name_nolock(const char *name) 2435 { 2436 struct reserve_mem_table *map; 2437 int i; 2438 2439 for (i = 0; i < reserved_mem_count; i++) { 2440 map = &reserved_mem_table[i]; 2441 if (!map->size) 2442 continue; 2443 if (strcmp(name, map->name) == 0) 2444 return map; 2445 } 2446 return NULL; 2447 } 2448 2449 /** 2450 * reserve_mem_find_by_name - Find reserved memory region with a given name 2451 * @name: The name that is attached to a reserved memory region 2452 * @start: If found, holds the start address 2453 * @size: If found, holds the size of the address. 2454 * 2455 * @start and @size are only updated if @name is found. 2456 * 2457 * Returns: 1 if found or 0 if not found. 2458 */ 2459 int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size) 2460 { 2461 struct reserve_mem_table *map; 2462 2463 guard(mutex)(&reserve_mem_lock); 2464 map = reserve_mem_find_by_name_nolock(name); 2465 if (!map) 2466 return 0; 2467 2468 *start = map->start; 2469 *size = map->size; 2470 return 1; 2471 } 2472 EXPORT_SYMBOL_GPL(reserve_mem_find_by_name); 2473 2474 /** 2475 * reserve_mem_release_by_name - Release reserved memory region with a given name 2476 * @name: The name that is attached to a reserved memory region 2477 * 2478 * Forcibly release the pages in the reserved memory region so that those memory 2479 * can be used as free memory. After released the reserved region size becomes 0. 2480 * 2481 * Returns: 1 if released or 0 if not found. 2482 */ 2483 int reserve_mem_release_by_name(const char *name) 2484 { 2485 char buf[RESERVE_MEM_NAME_SIZE + 12]; 2486 struct reserve_mem_table *map; 2487 void *start, *end; 2488 2489 guard(mutex)(&reserve_mem_lock); 2490 map = reserve_mem_find_by_name_nolock(name); 2491 if (!map) 2492 return 0; 2493 2494 start = phys_to_virt(map->start); 2495 end = start + map->size; 2496 snprintf(buf, sizeof(buf), "reserve_mem:%s", name); 2497 free_reserved_area(start, end, 0, buf); 2498 map->size = 0; 2499 2500 return 1; 2501 } 2502 2503 #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 2504 __init void memblock_set_kho_scratch_only(void) 2505 { 2506 kho_scratch_only = true; 2507 } 2508 2509 __init void memblock_clear_kho_scratch_only(void) 2510 { 2511 kho_scratch_only = false; 2512 } 2513 2514 bool __init_memblock memblock_is_kho_scratch_memory(phys_addr_t addr) 2515 { 2516 int i = memblock_search(&memblock.memory, addr); 2517 2518 if (i == -1) 2519 return false; 2520 2521 return memblock_is_kho_scratch(&memblock.memory.regions[i]); 2522 } 2523 #endif 2524 2525 #ifdef CONFIG_KEXEC_HANDOVER 2526 2527 static int __init reserved_mem_preserve(void) 2528 { 2529 unsigned int nr_preserved = 0; 2530 int err; 2531 2532 for (unsigned int i = 0; i < reserved_mem_count; i++, nr_preserved++) { 2533 struct reserve_mem_table *map = &reserved_mem_table[i]; 2534 struct page *page = phys_to_page(map->start); 2535 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2536 2537 err = kho_preserve_pages(page, nr_pages); 2538 if (err) 2539 goto err_unpreserve; 2540 } 2541 2542 return 0; 2543 2544 err_unpreserve: 2545 for (unsigned int i = 0; i < nr_preserved; i++) { 2546 struct reserve_mem_table *map = &reserved_mem_table[i]; 2547 struct page *page = phys_to_page(map->start); 2548 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2549 2550 kho_unpreserve_pages(page, nr_pages); 2551 } 2552 2553 return err; 2554 } 2555 2556 static int __init prepare_kho_fdt(void) 2557 { 2558 struct page *fdt_page; 2559 void *fdt; 2560 int err; 2561 2562 fdt_page = alloc_page(GFP_KERNEL); 2563 if (!fdt_page) { 2564 err = -ENOMEM; 2565 goto err_report; 2566 } 2567 2568 fdt = page_to_virt(fdt_page); 2569 err = kho_preserve_pages(fdt_page, 1); 2570 if (err) 2571 goto err_free_fdt; 2572 2573 err |= fdt_create(fdt, PAGE_SIZE); 2574 err |= fdt_finish_reservemap(fdt); 2575 err |= fdt_begin_node(fdt, ""); 2576 err |= fdt_property_string(fdt, "compatible", MEMBLOCK_KHO_NODE_COMPATIBLE); 2577 2578 for (unsigned int i = 0; !err && i < reserved_mem_count; i++) { 2579 struct reserve_mem_table *map = &reserved_mem_table[i]; 2580 2581 err |= fdt_begin_node(fdt, map->name); 2582 err |= fdt_property_string(fdt, "compatible", RESERVE_MEM_KHO_NODE_COMPATIBLE); 2583 err |= fdt_property(fdt, "start", &map->start, sizeof(map->start)); 2584 err |= fdt_property(fdt, "size", &map->size, sizeof(map->size)); 2585 err |= fdt_end_node(fdt); 2586 } 2587 err |= fdt_end_node(fdt); 2588 err |= fdt_finish(fdt); 2589 2590 if (err) 2591 goto err_unpreserve_fdt; 2592 2593 err = kho_add_subtree(MEMBLOCK_KHO_FDT, fdt, fdt_totalsize(fdt)); 2594 if (err) 2595 goto err_unpreserve_fdt; 2596 2597 err = reserved_mem_preserve(); 2598 if (err) 2599 goto err_remove_subtree; 2600 2601 return 0; 2602 2603 err_remove_subtree: 2604 kho_remove_subtree(fdt); 2605 err_unpreserve_fdt: 2606 kho_unpreserve_pages(fdt_page, 1); 2607 err_free_fdt: 2608 put_page(fdt_page); 2609 err_report: 2610 pr_err("failed to prepare memblock FDT for KHO: %d\n", err); 2611 2612 return err; 2613 } 2614 2615 static int __init reserve_mem_init(void) 2616 { 2617 int err; 2618 2619 if (!kho_is_enabled() || !reserved_mem_count) 2620 return 0; 2621 2622 err = prepare_kho_fdt(); 2623 if (err) 2624 return err; 2625 return err; 2626 } 2627 late_initcall(reserve_mem_init); 2628 2629 static void *__init reserve_mem_kho_retrieve_fdt(void) 2630 { 2631 phys_addr_t fdt_phys; 2632 static void *fdt; 2633 int err; 2634 2635 if (fdt) 2636 return fdt; 2637 2638 err = kho_retrieve_subtree(MEMBLOCK_KHO_FDT, &fdt_phys, NULL); 2639 if (err) { 2640 if (err != -ENOENT) 2641 pr_warn("failed to retrieve FDT '%s' from KHO: %d\n", 2642 MEMBLOCK_KHO_FDT, err); 2643 return NULL; 2644 } 2645 2646 fdt = phys_to_virt(fdt_phys); 2647 2648 err = fdt_node_check_compatible(fdt, 0, MEMBLOCK_KHO_NODE_COMPATIBLE); 2649 if (err) { 2650 pr_warn("FDT '%s' is incompatible with '%s': %d\n", 2651 MEMBLOCK_KHO_FDT, MEMBLOCK_KHO_NODE_COMPATIBLE, err); 2652 fdt = NULL; 2653 } 2654 2655 return fdt; 2656 } 2657 2658 static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2659 phys_addr_t align) 2660 { 2661 int err, len_start, len_size, offset; 2662 const phys_addr_t *p_start, *p_size; 2663 const void *fdt; 2664 2665 fdt = reserve_mem_kho_retrieve_fdt(); 2666 if (!fdt) 2667 return false; 2668 2669 offset = fdt_subnode_offset(fdt, 0, name); 2670 if (offset < 0) { 2671 pr_warn("FDT '%s' has no child '%s': %d\n", 2672 MEMBLOCK_KHO_FDT, name, offset); 2673 return false; 2674 } 2675 err = fdt_node_check_compatible(fdt, offset, RESERVE_MEM_KHO_NODE_COMPATIBLE); 2676 if (err) { 2677 pr_warn("Node '%s' is incompatible with '%s': %d\n", 2678 name, RESERVE_MEM_KHO_NODE_COMPATIBLE, err); 2679 return false; 2680 } 2681 2682 p_start = fdt_getprop(fdt, offset, "start", &len_start); 2683 p_size = fdt_getprop(fdt, offset, "size", &len_size); 2684 if (!p_start || len_start != sizeof(*p_start) || !p_size || 2685 len_size != sizeof(*p_size)) { 2686 return false; 2687 } 2688 2689 if (*p_start & (align - 1)) { 2690 pr_warn("KHO reserve-mem '%s' has wrong alignment (0x%lx, 0x%lx)\n", 2691 name, (long)align, (long)*p_start); 2692 return false; 2693 } 2694 2695 if (*p_size != size) { 2696 pr_warn("KHO reserve-mem '%s' has wrong size (0x%lx != 0x%lx)\n", 2697 name, (long)*p_size, (long)size); 2698 return false; 2699 } 2700 2701 reserved_mem_add(*p_start, size, name); 2702 pr_info("Revived memory reservation '%s' from KHO\n", name); 2703 2704 return true; 2705 } 2706 #else 2707 static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2708 phys_addr_t align) 2709 { 2710 return false; 2711 } 2712 #endif /* CONFIG_KEXEC_HANDOVER */ 2713 2714 /* 2715 * Parse reserve_mem=nn:align:name 2716 */ 2717 static int __init reserve_mem(char *p) 2718 { 2719 phys_addr_t start, size, align, tmp; 2720 char *name; 2721 char *oldp; 2722 int len; 2723 2724 if (!p) 2725 goto err_param; 2726 2727 /* Check if there's room for more reserved memory */ 2728 if (reserved_mem_count >= RESERVE_MEM_MAX_ENTRIES) { 2729 pr_err("reserve_mem: no more room for reserved memory\n"); 2730 return -EBUSY; 2731 } 2732 2733 oldp = p; 2734 size = memparse(p, &p); 2735 if (!size || p == oldp) 2736 goto err_param; 2737 2738 if (*p != ':') 2739 goto err_param; 2740 2741 align = memparse(p+1, &p); 2742 if (*p != ':') 2743 goto err_param; 2744 2745 /* 2746 * memblock_phys_alloc() doesn't like a zero size align, 2747 * but it is OK for this command to have it. 2748 */ 2749 if (align < SMP_CACHE_BYTES) 2750 align = SMP_CACHE_BYTES; 2751 2752 name = p + 1; 2753 len = strlen(name); 2754 2755 /* name needs to have length but not too big */ 2756 if (!len || len >= RESERVE_MEM_NAME_SIZE) 2757 goto err_param; 2758 2759 /* Make sure that name has text */ 2760 for (p = name; *p; p++) { 2761 if (!isspace(*p)) 2762 break; 2763 } 2764 if (!*p) 2765 goto err_param; 2766 2767 /* Make sure the name is not already used */ 2768 if (reserve_mem_find_by_name(name, &start, &tmp)) { 2769 pr_err("reserve_mem: name \"%s\" was already used\n", name); 2770 return -EBUSY; 2771 } 2772 2773 /* Pick previous allocations up from KHO if available */ 2774 if (reserve_mem_kho_revive(name, size, align)) 2775 return 1; 2776 2777 /* TODO: Allocation must be outside of scratch region */ 2778 start = memblock_phys_alloc(size, align); 2779 if (!start) { 2780 pr_err("reserve_mem: memblock allocation failed\n"); 2781 return -ENOMEM; 2782 } 2783 2784 reserved_mem_add(start, size, name); 2785 2786 return 1; 2787 err_param: 2788 pr_err("reserve_mem: empty or malformed parameter\n"); 2789 return -EINVAL; 2790 } 2791 __setup("reserve_mem=", reserve_mem); 2792 2793 #ifdef CONFIG_DEBUG_FS 2794 #ifdef CONFIG_ARCH_KEEP_MEMBLOCK 2795 static const char * const flagname[] = { 2796 [ilog2(MEMBLOCK_HOTPLUG)] = "HOTPLUG", 2797 [ilog2(MEMBLOCK_MIRROR)] = "MIRROR", 2798 [ilog2(MEMBLOCK_NOMAP)] = "NOMAP", 2799 [ilog2(MEMBLOCK_DRIVER_MANAGED)] = "DRV_MNG", 2800 [ilog2(MEMBLOCK_RSRV_NOINIT)] = "RSV_NIT", 2801 [ilog2(MEMBLOCK_RSRV_KERN)] = "RSV_KERN", 2802 [ilog2(MEMBLOCK_KHO_SCRATCH)] = "KHO_SCRATCH", 2803 }; 2804 2805 static int memblock_debug_show(struct seq_file *m, void *private) 2806 { 2807 struct memblock_type *type = m->private; 2808 struct memblock_region *reg; 2809 int i, j, nid; 2810 unsigned int count = ARRAY_SIZE(flagname); 2811 phys_addr_t end; 2812 2813 for (i = 0; i < type->cnt; i++) { 2814 reg = &type->regions[i]; 2815 end = reg->base + reg->size - 1; 2816 nid = memblock_get_region_node(reg); 2817 2818 seq_printf(m, "%4d: ", i); 2819 seq_printf(m, "%pa..%pa ", ®->base, &end); 2820 if (numa_valid_node(nid)) 2821 seq_printf(m, "%4d ", nid); 2822 else 2823 seq_printf(m, "%4c ", 'x'); 2824 if (reg->flags) { 2825 for (j = 0; j < count; j++) { 2826 if (reg->flags & (1U << j)) { 2827 seq_printf(m, "%s\n", flagname[j]); 2828 break; 2829 } 2830 } 2831 if (j == count) 2832 seq_printf(m, "%s\n", "UNKNOWN"); 2833 } else { 2834 seq_printf(m, "%s\n", "NONE"); 2835 } 2836 } 2837 return 0; 2838 } 2839 DEFINE_SHOW_ATTRIBUTE(memblock_debug); 2840 2841 static inline void memblock_debugfs_expose_arrays(struct dentry *root) 2842 { 2843 debugfs_create_file("memory", 0444, root, 2844 &memblock.memory, &memblock_debug_fops); 2845 debugfs_create_file("reserved", 0444, root, 2846 &memblock.reserved, &memblock_debug_fops); 2847 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2848 debugfs_create_file("physmem", 0444, root, &physmem, 2849 &memblock_debug_fops); 2850 #endif 2851 } 2852 2853 #else 2854 2855 static inline void memblock_debugfs_expose_arrays(struct dentry *root) { } 2856 2857 #endif /* CONFIG_ARCH_KEEP_MEMBLOCK */ 2858 2859 static int memblock_reserve_mem_show(struct seq_file *m, void *private) 2860 { 2861 struct reserve_mem_table *map; 2862 char txtsz[16]; 2863 2864 guard(mutex)(&reserve_mem_lock); 2865 for (int i = 0; i < reserved_mem_count; i++) { 2866 map = &reserved_mem_table[i]; 2867 if (!map->size) 2868 continue; 2869 2870 memset(txtsz, 0, sizeof(txtsz)); 2871 string_get_size(map->size, 1, STRING_UNITS_2, txtsz, sizeof(txtsz)); 2872 seq_printf(m, "%s\t\t(%s)\n", map->name, txtsz); 2873 } 2874 2875 return 0; 2876 } 2877 DEFINE_SHOW_ATTRIBUTE(memblock_reserve_mem); 2878 2879 static int __init memblock_init_debugfs(void) 2880 { 2881 struct dentry *root; 2882 2883 if (!IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK) && !reserved_mem_count) 2884 return 0; 2885 2886 root = debugfs_create_dir("memblock", NULL); 2887 2888 if (reserved_mem_count) 2889 debugfs_create_file("reserve_mem_param", 0444, root, NULL, 2890 &memblock_reserve_mem_fops); 2891 2892 memblock_debugfs_expose_arrays(root); 2893 return 0; 2894 } 2895 __initcall(memblock_init_debugfs); 2896 2897 #endif /* CONFIG_DEBUG_FS */ 2898