1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Procedures for maintaining information about logical memory blocks. 4 * 5 * Peter Bergner, IBM Corp. June 2001. 6 * Copyright (C) 2001 Peter Bergner. 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/slab.h> 11 #include <linux/init.h> 12 #include <linux/bitops.h> 13 #include <linux/poison.h> 14 #include <linux/pfn.h> 15 #include <linux/debugfs.h> 16 #include <linux/kmemleak.h> 17 #include <linux/seq_file.h> 18 #include <linux/memblock.h> 19 #include <linux/mutex.h> 20 #include <linux/string_helpers.h> 21 22 #ifdef CONFIG_KEXEC_HANDOVER 23 #include <linux/libfdt.h> 24 #include <linux/kexec_handover.h> 25 #include <linux/kho/abi/memblock.h> 26 #endif /* CONFIG_KEXEC_HANDOVER */ 27 28 #include <asm/sections.h> 29 #include <linux/io.h> 30 31 #include "internal.h" 32 33 #define INIT_MEMBLOCK_REGIONS 128 34 #define INIT_PHYSMEM_REGIONS 4 35 36 #ifndef INIT_MEMBLOCK_RESERVED_REGIONS 37 # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS 38 #endif 39 40 #ifndef INIT_MEMBLOCK_MEMORY_REGIONS 41 #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS 42 #endif 43 44 /** 45 * DOC: memblock overview 46 * 47 * Memblock is a method of managing memory regions during the early 48 * boot period when the usual kernel memory allocators are not up and 49 * running. 50 * 51 * Memblock views the system memory as collections of contiguous 52 * regions. There are several types of these collections: 53 * 54 * * ``memory`` - describes the physical memory available to the 55 * kernel; this may differ from the actual physical memory installed 56 * in the system, for instance when the memory is restricted with 57 * ``mem=`` command line parameter 58 * * ``reserved`` - describes the regions that were allocated 59 * * ``physmem`` - describes the actual physical memory available during 60 * boot regardless of the possible restrictions and memory hot(un)plug; 61 * the ``physmem`` type is only available on some architectures. 62 * 63 * Each region is represented by struct memblock_region that 64 * defines the region extents, its attributes and NUMA node id on NUMA 65 * systems. Every memory type is described by the struct memblock_type 66 * which contains an array of memory regions along with 67 * the allocator metadata. The "memory" and "reserved" types are nicely 68 * wrapped with struct memblock. This structure is statically 69 * initialized at build time. The region arrays are initially sized to 70 * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and 71 * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array 72 * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS. 73 * The memblock_allow_resize() enables automatic resizing of the region 74 * arrays during addition of new regions. This feature should be used 75 * with care so that memory allocated for the region array will not 76 * overlap with areas that should be reserved, for example initrd. 77 * 78 * The early architecture setup should tell memblock what the physical 79 * memory layout is by using memblock_add() or memblock_add_node() 80 * functions. The first function does not assign the region to a NUMA 81 * node and it is appropriate for UMA systems. Yet, it is possible to 82 * use it on NUMA systems as well and assign the region to a NUMA node 83 * later in the setup process using memblock_set_node(). The 84 * memblock_add_node() performs such an assignment directly. 85 * 86 * Once memblock is setup the memory can be allocated using one of the 87 * API variants: 88 * 89 * * memblock_phys_alloc*() - these functions return the **physical** 90 * address of the allocated memory 91 * * memblock_alloc*() - these functions return the **virtual** address 92 * of the allocated memory. 93 * 94 * Note, that both API variants use implicit assumptions about allowed 95 * memory ranges and the fallback methods. Consult the documentation 96 * of memblock_alloc_internal() and memblock_alloc_range_nid() 97 * functions for more elaborate description. 98 * 99 * As the system boot progresses, the architecture specific mem_init() 100 * function frees all the memory to the buddy page allocator. 101 * 102 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the 103 * memblock data structures (except "physmem") will be discarded after the 104 * system initialization completes. 105 */ 106 107 #ifndef CONFIG_NUMA 108 struct pglist_data __refdata contig_page_data; 109 EXPORT_SYMBOL(contig_page_data); 110 #endif 111 112 unsigned long max_low_pfn; 113 unsigned long min_low_pfn; 114 unsigned long max_pfn; 115 unsigned long long max_possible_pfn; 116 117 #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 118 /* When set to true, only allocate from MEMBLOCK_KHO_SCRATCH ranges */ 119 static bool kho_scratch_only; 120 #else 121 #define kho_scratch_only false 122 #endif 123 124 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock; 125 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock; 126 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 127 static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS]; 128 #endif 129 130 struct memblock memblock __initdata_memblock = { 131 .memory.regions = memblock_memory_init_regions, 132 .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS, 133 .memory.name = "memory", 134 135 .reserved.regions = memblock_reserved_init_regions, 136 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS, 137 .reserved.name = "reserved", 138 139 .bottom_up = false, 140 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 141 }; 142 143 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 144 struct memblock_type physmem = { 145 .regions = memblock_physmem_init_regions, 146 .max = INIT_PHYSMEM_REGIONS, 147 .name = "physmem", 148 }; 149 #endif 150 151 /* 152 * keep a pointer to &memblock.memory in the text section to use it in 153 * __next_mem_range() and its helpers. 154 * For architectures that do not keep memblock data after init, this 155 * pointer will be reset to NULL at memblock_discard() 156 */ 157 static __refdata struct memblock_type *memblock_memory = &memblock.memory; 158 159 #define for_each_memblock_type(i, memblock_type, rgn) \ 160 for (i = 0, rgn = &memblock_type->regions[0]; \ 161 i < memblock_type->cnt; \ 162 i++, rgn = &memblock_type->regions[i]) 163 164 #define memblock_dbg(fmt, ...) \ 165 do { \ 166 if (memblock_debug) \ 167 pr_info(fmt, ##__VA_ARGS__); \ 168 } while (0) 169 170 static int memblock_debug __initdata_memblock; 171 static bool system_has_some_mirror __initdata_memblock; 172 static int memblock_can_resize __initdata_memblock; 173 static int memblock_memory_in_slab __initdata_memblock; 174 static int memblock_reserved_in_slab __initdata_memblock; 175 176 bool __init_memblock memblock_has_mirror(void) 177 { 178 return system_has_some_mirror; 179 } 180 181 static enum memblock_flags __init_memblock choose_memblock_flags(void) 182 { 183 /* skip non-scratch memory for kho early boot allocations */ 184 if (kho_scratch_only) 185 return MEMBLOCK_KHO_SCRATCH; 186 187 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; 188 } 189 190 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 191 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 192 { 193 return *size = min(*size, PHYS_ADDR_MAX - base); 194 } 195 196 /* 197 * Address comparison utilities 198 */ 199 unsigned long __init_memblock 200 memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, phys_addr_t base2, 201 phys_addr_t size2) 202 { 203 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 204 } 205 206 bool __init_memblock memblock_overlaps_region(struct memblock_type *type, 207 phys_addr_t base, phys_addr_t size) 208 { 209 unsigned long i; 210 211 memblock_cap_size(base, &size); 212 213 for (i = 0; i < type->cnt; i++) 214 if (memblock_addrs_overlap(base, size, type->regions[i].base, 215 type->regions[i].size)) 216 return true; 217 return false; 218 } 219 220 /** 221 * __memblock_find_range_bottom_up - find free area utility in bottom-up 222 * @start: start of candidate range 223 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 224 * %MEMBLOCK_ALLOC_ACCESSIBLE 225 * @size: size of free area to find 226 * @align: alignment of free area to find 227 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 228 * @flags: pick from blocks based on memory attributes 229 * 230 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 231 * 232 * Return: 233 * Found address on success, 0 on failure. 234 */ 235 static phys_addr_t __init_memblock 236 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 237 phys_addr_t size, phys_addr_t align, int nid, 238 enum memblock_flags flags) 239 { 240 phys_addr_t this_start, this_end, cand; 241 u64 i; 242 243 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) { 244 this_start = clamp(this_start, start, end); 245 this_end = clamp(this_end, start, end); 246 247 cand = round_up(this_start, align); 248 if (cand < this_end && this_end - cand >= size) 249 return cand; 250 } 251 252 return 0; 253 } 254 255 /** 256 * __memblock_find_range_top_down - find free area utility, in top-down 257 * @start: start of candidate range 258 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 259 * %MEMBLOCK_ALLOC_ACCESSIBLE 260 * @size: size of free area to find 261 * @align: alignment of free area to find 262 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 263 * @flags: pick from blocks based on memory attributes 264 * 265 * Utility called from memblock_find_in_range_node(), find free area top-down. 266 * 267 * Return: 268 * Found address on success, 0 on failure. 269 */ 270 static phys_addr_t __init_memblock 271 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 272 phys_addr_t size, phys_addr_t align, int nid, 273 enum memblock_flags flags) 274 { 275 phys_addr_t this_start, this_end, cand; 276 u64 i; 277 278 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end, 279 NULL) { 280 this_start = clamp(this_start, start, end); 281 this_end = clamp(this_end, start, end); 282 283 if (this_end < size) 284 continue; 285 286 cand = round_down(this_end - size, align); 287 if (cand >= this_start) 288 return cand; 289 } 290 291 return 0; 292 } 293 294 /** 295 * memblock_find_in_range_node - find free area in given range and node 296 * @size: size of free area to find 297 * @align: alignment of free area to find 298 * @start: start of candidate range 299 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 300 * %MEMBLOCK_ALLOC_ACCESSIBLE 301 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 302 * @flags: pick from blocks based on memory attributes 303 * 304 * Find @size free area aligned to @align in the specified range and node. 305 * 306 * Return: 307 * Found address on success, 0 on failure. 308 */ 309 static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 310 phys_addr_t align, phys_addr_t start, 311 phys_addr_t end, int nid, 312 enum memblock_flags flags) 313 { 314 /* pump up @end */ 315 if (end == MEMBLOCK_ALLOC_ACCESSIBLE || 316 end == MEMBLOCK_ALLOC_NOLEAKTRACE) 317 end = memblock.current_limit; 318 319 /* avoid allocating the first page */ 320 start = max_t(phys_addr_t, start, PAGE_SIZE); 321 end = max(start, end); 322 323 if (memblock_bottom_up()) 324 return __memblock_find_range_bottom_up(start, end, size, align, 325 nid, flags); 326 else 327 return __memblock_find_range_top_down(start, end, size, align, 328 nid, flags); 329 } 330 331 /** 332 * memblock_find_in_range - find free area in given range 333 * @start: start of candidate range 334 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 335 * %MEMBLOCK_ALLOC_ACCESSIBLE 336 * @size: size of free area to find 337 * @align: alignment of free area to find 338 * 339 * Find @size free area aligned to @align in the specified range. 340 * 341 * Return: 342 * Found address on success, 0 on failure. 343 */ 344 static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 345 phys_addr_t end, phys_addr_t size, 346 phys_addr_t align) 347 { 348 phys_addr_t ret; 349 enum memblock_flags flags = choose_memblock_flags(); 350 351 again: 352 ret = memblock_find_in_range_node(size, align, start, end, 353 NUMA_NO_NODE, flags); 354 355 if (!ret && (flags & MEMBLOCK_MIRROR)) { 356 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 357 &size); 358 flags &= ~MEMBLOCK_MIRROR; 359 goto again; 360 } 361 362 return ret; 363 } 364 365 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 366 { 367 type->total_size -= type->regions[r].size; 368 memmove(&type->regions[r], &type->regions[r + 1], 369 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 370 type->cnt--; 371 372 /* Special case for empty arrays */ 373 if (type->cnt == 0) { 374 WARN_ON(type->total_size != 0); 375 type->regions[0].base = 0; 376 type->regions[0].size = 0; 377 type->regions[0].flags = 0; 378 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 379 } 380 } 381 382 #ifndef CONFIG_ARCH_KEEP_MEMBLOCK 383 /** 384 * memblock_discard - discard memory and reserved arrays if they were allocated 385 */ 386 void __init memblock_discard(void) 387 { 388 phys_addr_t size; 389 void *addr; 390 391 if (memblock.reserved.regions != memblock_reserved_init_regions) { 392 addr = memblock.reserved.regions; 393 size = PAGE_ALIGN(sizeof(struct memblock_region) * 394 memblock.reserved.max); 395 if (memblock_reserved_in_slab) 396 kfree(addr); 397 else 398 memblock_free(addr, size); 399 } 400 401 if (memblock.memory.regions != memblock_memory_init_regions) { 402 addr = memblock.memory.regions; 403 size = PAGE_ALIGN(sizeof(struct memblock_region) * 404 memblock.memory.max); 405 if (memblock_memory_in_slab) 406 kfree(addr); 407 else 408 memblock_free(addr, size); 409 } 410 411 memblock_memory = NULL; 412 } 413 #endif 414 415 /** 416 * memblock_double_array - double the size of the memblock regions array 417 * @type: memblock type of the regions array being doubled 418 * @new_area_start: starting address of memory range to avoid overlap with 419 * @new_area_size: size of memory range to avoid overlap with 420 * 421 * Double the size of the @type regions array. If memblock is being used to 422 * allocate memory for a new reserved regions array and there is a previously 423 * allocated memory range [@new_area_start, @new_area_start + @new_area_size] 424 * waiting to be reserved, ensure the memory used by the new array does 425 * not overlap. 426 * 427 * Return: 428 * 0 on success, -1 on failure. 429 */ 430 static int __init_memblock memblock_double_array(struct memblock_type *type, 431 phys_addr_t new_area_start, 432 phys_addr_t new_area_size) 433 { 434 struct memblock_region *new_array, *old_array; 435 phys_addr_t old_alloc_size, new_alloc_size; 436 phys_addr_t old_size, new_size, addr, new_end; 437 int use_slab = slab_is_available(); 438 int *in_slab; 439 440 /* We don't allow resizing until we know about the reserved regions 441 * of memory that aren't suitable for allocation 442 */ 443 if (!memblock_can_resize) 444 panic("memblock: cannot resize %s array\n", type->name); 445 446 /* Calculate new doubled size */ 447 old_size = type->max * sizeof(struct memblock_region); 448 new_size = old_size << 1; 449 /* 450 * We need to allocated new one align to PAGE_SIZE, 451 * so we can free them completely later. 452 */ 453 old_alloc_size = PAGE_ALIGN(old_size); 454 new_alloc_size = PAGE_ALIGN(new_size); 455 456 /* Retrieve the slab flag */ 457 if (type == &memblock.memory) 458 in_slab = &memblock_memory_in_slab; 459 else 460 in_slab = &memblock_reserved_in_slab; 461 462 /* Try to find some space for it */ 463 if (use_slab) { 464 new_array = kmalloc(new_size, GFP_KERNEL); 465 addr = new_array ? __pa(new_array) : 0; 466 } else { 467 /* only exclude range when trying to double reserved.regions */ 468 if (type != &memblock.reserved) 469 new_area_start = new_area_size = 0; 470 471 addr = memblock_find_in_range(new_area_start + new_area_size, 472 memblock.current_limit, 473 new_alloc_size, PAGE_SIZE); 474 if (!addr && new_area_size) 475 addr = memblock_find_in_range(0, 476 min(new_area_start, memblock.current_limit), 477 new_alloc_size, PAGE_SIZE); 478 479 if (addr) { 480 /* The memory may not have been accepted, yet. */ 481 accept_memory(addr, new_alloc_size); 482 483 new_array = __va(addr); 484 } else { 485 new_array = NULL; 486 } 487 } 488 if (!addr) { 489 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 490 type->name, type->max, type->max * 2); 491 return -1; 492 } 493 494 new_end = addr + new_size - 1; 495 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]", 496 type->name, type->max * 2, &addr, &new_end); 497 498 /* 499 * Found space, we now need to move the array over before we add the 500 * reserved region since it may be our reserved array itself that is 501 * full. 502 */ 503 memcpy(new_array, type->regions, old_size); 504 memset(new_array + type->max, 0, old_size); 505 old_array = type->regions; 506 type->regions = new_array; 507 type->max <<= 1; 508 509 /* Free old array. We needn't free it if the array is the static one */ 510 if (*in_slab) 511 kfree(old_array); 512 else if (old_array != memblock_memory_init_regions && 513 old_array != memblock_reserved_init_regions) 514 memblock_free(old_array, old_alloc_size); 515 516 /* 517 * Reserve the new array if that comes from the memblock. Otherwise, we 518 * needn't do it 519 */ 520 if (!use_slab) 521 BUG_ON(memblock_reserve_kern(addr, new_alloc_size)); 522 523 /* Update slab flag */ 524 *in_slab = use_slab; 525 526 return 0; 527 } 528 529 /** 530 * memblock_merge_regions - merge neighboring compatible regions 531 * @type: memblock type to scan 532 * @start_rgn: start scanning from (@start_rgn - 1) 533 * @end_rgn: end scanning at (@end_rgn - 1) 534 * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn) 535 */ 536 static void __init_memblock memblock_merge_regions(struct memblock_type *type, 537 unsigned long start_rgn, 538 unsigned long end_rgn) 539 { 540 int i = 0; 541 if (start_rgn) 542 i = start_rgn - 1; 543 end_rgn = min(end_rgn, type->cnt - 1); 544 while (i < end_rgn) { 545 struct memblock_region *this = &type->regions[i]; 546 struct memblock_region *next = &type->regions[i + 1]; 547 548 if (this->base + this->size != next->base || 549 memblock_get_region_node(this) != 550 memblock_get_region_node(next) || 551 this->flags != next->flags) { 552 BUG_ON(this->base + this->size > next->base); 553 i++; 554 continue; 555 } 556 557 this->size += next->size; 558 /* move forward from next + 1, index of which is i + 2 */ 559 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 560 type->cnt--; 561 end_rgn--; 562 } 563 } 564 565 /** 566 * memblock_insert_region - insert new memblock region 567 * @type: memblock type to insert into 568 * @idx: index for the insertion point 569 * @base: base address of the new region 570 * @size: size of the new region 571 * @nid: node id of the new region 572 * @flags: flags of the new region 573 * 574 * Insert new memblock region [@base, @base + @size) into @type at @idx. 575 * @type must already have extra room to accommodate the new region. 576 */ 577 static void __init_memblock memblock_insert_region(struct memblock_type *type, 578 int idx, phys_addr_t base, 579 phys_addr_t size, 580 int nid, 581 enum memblock_flags flags) 582 { 583 struct memblock_region *rgn = &type->regions[idx]; 584 585 BUG_ON(type->cnt >= type->max); 586 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 587 rgn->base = base; 588 rgn->size = size; 589 rgn->flags = flags; 590 memblock_set_region_node(rgn, nid); 591 type->cnt++; 592 type->total_size += size; 593 } 594 595 /** 596 * memblock_add_range - add new memblock region 597 * @type: memblock type to add new region into 598 * @base: base address of the new region 599 * @size: size of the new region 600 * @nid: nid of the new region 601 * @flags: flags of the new region 602 * 603 * Add new memblock region [@base, @base + @size) into @type. The new region 604 * is allowed to overlap with existing ones - overlaps don't affect already 605 * existing regions. @type is guaranteed to be minimal (all neighbouring 606 * compatible regions are merged) after the addition. 607 * 608 * Return: 609 * 0 on success, -errno on failure. 610 */ 611 static int __init_memblock memblock_add_range(struct memblock_type *type, 612 phys_addr_t base, phys_addr_t size, 613 int nid, enum memblock_flags flags) 614 { 615 bool insert = false; 616 phys_addr_t obase = base; 617 phys_addr_t end = base + memblock_cap_size(base, &size); 618 int idx, nr_new, start_rgn = -1, end_rgn; 619 struct memblock_region *rgn; 620 621 if (!size) 622 return 0; 623 624 /* special case for empty array */ 625 if (type->regions[0].size == 0) { 626 WARN_ON(type->cnt != 0 || type->total_size); 627 type->regions[0].base = base; 628 type->regions[0].size = size; 629 type->regions[0].flags = flags; 630 memblock_set_region_node(&type->regions[0], nid); 631 type->total_size = size; 632 type->cnt = 1; 633 return 0; 634 } 635 636 /* 637 * The worst case is when new range overlaps all existing regions, 638 * then we'll need type->cnt + 1 empty regions in @type. So if 639 * type->cnt * 2 + 1 is less than or equal to type->max, we know 640 * that there is enough empty regions in @type, and we can insert 641 * regions directly. 642 */ 643 if (type->cnt * 2 + 1 <= type->max) 644 insert = true; 645 646 repeat: 647 /* 648 * The following is executed twice. Once with %false @insert and 649 * then with %true. The first counts the number of regions needed 650 * to accommodate the new area. The second actually inserts them. 651 */ 652 base = obase; 653 nr_new = 0; 654 655 for_each_memblock_type(idx, type, rgn) { 656 phys_addr_t rbase = rgn->base; 657 phys_addr_t rend = rbase + rgn->size; 658 659 if (rbase >= end) 660 break; 661 if (rend <= base) 662 continue; 663 /* 664 * @rgn overlaps. If it separates the lower part of new 665 * area, insert that portion. 666 */ 667 if (rbase > base) { 668 #ifdef CONFIG_NUMA 669 WARN_ON(nid != memblock_get_region_node(rgn)); 670 #endif 671 WARN_ON(flags != MEMBLOCK_NONE && flags != rgn->flags); 672 nr_new++; 673 if (insert) { 674 if (start_rgn == -1) 675 start_rgn = idx; 676 end_rgn = idx + 1; 677 memblock_insert_region(type, idx++, base, 678 rbase - base, nid, 679 flags); 680 } 681 } 682 /* area below @rend is dealt with, forget about it */ 683 base = min(rend, end); 684 } 685 686 /* insert the remaining portion */ 687 if (base < end) { 688 nr_new++; 689 if (insert) { 690 if (start_rgn == -1) 691 start_rgn = idx; 692 end_rgn = idx + 1; 693 memblock_insert_region(type, idx, base, end - base, 694 nid, flags); 695 } 696 } 697 698 if (!nr_new) 699 return 0; 700 701 /* 702 * If this was the first round, resize array and repeat for actual 703 * insertions; otherwise, merge and return. 704 */ 705 if (!insert) { 706 while (type->cnt + nr_new > type->max) 707 if (memblock_double_array(type, obase, size) < 0) 708 return -ENOMEM; 709 insert = true; 710 goto repeat; 711 } else { 712 memblock_merge_regions(type, start_rgn, end_rgn); 713 return 0; 714 } 715 } 716 717 /** 718 * memblock_add_node - add new memblock region within a NUMA node 719 * @base: base address of the new region 720 * @size: size of the new region 721 * @nid: nid of the new region 722 * @flags: flags of the new region 723 * 724 * Add new memblock region [@base, @base + @size) to the "memory" 725 * type. See memblock_add_range() description for mode details 726 * 727 * Return: 728 * 0 on success, -errno on failure. 729 */ 730 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 731 int nid, enum memblock_flags flags) 732 { 733 phys_addr_t end = base + size - 1; 734 735 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 736 &base, &end, nid, flags, (void *)_RET_IP_); 737 738 return memblock_add_range(&memblock.memory, base, size, nid, flags); 739 } 740 741 /** 742 * memblock_add - add new memblock region 743 * @base: base address of the new region 744 * @size: size of the new region 745 * 746 * Add new memblock region [@base, @base + @size) to the "memory" 747 * type. See memblock_add_range() description for mode details 748 * 749 * Return: 750 * 0 on success, -errno on failure. 751 */ 752 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 753 { 754 phys_addr_t end = base + size - 1; 755 756 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 757 &base, &end, (void *)_RET_IP_); 758 759 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0); 760 } 761 762 /** 763 * memblock_validate_numa_coverage - check if amount of memory with 764 * no node ID assigned is less than a threshold 765 * @threshold_bytes: maximal memory size that can have unassigned node 766 * ID (in bytes). 767 * 768 * A buggy firmware may report memory that does not belong to any node. 769 * Check if amount of such memory is below @threshold_bytes. 770 * 771 * Return: true on success, false on failure. 772 */ 773 bool __init_memblock memblock_validate_numa_coverage(unsigned long threshold_bytes) 774 { 775 unsigned long nr_pages = 0; 776 unsigned long start_pfn, end_pfn, mem_size_mb; 777 int nid, i; 778 779 /* calculate lost page */ 780 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 781 if (!numa_valid_node(nid)) 782 nr_pages += end_pfn - start_pfn; 783 } 784 785 if ((nr_pages << PAGE_SHIFT) > threshold_bytes) { 786 mem_size_mb = memblock_phys_mem_size() / SZ_1M; 787 pr_err("NUMA: no nodes coverage for %luMB of %luMB RAM\n", 788 (nr_pages << PAGE_SHIFT) / SZ_1M, mem_size_mb); 789 return false; 790 } 791 792 return true; 793 } 794 795 796 /** 797 * memblock_isolate_range - isolate given range into disjoint memblocks 798 * @type: memblock type to isolate range for 799 * @base: base of range to isolate 800 * @size: size of range to isolate 801 * @start_rgn: out parameter for the start of isolated region 802 * @end_rgn: out parameter for the end of isolated region 803 * 804 * Walk @type and ensure that regions don't cross the boundaries defined by 805 * [@base, @base + @size). Crossing regions are split at the boundaries, 806 * which may create at most two more regions. The index of the first 807 * region inside the range is returned in *@start_rgn and the index of the 808 * first region after the range is returned in *@end_rgn. 809 * 810 * Return: 811 * 0 on success, -errno on failure. 812 */ 813 static int __init_memblock memblock_isolate_range(struct memblock_type *type, 814 phys_addr_t base, phys_addr_t size, 815 int *start_rgn, int *end_rgn) 816 { 817 phys_addr_t end = base + memblock_cap_size(base, &size); 818 int idx; 819 struct memblock_region *rgn; 820 821 *start_rgn = *end_rgn = 0; 822 823 if (!size) 824 return 0; 825 826 /* we'll create at most two more regions */ 827 while (type->cnt + 2 > type->max) 828 if (memblock_double_array(type, base, size) < 0) 829 return -ENOMEM; 830 831 for_each_memblock_type(idx, type, rgn) { 832 phys_addr_t rbase = rgn->base; 833 phys_addr_t rend = rbase + rgn->size; 834 835 if (rbase >= end) 836 break; 837 if (rend <= base) 838 continue; 839 840 if (rbase < base) { 841 /* 842 * @rgn intersects from below. Split and continue 843 * to process the next region - the new top half. 844 */ 845 rgn->base = base; 846 rgn->size -= base - rbase; 847 type->total_size -= base - rbase; 848 memblock_insert_region(type, idx, rbase, base - rbase, 849 memblock_get_region_node(rgn), 850 rgn->flags); 851 } else if (rend > end) { 852 /* 853 * @rgn intersects from above. Split and redo the 854 * current region - the new bottom half. 855 */ 856 rgn->base = end; 857 rgn->size -= end - rbase; 858 type->total_size -= end - rbase; 859 memblock_insert_region(type, idx--, rbase, end - rbase, 860 memblock_get_region_node(rgn), 861 rgn->flags); 862 } else { 863 /* @rgn is fully contained, record it */ 864 if (!*end_rgn) 865 *start_rgn = idx; 866 *end_rgn = idx + 1; 867 } 868 } 869 870 return 0; 871 } 872 873 static int __init_memblock memblock_remove_range(struct memblock_type *type, 874 phys_addr_t base, phys_addr_t size) 875 { 876 int start_rgn, end_rgn; 877 int i, ret; 878 879 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 880 if (ret) 881 return ret; 882 883 for (i = end_rgn - 1; i >= start_rgn; i--) 884 memblock_remove_region(type, i); 885 return 0; 886 } 887 888 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 889 { 890 phys_addr_t end = base + size - 1; 891 892 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 893 &base, &end, (void *)_RET_IP_); 894 895 return memblock_remove_range(&memblock.memory, base, size); 896 } 897 898 static unsigned long __free_reserved_area(phys_addr_t start, phys_addr_t end, 899 int poison) 900 { 901 unsigned long pages = 0, pfn; 902 903 if (deferred_pages_enabled()) { 904 WARN(1, "Cannot free reserved memory because of deferred initialization of the memory map"); 905 return 0; 906 } 907 908 for_each_valid_pfn(pfn, PFN_UP(start), PFN_DOWN(end)) { 909 struct page *page = pfn_to_page(pfn); 910 void *direct_map_addr; 911 912 /* 913 * 'direct_map_addr' might be different from the kernel virtual 914 * address because some architectures use aliases. 915 * Going via physical address, pfn_to_page() and page_address() 916 * ensures that we get a _writeable_ alias for the memset(). 917 */ 918 direct_map_addr = page_address(page); 919 /* 920 * Perform a kasan-unchecked memset() since this memory 921 * has not been initialized. 922 */ 923 direct_map_addr = kasan_reset_tag(direct_map_addr); 924 if ((unsigned int)poison <= 0xFF) 925 memset(direct_map_addr, poison, PAGE_SIZE); 926 927 free_reserved_page(page); 928 pages++; 929 } 930 return pages; 931 } 932 933 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 934 { 935 phys_addr_t start_pa, end_pa; 936 unsigned long pages; 937 938 /* 939 * end is the first address past the region and it may be beyond what 940 * __pa() or __pa_symbol() can handle. 941 * Use the address included in the range for the conversion and add back 942 * 1 afterwards. 943 */ 944 if (__is_kernel((unsigned long)start)) { 945 start_pa = __pa_symbol(start); 946 end_pa = __pa_symbol(end - 1) + 1; 947 } else { 948 start_pa = __pa(start); 949 end_pa = __pa(end - 1) + 1; 950 } 951 952 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) { 953 if (start_pa < end_pa) 954 memblock_remove_range(&memblock.reserved, 955 start_pa, end_pa - start_pa); 956 } 957 958 pages = __free_reserved_area(start_pa, end_pa, poison); 959 if (pages && s) 960 pr_info("Freeing %s memory: %ldK\n", s, K(pages)); 961 962 return pages; 963 } 964 965 /** 966 * memblock_free - free boot memory allocation 967 * @ptr: starting address of the boot memory allocation 968 * @size: size of the boot memory block in bytes 969 * 970 * Free boot memory block previously allocated by memblock_alloc_xx() API. 971 * If called after the buddy allocator is available, the memory is released to 972 * the buddy allocator. 973 */ 974 void __init_memblock memblock_free(void *ptr, size_t size) 975 { 976 if (ptr) 977 memblock_phys_free(__pa(ptr), size); 978 } 979 980 /** 981 * memblock_phys_free - free boot memory block 982 * @base: phys starting address of the boot memory block 983 * @size: size of the boot memory block in bytes 984 * 985 * Free boot memory block previously allocated by memblock_phys_alloc_xx() API. 986 * If called after the buddy allocator is available, the memory is released to 987 * the buddy allocator. 988 */ 989 int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size) 990 { 991 phys_addr_t end = base + size - 1; 992 int ret = 0; 993 994 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 995 &base, &end, (void *)_RET_IP_); 996 997 kmemleak_free_part_phys(base, size); 998 999 if (!slab_is_available() || IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) 1000 ret = memblock_remove_range(&memblock.reserved, base, size); 1001 1002 if (slab_is_available()) 1003 __free_reserved_area(base, base + size, -1); 1004 1005 return ret; 1006 } 1007 1008 int __init_memblock __memblock_reserve(phys_addr_t base, phys_addr_t size, 1009 int nid, enum memblock_flags flags) 1010 { 1011 phys_addr_t end = base + size - 1; 1012 1013 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 1014 &base, &end, nid, flags, (void *)_RET_IP_); 1015 1016 return memblock_add_range(&memblock.reserved, base, size, nid, flags); 1017 } 1018 1019 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 1020 int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) 1021 { 1022 phys_addr_t end = base + size - 1; 1023 1024 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 1025 &base, &end, (void *)_RET_IP_); 1026 1027 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0); 1028 } 1029 #endif 1030 1031 /** 1032 * memblock_setclr_flag - set or clear flag for a memory region 1033 * @type: memblock type to set/clear flag for 1034 * @base: base address of the region 1035 * @size: size of the region 1036 * @set: set or clear the flag 1037 * @flag: the flag to update 1038 * 1039 * This function isolates region [@base, @base + @size), and sets/clears flag 1040 * 1041 * Return: 0 on success, -errno on failure. 1042 */ 1043 static int __init_memblock memblock_setclr_flag(struct memblock_type *type, 1044 phys_addr_t base, phys_addr_t size, int set, int flag) 1045 { 1046 int i, ret, start_rgn, end_rgn; 1047 1048 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1049 if (ret) 1050 return ret; 1051 1052 for (i = start_rgn; i < end_rgn; i++) { 1053 struct memblock_region *r = &type->regions[i]; 1054 1055 if (set) 1056 r->flags |= flag; 1057 else 1058 r->flags &= ~flag; 1059 } 1060 1061 memblock_merge_regions(type, start_rgn, end_rgn); 1062 return 0; 1063 } 1064 1065 /** 1066 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 1067 * @base: the base phys addr of the region 1068 * @size: the size of the region 1069 * 1070 * Return: 0 on success, -errno on failure. 1071 */ 1072 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 1073 { 1074 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_HOTPLUG); 1075 } 1076 1077 /** 1078 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 1079 * @base: the base phys addr of the region 1080 * @size: the size of the region 1081 * 1082 * Return: 0 on success, -errno on failure. 1083 */ 1084 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 1085 { 1086 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_HOTPLUG); 1087 } 1088 1089 /** 1090 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR. 1091 * @base: the base phys addr of the region 1092 * @size: the size of the region 1093 * 1094 * Return: 0 on success, -errno on failure. 1095 */ 1096 int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size) 1097 { 1098 if (!mirrored_kernelcore) 1099 return 0; 1100 1101 system_has_some_mirror = true; 1102 1103 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_MIRROR); 1104 } 1105 1106 /** 1107 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP. 1108 * @base: the base phys addr of the region 1109 * @size: the size of the region 1110 * 1111 * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the 1112 * direct mapping of the physical memory. These regions will still be 1113 * covered by the memory map. The struct page representing NOMAP memory 1114 * frames in the memory map will be PageReserved() 1115 * 1116 * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from 1117 * memblock, the caller must inform kmemleak to ignore that memory 1118 * 1119 * Return: 0 on success, -errno on failure. 1120 */ 1121 int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size) 1122 { 1123 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_NOMAP); 1124 } 1125 1126 /** 1127 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region. 1128 * @base: the base phys addr of the region 1129 * @size: the size of the region 1130 * 1131 * Return: 0 on success, -errno on failure. 1132 */ 1133 int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size) 1134 { 1135 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_NOMAP); 1136 } 1137 1138 /** 1139 * memblock_reserved_mark_noinit - Mark a reserved memory region with flag 1140 * MEMBLOCK_RSRV_NOINIT 1141 * 1142 * @base: the base phys addr of the region 1143 * @size: the size of the region 1144 * 1145 * The struct pages for the reserved regions marked %MEMBLOCK_RSRV_NOINIT will 1146 * not be fully initialized to allow the caller optimize their initialization. 1147 * 1148 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled, setting this flag 1149 * completely bypasses the initialization of struct pages for such region. 1150 * 1151 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is disabled, struct pages in this 1152 * region will be initialized with default values but won't be marked as 1153 * reserved. 1154 * 1155 * Return: 0 on success, -errno on failure. 1156 */ 1157 int __init_memblock memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size) 1158 { 1159 return memblock_setclr_flag(&memblock.reserved, base, size, 1, 1160 MEMBLOCK_RSRV_NOINIT); 1161 } 1162 1163 /** 1164 * memblock_reserved_mark_kern - Mark a reserved memory region with flag 1165 * MEMBLOCK_RSRV_KERN 1166 * 1167 * @base: the base phys addr of the region 1168 * @size: the size of the region 1169 * 1170 * Return: 0 on success, -errno on failure. 1171 */ 1172 int __init_memblock memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t size) 1173 { 1174 return memblock_setclr_flag(&memblock.reserved, base, size, 1, 1175 MEMBLOCK_RSRV_KERN); 1176 } 1177 1178 /** 1179 * memblock_mark_kho_scratch - Mark a memory region as MEMBLOCK_KHO_SCRATCH. 1180 * @base: the base phys addr of the region 1181 * @size: the size of the region 1182 * 1183 * Only memory regions marked with %MEMBLOCK_KHO_SCRATCH will be considered 1184 * for allocations during early boot with kexec handover. 1185 * 1186 * Return: 0 on success, -errno on failure. 1187 */ 1188 __init int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size) 1189 { 1190 return memblock_setclr_flag(&memblock.memory, base, size, 1, 1191 MEMBLOCK_KHO_SCRATCH); 1192 } 1193 1194 /** 1195 * memblock_clear_kho_scratch - Clear MEMBLOCK_KHO_SCRATCH flag for a 1196 * specified region. 1197 * @base: the base phys addr of the region 1198 * @size: the size of the region 1199 * 1200 * Return: 0 on success, -errno on failure. 1201 */ 1202 __init int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size) 1203 { 1204 return memblock_setclr_flag(&memblock.memory, base, size, 0, 1205 MEMBLOCK_KHO_SCRATCH); 1206 } 1207 1208 static bool should_skip_region(struct memblock_type *type, 1209 struct memblock_region *m, 1210 int nid, int flags) 1211 { 1212 int m_nid = memblock_get_region_node(m); 1213 1214 /* we never skip regions when iterating memblock.reserved or physmem */ 1215 if (type != memblock_memory) 1216 return false; 1217 1218 /* only memory regions are associated with nodes, check it */ 1219 if (numa_valid_node(nid) && nid != m_nid) 1220 return true; 1221 1222 /* skip hotpluggable memory regions if needed */ 1223 if (movable_node_is_enabled() && memblock_is_hotpluggable(m) && 1224 !(flags & MEMBLOCK_HOTPLUG)) 1225 return true; 1226 1227 /* if we want mirror memory skip non-mirror memory regions */ 1228 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 1229 return true; 1230 1231 /* skip nomap memory unless we were asked for it explicitly */ 1232 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 1233 return true; 1234 1235 /* skip driver-managed memory unless we were asked for it explicitly */ 1236 if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m)) 1237 return true; 1238 1239 /* 1240 * In early alloc during kexec handover, we can only consider 1241 * MEMBLOCK_KHO_SCRATCH regions for the allocations 1242 */ 1243 if ((flags & MEMBLOCK_KHO_SCRATCH) && !memblock_is_kho_scratch(m)) 1244 return true; 1245 1246 return false; 1247 } 1248 1249 /** 1250 * __next_mem_range - next function for for_each_free_mem_range() etc. 1251 * @idx: pointer to u64 loop variable 1252 * @nid: node selector, %NUMA_NO_NODE for all nodes 1253 * @flags: pick from blocks based on memory attributes 1254 * @type_a: pointer to memblock_type from where the range is taken 1255 * @type_b: pointer to memblock_type which excludes memory from being taken 1256 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1257 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1258 * @out_nid: ptr to int for nid of the range, can be %NULL 1259 * 1260 * Find the first area from *@idx which matches @nid, fill the out 1261 * parameters, and update *@idx for the next iteration. The lower 32bit of 1262 * *@idx contains index into type_a and the upper 32bit indexes the 1263 * areas before each region in type_b. For example, if type_b regions 1264 * look like the following, 1265 * 1266 * 0:[0-16), 1:[32-48), 2:[128-130) 1267 * 1268 * The upper 32bit indexes the following regions. 1269 * 1270 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 1271 * 1272 * As both region arrays are sorted, the function advances the two indices 1273 * in lockstep and returns each intersection. 1274 */ 1275 void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags, 1276 struct memblock_type *type_a, 1277 struct memblock_type *type_b, phys_addr_t *out_start, 1278 phys_addr_t *out_end, int *out_nid) 1279 { 1280 int idx_a = *idx & 0xffffffff; 1281 int idx_b = *idx >> 32; 1282 1283 for (; idx_a < type_a->cnt; idx_a++) { 1284 struct memblock_region *m = &type_a->regions[idx_a]; 1285 1286 phys_addr_t m_start = m->base; 1287 phys_addr_t m_end = m->base + m->size; 1288 int m_nid = memblock_get_region_node(m); 1289 1290 if (should_skip_region(type_a, m, nid, flags)) 1291 continue; 1292 1293 if (!type_b) { 1294 if (out_start) 1295 *out_start = m_start; 1296 if (out_end) 1297 *out_end = m_end; 1298 if (out_nid) 1299 *out_nid = m_nid; 1300 idx_a++; 1301 *idx = (u32)idx_a | (u64)idx_b << 32; 1302 return; 1303 } 1304 1305 /* scan areas before each reservation */ 1306 for (; idx_b < type_b->cnt + 1; idx_b++) { 1307 struct memblock_region *r; 1308 phys_addr_t r_start; 1309 phys_addr_t r_end; 1310 1311 r = &type_b->regions[idx_b]; 1312 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1313 r_end = idx_b < type_b->cnt ? 1314 r->base : PHYS_ADDR_MAX; 1315 1316 /* 1317 * if idx_b advanced past idx_a, 1318 * break out to advance idx_a 1319 */ 1320 if (r_start >= m_end) 1321 break; 1322 /* if the two regions intersect, we're done */ 1323 if (m_start < r_end) { 1324 if (out_start) 1325 *out_start = 1326 max(m_start, r_start); 1327 if (out_end) 1328 *out_end = min(m_end, r_end); 1329 if (out_nid) 1330 *out_nid = m_nid; 1331 /* 1332 * The region which ends first is 1333 * advanced for the next iteration. 1334 */ 1335 if (m_end <= r_end) 1336 idx_a++; 1337 else 1338 idx_b++; 1339 *idx = (u32)idx_a | (u64)idx_b << 32; 1340 return; 1341 } 1342 } 1343 } 1344 1345 /* signal end of iteration */ 1346 *idx = ULLONG_MAX; 1347 } 1348 1349 /** 1350 * __next_mem_range_rev - generic next function for for_each_*_range_rev() 1351 * 1352 * @idx: pointer to u64 loop variable 1353 * @nid: node selector, %NUMA_NO_NODE for all nodes 1354 * @flags: pick from blocks based on memory attributes 1355 * @type_a: pointer to memblock_type from where the range is taken 1356 * @type_b: pointer to memblock_type which excludes memory from being taken 1357 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1358 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1359 * @out_nid: ptr to int for nid of the range, can be %NULL 1360 * 1361 * Finds the next range from type_a which is not marked as unsuitable 1362 * in type_b. 1363 * 1364 * Reverse of __next_mem_range(). 1365 */ 1366 void __init_memblock __next_mem_range_rev(u64 *idx, int nid, 1367 enum memblock_flags flags, 1368 struct memblock_type *type_a, 1369 struct memblock_type *type_b, 1370 phys_addr_t *out_start, 1371 phys_addr_t *out_end, int *out_nid) 1372 { 1373 int idx_a = *idx & 0xffffffff; 1374 int idx_b = *idx >> 32; 1375 1376 if (*idx == (u64)ULLONG_MAX) { 1377 idx_a = type_a->cnt - 1; 1378 if (type_b != NULL) 1379 idx_b = type_b->cnt; 1380 else 1381 idx_b = 0; 1382 } 1383 1384 for (; idx_a >= 0; idx_a--) { 1385 struct memblock_region *m = &type_a->regions[idx_a]; 1386 1387 phys_addr_t m_start = m->base; 1388 phys_addr_t m_end = m->base + m->size; 1389 int m_nid = memblock_get_region_node(m); 1390 1391 if (should_skip_region(type_a, m, nid, flags)) 1392 continue; 1393 1394 if (!type_b) { 1395 if (out_start) 1396 *out_start = m_start; 1397 if (out_end) 1398 *out_end = m_end; 1399 if (out_nid) 1400 *out_nid = m_nid; 1401 idx_a--; 1402 *idx = (u32)idx_a | (u64)idx_b << 32; 1403 return; 1404 } 1405 1406 /* scan areas before each reservation */ 1407 for (; idx_b >= 0; idx_b--) { 1408 struct memblock_region *r; 1409 phys_addr_t r_start; 1410 phys_addr_t r_end; 1411 1412 r = &type_b->regions[idx_b]; 1413 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1414 r_end = idx_b < type_b->cnt ? 1415 r->base : PHYS_ADDR_MAX; 1416 /* 1417 * if idx_b advanced past idx_a, 1418 * break out to advance idx_a 1419 */ 1420 1421 if (r_end <= m_start) 1422 break; 1423 /* if the two regions intersect, we're done */ 1424 if (m_end > r_start) { 1425 if (out_start) 1426 *out_start = max(m_start, r_start); 1427 if (out_end) 1428 *out_end = min(m_end, r_end); 1429 if (out_nid) 1430 *out_nid = m_nid; 1431 if (m_start >= r_start) 1432 idx_a--; 1433 else 1434 idx_b--; 1435 *idx = (u32)idx_a | (u64)idx_b << 32; 1436 return; 1437 } 1438 } 1439 } 1440 /* signal end of iteration */ 1441 *idx = ULLONG_MAX; 1442 } 1443 1444 /* 1445 * Common iterator interface used to define for_each_mem_pfn_range(). 1446 */ 1447 void __init_memblock __next_mem_pfn_range(int *idx, int nid, 1448 unsigned long *out_start_pfn, 1449 unsigned long *out_end_pfn, int *out_nid) 1450 { 1451 struct memblock_type *type = &memblock.memory; 1452 struct memblock_region *r; 1453 int r_nid; 1454 1455 while (++*idx < type->cnt) { 1456 r = &type->regions[*idx]; 1457 r_nid = memblock_get_region_node(r); 1458 1459 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 1460 continue; 1461 if (!numa_valid_node(nid) || nid == r_nid) 1462 break; 1463 } 1464 if (*idx >= type->cnt) { 1465 *idx = -1; 1466 return; 1467 } 1468 1469 if (out_start_pfn) 1470 *out_start_pfn = PFN_UP(r->base); 1471 if (out_end_pfn) 1472 *out_end_pfn = PFN_DOWN(r->base + r->size); 1473 if (out_nid) 1474 *out_nid = r_nid; 1475 } 1476 1477 /** 1478 * memblock_set_node - set node ID on memblock regions 1479 * @base: base of area to set node ID for 1480 * @size: size of area to set node ID for 1481 * @type: memblock type to set node ID for 1482 * @nid: node ID to set 1483 * 1484 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid. 1485 * Regions which cross the area boundaries are split as necessary. 1486 * 1487 * Return: 1488 * 0 on success, -errno on failure. 1489 */ 1490 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 1491 struct memblock_type *type, int nid) 1492 { 1493 #ifdef CONFIG_NUMA 1494 int start_rgn, end_rgn; 1495 int i, ret; 1496 1497 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1498 if (ret) 1499 return ret; 1500 1501 for (i = start_rgn; i < end_rgn; i++) 1502 memblock_set_region_node(&type->regions[i], nid); 1503 1504 memblock_merge_regions(type, start_rgn, end_rgn); 1505 #endif 1506 return 0; 1507 } 1508 1509 /** 1510 * memblock_alloc_range_nid - allocate boot memory block 1511 * @size: size of memory block to be allocated in bytes 1512 * @align: alignment of the region and block's size 1513 * @start: the lower bound of the memory region to allocate (phys address) 1514 * @end: the upper bound of the memory region to allocate (phys address) 1515 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1516 * @exact_nid: control the allocation fall back to other nodes 1517 * 1518 * The allocation is performed from memory region limited by 1519 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE. 1520 * 1521 * If the specified node can not hold the requested memory and @exact_nid 1522 * is false, the allocation falls back to any node in the system. 1523 * 1524 * For systems with memory mirroring, the allocation is attempted first 1525 * from the regions with mirroring enabled and then retried from any 1526 * memory region. 1527 * 1528 * In addition, function using kmemleak_alloc_phys for allocated boot 1529 * memory block, it is never reported as leaks. 1530 * 1531 * Return: 1532 * Physical address of allocated memory block on success, %0 on failure. 1533 */ 1534 phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, 1535 phys_addr_t align, phys_addr_t start, 1536 phys_addr_t end, int nid, 1537 bool exact_nid) 1538 { 1539 enum memblock_flags flags = choose_memblock_flags(); 1540 phys_addr_t found; 1541 1542 /* 1543 * Detect any accidental use of these APIs after slab is ready, as at 1544 * this moment memblock may be deinitialized already and its 1545 * internal data may be destroyed (after execution of memblock_free_all) 1546 */ 1547 if (WARN_ON_ONCE(slab_is_available())) { 1548 void *vaddr = kzalloc_node(size, GFP_NOWAIT, nid); 1549 1550 return vaddr ? virt_to_phys(vaddr) : 0; 1551 } 1552 1553 if (!align) { 1554 /* Can't use WARNs this early in boot on powerpc */ 1555 dump_stack(); 1556 align = SMP_CACHE_BYTES; 1557 } 1558 1559 again: 1560 found = memblock_find_in_range_node(size, align, start, end, nid, 1561 flags); 1562 if (found && !__memblock_reserve(found, size, nid, MEMBLOCK_RSRV_KERN)) 1563 goto done; 1564 1565 if (numa_valid_node(nid) && !exact_nid) { 1566 found = memblock_find_in_range_node(size, align, start, 1567 end, NUMA_NO_NODE, 1568 flags); 1569 if (found && !memblock_reserve_kern(found, size)) 1570 goto done; 1571 } 1572 1573 if (flags & MEMBLOCK_MIRROR) { 1574 flags &= ~MEMBLOCK_MIRROR; 1575 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 1576 &size); 1577 goto again; 1578 } 1579 1580 return 0; 1581 1582 done: 1583 /* 1584 * Skip kmemleak for those places like kasan_init() and 1585 * early_pgtable_alloc() due to high volume. 1586 */ 1587 if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) 1588 /* 1589 * Memblock allocated blocks are never reported as 1590 * leaks. This is because many of these blocks are 1591 * only referred via the physical address which is 1592 * not looked up by kmemleak. 1593 */ 1594 kmemleak_alloc_phys(found, size, 0); 1595 1596 /* 1597 * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, 1598 * require memory to be accepted before it can be used by the 1599 * guest. 1600 * 1601 * Accept the memory of the allocated buffer. 1602 */ 1603 accept_memory(found, size); 1604 1605 return found; 1606 } 1607 1608 /** 1609 * memblock_phys_alloc_range - allocate a memory block inside specified range 1610 * @size: size of memory block to be allocated in bytes 1611 * @align: alignment of the region and block's size 1612 * @start: the lower bound of the memory region to allocate (physical address) 1613 * @end: the upper bound of the memory region to allocate (physical address) 1614 * 1615 * Allocate @size bytes in the between @start and @end. 1616 * 1617 * Return: physical address of the allocated memory block on success, 1618 * %0 on failure. 1619 */ 1620 phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, 1621 phys_addr_t align, 1622 phys_addr_t start, 1623 phys_addr_t end) 1624 { 1625 memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n", 1626 __func__, (u64)size, (u64)align, &start, &end, 1627 (void *)_RET_IP_); 1628 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, 1629 false); 1630 } 1631 1632 /** 1633 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node 1634 * @size: size of memory block to be allocated in bytes 1635 * @align: alignment of the region and block's size 1636 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1637 * 1638 * Allocates memory block from the specified NUMA node. If the node 1639 * has no available memory, attempts to allocated from any node in the 1640 * system. 1641 * 1642 * Return: physical address of the allocated memory block on success, 1643 * %0 on failure. 1644 */ 1645 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1646 { 1647 return memblock_alloc_range_nid(size, align, 0, 1648 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false); 1649 } 1650 1651 /** 1652 * memblock_alloc_internal - allocate boot memory block 1653 * @size: size of memory block to be allocated in bytes 1654 * @align: alignment of the region and block's size 1655 * @min_addr: the lower bound of the memory region to allocate (phys address) 1656 * @max_addr: the upper bound of the memory region to allocate (phys address) 1657 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1658 * @exact_nid: control the allocation fall back to other nodes 1659 * 1660 * Allocates memory block using memblock_alloc_range_nid() and 1661 * converts the returned physical address to virtual. 1662 * 1663 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1664 * will fall back to memory below @min_addr. Other constraints, such 1665 * as node and mirrored memory will be handled again in 1666 * memblock_alloc_range_nid(). 1667 * 1668 * Return: 1669 * Virtual address of allocated memory block on success, NULL on failure. 1670 */ 1671 static void * __init memblock_alloc_internal( 1672 phys_addr_t size, phys_addr_t align, 1673 phys_addr_t min_addr, phys_addr_t max_addr, 1674 int nid, bool exact_nid) 1675 { 1676 phys_addr_t alloc; 1677 1678 1679 if (max_addr > memblock.current_limit) 1680 max_addr = memblock.current_limit; 1681 1682 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid, 1683 exact_nid); 1684 1685 /* retry allocation without lower limit */ 1686 if (!alloc && min_addr) 1687 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid, 1688 exact_nid); 1689 1690 if (!alloc) 1691 return NULL; 1692 1693 return phys_to_virt(alloc); 1694 } 1695 1696 /** 1697 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node 1698 * without zeroing memory 1699 * @size: size of memory block to be allocated in bytes 1700 * @align: alignment of the region and block's size 1701 * @min_addr: the lower bound of the memory region from where the allocation 1702 * is preferred (phys address) 1703 * @max_addr: the upper bound of the memory region from where the allocation 1704 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1705 * allocate only from memory limited by memblock.current_limit value 1706 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1707 * 1708 * Public function, provides additional debug information (including caller 1709 * info), if enabled. Does not zero allocated memory. 1710 * 1711 * Return: 1712 * Virtual address of allocated memory block on success, NULL on failure. 1713 */ 1714 void * __init memblock_alloc_exact_nid_raw( 1715 phys_addr_t size, phys_addr_t align, 1716 phys_addr_t min_addr, phys_addr_t max_addr, 1717 int nid) 1718 { 1719 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1720 __func__, (u64)size, (u64)align, nid, &min_addr, 1721 &max_addr, (void *)_RET_IP_); 1722 1723 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1724 true); 1725 } 1726 1727 /** 1728 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing 1729 * memory and without panicking 1730 * @size: size of memory block to be allocated in bytes 1731 * @align: alignment of the region and block's size 1732 * @min_addr: the lower bound of the memory region from where the allocation 1733 * is preferred (phys address) 1734 * @max_addr: the upper bound of the memory region from where the allocation 1735 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1736 * allocate only from memory limited by memblock.current_limit value 1737 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1738 * 1739 * Public function, provides additional debug information (including caller 1740 * info), if enabled. Does not zero allocated memory, does not panic if request 1741 * cannot be satisfied. 1742 * 1743 * Return: 1744 * Virtual address of allocated memory block on success, NULL on failure. 1745 */ 1746 void * __init memblock_alloc_try_nid_raw( 1747 phys_addr_t size, phys_addr_t align, 1748 phys_addr_t min_addr, phys_addr_t max_addr, 1749 int nid) 1750 { 1751 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1752 __func__, (u64)size, (u64)align, nid, &min_addr, 1753 &max_addr, (void *)_RET_IP_); 1754 1755 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1756 false); 1757 } 1758 1759 /** 1760 * memblock_alloc_try_nid - allocate boot memory block 1761 * @size: size of memory block to be allocated in bytes 1762 * @align: alignment of the region and block's size 1763 * @min_addr: the lower bound of the memory region from where the allocation 1764 * is preferred (phys address) 1765 * @max_addr: the upper bound of the memory region from where the allocation 1766 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1767 * allocate only from memory limited by memblock.current_limit value 1768 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1769 * 1770 * Public function, provides additional debug information (including caller 1771 * info), if enabled. This function zeroes the allocated memory. 1772 * 1773 * Return: 1774 * Virtual address of allocated memory block on success, NULL on failure. 1775 */ 1776 void * __init memblock_alloc_try_nid( 1777 phys_addr_t size, phys_addr_t align, 1778 phys_addr_t min_addr, phys_addr_t max_addr, 1779 int nid) 1780 { 1781 void *ptr; 1782 1783 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1784 __func__, (u64)size, (u64)align, nid, &min_addr, 1785 &max_addr, (void *)_RET_IP_); 1786 ptr = memblock_alloc_internal(size, align, 1787 min_addr, max_addr, nid, false); 1788 if (ptr) 1789 memset(ptr, 0, size); 1790 1791 return ptr; 1792 } 1793 1794 /** 1795 * __memblock_alloc_or_panic - Try to allocate memory and panic on failure 1796 * @size: size of memory block to be allocated in bytes 1797 * @align: alignment of the region and block's size 1798 * @func: caller func name 1799 * 1800 * This function attempts to allocate memory using memblock_alloc, 1801 * and in case of failure, it calls panic with the formatted message. 1802 * This function should not be used directly, please use the macro memblock_alloc_or_panic. 1803 */ 1804 void *__init __memblock_alloc_or_panic(phys_addr_t size, phys_addr_t align, 1805 const char *func) 1806 { 1807 void *addr = memblock_alloc(size, align); 1808 1809 if (unlikely(!addr)) 1810 panic("%s: Failed to allocate %pap bytes\n", func, &size); 1811 return addr; 1812 } 1813 1814 /* 1815 * Remaining API functions 1816 */ 1817 1818 phys_addr_t __init_memblock memblock_phys_mem_size(void) 1819 { 1820 return memblock.memory.total_size; 1821 } 1822 1823 phys_addr_t __init_memblock memblock_reserved_size(void) 1824 { 1825 return memblock.reserved.total_size; 1826 } 1827 1828 phys_addr_t __init_memblock memblock_reserved_kern_size(phys_addr_t limit, int nid) 1829 { 1830 struct memblock_region *r; 1831 phys_addr_t total = 0; 1832 1833 for_each_reserved_mem_region(r) { 1834 phys_addr_t size = r->size; 1835 1836 if (r->base > limit) 1837 break; 1838 1839 if (r->base + r->size > limit) 1840 size = limit - r->base; 1841 1842 if (nid == memblock_get_region_node(r) || !numa_valid_node(nid)) 1843 if (r->flags & MEMBLOCK_RSRV_KERN) 1844 total += size; 1845 } 1846 1847 return total; 1848 } 1849 1850 /** 1851 * memblock_estimated_nr_free_pages - return estimated number of free pages 1852 * from memblock point of view 1853 * 1854 * During bootup, subsystems might need a rough estimate of the number of free 1855 * pages in the whole system, before precise numbers are available from the 1856 * buddy. Especially with CONFIG_DEFERRED_STRUCT_PAGE_INIT, the numbers 1857 * obtained from the buddy might be very imprecise during bootup. 1858 * 1859 * Return: 1860 * An estimated number of free pages from memblock point of view. 1861 */ 1862 unsigned long __init memblock_estimated_nr_free_pages(void) 1863 { 1864 return PHYS_PFN(memblock_phys_mem_size() - 1865 memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, NUMA_NO_NODE)); 1866 } 1867 1868 /* lowest address */ 1869 phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1870 { 1871 return memblock.memory.regions[0].base; 1872 } 1873 1874 phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1875 { 1876 int idx = memblock.memory.cnt - 1; 1877 1878 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1879 } 1880 1881 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) 1882 { 1883 phys_addr_t max_addr = PHYS_ADDR_MAX; 1884 struct memblock_region *r; 1885 1886 /* 1887 * translate the memory @limit size into the max address within one of 1888 * the memory memblock regions, if the @limit exceeds the total size 1889 * of those regions, max_addr will keep original value PHYS_ADDR_MAX 1890 */ 1891 for_each_mem_region(r) { 1892 if (limit <= r->size) { 1893 max_addr = r->base + limit; 1894 break; 1895 } 1896 limit -= r->size; 1897 } 1898 1899 return max_addr; 1900 } 1901 1902 void __init memblock_enforce_memory_limit(phys_addr_t limit) 1903 { 1904 phys_addr_t max_addr; 1905 1906 if (!limit) 1907 return; 1908 1909 max_addr = __find_max_addr(limit); 1910 1911 /* @limit exceeds the total size of the memory, do nothing */ 1912 if (max_addr == PHYS_ADDR_MAX) 1913 return; 1914 1915 /* truncate both memory and reserved regions */ 1916 memblock_remove_range(&memblock.memory, max_addr, 1917 PHYS_ADDR_MAX); 1918 memblock_remove_range(&memblock.reserved, max_addr, 1919 PHYS_ADDR_MAX); 1920 } 1921 1922 void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size) 1923 { 1924 int start_rgn, end_rgn; 1925 int i, ret; 1926 1927 if (!size) 1928 return; 1929 1930 if (!memblock_memory->total_size) { 1931 pr_warn("%s: No memory registered yet\n", __func__); 1932 return; 1933 } 1934 1935 ret = memblock_isolate_range(&memblock.memory, base, size, 1936 &start_rgn, &end_rgn); 1937 if (ret) 1938 return; 1939 1940 /* remove all the MAP regions */ 1941 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--) 1942 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1943 memblock_remove_region(&memblock.memory, i); 1944 1945 for (i = start_rgn - 1; i >= 0; i--) 1946 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1947 memblock_remove_region(&memblock.memory, i); 1948 1949 /* truncate the reserved regions */ 1950 memblock_remove_range(&memblock.reserved, 0, base); 1951 memblock_remove_range(&memblock.reserved, 1952 base + size, PHYS_ADDR_MAX); 1953 } 1954 1955 void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1956 { 1957 phys_addr_t max_addr; 1958 1959 if (!limit) 1960 return; 1961 1962 max_addr = __find_max_addr(limit); 1963 1964 /* @limit exceeds the total size of the memory, do nothing */ 1965 if (max_addr == PHYS_ADDR_MAX) 1966 return; 1967 1968 memblock_cap_memory_range(0, max_addr); 1969 } 1970 1971 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1972 { 1973 unsigned int left = 0, right = type->cnt; 1974 1975 do { 1976 unsigned int mid = (right + left) / 2; 1977 1978 if (addr < type->regions[mid].base) 1979 right = mid; 1980 else if (addr >= (type->regions[mid].base + 1981 type->regions[mid].size)) 1982 left = mid + 1; 1983 else 1984 return mid; 1985 } while (left < right); 1986 return -1; 1987 } 1988 1989 bool __init_memblock memblock_is_reserved(phys_addr_t addr) 1990 { 1991 return memblock_search(&memblock.reserved, addr) != -1; 1992 } 1993 1994 bool __init_memblock memblock_is_memory(phys_addr_t addr) 1995 { 1996 return memblock_search(&memblock.memory, addr) != -1; 1997 } 1998 1999 bool __init_memblock memblock_is_map_memory(phys_addr_t addr) 2000 { 2001 int i = memblock_search(&memblock.memory, addr); 2002 2003 if (i == -1) 2004 return false; 2005 return !memblock_is_nomap(&memblock.memory.regions[i]); 2006 } 2007 2008 int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 2009 unsigned long *start_pfn, unsigned long *end_pfn) 2010 { 2011 struct memblock_type *type = &memblock.memory; 2012 int mid = memblock_search(type, PFN_PHYS(pfn)); 2013 2014 if (mid == -1) 2015 return NUMA_NO_NODE; 2016 2017 *start_pfn = PFN_DOWN(type->regions[mid].base); 2018 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size); 2019 2020 return memblock_get_region_node(&type->regions[mid]); 2021 } 2022 2023 /** 2024 * memblock_is_region_memory - check if a region is a subset of memory 2025 * @base: base of region to check 2026 * @size: size of region to check 2027 * 2028 * Check if the region [@base, @base + @size) is a subset of a memory block. 2029 * 2030 * Return: 2031 * 0 if false, non-zero if true 2032 */ 2033 bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 2034 { 2035 int idx = memblock_search(&memblock.memory, base); 2036 phys_addr_t end = base + memblock_cap_size(base, &size); 2037 2038 if (idx == -1) 2039 return false; 2040 return (memblock.memory.regions[idx].base + 2041 memblock.memory.regions[idx].size) >= end; 2042 } 2043 2044 /** 2045 * memblock_is_region_reserved - check if a region intersects reserved memory 2046 * @base: base of region to check 2047 * @size: size of region to check 2048 * 2049 * Check if the region [@base, @base + @size) intersects a reserved 2050 * memory block. 2051 * 2052 * Return: 2053 * True if they intersect, false if not. 2054 */ 2055 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 2056 { 2057 return memblock_overlaps_region(&memblock.reserved, base, size); 2058 } 2059 2060 void __init_memblock memblock_trim_memory(phys_addr_t align) 2061 { 2062 phys_addr_t start, end, orig_start, orig_end; 2063 struct memblock_region *r; 2064 2065 for_each_mem_region(r) { 2066 orig_start = r->base; 2067 orig_end = r->base + r->size; 2068 start = round_up(orig_start, align); 2069 end = round_down(orig_end, align); 2070 2071 if (start == orig_start && end == orig_end) 2072 continue; 2073 2074 if (start < end) { 2075 r->base = start; 2076 r->size = end - start; 2077 } else { 2078 memblock_remove_region(&memblock.memory, 2079 r - memblock.memory.regions); 2080 r--; 2081 } 2082 } 2083 } 2084 2085 void __init_memblock memblock_set_current_limit(phys_addr_t limit) 2086 { 2087 memblock.current_limit = limit; 2088 } 2089 2090 phys_addr_t __init_memblock memblock_get_current_limit(void) 2091 { 2092 return memblock.current_limit; 2093 } 2094 2095 static void __init_memblock memblock_dump(struct memblock_type *type) 2096 { 2097 phys_addr_t base, end, size; 2098 enum memblock_flags flags; 2099 int idx; 2100 struct memblock_region *rgn; 2101 2102 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt); 2103 2104 for_each_memblock_type(idx, type, rgn) { 2105 char nid_buf[32] = ""; 2106 2107 base = rgn->base; 2108 size = rgn->size; 2109 end = base + size - 1; 2110 flags = rgn->flags; 2111 #ifdef CONFIG_NUMA 2112 if (numa_valid_node(memblock_get_region_node(rgn))) 2113 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 2114 memblock_get_region_node(rgn)); 2115 #endif 2116 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n", 2117 type->name, idx, &base, &end, &size, nid_buf, flags); 2118 } 2119 } 2120 2121 static void __init_memblock __memblock_dump_all(void) 2122 { 2123 pr_info("MEMBLOCK configuration:\n"); 2124 pr_info(" memory size = %pa reserved size = %pa\n", 2125 &memblock.memory.total_size, 2126 &memblock.reserved.total_size); 2127 2128 memblock_dump(&memblock.memory); 2129 memblock_dump(&memblock.reserved); 2130 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2131 memblock_dump(&physmem); 2132 #endif 2133 } 2134 2135 void __init_memblock memblock_dump_all(void) 2136 { 2137 if (memblock_debug) 2138 __memblock_dump_all(); 2139 } 2140 2141 void __init memblock_allow_resize(void) 2142 { 2143 memblock_can_resize = 1; 2144 } 2145 2146 static int __init early_memblock(char *p) 2147 { 2148 if (p && strstr(p, "debug")) 2149 memblock_debug = 1; 2150 return 0; 2151 } 2152 early_param("memblock", early_memblock); 2153 2154 static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn) 2155 { 2156 struct page *start_pg, *end_pg; 2157 phys_addr_t pg, pgend; 2158 2159 /* 2160 * Convert start_pfn/end_pfn to a struct page pointer. 2161 */ 2162 start_pg = pfn_to_page(start_pfn - 1) + 1; 2163 end_pg = pfn_to_page(end_pfn - 1) + 1; 2164 2165 /* 2166 * Convert to physical addresses, and round start upwards and end 2167 * downwards. 2168 */ 2169 pg = PAGE_ALIGN(__pa(start_pg)); 2170 pgend = PAGE_ALIGN_DOWN(__pa(end_pg)); 2171 2172 /* 2173 * If there are free pages between these, free the section of the 2174 * memmap array. 2175 */ 2176 if (pg < pgend) 2177 memblock_phys_free(pg, pgend - pg); 2178 } 2179 2180 /* 2181 * The mem_map array can get very big. Free the unused area of the memory map. 2182 */ 2183 static void __init free_unused_memmap(void) 2184 { 2185 unsigned long start, end, prev_end = 0; 2186 int i; 2187 2188 if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) || 2189 IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 2190 return; 2191 2192 /* 2193 * This relies on each bank being in address order. 2194 * The banks are sorted previously in bootmem_init(). 2195 */ 2196 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { 2197 #ifdef CONFIG_SPARSEMEM 2198 /* 2199 * Take care not to free memmap entries that don't exist 2200 * due to SPARSEMEM sections which aren't present. 2201 */ 2202 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION)); 2203 #endif 2204 /* 2205 * Align down here since many operations in VM subsystem 2206 * presume that there are no holes in the memory map inside 2207 * a pageblock 2208 */ 2209 start = pageblock_start_pfn(start); 2210 2211 /* 2212 * If we had a previous bank, and there is a space 2213 * between the current bank and the previous, free it. 2214 */ 2215 if (prev_end && prev_end < start) 2216 free_memmap(prev_end, start); 2217 2218 /* 2219 * Align up here since many operations in VM subsystem 2220 * presume that there are no holes in the memory map inside 2221 * a pageblock 2222 */ 2223 prev_end = pageblock_align(end); 2224 } 2225 2226 #ifdef CONFIG_SPARSEMEM 2227 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) { 2228 prev_end = pageblock_align(end); 2229 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); 2230 } 2231 #endif 2232 } 2233 2234 static void __init __free_pages_memory(unsigned long start, unsigned long end) 2235 { 2236 int order; 2237 2238 while (start < end) { 2239 /* 2240 * Free the pages in the largest chunks alignment allows. 2241 * 2242 * __ffs() behaviour is undefined for 0. start == 0 is 2243 * MAX_PAGE_ORDER-aligned, set order to MAX_PAGE_ORDER for 2244 * the case. 2245 */ 2246 if (start) 2247 order = min_t(int, MAX_PAGE_ORDER, __ffs(start)); 2248 else 2249 order = MAX_PAGE_ORDER; 2250 2251 while (start + (1UL << order) > end) 2252 order--; 2253 2254 memblock_free_pages(start, order); 2255 2256 start += (1UL << order); 2257 } 2258 } 2259 2260 static unsigned long __init __free_memory_core(phys_addr_t start, 2261 phys_addr_t end) 2262 { 2263 unsigned long start_pfn = PFN_UP(start); 2264 unsigned long end_pfn = PFN_DOWN(end); 2265 2266 if (!IS_ENABLED(CONFIG_HIGHMEM) && end_pfn > max_low_pfn) 2267 end_pfn = max_low_pfn; 2268 2269 if (start_pfn >= end_pfn) 2270 return 0; 2271 2272 __free_pages_memory(start_pfn, end_pfn); 2273 2274 return end_pfn - start_pfn; 2275 } 2276 2277 /* 2278 * Initialised pages do not have PageReserved set. This function is called 2279 * for each reserved range and marks the pages PageReserved. 2280 * When deferred initialization of struct pages is enabled it also ensures 2281 * that struct pages are properly initialised. 2282 */ 2283 static void __init memmap_init_reserved_range(phys_addr_t start, 2284 phys_addr_t end, int nid) 2285 { 2286 unsigned long pfn; 2287 2288 for_each_valid_pfn(pfn, PFN_DOWN(start), PFN_UP(end)) { 2289 struct page *page = pfn_to_page(pfn); 2290 2291 init_deferred_page(pfn, nid); 2292 2293 /* 2294 * no need for atomic set_bit because the struct 2295 * page is not visible yet so nobody should 2296 * access it yet. 2297 */ 2298 __SetPageReserved(page); 2299 } 2300 } 2301 2302 static void __init memmap_init_reserved_pages(void) 2303 { 2304 struct memblock_region *region; 2305 phys_addr_t start, end; 2306 int nid; 2307 unsigned long max_reserved; 2308 2309 /* 2310 * set nid on all reserved pages and also treat struct 2311 * pages for the NOMAP regions as PageReserved 2312 */ 2313 repeat: 2314 max_reserved = memblock.reserved.max; 2315 for_each_mem_region(region) { 2316 nid = memblock_get_region_node(region); 2317 start = region->base; 2318 end = start + region->size; 2319 2320 if (memblock_is_nomap(region)) 2321 memmap_init_reserved_range(start, end, nid); 2322 2323 memblock_set_node(start, region->size, &memblock.reserved, nid); 2324 } 2325 /* 2326 * 'max' is changed means memblock.reserved has been doubled its 2327 * array, which may result a new reserved region before current 2328 * 'start'. Now we should repeat the procedure to set its node id. 2329 */ 2330 if (max_reserved != memblock.reserved.max) 2331 goto repeat; 2332 2333 /* 2334 * initialize struct pages for reserved regions that don't have 2335 * the MEMBLOCK_RSRV_NOINIT flag set 2336 */ 2337 for_each_reserved_mem_region(region) { 2338 if (!memblock_is_reserved_noinit(region)) { 2339 nid = memblock_get_region_node(region); 2340 start = region->base; 2341 end = start + region->size; 2342 2343 if (!numa_valid_node(nid)) 2344 nid = early_pfn_to_nid(PFN_DOWN(start)); 2345 2346 memmap_init_reserved_range(start, end, nid); 2347 } 2348 } 2349 } 2350 2351 static unsigned long __init free_low_memory_core_early(void) 2352 { 2353 unsigned long count = 0; 2354 phys_addr_t start, end; 2355 u64 i; 2356 2357 memblock_clear_hotplug(0, -1); 2358 2359 memmap_init_reserved_pages(); 2360 2361 /* 2362 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id 2363 * because in some case like Node0 doesn't have RAM installed 2364 * low ram will be on Node1 2365 */ 2366 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, 2367 NULL) 2368 count += __free_memory_core(start, end); 2369 2370 return count; 2371 } 2372 2373 static int reset_managed_pages_done __initdata; 2374 2375 static void __init reset_node_managed_pages(pg_data_t *pgdat) 2376 { 2377 struct zone *z; 2378 2379 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 2380 atomic_long_set(&z->managed_pages, 0); 2381 } 2382 2383 void __init reset_all_zones_managed_pages(void) 2384 { 2385 struct pglist_data *pgdat; 2386 2387 if (reset_managed_pages_done) 2388 return; 2389 2390 for_each_online_pgdat(pgdat) 2391 reset_node_managed_pages(pgdat); 2392 2393 reset_managed_pages_done = 1; 2394 } 2395 2396 /** 2397 * memblock_free_all - release free pages to the buddy allocator 2398 */ 2399 void __init memblock_free_all(void) 2400 { 2401 unsigned long pages; 2402 2403 free_unused_memmap(); 2404 reset_all_zones_managed_pages(); 2405 2406 memblock_clear_kho_scratch_only(); 2407 pages = free_low_memory_core_early(); 2408 totalram_pages_add(pages); 2409 } 2410 2411 /* Keep a table to reserve named memory */ 2412 #define RESERVE_MEM_MAX_ENTRIES 8 2413 #define RESERVE_MEM_NAME_SIZE 16 2414 struct reserve_mem_table { 2415 char name[RESERVE_MEM_NAME_SIZE]; 2416 phys_addr_t start; 2417 phys_addr_t size; 2418 }; 2419 static struct reserve_mem_table reserved_mem_table[RESERVE_MEM_MAX_ENTRIES]; 2420 static int reserved_mem_count; 2421 static DEFINE_MUTEX(reserve_mem_lock); 2422 2423 /* Add wildcard region with a lookup name */ 2424 static void __init reserved_mem_add(phys_addr_t start, phys_addr_t size, 2425 const char *name) 2426 { 2427 struct reserve_mem_table *map; 2428 2429 map = &reserved_mem_table[reserved_mem_count++]; 2430 map->start = start; 2431 map->size = size; 2432 strscpy(map->name, name); 2433 } 2434 2435 static struct reserve_mem_table *reserve_mem_find_by_name_nolock(const char *name) 2436 { 2437 struct reserve_mem_table *map; 2438 int i; 2439 2440 for (i = 0; i < reserved_mem_count; i++) { 2441 map = &reserved_mem_table[i]; 2442 if (!map->size) 2443 continue; 2444 if (strcmp(name, map->name) == 0) 2445 return map; 2446 } 2447 return NULL; 2448 } 2449 2450 /** 2451 * reserve_mem_find_by_name - Find reserved memory region with a given name 2452 * @name: The name that is attached to a reserved memory region 2453 * @start: If found, holds the start address 2454 * @size: If found, holds the size of the address. 2455 * 2456 * @start and @size are only updated if @name is found. 2457 * 2458 * Returns: 1 if found or 0 if not found. 2459 */ 2460 int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size) 2461 { 2462 struct reserve_mem_table *map; 2463 2464 guard(mutex)(&reserve_mem_lock); 2465 map = reserve_mem_find_by_name_nolock(name); 2466 if (!map) 2467 return 0; 2468 2469 *start = map->start; 2470 *size = map->size; 2471 return 1; 2472 } 2473 EXPORT_SYMBOL_GPL(reserve_mem_find_by_name); 2474 2475 /** 2476 * reserve_mem_release_by_name - Release reserved memory region with a given name 2477 * @name: The name that is attached to a reserved memory region 2478 * 2479 * Forcibly release the pages in the reserved memory region so that those memory 2480 * can be used as free memory. After released the reserved region size becomes 0. 2481 * 2482 * Returns: 1 if released or 0 if not found. 2483 */ 2484 int reserve_mem_release_by_name(const char *name) 2485 { 2486 char buf[RESERVE_MEM_NAME_SIZE + 12]; 2487 struct reserve_mem_table *map; 2488 void *start, *end; 2489 2490 guard(mutex)(&reserve_mem_lock); 2491 map = reserve_mem_find_by_name_nolock(name); 2492 if (!map) 2493 return 0; 2494 2495 start = phys_to_virt(map->start); 2496 end = start + map->size; 2497 snprintf(buf, sizeof(buf), "reserve_mem:%s", name); 2498 free_reserved_area(start, end, 0, buf); 2499 map->size = 0; 2500 2501 return 1; 2502 } 2503 2504 #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 2505 __init void memblock_set_kho_scratch_only(void) 2506 { 2507 kho_scratch_only = true; 2508 } 2509 2510 __init void memblock_clear_kho_scratch_only(void) 2511 { 2512 kho_scratch_only = false; 2513 } 2514 2515 bool __init_memblock memblock_is_kho_scratch_memory(phys_addr_t addr) 2516 { 2517 int i = memblock_search(&memblock.memory, addr); 2518 2519 if (i == -1) 2520 return false; 2521 2522 return memblock_is_kho_scratch(&memblock.memory.regions[i]); 2523 } 2524 #endif 2525 2526 #ifdef CONFIG_KEXEC_HANDOVER 2527 2528 static int __init reserved_mem_preserve(void) 2529 { 2530 unsigned int nr_preserved = 0; 2531 int err; 2532 2533 for (unsigned int i = 0; i < reserved_mem_count; i++, nr_preserved++) { 2534 struct reserve_mem_table *map = &reserved_mem_table[i]; 2535 struct page *page = phys_to_page(map->start); 2536 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2537 2538 err = kho_preserve_pages(page, nr_pages); 2539 if (err) 2540 goto err_unpreserve; 2541 } 2542 2543 return 0; 2544 2545 err_unpreserve: 2546 for (unsigned int i = 0; i < nr_preserved; i++) { 2547 struct reserve_mem_table *map = &reserved_mem_table[i]; 2548 struct page *page = phys_to_page(map->start); 2549 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2550 2551 kho_unpreserve_pages(page, nr_pages); 2552 } 2553 2554 return err; 2555 } 2556 2557 static int __init prepare_kho_fdt(void) 2558 { 2559 struct page *fdt_page; 2560 void *fdt; 2561 int err; 2562 2563 fdt_page = alloc_page(GFP_KERNEL); 2564 if (!fdt_page) { 2565 err = -ENOMEM; 2566 goto err_report; 2567 } 2568 2569 fdt = page_to_virt(fdt_page); 2570 err = kho_preserve_pages(fdt_page, 1); 2571 if (err) 2572 goto err_free_fdt; 2573 2574 err |= fdt_create(fdt, PAGE_SIZE); 2575 err |= fdt_finish_reservemap(fdt); 2576 err |= fdt_begin_node(fdt, ""); 2577 err |= fdt_property_string(fdt, "compatible", MEMBLOCK_KHO_NODE_COMPATIBLE); 2578 2579 for (unsigned int i = 0; !err && i < reserved_mem_count; i++) { 2580 struct reserve_mem_table *map = &reserved_mem_table[i]; 2581 2582 err |= fdt_begin_node(fdt, map->name); 2583 err |= fdt_property_string(fdt, "compatible", RESERVE_MEM_KHO_NODE_COMPATIBLE); 2584 err |= fdt_property(fdt, "start", &map->start, sizeof(map->start)); 2585 err |= fdt_property(fdt, "size", &map->size, sizeof(map->size)); 2586 err |= fdt_end_node(fdt); 2587 } 2588 err |= fdt_end_node(fdt); 2589 err |= fdt_finish(fdt); 2590 2591 if (err) 2592 goto err_unpreserve_fdt; 2593 2594 err = kho_add_subtree(MEMBLOCK_KHO_FDT, fdt, fdt_totalsize(fdt)); 2595 if (err) 2596 goto err_unpreserve_fdt; 2597 2598 err = reserved_mem_preserve(); 2599 if (err) 2600 goto err_remove_subtree; 2601 2602 return 0; 2603 2604 err_remove_subtree: 2605 kho_remove_subtree(fdt); 2606 err_unpreserve_fdt: 2607 kho_unpreserve_pages(fdt_page, 1); 2608 err_free_fdt: 2609 put_page(fdt_page); 2610 err_report: 2611 pr_err("failed to prepare memblock FDT for KHO: %d\n", err); 2612 2613 return err; 2614 } 2615 2616 static int __init reserve_mem_init(void) 2617 { 2618 int err; 2619 2620 if (!kho_is_enabled() || !reserved_mem_count) 2621 return 0; 2622 2623 err = prepare_kho_fdt(); 2624 if (err) 2625 return err; 2626 return err; 2627 } 2628 late_initcall(reserve_mem_init); 2629 2630 static void *__init reserve_mem_kho_retrieve_fdt(void) 2631 { 2632 phys_addr_t fdt_phys; 2633 static void *fdt; 2634 int err; 2635 2636 if (fdt) 2637 return fdt; 2638 2639 err = kho_retrieve_subtree(MEMBLOCK_KHO_FDT, &fdt_phys, NULL); 2640 if (err) { 2641 if (err != -ENOENT) 2642 pr_warn("failed to retrieve FDT '%s' from KHO: %d\n", 2643 MEMBLOCK_KHO_FDT, err); 2644 return NULL; 2645 } 2646 2647 fdt = phys_to_virt(fdt_phys); 2648 2649 err = fdt_node_check_compatible(fdt, 0, MEMBLOCK_KHO_NODE_COMPATIBLE); 2650 if (err) { 2651 pr_warn("FDT '%s' is incompatible with '%s': %d\n", 2652 MEMBLOCK_KHO_FDT, MEMBLOCK_KHO_NODE_COMPATIBLE, err); 2653 fdt = NULL; 2654 } 2655 2656 return fdt; 2657 } 2658 2659 static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2660 phys_addr_t align) 2661 { 2662 int err, len_start, len_size, offset; 2663 const phys_addr_t *p_start, *p_size; 2664 const void *fdt; 2665 2666 fdt = reserve_mem_kho_retrieve_fdt(); 2667 if (!fdt) 2668 return false; 2669 2670 offset = fdt_subnode_offset(fdt, 0, name); 2671 if (offset < 0) { 2672 pr_warn("FDT '%s' has no child '%s': %d\n", 2673 MEMBLOCK_KHO_FDT, name, offset); 2674 return false; 2675 } 2676 err = fdt_node_check_compatible(fdt, offset, RESERVE_MEM_KHO_NODE_COMPATIBLE); 2677 if (err) { 2678 pr_warn("Node '%s' is incompatible with '%s': %d\n", 2679 name, RESERVE_MEM_KHO_NODE_COMPATIBLE, err); 2680 return false; 2681 } 2682 2683 p_start = fdt_getprop(fdt, offset, "start", &len_start); 2684 p_size = fdt_getprop(fdt, offset, "size", &len_size); 2685 if (!p_start || len_start != sizeof(*p_start) || !p_size || 2686 len_size != sizeof(*p_size)) { 2687 return false; 2688 } 2689 2690 if (*p_start & (align - 1)) { 2691 pr_warn("KHO reserve-mem '%s' has wrong alignment (0x%lx, 0x%lx)\n", 2692 name, (long)align, (long)*p_start); 2693 return false; 2694 } 2695 2696 if (*p_size != size) { 2697 pr_warn("KHO reserve-mem '%s' has wrong size (0x%lx != 0x%lx)\n", 2698 name, (long)*p_size, (long)size); 2699 return false; 2700 } 2701 2702 reserved_mem_add(*p_start, size, name); 2703 pr_info("Revived memory reservation '%s' from KHO\n", name); 2704 2705 return true; 2706 } 2707 #else 2708 static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2709 phys_addr_t align) 2710 { 2711 return false; 2712 } 2713 #endif /* CONFIG_KEXEC_HANDOVER */ 2714 2715 /* 2716 * Parse reserve_mem=nn:align:name 2717 */ 2718 static int __init reserve_mem(char *p) 2719 { 2720 phys_addr_t start, size, align, tmp; 2721 char *name; 2722 char *oldp; 2723 int len; 2724 2725 if (!p) 2726 goto err_param; 2727 2728 /* Check if there's room for more reserved memory */ 2729 if (reserved_mem_count >= RESERVE_MEM_MAX_ENTRIES) { 2730 pr_err("reserve_mem: no more room for reserved memory\n"); 2731 return -EBUSY; 2732 } 2733 2734 oldp = p; 2735 size = memparse(p, &p); 2736 if (!size || p == oldp) 2737 goto err_param; 2738 2739 if (*p != ':') 2740 goto err_param; 2741 2742 align = memparse(p+1, &p); 2743 if (*p != ':') 2744 goto err_param; 2745 2746 /* 2747 * memblock_phys_alloc() doesn't like a zero size align, 2748 * but it is OK for this command to have it. 2749 */ 2750 if (align < SMP_CACHE_BYTES) 2751 align = SMP_CACHE_BYTES; 2752 2753 name = p + 1; 2754 len = strlen(name); 2755 2756 /* name needs to have length but not too big */ 2757 if (!len || len >= RESERVE_MEM_NAME_SIZE) 2758 goto err_param; 2759 2760 /* Make sure that name has text */ 2761 for (p = name; *p; p++) { 2762 if (!isspace(*p)) 2763 break; 2764 } 2765 if (!*p) 2766 goto err_param; 2767 2768 /* Make sure the name is not already used */ 2769 if (reserve_mem_find_by_name(name, &start, &tmp)) { 2770 pr_err("reserve_mem: name \"%s\" was already used\n", name); 2771 return -EBUSY; 2772 } 2773 2774 /* Pick previous allocations up from KHO if available */ 2775 if (reserve_mem_kho_revive(name, size, align)) 2776 return 1; 2777 2778 /* TODO: Allocation must be outside of scratch region */ 2779 start = memblock_phys_alloc(size, align); 2780 if (!start) { 2781 pr_err("reserve_mem: memblock allocation failed\n"); 2782 return -ENOMEM; 2783 } 2784 2785 reserved_mem_add(start, size, name); 2786 2787 return 1; 2788 err_param: 2789 pr_err("reserve_mem: empty or malformed parameter\n"); 2790 return -EINVAL; 2791 } 2792 __setup("reserve_mem=", reserve_mem); 2793 2794 #ifdef CONFIG_DEBUG_FS 2795 #ifdef CONFIG_ARCH_KEEP_MEMBLOCK 2796 static const char * const flagname[] = { 2797 [ilog2(MEMBLOCK_HOTPLUG)] = "HOTPLUG", 2798 [ilog2(MEMBLOCK_MIRROR)] = "MIRROR", 2799 [ilog2(MEMBLOCK_NOMAP)] = "NOMAP", 2800 [ilog2(MEMBLOCK_DRIVER_MANAGED)] = "DRV_MNG", 2801 [ilog2(MEMBLOCK_RSRV_NOINIT)] = "RSV_NIT", 2802 [ilog2(MEMBLOCK_RSRV_KERN)] = "RSV_KERN", 2803 [ilog2(MEMBLOCK_KHO_SCRATCH)] = "KHO_SCRATCH", 2804 }; 2805 2806 static int memblock_debug_show(struct seq_file *m, void *private) 2807 { 2808 struct memblock_type *type = m->private; 2809 struct memblock_region *reg; 2810 int i, j, nid; 2811 unsigned int count = ARRAY_SIZE(flagname); 2812 phys_addr_t end; 2813 2814 for (i = 0; i < type->cnt; i++) { 2815 reg = &type->regions[i]; 2816 end = reg->base + reg->size - 1; 2817 nid = memblock_get_region_node(reg); 2818 2819 seq_printf(m, "%4d: ", i); 2820 seq_printf(m, "%pa..%pa ", ®->base, &end); 2821 if (numa_valid_node(nid)) 2822 seq_printf(m, "%4d ", nid); 2823 else 2824 seq_printf(m, "%4c ", 'x'); 2825 if (reg->flags) { 2826 for (j = 0; j < count; j++) { 2827 if (reg->flags & (1U << j)) { 2828 seq_printf(m, "%s\n", flagname[j]); 2829 break; 2830 } 2831 } 2832 if (j == count) 2833 seq_printf(m, "%s\n", "UNKNOWN"); 2834 } else { 2835 seq_printf(m, "%s\n", "NONE"); 2836 } 2837 } 2838 return 0; 2839 } 2840 DEFINE_SHOW_ATTRIBUTE(memblock_debug); 2841 2842 static inline void memblock_debugfs_expose_arrays(struct dentry *root) 2843 { 2844 debugfs_create_file("memory", 0444, root, 2845 &memblock.memory, &memblock_debug_fops); 2846 debugfs_create_file("reserved", 0444, root, 2847 &memblock.reserved, &memblock_debug_fops); 2848 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2849 debugfs_create_file("physmem", 0444, root, &physmem, 2850 &memblock_debug_fops); 2851 #endif 2852 } 2853 2854 #else 2855 2856 static inline void memblock_debugfs_expose_arrays(struct dentry *root) { } 2857 2858 #endif /* CONFIG_ARCH_KEEP_MEMBLOCK */ 2859 2860 static int memblock_reserve_mem_show(struct seq_file *m, void *private) 2861 { 2862 struct reserve_mem_table *map; 2863 char txtsz[16]; 2864 2865 guard(mutex)(&reserve_mem_lock); 2866 for (int i = 0; i < reserved_mem_count; i++) { 2867 map = &reserved_mem_table[i]; 2868 if (!map->size) 2869 continue; 2870 2871 memset(txtsz, 0, sizeof(txtsz)); 2872 string_get_size(map->size, 1, STRING_UNITS_2, txtsz, sizeof(txtsz)); 2873 seq_printf(m, "%s\t\t(%s)\n", map->name, txtsz); 2874 } 2875 2876 return 0; 2877 } 2878 DEFINE_SHOW_ATTRIBUTE(memblock_reserve_mem); 2879 2880 static int __init memblock_init_debugfs(void) 2881 { 2882 struct dentry *root; 2883 2884 if (!IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK) && !reserved_mem_count) 2885 return 0; 2886 2887 root = debugfs_create_dir("memblock", NULL); 2888 2889 if (reserved_mem_count) 2890 debugfs_create_file("reserve_mem_param", 0444, root, NULL, 2891 &memblock_reserve_mem_fops); 2892 2893 memblock_debugfs_expose_arrays(root); 2894 return 0; 2895 } 2896 __initcall(memblock_init_debugfs); 2897 2898 #endif /* CONFIG_DEBUG_FS */ 2899