1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Procedures for maintaining information about logical memory blocks. 4 * 5 * Peter Bergner, IBM Corp. June 2001. 6 * Copyright (C) 2001 Peter Bergner. 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/slab.h> 11 #include <linux/init.h> 12 #include <linux/bitops.h> 13 #include <linux/poison.h> 14 #include <linux/pfn.h> 15 #include <linux/debugfs.h> 16 #include <linux/kmemleak.h> 17 #include <linux/seq_file.h> 18 #include <linux/memblock.h> 19 #include <linux/mutex.h> 20 #include <linux/string_helpers.h> 21 22 #ifdef CONFIG_KEXEC_HANDOVER 23 #include <linux/libfdt.h> 24 #include <linux/kexec_handover.h> 25 #include <linux/kho/abi/memblock.h> 26 #endif /* CONFIG_KEXEC_HANDOVER */ 27 28 #include <asm/sections.h> 29 #include <linux/io.h> 30 31 #include "internal.h" 32 33 #define INIT_MEMBLOCK_REGIONS 128 34 #define INIT_PHYSMEM_REGIONS 4 35 36 #ifndef INIT_MEMBLOCK_RESERVED_REGIONS 37 # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS 38 #endif 39 40 #ifndef INIT_MEMBLOCK_MEMORY_REGIONS 41 #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS 42 #endif 43 44 /** 45 * DOC: memblock overview 46 * 47 * Memblock is a method of managing memory regions during the early 48 * boot period when the usual kernel memory allocators are not up and 49 * running. 50 * 51 * Memblock views the system memory as collections of contiguous 52 * regions. There are several types of these collections: 53 * 54 * * ``memory`` - describes the physical memory available to the 55 * kernel; this may differ from the actual physical memory installed 56 * in the system, for instance when the memory is restricted with 57 * ``mem=`` command line parameter 58 * * ``reserved`` - describes the regions that were allocated 59 * * ``physmem`` - describes the actual physical memory available during 60 * boot regardless of the possible restrictions and memory hot(un)plug; 61 * the ``physmem`` type is only available on some architectures. 62 * 63 * Each region is represented by struct memblock_region that 64 * defines the region extents, its attributes and NUMA node id on NUMA 65 * systems. Every memory type is described by the struct memblock_type 66 * which contains an array of memory regions along with 67 * the allocator metadata. The "memory" and "reserved" types are nicely 68 * wrapped with struct memblock. This structure is statically 69 * initialized at build time. The region arrays are initially sized to 70 * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and 71 * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array 72 * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS. 73 * The memblock_allow_resize() enables automatic resizing of the region 74 * arrays during addition of new regions. This feature should be used 75 * with care so that memory allocated for the region array will not 76 * overlap with areas that should be reserved, for example initrd. 77 * 78 * The early architecture setup should tell memblock what the physical 79 * memory layout is by using memblock_add() or memblock_add_node() 80 * functions. The first function does not assign the region to a NUMA 81 * node and it is appropriate for UMA systems. Yet, it is possible to 82 * use it on NUMA systems as well and assign the region to a NUMA node 83 * later in the setup process using memblock_set_node(). The 84 * memblock_add_node() performs such an assignment directly. 85 * 86 * Once memblock is setup the memory can be allocated using one of the 87 * API variants: 88 * 89 * * memblock_phys_alloc*() - these functions return the **physical** 90 * address of the allocated memory 91 * * memblock_alloc*() - these functions return the **virtual** address 92 * of the allocated memory. 93 * 94 * Note, that both API variants use implicit assumptions about allowed 95 * memory ranges and the fallback methods. Consult the documentation 96 * of memblock_alloc_internal() and memblock_alloc_range_nid() 97 * functions for more elaborate description. 98 * 99 * As the system boot progresses, the architecture specific mem_init() 100 * function frees all the memory to the buddy page allocator. 101 * 102 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the 103 * memblock data structures (except "physmem") will be discarded after the 104 * system initialization completes. 105 */ 106 107 #ifndef CONFIG_NUMA 108 struct pglist_data __refdata contig_page_data; 109 EXPORT_SYMBOL(contig_page_data); 110 #endif 111 112 unsigned long max_low_pfn; 113 unsigned long min_low_pfn; 114 unsigned long max_pfn; 115 unsigned long long max_possible_pfn; 116 117 #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 118 /* When set to true, only allocate from MEMBLOCK_KHO_SCRATCH ranges */ 119 static bool kho_scratch_only; 120 #else 121 #define kho_scratch_only false 122 #endif 123 124 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock; 125 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock; 126 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 127 static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS]; 128 #endif 129 130 struct memblock memblock __initdata_memblock = { 131 .memory.regions = memblock_memory_init_regions, 132 .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS, 133 .memory.name = "memory", 134 135 .reserved.regions = memblock_reserved_init_regions, 136 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS, 137 .reserved.name = "reserved", 138 139 .bottom_up = false, 140 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 141 }; 142 143 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 144 struct memblock_type physmem = { 145 .regions = memblock_physmem_init_regions, 146 .max = INIT_PHYSMEM_REGIONS, 147 .name = "physmem", 148 }; 149 #endif 150 151 /* 152 * keep a pointer to &memblock.memory in the text section to use it in 153 * __next_mem_range() and its helpers. 154 * For architectures that do not keep memblock data after init, this 155 * pointer will be reset to NULL at memblock_discard() 156 */ 157 static __refdata struct memblock_type *memblock_memory = &memblock.memory; 158 159 #define for_each_memblock_type(i, memblock_type, rgn) \ 160 for (i = 0, rgn = &memblock_type->regions[0]; \ 161 i < memblock_type->cnt; \ 162 i++, rgn = &memblock_type->regions[i]) 163 164 #define memblock_dbg(fmt, ...) \ 165 do { \ 166 if (memblock_debug) \ 167 pr_info(fmt, ##__VA_ARGS__); \ 168 } while (0) 169 170 static int memblock_debug __initdata_memblock; 171 static bool system_has_some_mirror __initdata_memblock; 172 static int memblock_can_resize __initdata_memblock; 173 static int memblock_memory_in_slab __initdata_memblock; 174 static int memblock_reserved_in_slab __initdata_memblock; 175 176 bool __init_memblock memblock_has_mirror(void) 177 { 178 return system_has_some_mirror; 179 } 180 181 static enum memblock_flags __init_memblock choose_memblock_flags(void) 182 { 183 /* skip non-scratch memory for kho early boot allocations */ 184 if (kho_scratch_only) 185 return MEMBLOCK_KHO_SCRATCH; 186 187 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; 188 } 189 190 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 191 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 192 { 193 return *size = min(*size, PHYS_ADDR_MAX - base); 194 } 195 196 /* 197 * Address comparison utilities 198 */ 199 unsigned long __init_memblock 200 memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, phys_addr_t base2, 201 phys_addr_t size2) 202 { 203 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 204 } 205 206 bool __init_memblock memblock_overlaps_region(struct memblock_type *type, 207 phys_addr_t base, phys_addr_t size) 208 { 209 unsigned long i; 210 211 memblock_cap_size(base, &size); 212 213 for (i = 0; i < type->cnt; i++) 214 if (memblock_addrs_overlap(base, size, type->regions[i].base, 215 type->regions[i].size)) 216 return true; 217 return false; 218 } 219 220 /** 221 * __memblock_find_range_bottom_up - find free area utility in bottom-up 222 * @start: start of candidate range 223 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 224 * %MEMBLOCK_ALLOC_ACCESSIBLE 225 * @size: size of free area to find 226 * @align: alignment of free area to find 227 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 228 * @flags: pick from blocks based on memory attributes 229 * 230 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 231 * 232 * Return: 233 * Found address on success, 0 on failure. 234 */ 235 static phys_addr_t __init_memblock 236 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 237 phys_addr_t size, phys_addr_t align, int nid, 238 enum memblock_flags flags) 239 { 240 phys_addr_t this_start, this_end, cand; 241 u64 i; 242 243 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) { 244 this_start = clamp(this_start, start, end); 245 this_end = clamp(this_end, start, end); 246 247 cand = round_up(this_start, align); 248 if (cand < this_end && this_end - cand >= size) 249 return cand; 250 } 251 252 return 0; 253 } 254 255 /** 256 * __memblock_find_range_top_down - find free area utility, in top-down 257 * @start: start of candidate range 258 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 259 * %MEMBLOCK_ALLOC_ACCESSIBLE 260 * @size: size of free area to find 261 * @align: alignment of free area to find 262 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 263 * @flags: pick from blocks based on memory attributes 264 * 265 * Utility called from memblock_find_in_range_node(), find free area top-down. 266 * 267 * Return: 268 * Found address on success, 0 on failure. 269 */ 270 static phys_addr_t __init_memblock 271 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 272 phys_addr_t size, phys_addr_t align, int nid, 273 enum memblock_flags flags) 274 { 275 phys_addr_t this_start, this_end, cand; 276 u64 i; 277 278 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end, 279 NULL) { 280 this_start = clamp(this_start, start, end); 281 this_end = clamp(this_end, start, end); 282 283 if (this_end < size) 284 continue; 285 286 cand = round_down(this_end - size, align); 287 if (cand >= this_start) 288 return cand; 289 } 290 291 return 0; 292 } 293 294 /** 295 * memblock_find_in_range_node - find free area in given range and node 296 * @size: size of free area to find 297 * @align: alignment of free area to find 298 * @start: start of candidate range 299 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 300 * %MEMBLOCK_ALLOC_ACCESSIBLE 301 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 302 * @flags: pick from blocks based on memory attributes 303 * 304 * Find @size free area aligned to @align in the specified range and node. 305 * 306 * Return: 307 * Found address on success, 0 on failure. 308 */ 309 static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 310 phys_addr_t align, phys_addr_t start, 311 phys_addr_t end, int nid, 312 enum memblock_flags flags) 313 { 314 /* pump up @end */ 315 if (end == MEMBLOCK_ALLOC_ACCESSIBLE || 316 end == MEMBLOCK_ALLOC_NOLEAKTRACE) 317 end = memblock.current_limit; 318 319 /* avoid allocating the first page */ 320 start = max_t(phys_addr_t, start, PAGE_SIZE); 321 end = max(start, end); 322 323 if (memblock_bottom_up()) 324 return __memblock_find_range_bottom_up(start, end, size, align, 325 nid, flags); 326 else 327 return __memblock_find_range_top_down(start, end, size, align, 328 nid, flags); 329 } 330 331 /** 332 * memblock_find_in_range - find free area in given range 333 * @start: start of candidate range 334 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 335 * %MEMBLOCK_ALLOC_ACCESSIBLE 336 * @size: size of free area to find 337 * @align: alignment of free area to find 338 * 339 * Find @size free area aligned to @align in the specified range. 340 * 341 * Return: 342 * Found address on success, 0 on failure. 343 */ 344 static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 345 phys_addr_t end, phys_addr_t size, 346 phys_addr_t align) 347 { 348 phys_addr_t ret; 349 enum memblock_flags flags = choose_memblock_flags(); 350 351 again: 352 ret = memblock_find_in_range_node(size, align, start, end, 353 NUMA_NO_NODE, flags); 354 355 if (!ret && (flags & MEMBLOCK_MIRROR)) { 356 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 357 &size); 358 flags &= ~MEMBLOCK_MIRROR; 359 goto again; 360 } 361 362 return ret; 363 } 364 365 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 366 { 367 type->total_size -= type->regions[r].size; 368 memmove(&type->regions[r], &type->regions[r + 1], 369 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 370 type->cnt--; 371 372 /* Special case for empty arrays */ 373 if (type->cnt == 0) { 374 WARN_ON(type->total_size != 0); 375 type->regions[0].base = 0; 376 type->regions[0].size = 0; 377 type->regions[0].flags = 0; 378 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 379 } 380 } 381 382 #ifndef CONFIG_ARCH_KEEP_MEMBLOCK 383 /** 384 * memblock_discard - discard memory and reserved arrays if they were allocated 385 */ 386 void __init memblock_discard(void) 387 { 388 phys_addr_t size; 389 void *addr; 390 391 if (memblock.reserved.regions != memblock_reserved_init_regions) { 392 addr = memblock.reserved.regions; 393 size = PAGE_ALIGN(sizeof(struct memblock_region) * 394 memblock.reserved.max); 395 if (memblock_reserved_in_slab) 396 kfree(addr); 397 else 398 memblock_free(addr, size); 399 } 400 401 if (memblock.memory.regions != memblock_memory_init_regions) { 402 addr = memblock.memory.regions; 403 size = PAGE_ALIGN(sizeof(struct memblock_region) * 404 memblock.memory.max); 405 if (memblock_memory_in_slab) 406 kfree(addr); 407 else 408 memblock_free(addr, size); 409 } 410 411 memblock_memory = NULL; 412 } 413 #endif 414 415 /** 416 * memblock_double_array - double the size of the memblock regions array 417 * @type: memblock type of the regions array being doubled 418 * @new_area_start: starting address of memory range to avoid overlap with 419 * @new_area_size: size of memory range to avoid overlap with 420 * 421 * Double the size of the @type regions array. If memblock is being used to 422 * allocate memory for a new reserved regions array and there is a previously 423 * allocated memory range [@new_area_start, @new_area_start + @new_area_size] 424 * waiting to be reserved, ensure the memory used by the new array does 425 * not overlap. 426 * 427 * Return: 428 * 0 on success, -1 on failure. 429 */ 430 static int __init_memblock memblock_double_array(struct memblock_type *type, 431 phys_addr_t new_area_start, 432 phys_addr_t new_area_size) 433 { 434 struct memblock_region *new_array, *old_array; 435 phys_addr_t old_alloc_size, new_alloc_size; 436 phys_addr_t old_size, new_size, addr, new_end; 437 int use_slab = slab_is_available(); 438 int *in_slab; 439 440 /* We don't allow resizing until we know about the reserved regions 441 * of memory that aren't suitable for allocation 442 */ 443 if (!memblock_can_resize) 444 panic("memblock: cannot resize %s array\n", type->name); 445 446 /* Calculate new doubled size */ 447 old_size = type->max * sizeof(struct memblock_region); 448 new_size = old_size << 1; 449 /* 450 * We need to allocated new one align to PAGE_SIZE, 451 * so we can free them completely later. 452 */ 453 old_alloc_size = PAGE_ALIGN(old_size); 454 new_alloc_size = PAGE_ALIGN(new_size); 455 456 /* Retrieve the slab flag */ 457 if (type == &memblock.memory) 458 in_slab = &memblock_memory_in_slab; 459 else 460 in_slab = &memblock_reserved_in_slab; 461 462 /* Try to find some space for it */ 463 if (use_slab) { 464 new_array = kmalloc(new_size, GFP_KERNEL); 465 addr = new_array ? __pa(new_array) : 0; 466 } else { 467 /* only exclude range when trying to double reserved.regions */ 468 if (type != &memblock.reserved) 469 new_area_start = new_area_size = 0; 470 471 addr = memblock_find_in_range(new_area_start + new_area_size, 472 memblock.current_limit, 473 new_alloc_size, PAGE_SIZE); 474 if (!addr && new_area_size) 475 addr = memblock_find_in_range(0, 476 min(new_area_start, memblock.current_limit), 477 new_alloc_size, PAGE_SIZE); 478 479 if (addr) { 480 /* The memory may not have been accepted, yet. */ 481 accept_memory(addr, new_alloc_size); 482 483 new_array = __va(addr); 484 } else { 485 new_array = NULL; 486 } 487 } 488 if (!addr) { 489 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 490 type->name, type->max, type->max * 2); 491 return -1; 492 } 493 494 new_end = addr + new_size - 1; 495 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]", 496 type->name, type->max * 2, &addr, &new_end); 497 498 /* 499 * Found space, we now need to move the array over before we add the 500 * reserved region since it may be our reserved array itself that is 501 * full. 502 */ 503 memcpy(new_array, type->regions, old_size); 504 memset(new_array + type->max, 0, old_size); 505 old_array = type->regions; 506 type->regions = new_array; 507 type->max <<= 1; 508 509 /* Free old array. We needn't free it if the array is the static one */ 510 if (*in_slab) 511 kfree(old_array); 512 else if (old_array != memblock_memory_init_regions && 513 old_array != memblock_reserved_init_regions) 514 memblock_free(old_array, old_alloc_size); 515 516 /* 517 * Reserve the new array if that comes from the memblock. Otherwise, we 518 * needn't do it 519 */ 520 if (!use_slab) 521 BUG_ON(memblock_reserve_kern(addr, new_alloc_size)); 522 523 /* Update slab flag */ 524 *in_slab = use_slab; 525 526 return 0; 527 } 528 529 /** 530 * memblock_merge_regions - merge neighboring compatible regions 531 * @type: memblock type to scan 532 * @start_rgn: start scanning from (@start_rgn - 1) 533 * @end_rgn: end scanning at (@end_rgn - 1) 534 * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn) 535 */ 536 static void __init_memblock memblock_merge_regions(struct memblock_type *type, 537 unsigned long start_rgn, 538 unsigned long end_rgn) 539 { 540 int i = 0; 541 if (start_rgn) 542 i = start_rgn - 1; 543 end_rgn = min(end_rgn, type->cnt - 1); 544 while (i < end_rgn) { 545 struct memblock_region *this = &type->regions[i]; 546 struct memblock_region *next = &type->regions[i + 1]; 547 548 if (this->base + this->size != next->base || 549 memblock_get_region_node(this) != 550 memblock_get_region_node(next) || 551 this->flags != next->flags) { 552 BUG_ON(this->base + this->size > next->base); 553 i++; 554 continue; 555 } 556 557 this->size += next->size; 558 /* move forward from next + 1, index of which is i + 2 */ 559 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 560 type->cnt--; 561 end_rgn--; 562 } 563 } 564 565 /** 566 * memblock_insert_region - insert new memblock region 567 * @type: memblock type to insert into 568 * @idx: index for the insertion point 569 * @base: base address of the new region 570 * @size: size of the new region 571 * @nid: node id of the new region 572 * @flags: flags of the new region 573 * 574 * Insert new memblock region [@base, @base + @size) into @type at @idx. 575 * @type must already have extra room to accommodate the new region. 576 */ 577 static void __init_memblock memblock_insert_region(struct memblock_type *type, 578 int idx, phys_addr_t base, 579 phys_addr_t size, 580 int nid, 581 enum memblock_flags flags) 582 { 583 struct memblock_region *rgn = &type->regions[idx]; 584 585 BUG_ON(type->cnt >= type->max); 586 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 587 rgn->base = base; 588 rgn->size = size; 589 rgn->flags = flags; 590 memblock_set_region_node(rgn, nid); 591 type->cnt++; 592 type->total_size += size; 593 } 594 595 /** 596 * memblock_add_range - add new memblock region 597 * @type: memblock type to add new region into 598 * @base: base address of the new region 599 * @size: size of the new region 600 * @nid: nid of the new region 601 * @flags: flags of the new region 602 * 603 * Add new memblock region [@base, @base + @size) into @type. The new region 604 * is allowed to overlap with existing ones - overlaps don't affect already 605 * existing regions. @type is guaranteed to be minimal (all neighbouring 606 * compatible regions are merged) after the addition. 607 * 608 * Return: 609 * 0 on success, -errno on failure. 610 */ 611 static int __init_memblock memblock_add_range(struct memblock_type *type, 612 phys_addr_t base, phys_addr_t size, 613 int nid, enum memblock_flags flags) 614 { 615 bool insert = false; 616 phys_addr_t obase = base; 617 phys_addr_t end = base + memblock_cap_size(base, &size); 618 int idx, nr_new, start_rgn = -1, end_rgn; 619 struct memblock_region *rgn; 620 621 if (!size) 622 return 0; 623 624 /* special case for empty array */ 625 if (type->regions[0].size == 0) { 626 WARN_ON(type->cnt != 0 || type->total_size); 627 type->regions[0].base = base; 628 type->regions[0].size = size; 629 type->regions[0].flags = flags; 630 memblock_set_region_node(&type->regions[0], nid); 631 type->total_size = size; 632 type->cnt = 1; 633 return 0; 634 } 635 636 /* 637 * The worst case is when new range overlaps all existing regions, 638 * then we'll need type->cnt + 1 empty regions in @type. So if 639 * type->cnt * 2 + 1 is less than or equal to type->max, we know 640 * that there is enough empty regions in @type, and we can insert 641 * regions directly. 642 */ 643 if (type->cnt * 2 + 1 <= type->max) 644 insert = true; 645 646 repeat: 647 /* 648 * The following is executed twice. Once with %false @insert and 649 * then with %true. The first counts the number of regions needed 650 * to accommodate the new area. The second actually inserts them. 651 */ 652 base = obase; 653 nr_new = 0; 654 655 for_each_memblock_type(idx, type, rgn) { 656 phys_addr_t rbase = rgn->base; 657 phys_addr_t rend = rbase + rgn->size; 658 659 if (rbase >= end) 660 break; 661 if (rend <= base) 662 continue; 663 /* 664 * @rgn overlaps. If it separates the lower part of new 665 * area, insert that portion. 666 */ 667 if (rbase > base) { 668 #ifdef CONFIG_NUMA 669 WARN_ON(nid != memblock_get_region_node(rgn)); 670 #endif 671 WARN_ON(flags != MEMBLOCK_NONE && flags != rgn->flags); 672 nr_new++; 673 if (insert) { 674 if (start_rgn == -1) 675 start_rgn = idx; 676 end_rgn = idx + 1; 677 memblock_insert_region(type, idx++, base, 678 rbase - base, nid, 679 flags); 680 } 681 } 682 /* area below @rend is dealt with, forget about it */ 683 base = min(rend, end); 684 } 685 686 /* insert the remaining portion */ 687 if (base < end) { 688 nr_new++; 689 if (insert) { 690 if (start_rgn == -1) 691 start_rgn = idx; 692 end_rgn = idx + 1; 693 memblock_insert_region(type, idx, base, end - base, 694 nid, flags); 695 } 696 } 697 698 if (!nr_new) 699 return 0; 700 701 /* 702 * If this was the first round, resize array and repeat for actual 703 * insertions; otherwise, merge and return. 704 */ 705 if (!insert) { 706 while (type->cnt + nr_new > type->max) 707 if (memblock_double_array(type, obase, size) < 0) 708 return -ENOMEM; 709 insert = true; 710 goto repeat; 711 } else { 712 memblock_merge_regions(type, start_rgn, end_rgn); 713 return 0; 714 } 715 } 716 717 /** 718 * memblock_add_node - add new memblock region within a NUMA node 719 * @base: base address of the new region 720 * @size: size of the new region 721 * @nid: nid of the new region 722 * @flags: flags of the new region 723 * 724 * Add new memblock region [@base, @base + @size) to the "memory" 725 * type. See memblock_add_range() description for mode details 726 * 727 * Return: 728 * 0 on success, -errno on failure. 729 */ 730 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 731 int nid, enum memblock_flags flags) 732 { 733 phys_addr_t end = base + size - 1; 734 735 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 736 &base, &end, nid, flags, (void *)_RET_IP_); 737 738 return memblock_add_range(&memblock.memory, base, size, nid, flags); 739 } 740 741 /** 742 * memblock_add - add new memblock region 743 * @base: base address of the new region 744 * @size: size of the new region 745 * 746 * Add new memblock region [@base, @base + @size) to the "memory" 747 * type. See memblock_add_range() description for mode details 748 * 749 * Return: 750 * 0 on success, -errno on failure. 751 */ 752 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 753 { 754 phys_addr_t end = base + size - 1; 755 756 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 757 &base, &end, (void *)_RET_IP_); 758 759 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0); 760 } 761 762 /** 763 * memblock_validate_numa_coverage - check if amount of memory with 764 * no node ID assigned is less than a threshold 765 * @threshold_bytes: maximal memory size that can have unassigned node 766 * ID (in bytes). 767 * 768 * A buggy firmware may report memory that does not belong to any node. 769 * Check if amount of such memory is below @threshold_bytes. 770 * 771 * Return: true on success, false on failure. 772 */ 773 bool __init_memblock memblock_validate_numa_coverage(unsigned long threshold_bytes) 774 { 775 unsigned long nr_pages = 0; 776 unsigned long start_pfn, end_pfn, mem_size_mb; 777 int nid, i; 778 779 /* calculate lost page */ 780 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 781 if (!numa_valid_node(nid)) 782 nr_pages += end_pfn - start_pfn; 783 } 784 785 if ((nr_pages << PAGE_SHIFT) > threshold_bytes) { 786 mem_size_mb = memblock_phys_mem_size() / SZ_1M; 787 pr_err("NUMA: no nodes coverage for %luMB of %luMB RAM\n", 788 (nr_pages << PAGE_SHIFT) / SZ_1M, mem_size_mb); 789 return false; 790 } 791 792 return true; 793 } 794 795 796 /** 797 * memblock_isolate_range - isolate given range into disjoint memblocks 798 * @type: memblock type to isolate range for 799 * @base: base of range to isolate 800 * @size: size of range to isolate 801 * @start_rgn: out parameter for the start of isolated region 802 * @end_rgn: out parameter for the end of isolated region 803 * 804 * Walk @type and ensure that regions don't cross the boundaries defined by 805 * [@base, @base + @size). Crossing regions are split at the boundaries, 806 * which may create at most two more regions. The index of the first 807 * region inside the range is returned in *@start_rgn and the index of the 808 * first region after the range is returned in *@end_rgn. 809 * 810 * Return: 811 * 0 on success, -errno on failure. 812 */ 813 static int __init_memblock memblock_isolate_range(struct memblock_type *type, 814 phys_addr_t base, phys_addr_t size, 815 int *start_rgn, int *end_rgn) 816 { 817 phys_addr_t end = base + memblock_cap_size(base, &size); 818 int idx; 819 struct memblock_region *rgn; 820 821 *start_rgn = *end_rgn = 0; 822 823 if (!size) 824 return 0; 825 826 /* we'll create at most two more regions */ 827 while (type->cnt + 2 > type->max) 828 if (memblock_double_array(type, base, size) < 0) 829 return -ENOMEM; 830 831 for_each_memblock_type(idx, type, rgn) { 832 phys_addr_t rbase = rgn->base; 833 phys_addr_t rend = rbase + rgn->size; 834 835 if (rbase >= end) 836 break; 837 if (rend <= base) 838 continue; 839 840 if (rbase < base) { 841 /* 842 * @rgn intersects from below. Split and continue 843 * to process the next region - the new top half. 844 */ 845 rgn->base = base; 846 rgn->size -= base - rbase; 847 type->total_size -= base - rbase; 848 memblock_insert_region(type, idx, rbase, base - rbase, 849 memblock_get_region_node(rgn), 850 rgn->flags); 851 } else if (rend > end) { 852 /* 853 * @rgn intersects from above. Split and redo the 854 * current region - the new bottom half. 855 */ 856 rgn->base = end; 857 rgn->size -= end - rbase; 858 type->total_size -= end - rbase; 859 memblock_insert_region(type, idx--, rbase, end - rbase, 860 memblock_get_region_node(rgn), 861 rgn->flags); 862 } else { 863 /* @rgn is fully contained, record it */ 864 if (!*end_rgn) 865 *start_rgn = idx; 866 *end_rgn = idx + 1; 867 } 868 } 869 870 return 0; 871 } 872 873 static int __init_memblock memblock_remove_range(struct memblock_type *type, 874 phys_addr_t base, phys_addr_t size) 875 { 876 int start_rgn, end_rgn; 877 int i, ret; 878 879 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 880 if (ret) 881 return ret; 882 883 for (i = end_rgn - 1; i >= start_rgn; i--) 884 memblock_remove_region(type, i); 885 return 0; 886 } 887 888 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 889 { 890 phys_addr_t end = base + size - 1; 891 892 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 893 &base, &end, (void *)_RET_IP_); 894 895 return memblock_remove_range(&memblock.memory, base, size); 896 } 897 898 static unsigned long __free_reserved_area(phys_addr_t start, phys_addr_t end, 899 int poison) 900 { 901 unsigned long pages = 0, pfn; 902 903 if (deferred_pages_enabled()) { 904 WARN(1, "Cannot free reserved memory because of deferred initialization of the memory map"); 905 return 0; 906 } 907 908 for_each_valid_pfn(pfn, PFN_UP(start), PFN_DOWN(end)) { 909 struct page *page = pfn_to_page(pfn); 910 void *direct_map_addr; 911 912 /* 913 * 'direct_map_addr' might be different from the kernel virtual 914 * address because some architectures use aliases. 915 * Going via physical address, pfn_to_page() and page_address() 916 * ensures that we get a _writeable_ alias for the memset(). 917 */ 918 direct_map_addr = page_address(page); 919 /* 920 * Perform a kasan-unchecked memset() since this memory 921 * has not been initialized. 922 */ 923 direct_map_addr = kasan_reset_tag(direct_map_addr); 924 if ((unsigned int)poison <= 0xFF) 925 memset(direct_map_addr, poison, PAGE_SIZE); 926 927 free_reserved_page(page); 928 pages++; 929 } 930 return pages; 931 } 932 933 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 934 { 935 phys_addr_t start_pa, end_pa; 936 unsigned long pages; 937 938 /* 939 * end is the first address past the region and it may be beyond what 940 * __pa() or __pa_symbol() can handle. 941 * Use the address included in the range for the conversion and add back 942 * 1 afterwards. 943 */ 944 if (__is_kernel((unsigned long)start)) { 945 start_pa = __pa_symbol(start); 946 end_pa = __pa_symbol(end - 1) + 1; 947 } else { 948 start_pa = __pa(start); 949 end_pa = __pa(end - 1) + 1; 950 } 951 952 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) { 953 if (start_pa < end_pa) 954 memblock_remove_range(&memblock.reserved, 955 start_pa, end_pa - start_pa); 956 } 957 958 pages = __free_reserved_area(start_pa, end_pa, poison); 959 if (pages && s) 960 pr_info("Freeing %s memory: %ldK\n", s, K(pages)); 961 962 return pages; 963 } 964 965 /** 966 * memblock_free - free boot memory allocation 967 * @ptr: starting address of the boot memory allocation 968 * @size: size of the boot memory block in bytes 969 * 970 * Free boot memory block previously allocated by memblock_alloc_xx() API. 971 * If called after the buddy allocator is available, the memory is released to 972 * the buddy allocator. 973 */ 974 void __init_memblock memblock_free(void *ptr, size_t size) 975 { 976 if (ptr) 977 memblock_phys_free(__pa(ptr), size); 978 } 979 980 /** 981 * memblock_phys_free - free boot memory block 982 * @base: phys starting address of the boot memory block 983 * @size: size of the boot memory block in bytes 984 * 985 * Free boot memory block previously allocated by memblock_phys_alloc_xx() API. 986 * If called after the buddy allocator is available, the memory is released to 987 * the buddy allocator. 988 */ 989 int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size) 990 { 991 phys_addr_t end = base + size - 1; 992 int ret; 993 994 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 995 &base, &end, (void *)_RET_IP_); 996 997 kmemleak_free_part_phys(base, size); 998 ret = memblock_remove_range(&memblock.reserved, base, size); 999 1000 if (slab_is_available()) 1001 __free_reserved_area(base, base + size, -1); 1002 1003 return ret; 1004 } 1005 1006 int __init_memblock __memblock_reserve(phys_addr_t base, phys_addr_t size, 1007 int nid, enum memblock_flags flags) 1008 { 1009 phys_addr_t end = base + size - 1; 1010 1011 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 1012 &base, &end, nid, flags, (void *)_RET_IP_); 1013 1014 return memblock_add_range(&memblock.reserved, base, size, nid, flags); 1015 } 1016 1017 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 1018 int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) 1019 { 1020 phys_addr_t end = base + size - 1; 1021 1022 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 1023 &base, &end, (void *)_RET_IP_); 1024 1025 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0); 1026 } 1027 #endif 1028 1029 #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 1030 __init void memblock_set_kho_scratch_only(void) 1031 { 1032 kho_scratch_only = true; 1033 } 1034 1035 __init void memblock_clear_kho_scratch_only(void) 1036 { 1037 kho_scratch_only = false; 1038 } 1039 1040 __init void memmap_init_kho_scratch_pages(void) 1041 { 1042 phys_addr_t start, end; 1043 unsigned long pfn; 1044 int nid; 1045 u64 i; 1046 1047 if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) 1048 return; 1049 1050 /* 1051 * Initialize struct pages for free scratch memory. 1052 * The struct pages for reserved scratch memory will be set up in 1053 * memmap_init_reserved_pages() 1054 */ 1055 __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE, 1056 MEMBLOCK_KHO_SCRATCH, &start, &end, &nid) { 1057 for (pfn = PFN_UP(start); pfn < PFN_DOWN(end); pfn++) 1058 init_deferred_page(pfn, nid); 1059 } 1060 } 1061 #endif 1062 1063 /** 1064 * memblock_setclr_flag - set or clear flag for a memory region 1065 * @type: memblock type to set/clear flag for 1066 * @base: base address of the region 1067 * @size: size of the region 1068 * @set: set or clear the flag 1069 * @flag: the flag to update 1070 * 1071 * This function isolates region [@base, @base + @size), and sets/clears flag 1072 * 1073 * Return: 0 on success, -errno on failure. 1074 */ 1075 static int __init_memblock memblock_setclr_flag(struct memblock_type *type, 1076 phys_addr_t base, phys_addr_t size, int set, int flag) 1077 { 1078 int i, ret, start_rgn, end_rgn; 1079 1080 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1081 if (ret) 1082 return ret; 1083 1084 for (i = start_rgn; i < end_rgn; i++) { 1085 struct memblock_region *r = &type->regions[i]; 1086 1087 if (set) 1088 r->flags |= flag; 1089 else 1090 r->flags &= ~flag; 1091 } 1092 1093 memblock_merge_regions(type, start_rgn, end_rgn); 1094 return 0; 1095 } 1096 1097 /** 1098 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 1099 * @base: the base phys addr of the region 1100 * @size: the size of the region 1101 * 1102 * Return: 0 on success, -errno on failure. 1103 */ 1104 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 1105 { 1106 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_HOTPLUG); 1107 } 1108 1109 /** 1110 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 1111 * @base: the base phys addr of the region 1112 * @size: the size of the region 1113 * 1114 * Return: 0 on success, -errno on failure. 1115 */ 1116 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 1117 { 1118 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_HOTPLUG); 1119 } 1120 1121 /** 1122 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR. 1123 * @base: the base phys addr of the region 1124 * @size: the size of the region 1125 * 1126 * Return: 0 on success, -errno on failure. 1127 */ 1128 int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size) 1129 { 1130 if (!mirrored_kernelcore) 1131 return 0; 1132 1133 system_has_some_mirror = true; 1134 1135 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_MIRROR); 1136 } 1137 1138 /** 1139 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP. 1140 * @base: the base phys addr of the region 1141 * @size: the size of the region 1142 * 1143 * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the 1144 * direct mapping of the physical memory. These regions will still be 1145 * covered by the memory map. The struct page representing NOMAP memory 1146 * frames in the memory map will be PageReserved() 1147 * 1148 * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from 1149 * memblock, the caller must inform kmemleak to ignore that memory 1150 * 1151 * Return: 0 on success, -errno on failure. 1152 */ 1153 int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size) 1154 { 1155 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_NOMAP); 1156 } 1157 1158 /** 1159 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region. 1160 * @base: the base phys addr of the region 1161 * @size: the size of the region 1162 * 1163 * Return: 0 on success, -errno on failure. 1164 */ 1165 int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size) 1166 { 1167 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_NOMAP); 1168 } 1169 1170 /** 1171 * memblock_reserved_mark_noinit - Mark a reserved memory region with flag 1172 * MEMBLOCK_RSRV_NOINIT 1173 * 1174 * @base: the base phys addr of the region 1175 * @size: the size of the region 1176 * 1177 * The struct pages for the reserved regions marked %MEMBLOCK_RSRV_NOINIT will 1178 * not be fully initialized to allow the caller optimize their initialization. 1179 * 1180 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled, setting this flag 1181 * completely bypasses the initialization of struct pages for such region. 1182 * 1183 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is disabled, struct pages in this 1184 * region will be initialized with default values but won't be marked as 1185 * reserved. 1186 * 1187 * Return: 0 on success, -errno on failure. 1188 */ 1189 int __init_memblock memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size) 1190 { 1191 return memblock_setclr_flag(&memblock.reserved, base, size, 1, 1192 MEMBLOCK_RSRV_NOINIT); 1193 } 1194 1195 /** 1196 * memblock_reserved_mark_kern - Mark a reserved memory region with flag 1197 * MEMBLOCK_RSRV_KERN 1198 * 1199 * @base: the base phys addr of the region 1200 * @size: the size of the region 1201 * 1202 * Return: 0 on success, -errno on failure. 1203 */ 1204 int __init_memblock memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t size) 1205 { 1206 return memblock_setclr_flag(&memblock.reserved, base, size, 1, 1207 MEMBLOCK_RSRV_KERN); 1208 } 1209 1210 /** 1211 * memblock_mark_kho_scratch - Mark a memory region as MEMBLOCK_KHO_SCRATCH. 1212 * @base: the base phys addr of the region 1213 * @size: the size of the region 1214 * 1215 * Only memory regions marked with %MEMBLOCK_KHO_SCRATCH will be considered 1216 * for allocations during early boot with kexec handover. 1217 * 1218 * Return: 0 on success, -errno on failure. 1219 */ 1220 __init int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size) 1221 { 1222 return memblock_setclr_flag(&memblock.memory, base, size, 1, 1223 MEMBLOCK_KHO_SCRATCH); 1224 } 1225 1226 /** 1227 * memblock_clear_kho_scratch - Clear MEMBLOCK_KHO_SCRATCH flag for a 1228 * specified region. 1229 * @base: the base phys addr of the region 1230 * @size: the size of the region 1231 * 1232 * Return: 0 on success, -errno on failure. 1233 */ 1234 __init int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size) 1235 { 1236 return memblock_setclr_flag(&memblock.memory, base, size, 0, 1237 MEMBLOCK_KHO_SCRATCH); 1238 } 1239 1240 static bool should_skip_region(struct memblock_type *type, 1241 struct memblock_region *m, 1242 int nid, int flags) 1243 { 1244 int m_nid = memblock_get_region_node(m); 1245 1246 /* we never skip regions when iterating memblock.reserved or physmem */ 1247 if (type != memblock_memory) 1248 return false; 1249 1250 /* only memory regions are associated with nodes, check it */ 1251 if (numa_valid_node(nid) && nid != m_nid) 1252 return true; 1253 1254 /* skip hotpluggable memory regions if needed */ 1255 if (movable_node_is_enabled() && memblock_is_hotpluggable(m) && 1256 !(flags & MEMBLOCK_HOTPLUG)) 1257 return true; 1258 1259 /* if we want mirror memory skip non-mirror memory regions */ 1260 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 1261 return true; 1262 1263 /* skip nomap memory unless we were asked for it explicitly */ 1264 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 1265 return true; 1266 1267 /* skip driver-managed memory unless we were asked for it explicitly */ 1268 if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m)) 1269 return true; 1270 1271 /* 1272 * In early alloc during kexec handover, we can only consider 1273 * MEMBLOCK_KHO_SCRATCH regions for the allocations 1274 */ 1275 if ((flags & MEMBLOCK_KHO_SCRATCH) && !memblock_is_kho_scratch(m)) 1276 return true; 1277 1278 return false; 1279 } 1280 1281 /** 1282 * __next_mem_range - next function for for_each_free_mem_range() etc. 1283 * @idx: pointer to u64 loop variable 1284 * @nid: node selector, %NUMA_NO_NODE for all nodes 1285 * @flags: pick from blocks based on memory attributes 1286 * @type_a: pointer to memblock_type from where the range is taken 1287 * @type_b: pointer to memblock_type which excludes memory from being taken 1288 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1289 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1290 * @out_nid: ptr to int for nid of the range, can be %NULL 1291 * 1292 * Find the first area from *@idx which matches @nid, fill the out 1293 * parameters, and update *@idx for the next iteration. The lower 32bit of 1294 * *@idx contains index into type_a and the upper 32bit indexes the 1295 * areas before each region in type_b. For example, if type_b regions 1296 * look like the following, 1297 * 1298 * 0:[0-16), 1:[32-48), 2:[128-130) 1299 * 1300 * The upper 32bit indexes the following regions. 1301 * 1302 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 1303 * 1304 * As both region arrays are sorted, the function advances the two indices 1305 * in lockstep and returns each intersection. 1306 */ 1307 void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags, 1308 struct memblock_type *type_a, 1309 struct memblock_type *type_b, phys_addr_t *out_start, 1310 phys_addr_t *out_end, int *out_nid) 1311 { 1312 int idx_a = *idx & 0xffffffff; 1313 int idx_b = *idx >> 32; 1314 1315 for (; idx_a < type_a->cnt; idx_a++) { 1316 struct memblock_region *m = &type_a->regions[idx_a]; 1317 1318 phys_addr_t m_start = m->base; 1319 phys_addr_t m_end = m->base + m->size; 1320 int m_nid = memblock_get_region_node(m); 1321 1322 if (should_skip_region(type_a, m, nid, flags)) 1323 continue; 1324 1325 if (!type_b) { 1326 if (out_start) 1327 *out_start = m_start; 1328 if (out_end) 1329 *out_end = m_end; 1330 if (out_nid) 1331 *out_nid = m_nid; 1332 idx_a++; 1333 *idx = (u32)idx_a | (u64)idx_b << 32; 1334 return; 1335 } 1336 1337 /* scan areas before each reservation */ 1338 for (; idx_b < type_b->cnt + 1; idx_b++) { 1339 struct memblock_region *r; 1340 phys_addr_t r_start; 1341 phys_addr_t r_end; 1342 1343 r = &type_b->regions[idx_b]; 1344 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1345 r_end = idx_b < type_b->cnt ? 1346 r->base : PHYS_ADDR_MAX; 1347 1348 /* 1349 * if idx_b advanced past idx_a, 1350 * break out to advance idx_a 1351 */ 1352 if (r_start >= m_end) 1353 break; 1354 /* if the two regions intersect, we're done */ 1355 if (m_start < r_end) { 1356 if (out_start) 1357 *out_start = 1358 max(m_start, r_start); 1359 if (out_end) 1360 *out_end = min(m_end, r_end); 1361 if (out_nid) 1362 *out_nid = m_nid; 1363 /* 1364 * The region which ends first is 1365 * advanced for the next iteration. 1366 */ 1367 if (m_end <= r_end) 1368 idx_a++; 1369 else 1370 idx_b++; 1371 *idx = (u32)idx_a | (u64)idx_b << 32; 1372 return; 1373 } 1374 } 1375 } 1376 1377 /* signal end of iteration */ 1378 *idx = ULLONG_MAX; 1379 } 1380 1381 /** 1382 * __next_mem_range_rev - generic next function for for_each_*_range_rev() 1383 * 1384 * @idx: pointer to u64 loop variable 1385 * @nid: node selector, %NUMA_NO_NODE for all nodes 1386 * @flags: pick from blocks based on memory attributes 1387 * @type_a: pointer to memblock_type from where the range is taken 1388 * @type_b: pointer to memblock_type which excludes memory from being taken 1389 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1390 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1391 * @out_nid: ptr to int for nid of the range, can be %NULL 1392 * 1393 * Finds the next range from type_a which is not marked as unsuitable 1394 * in type_b. 1395 * 1396 * Reverse of __next_mem_range(). 1397 */ 1398 void __init_memblock __next_mem_range_rev(u64 *idx, int nid, 1399 enum memblock_flags flags, 1400 struct memblock_type *type_a, 1401 struct memblock_type *type_b, 1402 phys_addr_t *out_start, 1403 phys_addr_t *out_end, int *out_nid) 1404 { 1405 int idx_a = *idx & 0xffffffff; 1406 int idx_b = *idx >> 32; 1407 1408 if (*idx == (u64)ULLONG_MAX) { 1409 idx_a = type_a->cnt - 1; 1410 if (type_b != NULL) 1411 idx_b = type_b->cnt; 1412 else 1413 idx_b = 0; 1414 } 1415 1416 for (; idx_a >= 0; idx_a--) { 1417 struct memblock_region *m = &type_a->regions[idx_a]; 1418 1419 phys_addr_t m_start = m->base; 1420 phys_addr_t m_end = m->base + m->size; 1421 int m_nid = memblock_get_region_node(m); 1422 1423 if (should_skip_region(type_a, m, nid, flags)) 1424 continue; 1425 1426 if (!type_b) { 1427 if (out_start) 1428 *out_start = m_start; 1429 if (out_end) 1430 *out_end = m_end; 1431 if (out_nid) 1432 *out_nid = m_nid; 1433 idx_a--; 1434 *idx = (u32)idx_a | (u64)idx_b << 32; 1435 return; 1436 } 1437 1438 /* scan areas before each reservation */ 1439 for (; idx_b >= 0; idx_b--) { 1440 struct memblock_region *r; 1441 phys_addr_t r_start; 1442 phys_addr_t r_end; 1443 1444 r = &type_b->regions[idx_b]; 1445 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1446 r_end = idx_b < type_b->cnt ? 1447 r->base : PHYS_ADDR_MAX; 1448 /* 1449 * if idx_b advanced past idx_a, 1450 * break out to advance idx_a 1451 */ 1452 1453 if (r_end <= m_start) 1454 break; 1455 /* if the two regions intersect, we're done */ 1456 if (m_end > r_start) { 1457 if (out_start) 1458 *out_start = max(m_start, r_start); 1459 if (out_end) 1460 *out_end = min(m_end, r_end); 1461 if (out_nid) 1462 *out_nid = m_nid; 1463 if (m_start >= r_start) 1464 idx_a--; 1465 else 1466 idx_b--; 1467 *idx = (u32)idx_a | (u64)idx_b << 32; 1468 return; 1469 } 1470 } 1471 } 1472 /* signal end of iteration */ 1473 *idx = ULLONG_MAX; 1474 } 1475 1476 /* 1477 * Common iterator interface used to define for_each_mem_pfn_range(). 1478 */ 1479 void __init_memblock __next_mem_pfn_range(int *idx, int nid, 1480 unsigned long *out_start_pfn, 1481 unsigned long *out_end_pfn, int *out_nid) 1482 { 1483 struct memblock_type *type = &memblock.memory; 1484 struct memblock_region *r; 1485 int r_nid; 1486 1487 while (++*idx < type->cnt) { 1488 r = &type->regions[*idx]; 1489 r_nid = memblock_get_region_node(r); 1490 1491 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 1492 continue; 1493 if (!numa_valid_node(nid) || nid == r_nid) 1494 break; 1495 } 1496 if (*idx >= type->cnt) { 1497 *idx = -1; 1498 return; 1499 } 1500 1501 if (out_start_pfn) 1502 *out_start_pfn = PFN_UP(r->base); 1503 if (out_end_pfn) 1504 *out_end_pfn = PFN_DOWN(r->base + r->size); 1505 if (out_nid) 1506 *out_nid = r_nid; 1507 } 1508 1509 /** 1510 * memblock_set_node - set node ID on memblock regions 1511 * @base: base of area to set node ID for 1512 * @size: size of area to set node ID for 1513 * @type: memblock type to set node ID for 1514 * @nid: node ID to set 1515 * 1516 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid. 1517 * Regions which cross the area boundaries are split as necessary. 1518 * 1519 * Return: 1520 * 0 on success, -errno on failure. 1521 */ 1522 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 1523 struct memblock_type *type, int nid) 1524 { 1525 #ifdef CONFIG_NUMA 1526 int start_rgn, end_rgn; 1527 int i, ret; 1528 1529 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1530 if (ret) 1531 return ret; 1532 1533 for (i = start_rgn; i < end_rgn; i++) 1534 memblock_set_region_node(&type->regions[i], nid); 1535 1536 memblock_merge_regions(type, start_rgn, end_rgn); 1537 #endif 1538 return 0; 1539 } 1540 1541 /** 1542 * memblock_alloc_range_nid - allocate boot memory block 1543 * @size: size of memory block to be allocated in bytes 1544 * @align: alignment of the region and block's size 1545 * @start: the lower bound of the memory region to allocate (phys address) 1546 * @end: the upper bound of the memory region to allocate (phys address) 1547 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1548 * @exact_nid: control the allocation fall back to other nodes 1549 * 1550 * The allocation is performed from memory region limited by 1551 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE. 1552 * 1553 * If the specified node can not hold the requested memory and @exact_nid 1554 * is false, the allocation falls back to any node in the system. 1555 * 1556 * For systems with memory mirroring, the allocation is attempted first 1557 * from the regions with mirroring enabled and then retried from any 1558 * memory region. 1559 * 1560 * In addition, function using kmemleak_alloc_phys for allocated boot 1561 * memory block, it is never reported as leaks. 1562 * 1563 * Return: 1564 * Physical address of allocated memory block on success, %0 on failure. 1565 */ 1566 phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, 1567 phys_addr_t align, phys_addr_t start, 1568 phys_addr_t end, int nid, 1569 bool exact_nid) 1570 { 1571 enum memblock_flags flags = choose_memblock_flags(); 1572 phys_addr_t found; 1573 1574 /* 1575 * Detect any accidental use of these APIs after slab is ready, as at 1576 * this moment memblock may be deinitialized already and its 1577 * internal data may be destroyed (after execution of memblock_free_all) 1578 */ 1579 if (WARN_ON_ONCE(slab_is_available())) { 1580 void *vaddr = kzalloc_node(size, GFP_NOWAIT, nid); 1581 1582 return vaddr ? virt_to_phys(vaddr) : 0; 1583 } 1584 1585 if (!align) { 1586 /* Can't use WARNs this early in boot on powerpc */ 1587 dump_stack(); 1588 align = SMP_CACHE_BYTES; 1589 } 1590 1591 again: 1592 found = memblock_find_in_range_node(size, align, start, end, nid, 1593 flags); 1594 if (found && !__memblock_reserve(found, size, nid, MEMBLOCK_RSRV_KERN)) 1595 goto done; 1596 1597 if (numa_valid_node(nid) && !exact_nid) { 1598 found = memblock_find_in_range_node(size, align, start, 1599 end, NUMA_NO_NODE, 1600 flags); 1601 if (found && !memblock_reserve_kern(found, size)) 1602 goto done; 1603 } 1604 1605 if (flags & MEMBLOCK_MIRROR) { 1606 flags &= ~MEMBLOCK_MIRROR; 1607 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 1608 &size); 1609 goto again; 1610 } 1611 1612 return 0; 1613 1614 done: 1615 /* 1616 * Skip kmemleak for those places like kasan_init() and 1617 * early_pgtable_alloc() due to high volume. 1618 */ 1619 if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) 1620 /* 1621 * Memblock allocated blocks are never reported as 1622 * leaks. This is because many of these blocks are 1623 * only referred via the physical address which is 1624 * not looked up by kmemleak. 1625 */ 1626 kmemleak_alloc_phys(found, size, 0); 1627 1628 /* 1629 * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, 1630 * require memory to be accepted before it can be used by the 1631 * guest. 1632 * 1633 * Accept the memory of the allocated buffer. 1634 */ 1635 accept_memory(found, size); 1636 1637 return found; 1638 } 1639 1640 /** 1641 * memblock_phys_alloc_range - allocate a memory block inside specified range 1642 * @size: size of memory block to be allocated in bytes 1643 * @align: alignment of the region and block's size 1644 * @start: the lower bound of the memory region to allocate (physical address) 1645 * @end: the upper bound of the memory region to allocate (physical address) 1646 * 1647 * Allocate @size bytes in the between @start and @end. 1648 * 1649 * Return: physical address of the allocated memory block on success, 1650 * %0 on failure. 1651 */ 1652 phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, 1653 phys_addr_t align, 1654 phys_addr_t start, 1655 phys_addr_t end) 1656 { 1657 memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n", 1658 __func__, (u64)size, (u64)align, &start, &end, 1659 (void *)_RET_IP_); 1660 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, 1661 false); 1662 } 1663 1664 /** 1665 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node 1666 * @size: size of memory block to be allocated in bytes 1667 * @align: alignment of the region and block's size 1668 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1669 * 1670 * Allocates memory block from the specified NUMA node. If the node 1671 * has no available memory, attempts to allocated from any node in the 1672 * system. 1673 * 1674 * Return: physical address of the allocated memory block on success, 1675 * %0 on failure. 1676 */ 1677 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1678 { 1679 return memblock_alloc_range_nid(size, align, 0, 1680 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false); 1681 } 1682 1683 /** 1684 * memblock_alloc_internal - allocate boot memory block 1685 * @size: size of memory block to be allocated in bytes 1686 * @align: alignment of the region and block's size 1687 * @min_addr: the lower bound of the memory region to allocate (phys address) 1688 * @max_addr: the upper bound of the memory region to allocate (phys address) 1689 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1690 * @exact_nid: control the allocation fall back to other nodes 1691 * 1692 * Allocates memory block using memblock_alloc_range_nid() and 1693 * converts the returned physical address to virtual. 1694 * 1695 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1696 * will fall back to memory below @min_addr. Other constraints, such 1697 * as node and mirrored memory will be handled again in 1698 * memblock_alloc_range_nid(). 1699 * 1700 * Return: 1701 * Virtual address of allocated memory block on success, NULL on failure. 1702 */ 1703 static void * __init memblock_alloc_internal( 1704 phys_addr_t size, phys_addr_t align, 1705 phys_addr_t min_addr, phys_addr_t max_addr, 1706 int nid, bool exact_nid) 1707 { 1708 phys_addr_t alloc; 1709 1710 1711 if (max_addr > memblock.current_limit) 1712 max_addr = memblock.current_limit; 1713 1714 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid, 1715 exact_nid); 1716 1717 /* retry allocation without lower limit */ 1718 if (!alloc && min_addr) 1719 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid, 1720 exact_nid); 1721 1722 if (!alloc) 1723 return NULL; 1724 1725 return phys_to_virt(alloc); 1726 } 1727 1728 /** 1729 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node 1730 * without zeroing memory 1731 * @size: size of memory block to be allocated in bytes 1732 * @align: alignment of the region and block's size 1733 * @min_addr: the lower bound of the memory region from where the allocation 1734 * is preferred (phys address) 1735 * @max_addr: the upper bound of the memory region from where the allocation 1736 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1737 * allocate only from memory limited by memblock.current_limit value 1738 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1739 * 1740 * Public function, provides additional debug information (including caller 1741 * info), if enabled. Does not zero allocated memory. 1742 * 1743 * Return: 1744 * Virtual address of allocated memory block on success, NULL on failure. 1745 */ 1746 void * __init memblock_alloc_exact_nid_raw( 1747 phys_addr_t size, phys_addr_t align, 1748 phys_addr_t min_addr, phys_addr_t max_addr, 1749 int nid) 1750 { 1751 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1752 __func__, (u64)size, (u64)align, nid, &min_addr, 1753 &max_addr, (void *)_RET_IP_); 1754 1755 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1756 true); 1757 } 1758 1759 /** 1760 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing 1761 * memory and without panicking 1762 * @size: size of memory block to be allocated in bytes 1763 * @align: alignment of the region and block's size 1764 * @min_addr: the lower bound of the memory region from where the allocation 1765 * is preferred (phys address) 1766 * @max_addr: the upper bound of the memory region from where the allocation 1767 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1768 * allocate only from memory limited by memblock.current_limit value 1769 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1770 * 1771 * Public function, provides additional debug information (including caller 1772 * info), if enabled. Does not zero allocated memory, does not panic if request 1773 * cannot be satisfied. 1774 * 1775 * Return: 1776 * Virtual address of allocated memory block on success, NULL on failure. 1777 */ 1778 void * __init memblock_alloc_try_nid_raw( 1779 phys_addr_t size, phys_addr_t align, 1780 phys_addr_t min_addr, phys_addr_t max_addr, 1781 int nid) 1782 { 1783 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1784 __func__, (u64)size, (u64)align, nid, &min_addr, 1785 &max_addr, (void *)_RET_IP_); 1786 1787 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1788 false); 1789 } 1790 1791 /** 1792 * memblock_alloc_try_nid - allocate boot memory block 1793 * @size: size of memory block to be allocated in bytes 1794 * @align: alignment of the region and block's size 1795 * @min_addr: the lower bound of the memory region from where the allocation 1796 * is preferred (phys address) 1797 * @max_addr: the upper bound of the memory region from where the allocation 1798 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1799 * allocate only from memory limited by memblock.current_limit value 1800 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1801 * 1802 * Public function, provides additional debug information (including caller 1803 * info), if enabled. This function zeroes the allocated memory. 1804 * 1805 * Return: 1806 * Virtual address of allocated memory block on success, NULL on failure. 1807 */ 1808 void * __init memblock_alloc_try_nid( 1809 phys_addr_t size, phys_addr_t align, 1810 phys_addr_t min_addr, phys_addr_t max_addr, 1811 int nid) 1812 { 1813 void *ptr; 1814 1815 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1816 __func__, (u64)size, (u64)align, nid, &min_addr, 1817 &max_addr, (void *)_RET_IP_); 1818 ptr = memblock_alloc_internal(size, align, 1819 min_addr, max_addr, nid, false); 1820 if (ptr) 1821 memset(ptr, 0, size); 1822 1823 return ptr; 1824 } 1825 1826 /** 1827 * __memblock_alloc_or_panic - Try to allocate memory and panic on failure 1828 * @size: size of memory block to be allocated in bytes 1829 * @align: alignment of the region and block's size 1830 * @func: caller func name 1831 * 1832 * This function attempts to allocate memory using memblock_alloc, 1833 * and in case of failure, it calls panic with the formatted message. 1834 * This function should not be used directly, please use the macro memblock_alloc_or_panic. 1835 */ 1836 void *__init __memblock_alloc_or_panic(phys_addr_t size, phys_addr_t align, 1837 const char *func) 1838 { 1839 void *addr = memblock_alloc(size, align); 1840 1841 if (unlikely(!addr)) 1842 panic("%s: Failed to allocate %pap bytes\n", func, &size); 1843 return addr; 1844 } 1845 1846 /* 1847 * Remaining API functions 1848 */ 1849 1850 phys_addr_t __init_memblock memblock_phys_mem_size(void) 1851 { 1852 return memblock.memory.total_size; 1853 } 1854 1855 phys_addr_t __init_memblock memblock_reserved_size(void) 1856 { 1857 return memblock.reserved.total_size; 1858 } 1859 1860 phys_addr_t __init_memblock memblock_reserved_kern_size(phys_addr_t limit, int nid) 1861 { 1862 struct memblock_region *r; 1863 phys_addr_t total = 0; 1864 1865 for_each_reserved_mem_region(r) { 1866 phys_addr_t size = r->size; 1867 1868 if (r->base > limit) 1869 break; 1870 1871 if (r->base + r->size > limit) 1872 size = limit - r->base; 1873 1874 if (nid == memblock_get_region_node(r) || !numa_valid_node(nid)) 1875 if (r->flags & MEMBLOCK_RSRV_KERN) 1876 total += size; 1877 } 1878 1879 return total; 1880 } 1881 1882 /** 1883 * memblock_estimated_nr_free_pages - return estimated number of free pages 1884 * from memblock point of view 1885 * 1886 * During bootup, subsystems might need a rough estimate of the number of free 1887 * pages in the whole system, before precise numbers are available from the 1888 * buddy. Especially with CONFIG_DEFERRED_STRUCT_PAGE_INIT, the numbers 1889 * obtained from the buddy might be very imprecise during bootup. 1890 * 1891 * Return: 1892 * An estimated number of free pages from memblock point of view. 1893 */ 1894 unsigned long __init memblock_estimated_nr_free_pages(void) 1895 { 1896 return PHYS_PFN(memblock_phys_mem_size() - 1897 memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, NUMA_NO_NODE)); 1898 } 1899 1900 /* lowest address */ 1901 phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1902 { 1903 return memblock.memory.regions[0].base; 1904 } 1905 1906 phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1907 { 1908 int idx = memblock.memory.cnt - 1; 1909 1910 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1911 } 1912 1913 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) 1914 { 1915 phys_addr_t max_addr = PHYS_ADDR_MAX; 1916 struct memblock_region *r; 1917 1918 /* 1919 * translate the memory @limit size into the max address within one of 1920 * the memory memblock regions, if the @limit exceeds the total size 1921 * of those regions, max_addr will keep original value PHYS_ADDR_MAX 1922 */ 1923 for_each_mem_region(r) { 1924 if (limit <= r->size) { 1925 max_addr = r->base + limit; 1926 break; 1927 } 1928 limit -= r->size; 1929 } 1930 1931 return max_addr; 1932 } 1933 1934 void __init memblock_enforce_memory_limit(phys_addr_t limit) 1935 { 1936 phys_addr_t max_addr; 1937 1938 if (!limit) 1939 return; 1940 1941 max_addr = __find_max_addr(limit); 1942 1943 /* @limit exceeds the total size of the memory, do nothing */ 1944 if (max_addr == PHYS_ADDR_MAX) 1945 return; 1946 1947 /* truncate both memory and reserved regions */ 1948 memblock_remove_range(&memblock.memory, max_addr, 1949 PHYS_ADDR_MAX); 1950 memblock_remove_range(&memblock.reserved, max_addr, 1951 PHYS_ADDR_MAX); 1952 } 1953 1954 void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size) 1955 { 1956 int start_rgn, end_rgn; 1957 int i, ret; 1958 1959 if (!size) 1960 return; 1961 1962 if (!memblock_memory->total_size) { 1963 pr_warn("%s: No memory registered yet\n", __func__); 1964 return; 1965 } 1966 1967 ret = memblock_isolate_range(&memblock.memory, base, size, 1968 &start_rgn, &end_rgn); 1969 if (ret) 1970 return; 1971 1972 /* remove all the MAP regions */ 1973 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--) 1974 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1975 memblock_remove_region(&memblock.memory, i); 1976 1977 for (i = start_rgn - 1; i >= 0; i--) 1978 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1979 memblock_remove_region(&memblock.memory, i); 1980 1981 /* truncate the reserved regions */ 1982 memblock_remove_range(&memblock.reserved, 0, base); 1983 memblock_remove_range(&memblock.reserved, 1984 base + size, PHYS_ADDR_MAX); 1985 } 1986 1987 void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1988 { 1989 phys_addr_t max_addr; 1990 1991 if (!limit) 1992 return; 1993 1994 max_addr = __find_max_addr(limit); 1995 1996 /* @limit exceeds the total size of the memory, do nothing */ 1997 if (max_addr == PHYS_ADDR_MAX) 1998 return; 1999 2000 memblock_cap_memory_range(0, max_addr); 2001 } 2002 2003 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 2004 { 2005 unsigned int left = 0, right = type->cnt; 2006 2007 do { 2008 unsigned int mid = (right + left) / 2; 2009 2010 if (addr < type->regions[mid].base) 2011 right = mid; 2012 else if (addr >= (type->regions[mid].base + 2013 type->regions[mid].size)) 2014 left = mid + 1; 2015 else 2016 return mid; 2017 } while (left < right); 2018 return -1; 2019 } 2020 2021 bool __init_memblock memblock_is_reserved(phys_addr_t addr) 2022 { 2023 return memblock_search(&memblock.reserved, addr) != -1; 2024 } 2025 2026 bool __init_memblock memblock_is_memory(phys_addr_t addr) 2027 { 2028 return memblock_search(&memblock.memory, addr) != -1; 2029 } 2030 2031 bool __init_memblock memblock_is_map_memory(phys_addr_t addr) 2032 { 2033 int i = memblock_search(&memblock.memory, addr); 2034 2035 if (i == -1) 2036 return false; 2037 return !memblock_is_nomap(&memblock.memory.regions[i]); 2038 } 2039 2040 int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 2041 unsigned long *start_pfn, unsigned long *end_pfn) 2042 { 2043 struct memblock_type *type = &memblock.memory; 2044 int mid = memblock_search(type, PFN_PHYS(pfn)); 2045 2046 if (mid == -1) 2047 return NUMA_NO_NODE; 2048 2049 *start_pfn = PFN_DOWN(type->regions[mid].base); 2050 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size); 2051 2052 return memblock_get_region_node(&type->regions[mid]); 2053 } 2054 2055 /** 2056 * memblock_is_region_memory - check if a region is a subset of memory 2057 * @base: base of region to check 2058 * @size: size of region to check 2059 * 2060 * Check if the region [@base, @base + @size) is a subset of a memory block. 2061 * 2062 * Return: 2063 * 0 if false, non-zero if true 2064 */ 2065 bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 2066 { 2067 int idx = memblock_search(&memblock.memory, base); 2068 phys_addr_t end = base + memblock_cap_size(base, &size); 2069 2070 if (idx == -1) 2071 return false; 2072 return (memblock.memory.regions[idx].base + 2073 memblock.memory.regions[idx].size) >= end; 2074 } 2075 2076 /** 2077 * memblock_is_region_reserved - check if a region intersects reserved memory 2078 * @base: base of region to check 2079 * @size: size of region to check 2080 * 2081 * Check if the region [@base, @base + @size) intersects a reserved 2082 * memory block. 2083 * 2084 * Return: 2085 * True if they intersect, false if not. 2086 */ 2087 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 2088 { 2089 return memblock_overlaps_region(&memblock.reserved, base, size); 2090 } 2091 2092 void __init_memblock memblock_trim_memory(phys_addr_t align) 2093 { 2094 phys_addr_t start, end, orig_start, orig_end; 2095 struct memblock_region *r; 2096 2097 for_each_mem_region(r) { 2098 orig_start = r->base; 2099 orig_end = r->base + r->size; 2100 start = round_up(orig_start, align); 2101 end = round_down(orig_end, align); 2102 2103 if (start == orig_start && end == orig_end) 2104 continue; 2105 2106 if (start < end) { 2107 r->base = start; 2108 r->size = end - start; 2109 } else { 2110 memblock_remove_region(&memblock.memory, 2111 r - memblock.memory.regions); 2112 r--; 2113 } 2114 } 2115 } 2116 2117 void __init_memblock memblock_set_current_limit(phys_addr_t limit) 2118 { 2119 memblock.current_limit = limit; 2120 } 2121 2122 phys_addr_t __init_memblock memblock_get_current_limit(void) 2123 { 2124 return memblock.current_limit; 2125 } 2126 2127 static void __init_memblock memblock_dump(struct memblock_type *type) 2128 { 2129 phys_addr_t base, end, size; 2130 enum memblock_flags flags; 2131 int idx; 2132 struct memblock_region *rgn; 2133 2134 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt); 2135 2136 for_each_memblock_type(idx, type, rgn) { 2137 char nid_buf[32] = ""; 2138 2139 base = rgn->base; 2140 size = rgn->size; 2141 end = base + size - 1; 2142 flags = rgn->flags; 2143 #ifdef CONFIG_NUMA 2144 if (numa_valid_node(memblock_get_region_node(rgn))) 2145 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 2146 memblock_get_region_node(rgn)); 2147 #endif 2148 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n", 2149 type->name, idx, &base, &end, &size, nid_buf, flags); 2150 } 2151 } 2152 2153 static void __init_memblock __memblock_dump_all(void) 2154 { 2155 pr_info("MEMBLOCK configuration:\n"); 2156 pr_info(" memory size = %pa reserved size = %pa\n", 2157 &memblock.memory.total_size, 2158 &memblock.reserved.total_size); 2159 2160 memblock_dump(&memblock.memory); 2161 memblock_dump(&memblock.reserved); 2162 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2163 memblock_dump(&physmem); 2164 #endif 2165 } 2166 2167 void __init_memblock memblock_dump_all(void) 2168 { 2169 if (memblock_debug) 2170 __memblock_dump_all(); 2171 } 2172 2173 void __init memblock_allow_resize(void) 2174 { 2175 memblock_can_resize = 1; 2176 } 2177 2178 static int __init early_memblock(char *p) 2179 { 2180 if (p && strstr(p, "debug")) 2181 memblock_debug = 1; 2182 return 0; 2183 } 2184 early_param("memblock", early_memblock); 2185 2186 static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn) 2187 { 2188 struct page *start_pg, *end_pg; 2189 phys_addr_t pg, pgend; 2190 2191 /* 2192 * Convert start_pfn/end_pfn to a struct page pointer. 2193 */ 2194 start_pg = pfn_to_page(start_pfn - 1) + 1; 2195 end_pg = pfn_to_page(end_pfn - 1) + 1; 2196 2197 /* 2198 * Convert to physical addresses, and round start upwards and end 2199 * downwards. 2200 */ 2201 pg = PAGE_ALIGN(__pa(start_pg)); 2202 pgend = PAGE_ALIGN_DOWN(__pa(end_pg)); 2203 2204 /* 2205 * If there are free pages between these, free the section of the 2206 * memmap array. 2207 */ 2208 if (pg < pgend) 2209 memblock_phys_free(pg, pgend - pg); 2210 } 2211 2212 /* 2213 * The mem_map array can get very big. Free the unused area of the memory map. 2214 */ 2215 static void __init free_unused_memmap(void) 2216 { 2217 unsigned long start, end, prev_end = 0; 2218 int i; 2219 2220 if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) || 2221 IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 2222 return; 2223 2224 /* 2225 * This relies on each bank being in address order. 2226 * The banks are sorted previously in bootmem_init(). 2227 */ 2228 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { 2229 #ifdef CONFIG_SPARSEMEM 2230 /* 2231 * Take care not to free memmap entries that don't exist 2232 * due to SPARSEMEM sections which aren't present. 2233 */ 2234 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION)); 2235 #endif 2236 /* 2237 * Align down here since many operations in VM subsystem 2238 * presume that there are no holes in the memory map inside 2239 * a pageblock 2240 */ 2241 start = pageblock_start_pfn(start); 2242 2243 /* 2244 * If we had a previous bank, and there is a space 2245 * between the current bank and the previous, free it. 2246 */ 2247 if (prev_end && prev_end < start) 2248 free_memmap(prev_end, start); 2249 2250 /* 2251 * Align up here since many operations in VM subsystem 2252 * presume that there are no holes in the memory map inside 2253 * a pageblock 2254 */ 2255 prev_end = pageblock_align(end); 2256 } 2257 2258 #ifdef CONFIG_SPARSEMEM 2259 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) { 2260 prev_end = pageblock_align(end); 2261 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); 2262 } 2263 #endif 2264 } 2265 2266 static void __init __free_pages_memory(unsigned long start, unsigned long end) 2267 { 2268 int order; 2269 2270 while (start < end) { 2271 /* 2272 * Free the pages in the largest chunks alignment allows. 2273 * 2274 * __ffs() behaviour is undefined for 0. start == 0 is 2275 * MAX_PAGE_ORDER-aligned, set order to MAX_PAGE_ORDER for 2276 * the case. 2277 */ 2278 if (start) 2279 order = min_t(int, MAX_PAGE_ORDER, __ffs(start)); 2280 else 2281 order = MAX_PAGE_ORDER; 2282 2283 while (start + (1UL << order) > end) 2284 order--; 2285 2286 memblock_free_pages(start, order); 2287 2288 start += (1UL << order); 2289 } 2290 } 2291 2292 static unsigned long __init __free_memory_core(phys_addr_t start, 2293 phys_addr_t end) 2294 { 2295 unsigned long start_pfn = PFN_UP(start); 2296 unsigned long end_pfn = PFN_DOWN(end); 2297 2298 if (!IS_ENABLED(CONFIG_HIGHMEM) && end_pfn > max_low_pfn) 2299 end_pfn = max_low_pfn; 2300 2301 if (start_pfn >= end_pfn) 2302 return 0; 2303 2304 __free_pages_memory(start_pfn, end_pfn); 2305 2306 return end_pfn - start_pfn; 2307 } 2308 2309 /* 2310 * Initialised pages do not have PageReserved set. This function is called 2311 * for each reserved range and marks the pages PageReserved. 2312 * When deferred initialization of struct pages is enabled it also ensures 2313 * that struct pages are properly initialised. 2314 */ 2315 static void __init memmap_init_reserved_range(phys_addr_t start, 2316 phys_addr_t end, int nid) 2317 { 2318 unsigned long pfn; 2319 2320 for_each_valid_pfn(pfn, PFN_DOWN(start), PFN_UP(end)) { 2321 struct page *page = pfn_to_page(pfn); 2322 2323 init_deferred_page(pfn, nid); 2324 2325 /* 2326 * no need for atomic set_bit because the struct 2327 * page is not visible yet so nobody should 2328 * access it yet. 2329 */ 2330 __SetPageReserved(page); 2331 } 2332 } 2333 2334 static void __init memmap_init_reserved_pages(void) 2335 { 2336 struct memblock_region *region; 2337 phys_addr_t start, end; 2338 int nid; 2339 unsigned long max_reserved; 2340 2341 /* 2342 * set nid on all reserved pages and also treat struct 2343 * pages for the NOMAP regions as PageReserved 2344 */ 2345 repeat: 2346 max_reserved = memblock.reserved.max; 2347 for_each_mem_region(region) { 2348 nid = memblock_get_region_node(region); 2349 start = region->base; 2350 end = start + region->size; 2351 2352 if (memblock_is_nomap(region)) 2353 memmap_init_reserved_range(start, end, nid); 2354 2355 memblock_set_node(start, region->size, &memblock.reserved, nid); 2356 } 2357 /* 2358 * 'max' is changed means memblock.reserved has been doubled its 2359 * array, which may result a new reserved region before current 2360 * 'start'. Now we should repeat the procedure to set its node id. 2361 */ 2362 if (max_reserved != memblock.reserved.max) 2363 goto repeat; 2364 2365 /* 2366 * initialize struct pages for reserved regions that don't have 2367 * the MEMBLOCK_RSRV_NOINIT flag set 2368 */ 2369 for_each_reserved_mem_region(region) { 2370 if (!memblock_is_reserved_noinit(region)) { 2371 nid = memblock_get_region_node(region); 2372 start = region->base; 2373 end = start + region->size; 2374 2375 if (!numa_valid_node(nid)) 2376 nid = early_pfn_to_nid(PFN_DOWN(start)); 2377 2378 memmap_init_reserved_range(start, end, nid); 2379 } 2380 } 2381 } 2382 2383 static unsigned long __init free_low_memory_core_early(void) 2384 { 2385 unsigned long count = 0; 2386 phys_addr_t start, end; 2387 u64 i; 2388 2389 memblock_clear_hotplug(0, -1); 2390 2391 memmap_init_reserved_pages(); 2392 2393 /* 2394 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id 2395 * because in some case like Node0 doesn't have RAM installed 2396 * low ram will be on Node1 2397 */ 2398 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, 2399 NULL) 2400 count += __free_memory_core(start, end); 2401 2402 return count; 2403 } 2404 2405 static int reset_managed_pages_done __initdata; 2406 2407 static void __init reset_node_managed_pages(pg_data_t *pgdat) 2408 { 2409 struct zone *z; 2410 2411 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 2412 atomic_long_set(&z->managed_pages, 0); 2413 } 2414 2415 void __init reset_all_zones_managed_pages(void) 2416 { 2417 struct pglist_data *pgdat; 2418 2419 if (reset_managed_pages_done) 2420 return; 2421 2422 for_each_online_pgdat(pgdat) 2423 reset_node_managed_pages(pgdat); 2424 2425 reset_managed_pages_done = 1; 2426 } 2427 2428 /** 2429 * memblock_free_all - release free pages to the buddy allocator 2430 */ 2431 void __init memblock_free_all(void) 2432 { 2433 unsigned long pages; 2434 2435 free_unused_memmap(); 2436 reset_all_zones_managed_pages(); 2437 2438 memblock_clear_kho_scratch_only(); 2439 pages = free_low_memory_core_early(); 2440 totalram_pages_add(pages); 2441 } 2442 2443 /* Keep a table to reserve named memory */ 2444 #define RESERVE_MEM_MAX_ENTRIES 8 2445 #define RESERVE_MEM_NAME_SIZE 16 2446 struct reserve_mem_table { 2447 char name[RESERVE_MEM_NAME_SIZE]; 2448 phys_addr_t start; 2449 phys_addr_t size; 2450 }; 2451 static struct reserve_mem_table reserved_mem_table[RESERVE_MEM_MAX_ENTRIES]; 2452 static int reserved_mem_count; 2453 static DEFINE_MUTEX(reserve_mem_lock); 2454 2455 /* Add wildcard region with a lookup name */ 2456 static void __init reserved_mem_add(phys_addr_t start, phys_addr_t size, 2457 const char *name) 2458 { 2459 struct reserve_mem_table *map; 2460 2461 map = &reserved_mem_table[reserved_mem_count++]; 2462 map->start = start; 2463 map->size = size; 2464 strscpy(map->name, name); 2465 } 2466 2467 static struct reserve_mem_table *reserve_mem_find_by_name_nolock(const char *name) 2468 { 2469 struct reserve_mem_table *map; 2470 int i; 2471 2472 for (i = 0; i < reserved_mem_count; i++) { 2473 map = &reserved_mem_table[i]; 2474 if (!map->size) 2475 continue; 2476 if (strcmp(name, map->name) == 0) 2477 return map; 2478 } 2479 return NULL; 2480 } 2481 2482 /** 2483 * reserve_mem_find_by_name - Find reserved memory region with a given name 2484 * @name: The name that is attached to a reserved memory region 2485 * @start: If found, holds the start address 2486 * @size: If found, holds the size of the address. 2487 * 2488 * @start and @size are only updated if @name is found. 2489 * 2490 * Returns: 1 if found or 0 if not found. 2491 */ 2492 int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size) 2493 { 2494 struct reserve_mem_table *map; 2495 2496 guard(mutex)(&reserve_mem_lock); 2497 map = reserve_mem_find_by_name_nolock(name); 2498 if (!map) 2499 return 0; 2500 2501 *start = map->start; 2502 *size = map->size; 2503 return 1; 2504 } 2505 EXPORT_SYMBOL_GPL(reserve_mem_find_by_name); 2506 2507 /** 2508 * reserve_mem_release_by_name - Release reserved memory region with a given name 2509 * @name: The name that is attached to a reserved memory region 2510 * 2511 * Forcibly release the pages in the reserved memory region so that those memory 2512 * can be used as free memory. After released the reserved region size becomes 0. 2513 * 2514 * Returns: 1 if released or 0 if not found. 2515 */ 2516 int reserve_mem_release_by_name(const char *name) 2517 { 2518 char buf[RESERVE_MEM_NAME_SIZE + 12]; 2519 struct reserve_mem_table *map; 2520 void *start, *end; 2521 2522 guard(mutex)(&reserve_mem_lock); 2523 map = reserve_mem_find_by_name_nolock(name); 2524 if (!map) 2525 return 0; 2526 2527 start = phys_to_virt(map->start); 2528 end = start + map->size; 2529 snprintf(buf, sizeof(buf), "reserve_mem:%s", name); 2530 free_reserved_area(start, end, 0, buf); 2531 map->size = 0; 2532 2533 return 1; 2534 } 2535 2536 #ifdef CONFIG_KEXEC_HANDOVER 2537 2538 static int __init reserved_mem_preserve(void) 2539 { 2540 unsigned int nr_preserved = 0; 2541 int err; 2542 2543 for (unsigned int i = 0; i < reserved_mem_count; i++, nr_preserved++) { 2544 struct reserve_mem_table *map = &reserved_mem_table[i]; 2545 struct page *page = phys_to_page(map->start); 2546 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2547 2548 err = kho_preserve_pages(page, nr_pages); 2549 if (err) 2550 goto err_unpreserve; 2551 } 2552 2553 return 0; 2554 2555 err_unpreserve: 2556 for (unsigned int i = 0; i < nr_preserved; i++) { 2557 struct reserve_mem_table *map = &reserved_mem_table[i]; 2558 struct page *page = phys_to_page(map->start); 2559 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2560 2561 kho_unpreserve_pages(page, nr_pages); 2562 } 2563 2564 return err; 2565 } 2566 2567 static int __init prepare_kho_fdt(void) 2568 { 2569 struct page *fdt_page; 2570 void *fdt; 2571 int err; 2572 2573 fdt_page = alloc_page(GFP_KERNEL); 2574 if (!fdt_page) { 2575 err = -ENOMEM; 2576 goto err_report; 2577 } 2578 2579 fdt = page_to_virt(fdt_page); 2580 err = kho_preserve_pages(fdt_page, 1); 2581 if (err) 2582 goto err_free_fdt; 2583 2584 err |= fdt_create(fdt, PAGE_SIZE); 2585 err |= fdt_finish_reservemap(fdt); 2586 err |= fdt_begin_node(fdt, ""); 2587 err |= fdt_property_string(fdt, "compatible", MEMBLOCK_KHO_NODE_COMPATIBLE); 2588 2589 for (unsigned int i = 0; !err && i < reserved_mem_count; i++) { 2590 struct reserve_mem_table *map = &reserved_mem_table[i]; 2591 2592 err |= fdt_begin_node(fdt, map->name); 2593 err |= fdt_property_string(fdt, "compatible", RESERVE_MEM_KHO_NODE_COMPATIBLE); 2594 err |= fdt_property(fdt, "start", &map->start, sizeof(map->start)); 2595 err |= fdt_property(fdt, "size", &map->size, sizeof(map->size)); 2596 err |= fdt_end_node(fdt); 2597 } 2598 err |= fdt_end_node(fdt); 2599 err |= fdt_finish(fdt); 2600 2601 if (err) 2602 goto err_unpreserve_fdt; 2603 2604 err = kho_add_subtree(MEMBLOCK_KHO_FDT, fdt, fdt_totalsize(fdt)); 2605 if (err) 2606 goto err_unpreserve_fdt; 2607 2608 err = reserved_mem_preserve(); 2609 if (err) 2610 goto err_remove_subtree; 2611 2612 return 0; 2613 2614 err_remove_subtree: 2615 kho_remove_subtree(fdt); 2616 err_unpreserve_fdt: 2617 kho_unpreserve_pages(fdt_page, 1); 2618 err_free_fdt: 2619 put_page(fdt_page); 2620 err_report: 2621 pr_err("failed to prepare memblock FDT for KHO: %d\n", err); 2622 2623 return err; 2624 } 2625 2626 static int __init reserve_mem_init(void) 2627 { 2628 int err; 2629 2630 if (!kho_is_enabled() || !reserved_mem_count) 2631 return 0; 2632 2633 err = prepare_kho_fdt(); 2634 if (err) 2635 return err; 2636 return err; 2637 } 2638 late_initcall(reserve_mem_init); 2639 2640 static void *__init reserve_mem_kho_retrieve_fdt(void) 2641 { 2642 phys_addr_t fdt_phys; 2643 static void *fdt; 2644 int err; 2645 2646 if (fdt) 2647 return fdt; 2648 2649 err = kho_retrieve_subtree(MEMBLOCK_KHO_FDT, &fdt_phys, NULL); 2650 if (err) { 2651 if (err != -ENOENT) 2652 pr_warn("failed to retrieve FDT '%s' from KHO: %d\n", 2653 MEMBLOCK_KHO_FDT, err); 2654 return NULL; 2655 } 2656 2657 fdt = phys_to_virt(fdt_phys); 2658 2659 err = fdt_node_check_compatible(fdt, 0, MEMBLOCK_KHO_NODE_COMPATIBLE); 2660 if (err) { 2661 pr_warn("FDT '%s' is incompatible with '%s': %d\n", 2662 MEMBLOCK_KHO_FDT, MEMBLOCK_KHO_NODE_COMPATIBLE, err); 2663 fdt = NULL; 2664 } 2665 2666 return fdt; 2667 } 2668 2669 static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2670 phys_addr_t align) 2671 { 2672 int err, len_start, len_size, offset; 2673 const phys_addr_t *p_start, *p_size; 2674 const void *fdt; 2675 2676 fdt = reserve_mem_kho_retrieve_fdt(); 2677 if (!fdt) 2678 return false; 2679 2680 offset = fdt_subnode_offset(fdt, 0, name); 2681 if (offset < 0) { 2682 pr_warn("FDT '%s' has no child '%s': %d\n", 2683 MEMBLOCK_KHO_FDT, name, offset); 2684 return false; 2685 } 2686 err = fdt_node_check_compatible(fdt, offset, RESERVE_MEM_KHO_NODE_COMPATIBLE); 2687 if (err) { 2688 pr_warn("Node '%s' is incompatible with '%s': %d\n", 2689 name, RESERVE_MEM_KHO_NODE_COMPATIBLE, err); 2690 return false; 2691 } 2692 2693 p_start = fdt_getprop(fdt, offset, "start", &len_start); 2694 p_size = fdt_getprop(fdt, offset, "size", &len_size); 2695 if (!p_start || len_start != sizeof(*p_start) || !p_size || 2696 len_size != sizeof(*p_size)) { 2697 return false; 2698 } 2699 2700 if (*p_start & (align - 1)) { 2701 pr_warn("KHO reserve-mem '%s' has wrong alignment (0x%lx, 0x%lx)\n", 2702 name, (long)align, (long)*p_start); 2703 return false; 2704 } 2705 2706 if (*p_size != size) { 2707 pr_warn("KHO reserve-mem '%s' has wrong size (0x%lx != 0x%lx)\n", 2708 name, (long)*p_size, (long)size); 2709 return false; 2710 } 2711 2712 reserved_mem_add(*p_start, size, name); 2713 pr_info("Revived memory reservation '%s' from KHO\n", name); 2714 2715 return true; 2716 } 2717 #else 2718 static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2719 phys_addr_t align) 2720 { 2721 return false; 2722 } 2723 #endif /* CONFIG_KEXEC_HANDOVER */ 2724 2725 /* 2726 * Parse reserve_mem=nn:align:name 2727 */ 2728 static int __init reserve_mem(char *p) 2729 { 2730 phys_addr_t start, size, align, tmp; 2731 char *name; 2732 char *oldp; 2733 int len; 2734 2735 if (!p) 2736 goto err_param; 2737 2738 /* Check if there's room for more reserved memory */ 2739 if (reserved_mem_count >= RESERVE_MEM_MAX_ENTRIES) { 2740 pr_err("reserve_mem: no more room for reserved memory\n"); 2741 return -EBUSY; 2742 } 2743 2744 oldp = p; 2745 size = memparse(p, &p); 2746 if (!size || p == oldp) 2747 goto err_param; 2748 2749 if (*p != ':') 2750 goto err_param; 2751 2752 align = memparse(p+1, &p); 2753 if (*p != ':') 2754 goto err_param; 2755 2756 /* 2757 * memblock_phys_alloc() doesn't like a zero size align, 2758 * but it is OK for this command to have it. 2759 */ 2760 if (align < SMP_CACHE_BYTES) 2761 align = SMP_CACHE_BYTES; 2762 2763 name = p + 1; 2764 len = strlen(name); 2765 2766 /* name needs to have length but not too big */ 2767 if (!len || len >= RESERVE_MEM_NAME_SIZE) 2768 goto err_param; 2769 2770 /* Make sure that name has text */ 2771 for (p = name; *p; p++) { 2772 if (!isspace(*p)) 2773 break; 2774 } 2775 if (!*p) 2776 goto err_param; 2777 2778 /* Make sure the name is not already used */ 2779 if (reserve_mem_find_by_name(name, &start, &tmp)) { 2780 pr_err("reserve_mem: name \"%s\" was already used\n", name); 2781 return -EBUSY; 2782 } 2783 2784 /* Pick previous allocations up from KHO if available */ 2785 if (reserve_mem_kho_revive(name, size, align)) 2786 return 1; 2787 2788 /* TODO: Allocation must be outside of scratch region */ 2789 start = memblock_phys_alloc(size, align); 2790 if (!start) { 2791 pr_err("reserve_mem: memblock allocation failed\n"); 2792 return -ENOMEM; 2793 } 2794 2795 reserved_mem_add(start, size, name); 2796 2797 return 1; 2798 err_param: 2799 pr_err("reserve_mem: empty or malformed parameter\n"); 2800 return -EINVAL; 2801 } 2802 __setup("reserve_mem=", reserve_mem); 2803 2804 #ifdef CONFIG_DEBUG_FS 2805 #ifdef CONFIG_ARCH_KEEP_MEMBLOCK 2806 static const char * const flagname[] = { 2807 [ilog2(MEMBLOCK_HOTPLUG)] = "HOTPLUG", 2808 [ilog2(MEMBLOCK_MIRROR)] = "MIRROR", 2809 [ilog2(MEMBLOCK_NOMAP)] = "NOMAP", 2810 [ilog2(MEMBLOCK_DRIVER_MANAGED)] = "DRV_MNG", 2811 [ilog2(MEMBLOCK_RSRV_NOINIT)] = "RSV_NIT", 2812 [ilog2(MEMBLOCK_RSRV_KERN)] = "RSV_KERN", 2813 [ilog2(MEMBLOCK_KHO_SCRATCH)] = "KHO_SCRATCH", 2814 }; 2815 2816 static int memblock_debug_show(struct seq_file *m, void *private) 2817 { 2818 struct memblock_type *type = m->private; 2819 struct memblock_region *reg; 2820 int i, j, nid; 2821 unsigned int count = ARRAY_SIZE(flagname); 2822 phys_addr_t end; 2823 2824 for (i = 0; i < type->cnt; i++) { 2825 reg = &type->regions[i]; 2826 end = reg->base + reg->size - 1; 2827 nid = memblock_get_region_node(reg); 2828 2829 seq_printf(m, "%4d: ", i); 2830 seq_printf(m, "%pa..%pa ", ®->base, &end); 2831 if (numa_valid_node(nid)) 2832 seq_printf(m, "%4d ", nid); 2833 else 2834 seq_printf(m, "%4c ", 'x'); 2835 if (reg->flags) { 2836 for (j = 0; j < count; j++) { 2837 if (reg->flags & (1U << j)) { 2838 seq_printf(m, "%s\n", flagname[j]); 2839 break; 2840 } 2841 } 2842 if (j == count) 2843 seq_printf(m, "%s\n", "UNKNOWN"); 2844 } else { 2845 seq_printf(m, "%s\n", "NONE"); 2846 } 2847 } 2848 return 0; 2849 } 2850 DEFINE_SHOW_ATTRIBUTE(memblock_debug); 2851 2852 static inline void memblock_debugfs_expose_arrays(struct dentry *root) 2853 { 2854 debugfs_create_file("memory", 0444, root, 2855 &memblock.memory, &memblock_debug_fops); 2856 debugfs_create_file("reserved", 0444, root, 2857 &memblock.reserved, &memblock_debug_fops); 2858 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2859 debugfs_create_file("physmem", 0444, root, &physmem, 2860 &memblock_debug_fops); 2861 #endif 2862 } 2863 2864 #else 2865 2866 static inline void memblock_debugfs_expose_arrays(struct dentry *root) { } 2867 2868 #endif /* CONFIG_ARCH_KEEP_MEMBLOCK */ 2869 2870 static int memblock_reserve_mem_show(struct seq_file *m, void *private) 2871 { 2872 struct reserve_mem_table *map; 2873 char txtsz[16]; 2874 2875 guard(mutex)(&reserve_mem_lock); 2876 for (int i = 0; i < reserved_mem_count; i++) { 2877 map = &reserved_mem_table[i]; 2878 if (!map->size) 2879 continue; 2880 2881 memset(txtsz, 0, sizeof(txtsz)); 2882 string_get_size(map->size, 1, STRING_UNITS_2, txtsz, sizeof(txtsz)); 2883 seq_printf(m, "%s\t\t(%s)\n", map->name, txtsz); 2884 } 2885 2886 return 0; 2887 } 2888 DEFINE_SHOW_ATTRIBUTE(memblock_reserve_mem); 2889 2890 static int __init memblock_init_debugfs(void) 2891 { 2892 struct dentry *root; 2893 2894 if (!IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK) && !reserved_mem_count) 2895 return 0; 2896 2897 root = debugfs_create_dir("memblock", NULL); 2898 2899 if (reserved_mem_count) 2900 debugfs_create_file("reserve_mem_param", 0444, root, NULL, 2901 &memblock_reserve_mem_fops); 2902 2903 memblock_debugfs_expose_arrays(root); 2904 return 0; 2905 } 2906 __initcall(memblock_init_debugfs); 2907 2908 #endif /* CONFIG_DEBUG_FS */ 2909