1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Procedures for maintaining information about logical memory blocks. 4 * 5 * Peter Bergner, IBM Corp. June 2001. 6 * Copyright (C) 2001 Peter Bergner. 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/slab.h> 11 #include <linux/init.h> 12 #include <linux/bitops.h> 13 #include <linux/poison.h> 14 #include <linux/pfn.h> 15 #include <linux/debugfs.h> 16 #include <linux/kmemleak.h> 17 #include <linux/seq_file.h> 18 #include <linux/memblock.h> 19 20 #include <asm/sections.h> 21 #include <linux/io.h> 22 23 #include "internal.h" 24 25 #define INIT_MEMBLOCK_REGIONS 128 26 #define INIT_PHYSMEM_REGIONS 4 27 28 #ifndef INIT_MEMBLOCK_RESERVED_REGIONS 29 # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS 30 #endif 31 32 #ifndef INIT_MEMBLOCK_MEMORY_REGIONS 33 #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS 34 #endif 35 36 /** 37 * DOC: memblock overview 38 * 39 * Memblock is a method of managing memory regions during the early 40 * boot period when the usual kernel memory allocators are not up and 41 * running. 42 * 43 * Memblock views the system memory as collections of contiguous 44 * regions. There are several types of these collections: 45 * 46 * * ``memory`` - describes the physical memory available to the 47 * kernel; this may differ from the actual physical memory installed 48 * in the system, for instance when the memory is restricted with 49 * ``mem=`` command line parameter 50 * * ``reserved`` - describes the regions that were allocated 51 * * ``physmem`` - describes the actual physical memory available during 52 * boot regardless of the possible restrictions and memory hot(un)plug; 53 * the ``physmem`` type is only available on some architectures. 54 * 55 * Each region is represented by struct memblock_region that 56 * defines the region extents, its attributes and NUMA node id on NUMA 57 * systems. Every memory type is described by the struct memblock_type 58 * which contains an array of memory regions along with 59 * the allocator metadata. The "memory" and "reserved" types are nicely 60 * wrapped with struct memblock. This structure is statically 61 * initialized at build time. The region arrays are initially sized to 62 * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and 63 * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array 64 * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS. 65 * The memblock_allow_resize() enables automatic resizing of the region 66 * arrays during addition of new regions. This feature should be used 67 * with care so that memory allocated for the region array will not 68 * overlap with areas that should be reserved, for example initrd. 69 * 70 * The early architecture setup should tell memblock what the physical 71 * memory layout is by using memblock_add() or memblock_add_node() 72 * functions. The first function does not assign the region to a NUMA 73 * node and it is appropriate for UMA systems. Yet, it is possible to 74 * use it on NUMA systems as well and assign the region to a NUMA node 75 * later in the setup process using memblock_set_node(). The 76 * memblock_add_node() performs such an assignment directly. 77 * 78 * Once memblock is setup the memory can be allocated using one of the 79 * API variants: 80 * 81 * * memblock_phys_alloc*() - these functions return the **physical** 82 * address of the allocated memory 83 * * memblock_alloc*() - these functions return the **virtual** address 84 * of the allocated memory. 85 * 86 * Note, that both API variants use implicit assumptions about allowed 87 * memory ranges and the fallback methods. Consult the documentation 88 * of memblock_alloc_internal() and memblock_alloc_range_nid() 89 * functions for more elaborate description. 90 * 91 * As the system boot progresses, the architecture specific mem_init() 92 * function frees all the memory to the buddy page allocator. 93 * 94 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the 95 * memblock data structures (except "physmem") will be discarded after the 96 * system initialization completes. 97 */ 98 99 #ifndef CONFIG_NUMA 100 struct pglist_data __refdata contig_page_data; 101 EXPORT_SYMBOL(contig_page_data); 102 #endif 103 104 unsigned long max_low_pfn; 105 unsigned long min_low_pfn; 106 unsigned long max_pfn; 107 unsigned long long max_possible_pfn; 108 109 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock; 110 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock; 111 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 112 static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS]; 113 #endif 114 115 struct memblock memblock __initdata_memblock = { 116 .memory.regions = memblock_memory_init_regions, 117 .memory.cnt = 1, /* empty dummy entry */ 118 .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS, 119 .memory.name = "memory", 120 121 .reserved.regions = memblock_reserved_init_regions, 122 .reserved.cnt = 1, /* empty dummy entry */ 123 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS, 124 .reserved.name = "reserved", 125 126 .bottom_up = false, 127 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 128 }; 129 130 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 131 struct memblock_type physmem = { 132 .regions = memblock_physmem_init_regions, 133 .cnt = 1, /* empty dummy entry */ 134 .max = INIT_PHYSMEM_REGIONS, 135 .name = "physmem", 136 }; 137 #endif 138 139 /* 140 * keep a pointer to &memblock.memory in the text section to use it in 141 * __next_mem_range() and its helpers. 142 * For architectures that do not keep memblock data after init, this 143 * pointer will be reset to NULL at memblock_discard() 144 */ 145 static __refdata struct memblock_type *memblock_memory = &memblock.memory; 146 147 #define for_each_memblock_type(i, memblock_type, rgn) \ 148 for (i = 0, rgn = &memblock_type->regions[0]; \ 149 i < memblock_type->cnt; \ 150 i++, rgn = &memblock_type->regions[i]) 151 152 #define memblock_dbg(fmt, ...) \ 153 do { \ 154 if (memblock_debug) \ 155 pr_info(fmt, ##__VA_ARGS__); \ 156 } while (0) 157 158 static int memblock_debug __initdata_memblock; 159 static bool system_has_some_mirror __initdata_memblock = false; 160 static int memblock_can_resize __initdata_memblock; 161 static int memblock_memory_in_slab __initdata_memblock = 0; 162 static int memblock_reserved_in_slab __initdata_memblock = 0; 163 164 static enum memblock_flags __init_memblock choose_memblock_flags(void) 165 { 166 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; 167 } 168 169 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 170 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 171 { 172 return *size = min(*size, PHYS_ADDR_MAX - base); 173 } 174 175 /* 176 * Address comparison utilities 177 */ 178 static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, 179 phys_addr_t base2, phys_addr_t size2) 180 { 181 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 182 } 183 184 bool __init_memblock memblock_overlaps_region(struct memblock_type *type, 185 phys_addr_t base, phys_addr_t size) 186 { 187 unsigned long i; 188 189 memblock_cap_size(base, &size); 190 191 for (i = 0; i < type->cnt; i++) 192 if (memblock_addrs_overlap(base, size, type->regions[i].base, 193 type->regions[i].size)) 194 break; 195 return i < type->cnt; 196 } 197 198 /** 199 * __memblock_find_range_bottom_up - find free area utility in bottom-up 200 * @start: start of candidate range 201 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 202 * %MEMBLOCK_ALLOC_ACCESSIBLE 203 * @size: size of free area to find 204 * @align: alignment of free area to find 205 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 206 * @flags: pick from blocks based on memory attributes 207 * 208 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 209 * 210 * Return: 211 * Found address on success, 0 on failure. 212 */ 213 static phys_addr_t __init_memblock 214 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 215 phys_addr_t size, phys_addr_t align, int nid, 216 enum memblock_flags flags) 217 { 218 phys_addr_t this_start, this_end, cand; 219 u64 i; 220 221 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) { 222 this_start = clamp(this_start, start, end); 223 this_end = clamp(this_end, start, end); 224 225 cand = round_up(this_start, align); 226 if (cand < this_end && this_end - cand >= size) 227 return cand; 228 } 229 230 return 0; 231 } 232 233 /** 234 * __memblock_find_range_top_down - find free area utility, in top-down 235 * @start: start of candidate range 236 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 237 * %MEMBLOCK_ALLOC_ACCESSIBLE 238 * @size: size of free area to find 239 * @align: alignment of free area to find 240 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 241 * @flags: pick from blocks based on memory attributes 242 * 243 * Utility called from memblock_find_in_range_node(), find free area top-down. 244 * 245 * Return: 246 * Found address on success, 0 on failure. 247 */ 248 static phys_addr_t __init_memblock 249 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 250 phys_addr_t size, phys_addr_t align, int nid, 251 enum memblock_flags flags) 252 { 253 phys_addr_t this_start, this_end, cand; 254 u64 i; 255 256 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end, 257 NULL) { 258 this_start = clamp(this_start, start, end); 259 this_end = clamp(this_end, start, end); 260 261 if (this_end < size) 262 continue; 263 264 cand = round_down(this_end - size, align); 265 if (cand >= this_start) 266 return cand; 267 } 268 269 return 0; 270 } 271 272 /** 273 * memblock_find_in_range_node - find free area in given range and node 274 * @size: size of free area to find 275 * @align: alignment of free area to find 276 * @start: start of candidate range 277 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 278 * %MEMBLOCK_ALLOC_ACCESSIBLE 279 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 280 * @flags: pick from blocks based on memory attributes 281 * 282 * Find @size free area aligned to @align in the specified range and node. 283 * 284 * Return: 285 * Found address on success, 0 on failure. 286 */ 287 static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 288 phys_addr_t align, phys_addr_t start, 289 phys_addr_t end, int nid, 290 enum memblock_flags flags) 291 { 292 /* pump up @end */ 293 if (end == MEMBLOCK_ALLOC_ACCESSIBLE || 294 end == MEMBLOCK_ALLOC_NOLEAKTRACE) 295 end = memblock.current_limit; 296 297 /* avoid allocating the first page */ 298 start = max_t(phys_addr_t, start, PAGE_SIZE); 299 end = max(start, end); 300 301 if (memblock_bottom_up()) 302 return __memblock_find_range_bottom_up(start, end, size, align, 303 nid, flags); 304 else 305 return __memblock_find_range_top_down(start, end, size, align, 306 nid, flags); 307 } 308 309 /** 310 * memblock_find_in_range - find free area in given range 311 * @start: start of candidate range 312 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 313 * %MEMBLOCK_ALLOC_ACCESSIBLE 314 * @size: size of free area to find 315 * @align: alignment of free area to find 316 * 317 * Find @size free area aligned to @align in the specified range. 318 * 319 * Return: 320 * Found address on success, 0 on failure. 321 */ 322 static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 323 phys_addr_t end, phys_addr_t size, 324 phys_addr_t align) 325 { 326 phys_addr_t ret; 327 enum memblock_flags flags = choose_memblock_flags(); 328 329 again: 330 ret = memblock_find_in_range_node(size, align, start, end, 331 NUMA_NO_NODE, flags); 332 333 if (!ret && (flags & MEMBLOCK_MIRROR)) { 334 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 335 &size); 336 flags &= ~MEMBLOCK_MIRROR; 337 goto again; 338 } 339 340 return ret; 341 } 342 343 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 344 { 345 type->total_size -= type->regions[r].size; 346 memmove(&type->regions[r], &type->regions[r + 1], 347 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 348 type->cnt--; 349 350 /* Special case for empty arrays */ 351 if (type->cnt == 0) { 352 WARN_ON(type->total_size != 0); 353 type->cnt = 1; 354 type->regions[0].base = 0; 355 type->regions[0].size = 0; 356 type->regions[0].flags = 0; 357 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 358 } 359 } 360 361 #ifndef CONFIG_ARCH_KEEP_MEMBLOCK 362 /** 363 * memblock_discard - discard memory and reserved arrays if they were allocated 364 */ 365 void __init memblock_discard(void) 366 { 367 phys_addr_t addr, size; 368 369 if (memblock.reserved.regions != memblock_reserved_init_regions) { 370 addr = __pa(memblock.reserved.regions); 371 size = PAGE_ALIGN(sizeof(struct memblock_region) * 372 memblock.reserved.max); 373 if (memblock_reserved_in_slab) 374 kfree(memblock.reserved.regions); 375 else 376 memblock_free_late(addr, size); 377 } 378 379 if (memblock.memory.regions != memblock_memory_init_regions) { 380 addr = __pa(memblock.memory.regions); 381 size = PAGE_ALIGN(sizeof(struct memblock_region) * 382 memblock.memory.max); 383 if (memblock_memory_in_slab) 384 kfree(memblock.memory.regions); 385 else 386 memblock_free_late(addr, size); 387 } 388 389 memblock_memory = NULL; 390 } 391 #endif 392 393 /** 394 * memblock_double_array - double the size of the memblock regions array 395 * @type: memblock type of the regions array being doubled 396 * @new_area_start: starting address of memory range to avoid overlap with 397 * @new_area_size: size of memory range to avoid overlap with 398 * 399 * Double the size of the @type regions array. If memblock is being used to 400 * allocate memory for a new reserved regions array and there is a previously 401 * allocated memory range [@new_area_start, @new_area_start + @new_area_size] 402 * waiting to be reserved, ensure the memory used by the new array does 403 * not overlap. 404 * 405 * Return: 406 * 0 on success, -1 on failure. 407 */ 408 static int __init_memblock memblock_double_array(struct memblock_type *type, 409 phys_addr_t new_area_start, 410 phys_addr_t new_area_size) 411 { 412 struct memblock_region *new_array, *old_array; 413 phys_addr_t old_alloc_size, new_alloc_size; 414 phys_addr_t old_size, new_size, addr, new_end; 415 int use_slab = slab_is_available(); 416 int *in_slab; 417 418 /* We don't allow resizing until we know about the reserved regions 419 * of memory that aren't suitable for allocation 420 */ 421 if (!memblock_can_resize) 422 return -1; 423 424 /* Calculate new doubled size */ 425 old_size = type->max * sizeof(struct memblock_region); 426 new_size = old_size << 1; 427 /* 428 * We need to allocated new one align to PAGE_SIZE, 429 * so we can free them completely later. 430 */ 431 old_alloc_size = PAGE_ALIGN(old_size); 432 new_alloc_size = PAGE_ALIGN(new_size); 433 434 /* Retrieve the slab flag */ 435 if (type == &memblock.memory) 436 in_slab = &memblock_memory_in_slab; 437 else 438 in_slab = &memblock_reserved_in_slab; 439 440 /* Try to find some space for it */ 441 if (use_slab) { 442 new_array = kmalloc(new_size, GFP_KERNEL); 443 addr = new_array ? __pa(new_array) : 0; 444 } else { 445 /* only exclude range when trying to double reserved.regions */ 446 if (type != &memblock.reserved) 447 new_area_start = new_area_size = 0; 448 449 addr = memblock_find_in_range(new_area_start + new_area_size, 450 memblock.current_limit, 451 new_alloc_size, PAGE_SIZE); 452 if (!addr && new_area_size) 453 addr = memblock_find_in_range(0, 454 min(new_area_start, memblock.current_limit), 455 new_alloc_size, PAGE_SIZE); 456 457 new_array = addr ? __va(addr) : NULL; 458 } 459 if (!addr) { 460 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 461 type->name, type->max, type->max * 2); 462 return -1; 463 } 464 465 new_end = addr + new_size - 1; 466 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]", 467 type->name, type->max * 2, &addr, &new_end); 468 469 /* 470 * Found space, we now need to move the array over before we add the 471 * reserved region since it may be our reserved array itself that is 472 * full. 473 */ 474 memcpy(new_array, type->regions, old_size); 475 memset(new_array + type->max, 0, old_size); 476 old_array = type->regions; 477 type->regions = new_array; 478 type->max <<= 1; 479 480 /* Free old array. We needn't free it if the array is the static one */ 481 if (*in_slab) 482 kfree(old_array); 483 else if (old_array != memblock_memory_init_regions && 484 old_array != memblock_reserved_init_regions) 485 memblock_free(old_array, old_alloc_size); 486 487 /* 488 * Reserve the new array if that comes from the memblock. Otherwise, we 489 * needn't do it 490 */ 491 if (!use_slab) 492 BUG_ON(memblock_reserve(addr, new_alloc_size)); 493 494 /* Update slab flag */ 495 *in_slab = use_slab; 496 497 return 0; 498 } 499 500 /** 501 * memblock_merge_regions - merge neighboring compatible regions 502 * @type: memblock type to scan 503 * @start_rgn: start scanning from (@start_rgn - 1) 504 * @end_rgn: end scanning at (@end_rgn - 1) 505 * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn) 506 */ 507 static void __init_memblock memblock_merge_regions(struct memblock_type *type, 508 unsigned long start_rgn, 509 unsigned long end_rgn) 510 { 511 int i = 0; 512 if (start_rgn) 513 i = start_rgn - 1; 514 end_rgn = min(end_rgn, type->cnt - 1); 515 while (i < end_rgn) { 516 struct memblock_region *this = &type->regions[i]; 517 struct memblock_region *next = &type->regions[i + 1]; 518 519 if (this->base + this->size != next->base || 520 memblock_get_region_node(this) != 521 memblock_get_region_node(next) || 522 this->flags != next->flags) { 523 BUG_ON(this->base + this->size > next->base); 524 i++; 525 continue; 526 } 527 528 this->size += next->size; 529 /* move forward from next + 1, index of which is i + 2 */ 530 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 531 type->cnt--; 532 end_rgn--; 533 } 534 } 535 536 /** 537 * memblock_insert_region - insert new memblock region 538 * @type: memblock type to insert into 539 * @idx: index for the insertion point 540 * @base: base address of the new region 541 * @size: size of the new region 542 * @nid: node id of the new region 543 * @flags: flags of the new region 544 * 545 * Insert new memblock region [@base, @base + @size) into @type at @idx. 546 * @type must already have extra room to accommodate the new region. 547 */ 548 static void __init_memblock memblock_insert_region(struct memblock_type *type, 549 int idx, phys_addr_t base, 550 phys_addr_t size, 551 int nid, 552 enum memblock_flags flags) 553 { 554 struct memblock_region *rgn = &type->regions[idx]; 555 556 BUG_ON(type->cnt >= type->max); 557 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 558 rgn->base = base; 559 rgn->size = size; 560 rgn->flags = flags; 561 memblock_set_region_node(rgn, nid); 562 type->cnt++; 563 type->total_size += size; 564 } 565 566 /** 567 * memblock_add_range - add new memblock region 568 * @type: memblock type to add new region into 569 * @base: base address of the new region 570 * @size: size of the new region 571 * @nid: nid of the new region 572 * @flags: flags of the new region 573 * 574 * Add new memblock region [@base, @base + @size) into @type. The new region 575 * is allowed to overlap with existing ones - overlaps don't affect already 576 * existing regions. @type is guaranteed to be minimal (all neighbouring 577 * compatible regions are merged) after the addition. 578 * 579 * Return: 580 * 0 on success, -errno on failure. 581 */ 582 static int __init_memblock memblock_add_range(struct memblock_type *type, 583 phys_addr_t base, phys_addr_t size, 584 int nid, enum memblock_flags flags) 585 { 586 bool insert = false; 587 phys_addr_t obase = base; 588 phys_addr_t end = base + memblock_cap_size(base, &size); 589 int idx, nr_new, start_rgn = -1, end_rgn; 590 struct memblock_region *rgn; 591 592 if (!size) 593 return 0; 594 595 /* special case for empty array */ 596 if (type->regions[0].size == 0) { 597 WARN_ON(type->cnt != 1 || type->total_size); 598 type->regions[0].base = base; 599 type->regions[0].size = size; 600 type->regions[0].flags = flags; 601 memblock_set_region_node(&type->regions[0], nid); 602 type->total_size = size; 603 return 0; 604 } 605 606 /* 607 * The worst case is when new range overlaps all existing regions, 608 * then we'll need type->cnt + 1 empty regions in @type. So if 609 * type->cnt * 2 + 1 is less than or equal to type->max, we know 610 * that there is enough empty regions in @type, and we can insert 611 * regions directly. 612 */ 613 if (type->cnt * 2 + 1 <= type->max) 614 insert = true; 615 616 repeat: 617 /* 618 * The following is executed twice. Once with %false @insert and 619 * then with %true. The first counts the number of regions needed 620 * to accommodate the new area. The second actually inserts them. 621 */ 622 base = obase; 623 nr_new = 0; 624 625 for_each_memblock_type(idx, type, rgn) { 626 phys_addr_t rbase = rgn->base; 627 phys_addr_t rend = rbase + rgn->size; 628 629 if (rbase >= end) 630 break; 631 if (rend <= base) 632 continue; 633 /* 634 * @rgn overlaps. If it separates the lower part of new 635 * area, insert that portion. 636 */ 637 if (rbase > base) { 638 #ifdef CONFIG_NUMA 639 WARN_ON(nid != memblock_get_region_node(rgn)); 640 #endif 641 WARN_ON(flags != rgn->flags); 642 nr_new++; 643 if (insert) { 644 if (start_rgn == -1) 645 start_rgn = idx; 646 end_rgn = idx + 1; 647 memblock_insert_region(type, idx++, base, 648 rbase - base, nid, 649 flags); 650 } 651 } 652 /* area below @rend is dealt with, forget about it */ 653 base = min(rend, end); 654 } 655 656 /* insert the remaining portion */ 657 if (base < end) { 658 nr_new++; 659 if (insert) { 660 if (start_rgn == -1) 661 start_rgn = idx; 662 end_rgn = idx + 1; 663 memblock_insert_region(type, idx, base, end - base, 664 nid, flags); 665 } 666 } 667 668 if (!nr_new) 669 return 0; 670 671 /* 672 * If this was the first round, resize array and repeat for actual 673 * insertions; otherwise, merge and return. 674 */ 675 if (!insert) { 676 while (type->cnt + nr_new > type->max) 677 if (memblock_double_array(type, obase, size) < 0) 678 return -ENOMEM; 679 insert = true; 680 goto repeat; 681 } else { 682 memblock_merge_regions(type, start_rgn, end_rgn); 683 return 0; 684 } 685 } 686 687 /** 688 * memblock_add_node - add new memblock region within a NUMA node 689 * @base: base address of the new region 690 * @size: size of the new region 691 * @nid: nid of the new region 692 * @flags: flags of the new region 693 * 694 * Add new memblock region [@base, @base + @size) to the "memory" 695 * type. See memblock_add_range() description for mode details 696 * 697 * Return: 698 * 0 on success, -errno on failure. 699 */ 700 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 701 int nid, enum memblock_flags flags) 702 { 703 phys_addr_t end = base + size - 1; 704 705 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 706 &base, &end, nid, flags, (void *)_RET_IP_); 707 708 return memblock_add_range(&memblock.memory, base, size, nid, flags); 709 } 710 711 /** 712 * memblock_add - add new memblock region 713 * @base: base address of the new region 714 * @size: size of the new region 715 * 716 * Add new memblock region [@base, @base + @size) to the "memory" 717 * type. See memblock_add_range() description for mode details 718 * 719 * Return: 720 * 0 on success, -errno on failure. 721 */ 722 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 723 { 724 phys_addr_t end = base + size - 1; 725 726 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 727 &base, &end, (void *)_RET_IP_); 728 729 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0); 730 } 731 732 /** 733 * memblock_isolate_range - isolate given range into disjoint memblocks 734 * @type: memblock type to isolate range for 735 * @base: base of range to isolate 736 * @size: size of range to isolate 737 * @start_rgn: out parameter for the start of isolated region 738 * @end_rgn: out parameter for the end of isolated region 739 * 740 * Walk @type and ensure that regions don't cross the boundaries defined by 741 * [@base, @base + @size). Crossing regions are split at the boundaries, 742 * which may create at most two more regions. The index of the first 743 * region inside the range is returned in *@start_rgn and end in *@end_rgn. 744 * 745 * Return: 746 * 0 on success, -errno on failure. 747 */ 748 static int __init_memblock memblock_isolate_range(struct memblock_type *type, 749 phys_addr_t base, phys_addr_t size, 750 int *start_rgn, int *end_rgn) 751 { 752 phys_addr_t end = base + memblock_cap_size(base, &size); 753 int idx; 754 struct memblock_region *rgn; 755 756 *start_rgn = *end_rgn = 0; 757 758 if (!size) 759 return 0; 760 761 /* we'll create at most two more regions */ 762 while (type->cnt + 2 > type->max) 763 if (memblock_double_array(type, base, size) < 0) 764 return -ENOMEM; 765 766 for_each_memblock_type(idx, type, rgn) { 767 phys_addr_t rbase = rgn->base; 768 phys_addr_t rend = rbase + rgn->size; 769 770 if (rbase >= end) 771 break; 772 if (rend <= base) 773 continue; 774 775 if (rbase < base) { 776 /* 777 * @rgn intersects from below. Split and continue 778 * to process the next region - the new top half. 779 */ 780 rgn->base = base; 781 rgn->size -= base - rbase; 782 type->total_size -= base - rbase; 783 memblock_insert_region(type, idx, rbase, base - rbase, 784 memblock_get_region_node(rgn), 785 rgn->flags); 786 } else if (rend > end) { 787 /* 788 * @rgn intersects from above. Split and redo the 789 * current region - the new bottom half. 790 */ 791 rgn->base = end; 792 rgn->size -= end - rbase; 793 type->total_size -= end - rbase; 794 memblock_insert_region(type, idx--, rbase, end - rbase, 795 memblock_get_region_node(rgn), 796 rgn->flags); 797 } else { 798 /* @rgn is fully contained, record it */ 799 if (!*end_rgn) 800 *start_rgn = idx; 801 *end_rgn = idx + 1; 802 } 803 } 804 805 return 0; 806 } 807 808 static int __init_memblock memblock_remove_range(struct memblock_type *type, 809 phys_addr_t base, phys_addr_t size) 810 { 811 int start_rgn, end_rgn; 812 int i, ret; 813 814 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 815 if (ret) 816 return ret; 817 818 for (i = end_rgn - 1; i >= start_rgn; i--) 819 memblock_remove_region(type, i); 820 return 0; 821 } 822 823 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 824 { 825 phys_addr_t end = base + size - 1; 826 827 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 828 &base, &end, (void *)_RET_IP_); 829 830 return memblock_remove_range(&memblock.memory, base, size); 831 } 832 833 /** 834 * memblock_free - free boot memory allocation 835 * @ptr: starting address of the boot memory allocation 836 * @size: size of the boot memory block in bytes 837 * 838 * Free boot memory block previously allocated by memblock_alloc_xx() API. 839 * The freeing memory will not be released to the buddy allocator. 840 */ 841 void __init_memblock memblock_free(void *ptr, size_t size) 842 { 843 if (ptr) 844 memblock_phys_free(__pa(ptr), size); 845 } 846 847 /** 848 * memblock_phys_free - free boot memory block 849 * @base: phys starting address of the boot memory block 850 * @size: size of the boot memory block in bytes 851 * 852 * Free boot memory block previously allocated by memblock_phys_alloc_xx() API. 853 * The freeing memory will not be released to the buddy allocator. 854 */ 855 int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size) 856 { 857 phys_addr_t end = base + size - 1; 858 859 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 860 &base, &end, (void *)_RET_IP_); 861 862 kmemleak_free_part_phys(base, size); 863 return memblock_remove_range(&memblock.reserved, base, size); 864 } 865 866 int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size) 867 { 868 phys_addr_t end = base + size - 1; 869 870 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 871 &base, &end, (void *)_RET_IP_); 872 873 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0); 874 } 875 876 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 877 int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) 878 { 879 phys_addr_t end = base + size - 1; 880 881 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 882 &base, &end, (void *)_RET_IP_); 883 884 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0); 885 } 886 #endif 887 888 /** 889 * memblock_setclr_flag - set or clear flag for a memory region 890 * @base: base address of the region 891 * @size: size of the region 892 * @set: set or clear the flag 893 * @flag: the flag to update 894 * 895 * This function isolates region [@base, @base + @size), and sets/clears flag 896 * 897 * Return: 0 on success, -errno on failure. 898 */ 899 static int __init_memblock memblock_setclr_flag(phys_addr_t base, 900 phys_addr_t size, int set, int flag) 901 { 902 struct memblock_type *type = &memblock.memory; 903 int i, ret, start_rgn, end_rgn; 904 905 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 906 if (ret) 907 return ret; 908 909 for (i = start_rgn; i < end_rgn; i++) { 910 struct memblock_region *r = &type->regions[i]; 911 912 if (set) 913 r->flags |= flag; 914 else 915 r->flags &= ~flag; 916 } 917 918 memblock_merge_regions(type, start_rgn, end_rgn); 919 return 0; 920 } 921 922 /** 923 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 924 * @base: the base phys addr of the region 925 * @size: the size of the region 926 * 927 * Return: 0 on success, -errno on failure. 928 */ 929 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 930 { 931 return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG); 932 } 933 934 /** 935 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 936 * @base: the base phys addr of the region 937 * @size: the size of the region 938 * 939 * Return: 0 on success, -errno on failure. 940 */ 941 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 942 { 943 return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG); 944 } 945 946 /** 947 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR. 948 * @base: the base phys addr of the region 949 * @size: the size of the region 950 * 951 * Return: 0 on success, -errno on failure. 952 */ 953 int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size) 954 { 955 if (!mirrored_kernelcore) 956 return 0; 957 958 system_has_some_mirror = true; 959 960 return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR); 961 } 962 963 /** 964 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP. 965 * @base: the base phys addr of the region 966 * @size: the size of the region 967 * 968 * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the 969 * direct mapping of the physical memory. These regions will still be 970 * covered by the memory map. The struct page representing NOMAP memory 971 * frames in the memory map will be PageReserved() 972 * 973 * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from 974 * memblock, the caller must inform kmemleak to ignore that memory 975 * 976 * Return: 0 on success, -errno on failure. 977 */ 978 int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size) 979 { 980 return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP); 981 } 982 983 /** 984 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region. 985 * @base: the base phys addr of the region 986 * @size: the size of the region 987 * 988 * Return: 0 on success, -errno on failure. 989 */ 990 int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size) 991 { 992 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP); 993 } 994 995 static bool should_skip_region(struct memblock_type *type, 996 struct memblock_region *m, 997 int nid, int flags) 998 { 999 int m_nid = memblock_get_region_node(m); 1000 1001 /* we never skip regions when iterating memblock.reserved or physmem */ 1002 if (type != memblock_memory) 1003 return false; 1004 1005 /* only memory regions are associated with nodes, check it */ 1006 if (nid != NUMA_NO_NODE && nid != m_nid) 1007 return true; 1008 1009 /* skip hotpluggable memory regions if needed */ 1010 if (movable_node_is_enabled() && memblock_is_hotpluggable(m) && 1011 !(flags & MEMBLOCK_HOTPLUG)) 1012 return true; 1013 1014 /* if we want mirror memory skip non-mirror memory regions */ 1015 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 1016 return true; 1017 1018 /* skip nomap memory unless we were asked for it explicitly */ 1019 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 1020 return true; 1021 1022 /* skip driver-managed memory unless we were asked for it explicitly */ 1023 if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m)) 1024 return true; 1025 1026 return false; 1027 } 1028 1029 /** 1030 * __next_mem_range - next function for for_each_free_mem_range() etc. 1031 * @idx: pointer to u64 loop variable 1032 * @nid: node selector, %NUMA_NO_NODE for all nodes 1033 * @flags: pick from blocks based on memory attributes 1034 * @type_a: pointer to memblock_type from where the range is taken 1035 * @type_b: pointer to memblock_type which excludes memory from being taken 1036 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1037 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1038 * @out_nid: ptr to int for nid of the range, can be %NULL 1039 * 1040 * Find the first area from *@idx which matches @nid, fill the out 1041 * parameters, and update *@idx for the next iteration. The lower 32bit of 1042 * *@idx contains index into type_a and the upper 32bit indexes the 1043 * areas before each region in type_b. For example, if type_b regions 1044 * look like the following, 1045 * 1046 * 0:[0-16), 1:[32-48), 2:[128-130) 1047 * 1048 * The upper 32bit indexes the following regions. 1049 * 1050 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 1051 * 1052 * As both region arrays are sorted, the function advances the two indices 1053 * in lockstep and returns each intersection. 1054 */ 1055 void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags, 1056 struct memblock_type *type_a, 1057 struct memblock_type *type_b, phys_addr_t *out_start, 1058 phys_addr_t *out_end, int *out_nid) 1059 { 1060 int idx_a = *idx & 0xffffffff; 1061 int idx_b = *idx >> 32; 1062 1063 if (WARN_ONCE(nid == MAX_NUMNODES, 1064 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1065 nid = NUMA_NO_NODE; 1066 1067 for (; idx_a < type_a->cnt; idx_a++) { 1068 struct memblock_region *m = &type_a->regions[idx_a]; 1069 1070 phys_addr_t m_start = m->base; 1071 phys_addr_t m_end = m->base + m->size; 1072 int m_nid = memblock_get_region_node(m); 1073 1074 if (should_skip_region(type_a, m, nid, flags)) 1075 continue; 1076 1077 if (!type_b) { 1078 if (out_start) 1079 *out_start = m_start; 1080 if (out_end) 1081 *out_end = m_end; 1082 if (out_nid) 1083 *out_nid = m_nid; 1084 idx_a++; 1085 *idx = (u32)idx_a | (u64)idx_b << 32; 1086 return; 1087 } 1088 1089 /* scan areas before each reservation */ 1090 for (; idx_b < type_b->cnt + 1; idx_b++) { 1091 struct memblock_region *r; 1092 phys_addr_t r_start; 1093 phys_addr_t r_end; 1094 1095 r = &type_b->regions[idx_b]; 1096 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1097 r_end = idx_b < type_b->cnt ? 1098 r->base : PHYS_ADDR_MAX; 1099 1100 /* 1101 * if idx_b advanced past idx_a, 1102 * break out to advance idx_a 1103 */ 1104 if (r_start >= m_end) 1105 break; 1106 /* if the two regions intersect, we're done */ 1107 if (m_start < r_end) { 1108 if (out_start) 1109 *out_start = 1110 max(m_start, r_start); 1111 if (out_end) 1112 *out_end = min(m_end, r_end); 1113 if (out_nid) 1114 *out_nid = m_nid; 1115 /* 1116 * The region which ends first is 1117 * advanced for the next iteration. 1118 */ 1119 if (m_end <= r_end) 1120 idx_a++; 1121 else 1122 idx_b++; 1123 *idx = (u32)idx_a | (u64)idx_b << 32; 1124 return; 1125 } 1126 } 1127 } 1128 1129 /* signal end of iteration */ 1130 *idx = ULLONG_MAX; 1131 } 1132 1133 /** 1134 * __next_mem_range_rev - generic next function for for_each_*_range_rev() 1135 * 1136 * @idx: pointer to u64 loop variable 1137 * @nid: node selector, %NUMA_NO_NODE for all nodes 1138 * @flags: pick from blocks based on memory attributes 1139 * @type_a: pointer to memblock_type from where the range is taken 1140 * @type_b: pointer to memblock_type which excludes memory from being taken 1141 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1142 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1143 * @out_nid: ptr to int for nid of the range, can be %NULL 1144 * 1145 * Finds the next range from type_a which is not marked as unsuitable 1146 * in type_b. 1147 * 1148 * Reverse of __next_mem_range(). 1149 */ 1150 void __init_memblock __next_mem_range_rev(u64 *idx, int nid, 1151 enum memblock_flags flags, 1152 struct memblock_type *type_a, 1153 struct memblock_type *type_b, 1154 phys_addr_t *out_start, 1155 phys_addr_t *out_end, int *out_nid) 1156 { 1157 int idx_a = *idx & 0xffffffff; 1158 int idx_b = *idx >> 32; 1159 1160 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1161 nid = NUMA_NO_NODE; 1162 1163 if (*idx == (u64)ULLONG_MAX) { 1164 idx_a = type_a->cnt - 1; 1165 if (type_b != NULL) 1166 idx_b = type_b->cnt; 1167 else 1168 idx_b = 0; 1169 } 1170 1171 for (; idx_a >= 0; idx_a--) { 1172 struct memblock_region *m = &type_a->regions[idx_a]; 1173 1174 phys_addr_t m_start = m->base; 1175 phys_addr_t m_end = m->base + m->size; 1176 int m_nid = memblock_get_region_node(m); 1177 1178 if (should_skip_region(type_a, m, nid, flags)) 1179 continue; 1180 1181 if (!type_b) { 1182 if (out_start) 1183 *out_start = m_start; 1184 if (out_end) 1185 *out_end = m_end; 1186 if (out_nid) 1187 *out_nid = m_nid; 1188 idx_a--; 1189 *idx = (u32)idx_a | (u64)idx_b << 32; 1190 return; 1191 } 1192 1193 /* scan areas before each reservation */ 1194 for (; idx_b >= 0; idx_b--) { 1195 struct memblock_region *r; 1196 phys_addr_t r_start; 1197 phys_addr_t r_end; 1198 1199 r = &type_b->regions[idx_b]; 1200 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1201 r_end = idx_b < type_b->cnt ? 1202 r->base : PHYS_ADDR_MAX; 1203 /* 1204 * if idx_b advanced past idx_a, 1205 * break out to advance idx_a 1206 */ 1207 1208 if (r_end <= m_start) 1209 break; 1210 /* if the two regions intersect, we're done */ 1211 if (m_end > r_start) { 1212 if (out_start) 1213 *out_start = max(m_start, r_start); 1214 if (out_end) 1215 *out_end = min(m_end, r_end); 1216 if (out_nid) 1217 *out_nid = m_nid; 1218 if (m_start >= r_start) 1219 idx_a--; 1220 else 1221 idx_b--; 1222 *idx = (u32)idx_a | (u64)idx_b << 32; 1223 return; 1224 } 1225 } 1226 } 1227 /* signal end of iteration */ 1228 *idx = ULLONG_MAX; 1229 } 1230 1231 /* 1232 * Common iterator interface used to define for_each_mem_pfn_range(). 1233 */ 1234 void __init_memblock __next_mem_pfn_range(int *idx, int nid, 1235 unsigned long *out_start_pfn, 1236 unsigned long *out_end_pfn, int *out_nid) 1237 { 1238 struct memblock_type *type = &memblock.memory; 1239 struct memblock_region *r; 1240 int r_nid; 1241 1242 while (++*idx < type->cnt) { 1243 r = &type->regions[*idx]; 1244 r_nid = memblock_get_region_node(r); 1245 1246 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 1247 continue; 1248 if (nid == MAX_NUMNODES || nid == r_nid) 1249 break; 1250 } 1251 if (*idx >= type->cnt) { 1252 *idx = -1; 1253 return; 1254 } 1255 1256 if (out_start_pfn) 1257 *out_start_pfn = PFN_UP(r->base); 1258 if (out_end_pfn) 1259 *out_end_pfn = PFN_DOWN(r->base + r->size); 1260 if (out_nid) 1261 *out_nid = r_nid; 1262 } 1263 1264 /** 1265 * memblock_set_node - set node ID on memblock regions 1266 * @base: base of area to set node ID for 1267 * @size: size of area to set node ID for 1268 * @type: memblock type to set node ID for 1269 * @nid: node ID to set 1270 * 1271 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid. 1272 * Regions which cross the area boundaries are split as necessary. 1273 * 1274 * Return: 1275 * 0 on success, -errno on failure. 1276 */ 1277 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 1278 struct memblock_type *type, int nid) 1279 { 1280 #ifdef CONFIG_NUMA 1281 int start_rgn, end_rgn; 1282 int i, ret; 1283 1284 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1285 if (ret) 1286 return ret; 1287 1288 for (i = start_rgn; i < end_rgn; i++) 1289 memblock_set_region_node(&type->regions[i], nid); 1290 1291 memblock_merge_regions(type, start_rgn, end_rgn); 1292 #endif 1293 return 0; 1294 } 1295 1296 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1297 /** 1298 * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone() 1299 * 1300 * @idx: pointer to u64 loop variable 1301 * @zone: zone in which all of the memory blocks reside 1302 * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL 1303 * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL 1304 * 1305 * This function is meant to be a zone/pfn specific wrapper for the 1306 * for_each_mem_range type iterators. Specifically they are used in the 1307 * deferred memory init routines and as such we were duplicating much of 1308 * this logic throughout the code. So instead of having it in multiple 1309 * locations it seemed like it would make more sense to centralize this to 1310 * one new iterator that does everything they need. 1311 */ 1312 void __init_memblock 1313 __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, 1314 unsigned long *out_spfn, unsigned long *out_epfn) 1315 { 1316 int zone_nid = zone_to_nid(zone); 1317 phys_addr_t spa, epa; 1318 1319 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE, 1320 &memblock.memory, &memblock.reserved, 1321 &spa, &epa, NULL); 1322 1323 while (*idx != U64_MAX) { 1324 unsigned long epfn = PFN_DOWN(epa); 1325 unsigned long spfn = PFN_UP(spa); 1326 1327 /* 1328 * Verify the end is at least past the start of the zone and 1329 * that we have at least one PFN to initialize. 1330 */ 1331 if (zone->zone_start_pfn < epfn && spfn < epfn) { 1332 /* if we went too far just stop searching */ 1333 if (zone_end_pfn(zone) <= spfn) { 1334 *idx = U64_MAX; 1335 break; 1336 } 1337 1338 if (out_spfn) 1339 *out_spfn = max(zone->zone_start_pfn, spfn); 1340 if (out_epfn) 1341 *out_epfn = min(zone_end_pfn(zone), epfn); 1342 1343 return; 1344 } 1345 1346 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE, 1347 &memblock.memory, &memblock.reserved, 1348 &spa, &epa, NULL); 1349 } 1350 1351 /* signal end of iteration */ 1352 if (out_spfn) 1353 *out_spfn = ULONG_MAX; 1354 if (out_epfn) 1355 *out_epfn = 0; 1356 } 1357 1358 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1359 1360 /** 1361 * memblock_alloc_range_nid - allocate boot memory block 1362 * @size: size of memory block to be allocated in bytes 1363 * @align: alignment of the region and block's size 1364 * @start: the lower bound of the memory region to allocate (phys address) 1365 * @end: the upper bound of the memory region to allocate (phys address) 1366 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1367 * @exact_nid: control the allocation fall back to other nodes 1368 * 1369 * The allocation is performed from memory region limited by 1370 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE. 1371 * 1372 * If the specified node can not hold the requested memory and @exact_nid 1373 * is false, the allocation falls back to any node in the system. 1374 * 1375 * For systems with memory mirroring, the allocation is attempted first 1376 * from the regions with mirroring enabled and then retried from any 1377 * memory region. 1378 * 1379 * In addition, function using kmemleak_alloc_phys for allocated boot 1380 * memory block, it is never reported as leaks. 1381 * 1382 * Return: 1383 * Physical address of allocated memory block on success, %0 on failure. 1384 */ 1385 phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, 1386 phys_addr_t align, phys_addr_t start, 1387 phys_addr_t end, int nid, 1388 bool exact_nid) 1389 { 1390 enum memblock_flags flags = choose_memblock_flags(); 1391 phys_addr_t found; 1392 1393 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1394 nid = NUMA_NO_NODE; 1395 1396 if (!align) { 1397 /* Can't use WARNs this early in boot on powerpc */ 1398 dump_stack(); 1399 align = SMP_CACHE_BYTES; 1400 } 1401 1402 again: 1403 found = memblock_find_in_range_node(size, align, start, end, nid, 1404 flags); 1405 if (found && !memblock_reserve(found, size)) 1406 goto done; 1407 1408 if (nid != NUMA_NO_NODE && !exact_nid) { 1409 found = memblock_find_in_range_node(size, align, start, 1410 end, NUMA_NO_NODE, 1411 flags); 1412 if (found && !memblock_reserve(found, size)) 1413 goto done; 1414 } 1415 1416 if (flags & MEMBLOCK_MIRROR) { 1417 flags &= ~MEMBLOCK_MIRROR; 1418 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 1419 &size); 1420 goto again; 1421 } 1422 1423 return 0; 1424 1425 done: 1426 /* 1427 * Skip kmemleak for those places like kasan_init() and 1428 * early_pgtable_alloc() due to high volume. 1429 */ 1430 if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) 1431 /* 1432 * Memblock allocated blocks are never reported as 1433 * leaks. This is because many of these blocks are 1434 * only referred via the physical address which is 1435 * not looked up by kmemleak. 1436 */ 1437 kmemleak_alloc_phys(found, size, 0); 1438 1439 return found; 1440 } 1441 1442 /** 1443 * memblock_phys_alloc_range - allocate a memory block inside specified range 1444 * @size: size of memory block to be allocated in bytes 1445 * @align: alignment of the region and block's size 1446 * @start: the lower bound of the memory region to allocate (physical address) 1447 * @end: the upper bound of the memory region to allocate (physical address) 1448 * 1449 * Allocate @size bytes in the between @start and @end. 1450 * 1451 * Return: physical address of the allocated memory block on success, 1452 * %0 on failure. 1453 */ 1454 phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, 1455 phys_addr_t align, 1456 phys_addr_t start, 1457 phys_addr_t end) 1458 { 1459 memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n", 1460 __func__, (u64)size, (u64)align, &start, &end, 1461 (void *)_RET_IP_); 1462 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, 1463 false); 1464 } 1465 1466 /** 1467 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node 1468 * @size: size of memory block to be allocated in bytes 1469 * @align: alignment of the region and block's size 1470 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1471 * 1472 * Allocates memory block from the specified NUMA node. If the node 1473 * has no available memory, attempts to allocated from any node in the 1474 * system. 1475 * 1476 * Return: physical address of the allocated memory block on success, 1477 * %0 on failure. 1478 */ 1479 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1480 { 1481 return memblock_alloc_range_nid(size, align, 0, 1482 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false); 1483 } 1484 1485 /** 1486 * memblock_alloc_internal - allocate boot memory block 1487 * @size: size of memory block to be allocated in bytes 1488 * @align: alignment of the region and block's size 1489 * @min_addr: the lower bound of the memory region to allocate (phys address) 1490 * @max_addr: the upper bound of the memory region to allocate (phys address) 1491 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1492 * @exact_nid: control the allocation fall back to other nodes 1493 * 1494 * Allocates memory block using memblock_alloc_range_nid() and 1495 * converts the returned physical address to virtual. 1496 * 1497 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1498 * will fall back to memory below @min_addr. Other constraints, such 1499 * as node and mirrored memory will be handled again in 1500 * memblock_alloc_range_nid(). 1501 * 1502 * Return: 1503 * Virtual address of allocated memory block on success, NULL on failure. 1504 */ 1505 static void * __init memblock_alloc_internal( 1506 phys_addr_t size, phys_addr_t align, 1507 phys_addr_t min_addr, phys_addr_t max_addr, 1508 int nid, bool exact_nid) 1509 { 1510 phys_addr_t alloc; 1511 1512 /* 1513 * Detect any accidental use of these APIs after slab is ready, as at 1514 * this moment memblock may be deinitialized already and its 1515 * internal data may be destroyed (after execution of memblock_free_all) 1516 */ 1517 if (WARN_ON_ONCE(slab_is_available())) 1518 return kzalloc_node(size, GFP_NOWAIT, nid); 1519 1520 if (max_addr > memblock.current_limit) 1521 max_addr = memblock.current_limit; 1522 1523 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid, 1524 exact_nid); 1525 1526 /* retry allocation without lower limit */ 1527 if (!alloc && min_addr) 1528 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid, 1529 exact_nid); 1530 1531 if (!alloc) 1532 return NULL; 1533 1534 return phys_to_virt(alloc); 1535 } 1536 1537 /** 1538 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node 1539 * without zeroing memory 1540 * @size: size of memory block to be allocated in bytes 1541 * @align: alignment of the region and block's size 1542 * @min_addr: the lower bound of the memory region from where the allocation 1543 * is preferred (phys address) 1544 * @max_addr: the upper bound of the memory region from where the allocation 1545 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1546 * allocate only from memory limited by memblock.current_limit value 1547 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1548 * 1549 * Public function, provides additional debug information (including caller 1550 * info), if enabled. Does not zero allocated memory. 1551 * 1552 * Return: 1553 * Virtual address of allocated memory block on success, NULL on failure. 1554 */ 1555 void * __init memblock_alloc_exact_nid_raw( 1556 phys_addr_t size, phys_addr_t align, 1557 phys_addr_t min_addr, phys_addr_t max_addr, 1558 int nid) 1559 { 1560 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1561 __func__, (u64)size, (u64)align, nid, &min_addr, 1562 &max_addr, (void *)_RET_IP_); 1563 1564 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1565 true); 1566 } 1567 1568 /** 1569 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing 1570 * memory and without panicking 1571 * @size: size of memory block to be allocated in bytes 1572 * @align: alignment of the region and block's size 1573 * @min_addr: the lower bound of the memory region from where the allocation 1574 * is preferred (phys address) 1575 * @max_addr: the upper bound of the memory region from where the allocation 1576 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1577 * allocate only from memory limited by memblock.current_limit value 1578 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1579 * 1580 * Public function, provides additional debug information (including caller 1581 * info), if enabled. Does not zero allocated memory, does not panic if request 1582 * cannot be satisfied. 1583 * 1584 * Return: 1585 * Virtual address of allocated memory block on success, NULL on failure. 1586 */ 1587 void * __init memblock_alloc_try_nid_raw( 1588 phys_addr_t size, phys_addr_t align, 1589 phys_addr_t min_addr, phys_addr_t max_addr, 1590 int nid) 1591 { 1592 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1593 __func__, (u64)size, (u64)align, nid, &min_addr, 1594 &max_addr, (void *)_RET_IP_); 1595 1596 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1597 false); 1598 } 1599 1600 /** 1601 * memblock_alloc_try_nid - allocate boot memory block 1602 * @size: size of memory block to be allocated in bytes 1603 * @align: alignment of the region and block's size 1604 * @min_addr: the lower bound of the memory region from where the allocation 1605 * is preferred (phys address) 1606 * @max_addr: the upper bound of the memory region from where the allocation 1607 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1608 * allocate only from memory limited by memblock.current_limit value 1609 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1610 * 1611 * Public function, provides additional debug information (including caller 1612 * info), if enabled. This function zeroes the allocated memory. 1613 * 1614 * Return: 1615 * Virtual address of allocated memory block on success, NULL on failure. 1616 */ 1617 void * __init memblock_alloc_try_nid( 1618 phys_addr_t size, phys_addr_t align, 1619 phys_addr_t min_addr, phys_addr_t max_addr, 1620 int nid) 1621 { 1622 void *ptr; 1623 1624 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1625 __func__, (u64)size, (u64)align, nid, &min_addr, 1626 &max_addr, (void *)_RET_IP_); 1627 ptr = memblock_alloc_internal(size, align, 1628 min_addr, max_addr, nid, false); 1629 if (ptr) 1630 memset(ptr, 0, size); 1631 1632 return ptr; 1633 } 1634 1635 /** 1636 * memblock_free_late - free pages directly to buddy allocator 1637 * @base: phys starting address of the boot memory block 1638 * @size: size of the boot memory block in bytes 1639 * 1640 * This is only useful when the memblock allocator has already been torn 1641 * down, but we are still initializing the system. Pages are released directly 1642 * to the buddy allocator. 1643 */ 1644 void __init memblock_free_late(phys_addr_t base, phys_addr_t size) 1645 { 1646 phys_addr_t cursor, end; 1647 1648 end = base + size - 1; 1649 memblock_dbg("%s: [%pa-%pa] %pS\n", 1650 __func__, &base, &end, (void *)_RET_IP_); 1651 kmemleak_free_part_phys(base, size); 1652 cursor = PFN_UP(base); 1653 end = PFN_DOWN(base + size); 1654 1655 for (; cursor < end; cursor++) { 1656 memblock_free_pages(pfn_to_page(cursor), cursor, 0); 1657 totalram_pages_inc(); 1658 } 1659 } 1660 1661 /* 1662 * Remaining API functions 1663 */ 1664 1665 phys_addr_t __init_memblock memblock_phys_mem_size(void) 1666 { 1667 return memblock.memory.total_size; 1668 } 1669 1670 phys_addr_t __init_memblock memblock_reserved_size(void) 1671 { 1672 return memblock.reserved.total_size; 1673 } 1674 1675 /* lowest address */ 1676 phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1677 { 1678 return memblock.memory.regions[0].base; 1679 } 1680 1681 phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1682 { 1683 int idx = memblock.memory.cnt - 1; 1684 1685 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1686 } 1687 1688 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) 1689 { 1690 phys_addr_t max_addr = PHYS_ADDR_MAX; 1691 struct memblock_region *r; 1692 1693 /* 1694 * translate the memory @limit size into the max address within one of 1695 * the memory memblock regions, if the @limit exceeds the total size 1696 * of those regions, max_addr will keep original value PHYS_ADDR_MAX 1697 */ 1698 for_each_mem_region(r) { 1699 if (limit <= r->size) { 1700 max_addr = r->base + limit; 1701 break; 1702 } 1703 limit -= r->size; 1704 } 1705 1706 return max_addr; 1707 } 1708 1709 void __init memblock_enforce_memory_limit(phys_addr_t limit) 1710 { 1711 phys_addr_t max_addr; 1712 1713 if (!limit) 1714 return; 1715 1716 max_addr = __find_max_addr(limit); 1717 1718 /* @limit exceeds the total size of the memory, do nothing */ 1719 if (max_addr == PHYS_ADDR_MAX) 1720 return; 1721 1722 /* truncate both memory and reserved regions */ 1723 memblock_remove_range(&memblock.memory, max_addr, 1724 PHYS_ADDR_MAX); 1725 memblock_remove_range(&memblock.reserved, max_addr, 1726 PHYS_ADDR_MAX); 1727 } 1728 1729 void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size) 1730 { 1731 int start_rgn, end_rgn; 1732 int i, ret; 1733 1734 if (!size) 1735 return; 1736 1737 if (!memblock_memory->total_size) { 1738 pr_warn("%s: No memory registered yet\n", __func__); 1739 return; 1740 } 1741 1742 ret = memblock_isolate_range(&memblock.memory, base, size, 1743 &start_rgn, &end_rgn); 1744 if (ret) 1745 return; 1746 1747 /* remove all the MAP regions */ 1748 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--) 1749 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1750 memblock_remove_region(&memblock.memory, i); 1751 1752 for (i = start_rgn - 1; i >= 0; i--) 1753 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1754 memblock_remove_region(&memblock.memory, i); 1755 1756 /* truncate the reserved regions */ 1757 memblock_remove_range(&memblock.reserved, 0, base); 1758 memblock_remove_range(&memblock.reserved, 1759 base + size, PHYS_ADDR_MAX); 1760 } 1761 1762 void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1763 { 1764 phys_addr_t max_addr; 1765 1766 if (!limit) 1767 return; 1768 1769 max_addr = __find_max_addr(limit); 1770 1771 /* @limit exceeds the total size of the memory, do nothing */ 1772 if (max_addr == PHYS_ADDR_MAX) 1773 return; 1774 1775 memblock_cap_memory_range(0, max_addr); 1776 } 1777 1778 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1779 { 1780 unsigned int left = 0, right = type->cnt; 1781 1782 do { 1783 unsigned int mid = (right + left) / 2; 1784 1785 if (addr < type->regions[mid].base) 1786 right = mid; 1787 else if (addr >= (type->regions[mid].base + 1788 type->regions[mid].size)) 1789 left = mid + 1; 1790 else 1791 return mid; 1792 } while (left < right); 1793 return -1; 1794 } 1795 1796 bool __init_memblock memblock_is_reserved(phys_addr_t addr) 1797 { 1798 return memblock_search(&memblock.reserved, addr) != -1; 1799 } 1800 1801 bool __init_memblock memblock_is_memory(phys_addr_t addr) 1802 { 1803 return memblock_search(&memblock.memory, addr) != -1; 1804 } 1805 1806 bool __init_memblock memblock_is_map_memory(phys_addr_t addr) 1807 { 1808 int i = memblock_search(&memblock.memory, addr); 1809 1810 if (i == -1) 1811 return false; 1812 return !memblock_is_nomap(&memblock.memory.regions[i]); 1813 } 1814 1815 int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 1816 unsigned long *start_pfn, unsigned long *end_pfn) 1817 { 1818 struct memblock_type *type = &memblock.memory; 1819 int mid = memblock_search(type, PFN_PHYS(pfn)); 1820 1821 if (mid == -1) 1822 return -1; 1823 1824 *start_pfn = PFN_DOWN(type->regions[mid].base); 1825 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size); 1826 1827 return memblock_get_region_node(&type->regions[mid]); 1828 } 1829 1830 /** 1831 * memblock_is_region_memory - check if a region is a subset of memory 1832 * @base: base of region to check 1833 * @size: size of region to check 1834 * 1835 * Check if the region [@base, @base + @size) is a subset of a memory block. 1836 * 1837 * Return: 1838 * 0 if false, non-zero if true 1839 */ 1840 bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 1841 { 1842 int idx = memblock_search(&memblock.memory, base); 1843 phys_addr_t end = base + memblock_cap_size(base, &size); 1844 1845 if (idx == -1) 1846 return false; 1847 return (memblock.memory.regions[idx].base + 1848 memblock.memory.regions[idx].size) >= end; 1849 } 1850 1851 /** 1852 * memblock_is_region_reserved - check if a region intersects reserved memory 1853 * @base: base of region to check 1854 * @size: size of region to check 1855 * 1856 * Check if the region [@base, @base + @size) intersects a reserved 1857 * memory block. 1858 * 1859 * Return: 1860 * True if they intersect, false if not. 1861 */ 1862 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 1863 { 1864 return memblock_overlaps_region(&memblock.reserved, base, size); 1865 } 1866 1867 void __init_memblock memblock_trim_memory(phys_addr_t align) 1868 { 1869 phys_addr_t start, end, orig_start, orig_end; 1870 struct memblock_region *r; 1871 1872 for_each_mem_region(r) { 1873 orig_start = r->base; 1874 orig_end = r->base + r->size; 1875 start = round_up(orig_start, align); 1876 end = round_down(orig_end, align); 1877 1878 if (start == orig_start && end == orig_end) 1879 continue; 1880 1881 if (start < end) { 1882 r->base = start; 1883 r->size = end - start; 1884 } else { 1885 memblock_remove_region(&memblock.memory, 1886 r - memblock.memory.regions); 1887 r--; 1888 } 1889 } 1890 } 1891 1892 void __init_memblock memblock_set_current_limit(phys_addr_t limit) 1893 { 1894 memblock.current_limit = limit; 1895 } 1896 1897 phys_addr_t __init_memblock memblock_get_current_limit(void) 1898 { 1899 return memblock.current_limit; 1900 } 1901 1902 static void __init_memblock memblock_dump(struct memblock_type *type) 1903 { 1904 phys_addr_t base, end, size; 1905 enum memblock_flags flags; 1906 int idx; 1907 struct memblock_region *rgn; 1908 1909 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt); 1910 1911 for_each_memblock_type(idx, type, rgn) { 1912 char nid_buf[32] = ""; 1913 1914 base = rgn->base; 1915 size = rgn->size; 1916 end = base + size - 1; 1917 flags = rgn->flags; 1918 #ifdef CONFIG_NUMA 1919 if (memblock_get_region_node(rgn) != MAX_NUMNODES) 1920 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 1921 memblock_get_region_node(rgn)); 1922 #endif 1923 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n", 1924 type->name, idx, &base, &end, &size, nid_buf, flags); 1925 } 1926 } 1927 1928 static void __init_memblock __memblock_dump_all(void) 1929 { 1930 pr_info("MEMBLOCK configuration:\n"); 1931 pr_info(" memory size = %pa reserved size = %pa\n", 1932 &memblock.memory.total_size, 1933 &memblock.reserved.total_size); 1934 1935 memblock_dump(&memblock.memory); 1936 memblock_dump(&memblock.reserved); 1937 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 1938 memblock_dump(&physmem); 1939 #endif 1940 } 1941 1942 void __init_memblock memblock_dump_all(void) 1943 { 1944 if (memblock_debug) 1945 __memblock_dump_all(); 1946 } 1947 1948 void __init memblock_allow_resize(void) 1949 { 1950 memblock_can_resize = 1; 1951 } 1952 1953 static int __init early_memblock(char *p) 1954 { 1955 if (p && strstr(p, "debug")) 1956 memblock_debug = 1; 1957 return 0; 1958 } 1959 early_param("memblock", early_memblock); 1960 1961 static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn) 1962 { 1963 struct page *start_pg, *end_pg; 1964 phys_addr_t pg, pgend; 1965 1966 /* 1967 * Convert start_pfn/end_pfn to a struct page pointer. 1968 */ 1969 start_pg = pfn_to_page(start_pfn - 1) + 1; 1970 end_pg = pfn_to_page(end_pfn - 1) + 1; 1971 1972 /* 1973 * Convert to physical addresses, and round start upwards and end 1974 * downwards. 1975 */ 1976 pg = PAGE_ALIGN(__pa(start_pg)); 1977 pgend = __pa(end_pg) & PAGE_MASK; 1978 1979 /* 1980 * If there are free pages between these, free the section of the 1981 * memmap array. 1982 */ 1983 if (pg < pgend) 1984 memblock_phys_free(pg, pgend - pg); 1985 } 1986 1987 /* 1988 * The mem_map array can get very big. Free the unused area of the memory map. 1989 */ 1990 static void __init free_unused_memmap(void) 1991 { 1992 unsigned long start, end, prev_end = 0; 1993 int i; 1994 1995 if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) || 1996 IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 1997 return; 1998 1999 /* 2000 * This relies on each bank being in address order. 2001 * The banks are sorted previously in bootmem_init(). 2002 */ 2003 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { 2004 #ifdef CONFIG_SPARSEMEM 2005 /* 2006 * Take care not to free memmap entries that don't exist 2007 * due to SPARSEMEM sections which aren't present. 2008 */ 2009 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION)); 2010 #endif 2011 /* 2012 * Align down here since many operations in VM subsystem 2013 * presume that there are no holes in the memory map inside 2014 * a pageblock 2015 */ 2016 start = pageblock_start_pfn(start); 2017 2018 /* 2019 * If we had a previous bank, and there is a space 2020 * between the current bank and the previous, free it. 2021 */ 2022 if (prev_end && prev_end < start) 2023 free_memmap(prev_end, start); 2024 2025 /* 2026 * Align up here since many operations in VM subsystem 2027 * presume that there are no holes in the memory map inside 2028 * a pageblock 2029 */ 2030 prev_end = pageblock_align(end); 2031 } 2032 2033 #ifdef CONFIG_SPARSEMEM 2034 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) { 2035 prev_end = pageblock_align(end); 2036 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); 2037 } 2038 #endif 2039 } 2040 2041 static void __init __free_pages_memory(unsigned long start, unsigned long end) 2042 { 2043 int order; 2044 2045 while (start < end) { 2046 /* 2047 * Free the pages in the largest chunks alignment allows. 2048 * 2049 * __ffs() behaviour is undefined for 0. start == 0 is 2050 * MAX_ORDER-aligned, set order to MAX_ORDER for the case. 2051 */ 2052 if (start) 2053 order = min_t(int, MAX_ORDER, __ffs(start)); 2054 else 2055 order = MAX_ORDER; 2056 2057 while (start + (1UL << order) > end) 2058 order--; 2059 2060 memblock_free_pages(pfn_to_page(start), start, order); 2061 2062 start += (1UL << order); 2063 } 2064 } 2065 2066 static unsigned long __init __free_memory_core(phys_addr_t start, 2067 phys_addr_t end) 2068 { 2069 unsigned long start_pfn = PFN_UP(start); 2070 unsigned long end_pfn = min_t(unsigned long, 2071 PFN_DOWN(end), max_low_pfn); 2072 2073 if (start_pfn >= end_pfn) 2074 return 0; 2075 2076 __free_pages_memory(start_pfn, end_pfn); 2077 2078 return end_pfn - start_pfn; 2079 } 2080 2081 static void __init memmap_init_reserved_pages(void) 2082 { 2083 struct memblock_region *region; 2084 phys_addr_t start, end; 2085 u64 i; 2086 2087 /* initialize struct pages for the reserved regions */ 2088 for_each_reserved_mem_range(i, &start, &end) 2089 reserve_bootmem_region(start, end); 2090 2091 /* and also treat struct pages for the NOMAP regions as PageReserved */ 2092 for_each_mem_region(region) { 2093 if (memblock_is_nomap(region)) { 2094 start = region->base; 2095 end = start + region->size; 2096 reserve_bootmem_region(start, end); 2097 } 2098 } 2099 } 2100 2101 static unsigned long __init free_low_memory_core_early(void) 2102 { 2103 unsigned long count = 0; 2104 phys_addr_t start, end; 2105 u64 i; 2106 2107 memblock_clear_hotplug(0, -1); 2108 2109 memmap_init_reserved_pages(); 2110 2111 /* 2112 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id 2113 * because in some case like Node0 doesn't have RAM installed 2114 * low ram will be on Node1 2115 */ 2116 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, 2117 NULL) 2118 count += __free_memory_core(start, end); 2119 2120 return count; 2121 } 2122 2123 static int reset_managed_pages_done __initdata; 2124 2125 void reset_node_managed_pages(pg_data_t *pgdat) 2126 { 2127 struct zone *z; 2128 2129 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 2130 atomic_long_set(&z->managed_pages, 0); 2131 } 2132 2133 void __init reset_all_zones_managed_pages(void) 2134 { 2135 struct pglist_data *pgdat; 2136 2137 if (reset_managed_pages_done) 2138 return; 2139 2140 for_each_online_pgdat(pgdat) 2141 reset_node_managed_pages(pgdat); 2142 2143 reset_managed_pages_done = 1; 2144 } 2145 2146 /** 2147 * memblock_free_all - release free pages to the buddy allocator 2148 */ 2149 void __init memblock_free_all(void) 2150 { 2151 unsigned long pages; 2152 2153 free_unused_memmap(); 2154 reset_all_zones_managed_pages(); 2155 2156 pages = free_low_memory_core_early(); 2157 totalram_pages_add(pages); 2158 } 2159 2160 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK) 2161 2162 static int memblock_debug_show(struct seq_file *m, void *private) 2163 { 2164 struct memblock_type *type = m->private; 2165 struct memblock_region *reg; 2166 int i; 2167 phys_addr_t end; 2168 2169 for (i = 0; i < type->cnt; i++) { 2170 reg = &type->regions[i]; 2171 end = reg->base + reg->size - 1; 2172 2173 seq_printf(m, "%4d: ", i); 2174 seq_printf(m, "%pa..%pa\n", ®->base, &end); 2175 } 2176 return 0; 2177 } 2178 DEFINE_SHOW_ATTRIBUTE(memblock_debug); 2179 2180 static int __init memblock_init_debugfs(void) 2181 { 2182 struct dentry *root = debugfs_create_dir("memblock", NULL); 2183 2184 debugfs_create_file("memory", 0444, root, 2185 &memblock.memory, &memblock_debug_fops); 2186 debugfs_create_file("reserved", 0444, root, 2187 &memblock.reserved, &memblock_debug_fops); 2188 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2189 debugfs_create_file("physmem", 0444, root, &physmem, 2190 &memblock_debug_fops); 2191 #endif 2192 2193 return 0; 2194 } 2195 __initcall(memblock_init_debugfs); 2196 2197 #endif /* CONFIG_DEBUG_FS */ 2198