1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Low level x86 E820 memory map handling functions. 4 * 5 * The firmware and bootloader passes us the "E820 table", which is the primary 6 * physical memory layout description available about x86 systems. 7 * 8 * The kernel takes the E820 memory layout and optionally modifies it with 9 * quirks and other tweaks, and feeds that into the generic Linux memory 10 * allocation code routines via a platform independent interface (memblock, etc.). 11 */ 12 #include <linux/crash_dump.h> 13 #include <linux/memblock.h> 14 #include <linux/suspend.h> 15 #include <linux/acpi.h> 16 #include <linux/firmware-map.h> 17 #include <linux/sort.h> 18 #include <linux/memory_hotplug.h> 19 20 #include <asm/e820/api.h> 21 #include <asm/setup.h> 22 23 /* 24 * We organize the E820 table into three main data structures: 25 * 26 * - 'e820_table_firmware': the original firmware version passed to us by the 27 * bootloader - not modified by the kernel. It is composed of two parts: 28 * the first 128 E820 memory entries in boot_params.e820_table and the remaining 29 * (if any) entries of the SETUP_E820_EXT nodes. We use this to: 30 * 31 * - the hibernation code uses it to generate a kernel-independent CRC32 32 * checksum of the physical memory layout of a system. 33 * 34 * - 'e820_table_kexec': a slightly modified (by the kernel) firmware version 35 * passed to us by the bootloader - the major difference between 36 * e820_table_firmware[] and this one is that e820_table_kexec[] 37 * might be modified by the kexec itself to fake an mptable. 38 * We use this to: 39 * 40 * - kexec, which is a bootloader in disguise, uses the original E820 41 * layout to pass to the kexec-ed kernel. This way the original kernel 42 * can have a restricted E820 map while the kexec()-ed kexec-kernel 43 * can have access to full memory - etc. 44 * 45 * Export the memory layout via /sys/firmware/memmap. kexec-tools uses 46 * the entries to create an E820 table for the kexec kernel. 47 * 48 * kexec_file_load in-kernel code uses the table for the kexec kernel. 49 * 50 * - 'e820_table': this is the main E820 table that is massaged by the 51 * low level x86 platform code, or modified by boot parameters, before 52 * passed on to higher level MM layers. 53 * 54 * Once the E820 map has been converted to the standard Linux memory layout 55 * information its role stops - modifying it has no effect and does not get 56 * re-propagated. So its main role is a temporary bootstrap storage of firmware 57 * specific memory layout data during early bootup. 58 */ 59 static struct e820_table e820_table_init __initdata; 60 static struct e820_table e820_table_kexec_init __initdata; 61 static struct e820_table e820_table_firmware_init __initdata; 62 63 struct e820_table *e820_table __refdata = &e820_table_init; 64 struct e820_table *e820_table_kexec __refdata = &e820_table_kexec_init; 65 struct e820_table *e820_table_firmware __refdata = &e820_table_firmware_init; 66 67 /* For PCI or other memory-mapped resources */ 68 unsigned long pci_mem_start = 0xaeedbabe; 69 #ifdef CONFIG_PCI 70 EXPORT_SYMBOL(pci_mem_start); 71 #endif 72 73 /* 74 * This function checks if any part of the range <start,end> is mapped 75 * with type. 76 */ 77 static bool _e820__mapped_any(struct e820_table *table, 78 u64 start, u64 end, enum e820_type type) 79 { 80 int i; 81 82 for (i = 0; i < table->nr_entries; i++) { 83 struct e820_entry *entry = &table->entries[i]; 84 85 if (type && entry->type != type) 86 continue; 87 if (entry->addr >= end || entry->addr + entry->size <= start) 88 continue; 89 return true; 90 } 91 return false; 92 } 93 94 bool e820__mapped_raw_any(u64 start, u64 end, enum e820_type type) 95 { 96 return _e820__mapped_any(e820_table_firmware, start, end, type); 97 } 98 EXPORT_SYMBOL_GPL(e820__mapped_raw_any); 99 100 bool e820__mapped_any(u64 start, u64 end, enum e820_type type) 101 { 102 return _e820__mapped_any(e820_table, start, end, type); 103 } 104 EXPORT_SYMBOL_GPL(e820__mapped_any); 105 106 /* 107 * This function checks if the entire <start,end> range is mapped with 'type'. 108 * 109 * Note: this function only works correctly once the E820 table is sorted and 110 * not-overlapping (at least for the range specified), which is the case normally. 111 */ 112 static struct e820_entry *__e820__mapped_all(u64 start, u64 end, 113 enum e820_type type) 114 { 115 int i; 116 117 for (i = 0; i < e820_table->nr_entries; i++) { 118 struct e820_entry *entry = &e820_table->entries[i]; 119 120 if (type && entry->type != type) 121 continue; 122 123 /* Is the region (part) in overlap with the current region? */ 124 if (entry->addr >= end || entry->addr + entry->size <= start) 125 continue; 126 127 /* 128 * If the region is at the beginning of <start,end> we move 129 * 'start' to the end of the region since it's ok until there 130 */ 131 if (entry->addr <= start) 132 start = entry->addr + entry->size; 133 134 /* 135 * If 'start' is now at or beyond 'end', we're done, full 136 * coverage of the desired range exists: 137 */ 138 if (start >= end) 139 return entry; 140 } 141 142 return NULL; 143 } 144 145 /* 146 * This function checks if the entire range <start,end> is mapped with type. 147 */ 148 bool __init e820__mapped_all(u64 start, u64 end, enum e820_type type) 149 { 150 return __e820__mapped_all(start, end, type); 151 } 152 153 /* 154 * This function returns the type associated with the range <start,end>. 155 */ 156 int e820__get_entry_type(u64 start, u64 end) 157 { 158 struct e820_entry *entry = __e820__mapped_all(start, end, 0); 159 160 return entry ? entry->type : -EINVAL; 161 } 162 163 /* 164 * Add a memory region to the kernel E820 map. 165 */ 166 static void __init __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type) 167 { 168 int x = table->nr_entries; 169 170 if (x >= ARRAY_SIZE(table->entries)) { 171 pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n", 172 start, start + size - 1); 173 return; 174 } 175 176 table->entries[x].addr = start; 177 table->entries[x].size = size; 178 table->entries[x].type = type; 179 table->nr_entries++; 180 } 181 182 void __init e820__range_add(u64 start, u64 size, enum e820_type type) 183 { 184 __e820__range_add(e820_table, start, size, type); 185 } 186 187 static void __init e820_print_type(enum e820_type type) 188 { 189 switch (type) { 190 case E820_TYPE_RAM: pr_cont("usable"); break; 191 case E820_TYPE_RESERVED: pr_cont("reserved"); break; 192 case E820_TYPE_SOFT_RESERVED: pr_cont("soft reserved"); break; 193 case E820_TYPE_ACPI: pr_cont("ACPI data"); break; 194 case E820_TYPE_NVS: pr_cont("ACPI NVS"); break; 195 case E820_TYPE_UNUSABLE: pr_cont("unusable"); break; 196 case E820_TYPE_PMEM: /* Fall through: */ 197 case E820_TYPE_PRAM: pr_cont("persistent (type %u)", type); break; 198 default: pr_cont("type %u", type); break; 199 } 200 } 201 202 void __init e820__print_table(char *who) 203 { 204 int i; 205 206 for (i = 0; i < e820_table->nr_entries; i++) { 207 pr_info("%s: [mem %#018Lx-%#018Lx] ", 208 who, 209 e820_table->entries[i].addr, 210 e820_table->entries[i].addr + e820_table->entries[i].size - 1); 211 212 e820_print_type(e820_table->entries[i].type); 213 pr_cont("\n"); 214 } 215 } 216 217 /* 218 * Sanitize an E820 map. 219 * 220 * Some E820 layouts include overlapping entries. The following 221 * replaces the original E820 map with a new one, removing overlaps, 222 * and resolving conflicting memory types in favor of highest 223 * numbered type. 224 * 225 * The input parameter 'entries' points to an array of 'struct 226 * e820_entry' which on entry has elements in the range [0, *nr_entries) 227 * valid, and which has space for up to max_nr_entries entries. 228 * On return, the resulting sanitized E820 map entries will be in 229 * overwritten in the same location, starting at 'entries'. 230 * 231 * The integer pointed to by nr_entries must be valid on entry (the 232 * current number of valid entries located at 'entries'). If the 233 * sanitizing succeeds the *nr_entries will be updated with the new 234 * number of valid entries (something no more than max_nr_entries). 235 * 236 * The return value from e820__update_table() is zero if it 237 * successfully 'sanitized' the map entries passed in, and is -1 238 * if it did nothing, which can happen if either of (1) it was 239 * only passed one map entry, or (2) any of the input map entries 240 * were invalid (start + size < start, meaning that the size was 241 * so big the described memory range wrapped around through zero.) 242 * 243 * Visually we're performing the following 244 * (1,2,3,4 = memory types)... 245 * 246 * Sample memory map (w/overlaps): 247 * ____22__________________ 248 * ______________________4_ 249 * ____1111________________ 250 * _44_____________________ 251 * 11111111________________ 252 * ____________________33__ 253 * ___________44___________ 254 * __________33333_________ 255 * ______________22________ 256 * ___________________2222_ 257 * _________111111111______ 258 * _____________________11_ 259 * _________________4______ 260 * 261 * Sanitized equivalent (no overlap): 262 * 1_______________________ 263 * _44_____________________ 264 * ___1____________________ 265 * ____22__________________ 266 * ______11________________ 267 * _________1______________ 268 * __________3_____________ 269 * ___________44___________ 270 * _____________33_________ 271 * _______________2________ 272 * ________________1_______ 273 * _________________4______ 274 * ___________________2____ 275 * ____________________33__ 276 * ______________________4_ 277 */ 278 struct change_member { 279 /* Pointer to the original entry: */ 280 struct e820_entry *entry; 281 /* Address for this change point: */ 282 unsigned long long addr; 283 }; 284 285 static struct change_member change_point_list[2*E820_MAX_ENTRIES] __initdata; 286 static struct change_member *change_point[2*E820_MAX_ENTRIES] __initdata; 287 static struct e820_entry *overlap_list[E820_MAX_ENTRIES] __initdata; 288 static struct e820_entry new_entries[E820_MAX_ENTRIES] __initdata; 289 290 static int __init cpcompare(const void *a, const void *b) 291 { 292 struct change_member * const *app = a, * const *bpp = b; 293 const struct change_member *ap = *app, *bp = *bpp; 294 295 /* 296 * Inputs are pointers to two elements of change_point[]. If their 297 * addresses are not equal, their difference dominates. If the addresses 298 * are equal, then consider one that represents the end of its region 299 * to be greater than one that does not. 300 */ 301 if (ap->addr != bp->addr) 302 return ap->addr > bp->addr ? 1 : -1; 303 304 return (ap->addr != ap->entry->addr) - (bp->addr != bp->entry->addr); 305 } 306 307 static bool e820_nomerge(enum e820_type type) 308 { 309 /* 310 * These types may indicate distinct platform ranges aligned to 311 * numa node, protection domain, performance domain, or other 312 * boundaries. Do not merge them. 313 */ 314 if (type == E820_TYPE_PRAM) 315 return true; 316 if (type == E820_TYPE_SOFT_RESERVED) 317 return true; 318 return false; 319 } 320 321 int __init e820__update_table(struct e820_table *table) 322 { 323 struct e820_entry *entries = table->entries; 324 u32 max_nr_entries = ARRAY_SIZE(table->entries); 325 enum e820_type current_type, last_type; 326 unsigned long long last_addr; 327 u32 new_nr_entries, overlap_entries; 328 u32 i, chg_idx, chg_nr; 329 330 /* If there's only one memory region, don't bother: */ 331 if (table->nr_entries < 2) 332 return -1; 333 334 BUG_ON(table->nr_entries > max_nr_entries); 335 336 /* Bail out if we find any unreasonable addresses in the map: */ 337 for (i = 0; i < table->nr_entries; i++) { 338 if (entries[i].addr + entries[i].size < entries[i].addr) 339 return -1; 340 } 341 342 /* Create pointers for initial change-point information (for sorting): */ 343 for (i = 0; i < 2 * table->nr_entries; i++) 344 change_point[i] = &change_point_list[i]; 345 346 /* 347 * Record all known change-points (starting and ending addresses), 348 * omitting empty memory regions: 349 */ 350 chg_idx = 0; 351 for (i = 0; i < table->nr_entries; i++) { 352 if (entries[i].size != 0) { 353 change_point[chg_idx]->addr = entries[i].addr; 354 change_point[chg_idx++]->entry = &entries[i]; 355 change_point[chg_idx]->addr = entries[i].addr + entries[i].size; 356 change_point[chg_idx++]->entry = &entries[i]; 357 } 358 } 359 chg_nr = chg_idx; 360 361 /* Sort change-point list by memory addresses (low -> high): */ 362 sort(change_point, chg_nr, sizeof(*change_point), cpcompare, NULL); 363 364 /* Create a new memory map, removing overlaps: */ 365 overlap_entries = 0; /* Number of entries in the overlap table */ 366 new_nr_entries = 0; /* Index for creating new map entries */ 367 last_type = 0; /* Start with undefined memory type */ 368 last_addr = 0; /* Start with 0 as last starting address */ 369 370 /* Loop through change-points, determining effect on the new map: */ 371 for (chg_idx = 0; chg_idx < chg_nr; chg_idx++) { 372 /* Keep track of all overlapping entries */ 373 if (change_point[chg_idx]->addr == change_point[chg_idx]->entry->addr) { 374 /* Add map entry to overlap list (> 1 entry implies an overlap) */ 375 overlap_list[overlap_entries++] = change_point[chg_idx]->entry; 376 } else { 377 /* Remove entry from list (order independent, so swap with last): */ 378 for (i = 0; i < overlap_entries; i++) { 379 if (overlap_list[i] == change_point[chg_idx]->entry) 380 overlap_list[i] = overlap_list[overlap_entries-1]; 381 } 382 overlap_entries--; 383 } 384 /* 385 * If there are overlapping entries, decide which 386 * "type" to use (larger value takes precedence -- 387 * 1=usable, 2,3,4,4+=unusable) 388 */ 389 current_type = 0; 390 for (i = 0; i < overlap_entries; i++) { 391 if (overlap_list[i]->type > current_type) 392 current_type = overlap_list[i]->type; 393 } 394 395 /* Continue building up new map based on this information: */ 396 if (current_type != last_type || e820_nomerge(current_type)) { 397 if (last_type) { 398 new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr; 399 /* Move forward only if the new size was non-zero: */ 400 if (new_entries[new_nr_entries].size != 0) 401 /* No more space left for new entries? */ 402 if (++new_nr_entries >= max_nr_entries) 403 break; 404 } 405 if (current_type) { 406 new_entries[new_nr_entries].addr = change_point[chg_idx]->addr; 407 new_entries[new_nr_entries].type = current_type; 408 last_addr = change_point[chg_idx]->addr; 409 } 410 last_type = current_type; 411 } 412 } 413 414 /* Copy the new entries into the original location: */ 415 memcpy(entries, new_entries, new_nr_entries*sizeof(*entries)); 416 table->nr_entries = new_nr_entries; 417 418 return 0; 419 } 420 421 static int __init __append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) 422 { 423 struct boot_e820_entry *entry = entries; 424 425 while (nr_entries) { 426 u64 start = entry->addr; 427 u64 size = entry->size; 428 u64 end = start + size - 1; 429 u32 type = entry->type; 430 431 /* Ignore the entry on 64-bit overflow: */ 432 if (start > end && likely(size)) 433 return -1; 434 435 e820__range_add(start, size, type); 436 437 entry++; 438 nr_entries--; 439 } 440 return 0; 441 } 442 443 /* 444 * Copy the BIOS E820 map into a safe place. 445 * 446 * Sanity-check it while we're at it.. 447 * 448 * If we're lucky and live on a modern system, the setup code 449 * will have given us a memory map that we can use to properly 450 * set up memory. If we aren't, we'll fake a memory map. 451 */ 452 static int __init append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) 453 { 454 /* Only one memory region (or negative)? Ignore it */ 455 if (nr_entries < 2) 456 return -1; 457 458 return __append_e820_table(entries, nr_entries); 459 } 460 461 static u64 __init 462 __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) 463 { 464 u64 end; 465 unsigned int i; 466 u64 real_updated_size = 0; 467 468 BUG_ON(old_type == new_type); 469 470 if (size > (ULLONG_MAX - start)) 471 size = ULLONG_MAX - start; 472 473 end = start + size; 474 printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx] ", start, end - 1); 475 e820_print_type(old_type); 476 pr_cont(" ==> "); 477 e820_print_type(new_type); 478 pr_cont("\n"); 479 480 for (i = 0; i < table->nr_entries; i++) { 481 struct e820_entry *entry = &table->entries[i]; 482 u64 final_start, final_end; 483 u64 entry_end; 484 485 if (entry->type != old_type) 486 continue; 487 488 entry_end = entry->addr + entry->size; 489 490 /* Completely covered by new range? */ 491 if (entry->addr >= start && entry_end <= end) { 492 entry->type = new_type; 493 real_updated_size += entry->size; 494 continue; 495 } 496 497 /* New range is completely covered? */ 498 if (entry->addr < start && entry_end > end) { 499 __e820__range_add(table, start, size, new_type); 500 __e820__range_add(table, end, entry_end - end, entry->type); 501 entry->size = start - entry->addr; 502 real_updated_size += size; 503 continue; 504 } 505 506 /* Partially covered: */ 507 final_start = max(start, entry->addr); 508 final_end = min(end, entry_end); 509 if (final_start >= final_end) 510 continue; 511 512 __e820__range_add(table, final_start, final_end - final_start, new_type); 513 514 real_updated_size += final_end - final_start; 515 516 /* 517 * Left range could be head or tail, so need to update 518 * its size first: 519 */ 520 entry->size -= final_end - final_start; 521 if (entry->addr < final_start) 522 continue; 523 524 entry->addr = final_end; 525 } 526 return real_updated_size; 527 } 528 529 u64 __init e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) 530 { 531 return __e820__range_update(e820_table, start, size, old_type, new_type); 532 } 533 534 u64 __init e820__range_update_table(struct e820_table *t, u64 start, u64 size, 535 enum e820_type old_type, enum e820_type new_type) 536 { 537 return __e820__range_update(t, start, size, old_type, new_type); 538 } 539 540 /* Remove a range of memory from the E820 table: */ 541 u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type) 542 { 543 int i; 544 u64 end; 545 u64 real_removed_size = 0; 546 547 if (size > (ULLONG_MAX - start)) 548 size = ULLONG_MAX - start; 549 550 end = start + size; 551 printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx] ", start, end - 1); 552 if (check_type) 553 e820_print_type(old_type); 554 pr_cont("\n"); 555 556 for (i = 0; i < e820_table->nr_entries; i++) { 557 struct e820_entry *entry = &e820_table->entries[i]; 558 u64 final_start, final_end; 559 u64 entry_end; 560 561 if (check_type && entry->type != old_type) 562 continue; 563 564 entry_end = entry->addr + entry->size; 565 566 /* Completely covered? */ 567 if (entry->addr >= start && entry_end <= end) { 568 real_removed_size += entry->size; 569 memset(entry, 0, sizeof(*entry)); 570 continue; 571 } 572 573 /* Is the new range completely covered? */ 574 if (entry->addr < start && entry_end > end) { 575 e820__range_add(end, entry_end - end, entry->type); 576 entry->size = start - entry->addr; 577 real_removed_size += size; 578 continue; 579 } 580 581 /* Partially covered: */ 582 final_start = max(start, entry->addr); 583 final_end = min(end, entry_end); 584 if (final_start >= final_end) 585 continue; 586 587 real_removed_size += final_end - final_start; 588 589 /* 590 * Left range could be head or tail, so need to update 591 * the size first: 592 */ 593 entry->size -= final_end - final_start; 594 if (entry->addr < final_start) 595 continue; 596 597 entry->addr = final_end; 598 } 599 return real_removed_size; 600 } 601 602 void __init e820__update_table_print(void) 603 { 604 if (e820__update_table(e820_table)) 605 return; 606 607 pr_info("modified physical RAM map:\n"); 608 e820__print_table("modified"); 609 } 610 611 static void __init e820__update_table_kexec(void) 612 { 613 e820__update_table(e820_table_kexec); 614 } 615 616 #define MAX_GAP_END 0x100000000ull 617 618 /* 619 * Search for a gap in the E820 memory space from 0 to MAX_GAP_END (4GB). 620 */ 621 static int __init e820_search_gap(unsigned long *gapstart, unsigned long *gapsize) 622 { 623 unsigned long long last = MAX_GAP_END; 624 int i = e820_table->nr_entries; 625 int found = 0; 626 627 while (--i >= 0) { 628 unsigned long long start = e820_table->entries[i].addr; 629 unsigned long long end = start + e820_table->entries[i].size; 630 631 /* 632 * Since "last" is at most 4GB, we know we'll 633 * fit in 32 bits if this condition is true: 634 */ 635 if (last > end) { 636 unsigned long gap = last - end; 637 638 if (gap >= *gapsize) { 639 *gapsize = gap; 640 *gapstart = end; 641 found = 1; 642 } 643 } 644 if (start < last) 645 last = start; 646 } 647 return found; 648 } 649 650 /* 651 * Search for the biggest gap in the low 32 bits of the E820 652 * memory space. We pass this space to the PCI subsystem, so 653 * that it can assign MMIO resources for hotplug or 654 * unconfigured devices in. 655 * 656 * Hopefully the BIOS let enough space left. 657 */ 658 __init void e820__setup_pci_gap(void) 659 { 660 unsigned long gapstart, gapsize; 661 int found; 662 663 gapsize = 0x400000; 664 found = e820_search_gap(&gapstart, &gapsize); 665 666 if (!found) { 667 #ifdef CONFIG_X86_64 668 gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; 669 pr_err("Cannot find an available gap in the 32-bit address range\n"); 670 pr_err("PCI devices with unassigned 32-bit BARs may not work!\n"); 671 #else 672 gapstart = 0x10000000; 673 #endif 674 } 675 676 /* 677 * e820__reserve_resources_late() protects stolen RAM already: 678 */ 679 pci_mem_start = gapstart; 680 681 pr_info("[mem %#010lx-%#010lx] available for PCI devices\n", 682 gapstart, gapstart + gapsize - 1); 683 } 684 685 /* 686 * Called late during init, in free_initmem(). 687 * 688 * Initial e820_table and e820_table_kexec are largish __initdata arrays. 689 * 690 * Copy them to a (usually much smaller) dynamically allocated area that is 691 * sized precisely after the number of e820 entries. 692 * 693 * This is done after we've performed all the fixes and tweaks to the tables. 694 * All functions which modify them are __init functions, which won't exist 695 * after free_initmem(). 696 */ 697 __init void e820__reallocate_tables(void) 698 { 699 struct e820_table *n; 700 int size; 701 702 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table->nr_entries; 703 n = kmemdup(e820_table, size, GFP_KERNEL); 704 BUG_ON(!n); 705 e820_table = n; 706 707 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_kexec->nr_entries; 708 n = kmemdup(e820_table_kexec, size, GFP_KERNEL); 709 BUG_ON(!n); 710 e820_table_kexec = n; 711 712 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_firmware->nr_entries; 713 n = kmemdup(e820_table_firmware, size, GFP_KERNEL); 714 BUG_ON(!n); 715 e820_table_firmware = n; 716 } 717 718 /* 719 * Because of the small fixed size of struct boot_params, only the first 720 * 128 E820 memory entries are passed to the kernel via boot_params.e820_table, 721 * the remaining (if any) entries are passed via the SETUP_E820_EXT node of 722 * struct setup_data, which is parsed here. 723 */ 724 void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len) 725 { 726 int entries; 727 struct boot_e820_entry *extmap; 728 struct setup_data *sdata; 729 730 sdata = early_memremap(phys_addr, data_len); 731 entries = sdata->len / sizeof(*extmap); 732 extmap = (struct boot_e820_entry *)(sdata->data); 733 734 __append_e820_table(extmap, entries); 735 e820__update_table(e820_table); 736 737 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); 738 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); 739 740 early_memunmap(sdata, data_len); 741 pr_info("extended physical RAM map:\n"); 742 e820__print_table("extended"); 743 } 744 745 /* 746 * Find the ranges of physical addresses that do not correspond to 747 * E820 RAM areas and register the corresponding pages as 'nosave' for 748 * hibernation (32-bit) or software suspend and suspend to RAM (64-bit). 749 * 750 * This function requires the E820 map to be sorted and without any 751 * overlapping entries. 752 */ 753 void __init e820__register_nosave_regions(unsigned long limit_pfn) 754 { 755 int i; 756 unsigned long pfn = 0; 757 758 for (i = 0; i < e820_table->nr_entries; i++) { 759 struct e820_entry *entry = &e820_table->entries[i]; 760 761 if (pfn < PFN_UP(entry->addr)) 762 register_nosave_region(pfn, PFN_UP(entry->addr)); 763 764 pfn = PFN_DOWN(entry->addr + entry->size); 765 766 if (entry->type != E820_TYPE_RAM) 767 register_nosave_region(PFN_UP(entry->addr), pfn); 768 769 if (pfn >= limit_pfn) 770 break; 771 } 772 } 773 774 #ifdef CONFIG_ACPI 775 /* 776 * Register ACPI NVS memory regions, so that we can save/restore them during 777 * hibernation and the subsequent resume: 778 */ 779 static int __init e820__register_nvs_regions(void) 780 { 781 int i; 782 783 for (i = 0; i < e820_table->nr_entries; i++) { 784 struct e820_entry *entry = &e820_table->entries[i]; 785 786 if (entry->type == E820_TYPE_NVS) 787 acpi_nvs_register(entry->addr, entry->size); 788 } 789 790 return 0; 791 } 792 core_initcall(e820__register_nvs_regions); 793 #endif 794 795 /* 796 * Allocate the requested number of bytes with the requested alignment 797 * and return (the physical address) to the caller. Also register this 798 * range in the 'kexec' E820 table as a reserved range. 799 * 800 * This allows kexec to fake a new mptable, as if it came from the real 801 * system. 802 */ 803 u64 __init e820__memblock_alloc_reserved(u64 size, u64 align) 804 { 805 u64 addr; 806 807 addr = memblock_phys_alloc(size, align); 808 if (addr) { 809 e820__range_update_table(e820_table_kexec, addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); 810 pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); 811 e820__update_table_kexec(); 812 } 813 814 return addr; 815 } 816 817 #ifdef CONFIG_X86_32 818 # ifdef CONFIG_X86_PAE 819 # define MAX_ARCH_PFN (1ULL<<(36-PAGE_SHIFT)) 820 # else 821 # define MAX_ARCH_PFN (1ULL<<(32-PAGE_SHIFT)) 822 # endif 823 #else /* CONFIG_X86_32 */ 824 # define MAX_ARCH_PFN MAXMEM>>PAGE_SHIFT 825 #endif 826 827 /* 828 * Find the highest page frame number we have available 829 */ 830 static unsigned long __init e820__end_ram_pfn(unsigned long limit_pfn) 831 { 832 int i; 833 unsigned long last_pfn = 0; 834 unsigned long max_arch_pfn = MAX_ARCH_PFN; 835 836 for (i = 0; i < e820_table->nr_entries; i++) { 837 struct e820_entry *entry = &e820_table->entries[i]; 838 unsigned long start_pfn; 839 unsigned long end_pfn; 840 841 if (entry->type != E820_TYPE_RAM && 842 entry->type != E820_TYPE_ACPI) 843 continue; 844 845 start_pfn = entry->addr >> PAGE_SHIFT; 846 end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT; 847 848 if (start_pfn >= limit_pfn) 849 continue; 850 if (end_pfn > limit_pfn) { 851 last_pfn = limit_pfn; 852 break; 853 } 854 if (end_pfn > last_pfn) 855 last_pfn = end_pfn; 856 } 857 858 if (last_pfn > max_arch_pfn) 859 last_pfn = max_arch_pfn; 860 861 pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n", 862 last_pfn, max_arch_pfn); 863 return last_pfn; 864 } 865 866 unsigned long __init e820__end_of_ram_pfn(void) 867 { 868 return e820__end_ram_pfn(MAX_ARCH_PFN); 869 } 870 871 unsigned long __init e820__end_of_low_ram_pfn(void) 872 { 873 return e820__end_ram_pfn(1UL << (32 - PAGE_SHIFT)); 874 } 875 876 static void __init early_panic(char *msg) 877 { 878 early_printk(msg); 879 panic(msg); 880 } 881 882 static int userdef __initdata; 883 884 /* The "mem=nopentium" boot option disables 4MB page tables on 32-bit kernels: */ 885 static int __init parse_memopt(char *p) 886 { 887 u64 mem_size; 888 889 if (!p) 890 return -EINVAL; 891 892 if (!strcmp(p, "nopentium")) { 893 #ifdef CONFIG_X86_32 894 setup_clear_cpu_cap(X86_FEATURE_PSE); 895 return 0; 896 #else 897 pr_warn("mem=nopentium ignored! (only supported on x86_32)\n"); 898 return -EINVAL; 899 #endif 900 } 901 902 userdef = 1; 903 mem_size = memparse(p, &p); 904 905 /* Don't remove all memory when getting "mem={invalid}" parameter: */ 906 if (mem_size == 0) 907 return -EINVAL; 908 909 e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); 910 911 #ifdef CONFIG_MEMORY_HOTPLUG 912 max_mem_size = mem_size; 913 #endif 914 915 return 0; 916 } 917 early_param("mem", parse_memopt); 918 919 static int __init parse_memmap_one(char *p) 920 { 921 char *oldp; 922 u64 start_at, mem_size; 923 924 if (!p) 925 return -EINVAL; 926 927 if (!strncmp(p, "exactmap", 8)) { 928 e820_table->nr_entries = 0; 929 userdef = 1; 930 return 0; 931 } 932 933 oldp = p; 934 mem_size = memparse(p, &p); 935 if (p == oldp) 936 return -EINVAL; 937 938 userdef = 1; 939 if (*p == '@') { 940 start_at = memparse(p+1, &p); 941 e820__range_add(start_at, mem_size, E820_TYPE_RAM); 942 } else if (*p == '#') { 943 start_at = memparse(p+1, &p); 944 e820__range_add(start_at, mem_size, E820_TYPE_ACPI); 945 } else if (*p == '$') { 946 start_at = memparse(p+1, &p); 947 e820__range_add(start_at, mem_size, E820_TYPE_RESERVED); 948 } else if (*p == '!') { 949 start_at = memparse(p+1, &p); 950 e820__range_add(start_at, mem_size, E820_TYPE_PRAM); 951 } else if (*p == '%') { 952 enum e820_type from = 0, to = 0; 953 954 start_at = memparse(p + 1, &p); 955 if (*p == '-') 956 from = simple_strtoull(p + 1, &p, 0); 957 if (*p == '+') 958 to = simple_strtoull(p + 1, &p, 0); 959 if (*p != '\0') 960 return -EINVAL; 961 if (from && to) 962 e820__range_update(start_at, mem_size, from, to); 963 else if (to) 964 e820__range_add(start_at, mem_size, to); 965 else if (from) 966 e820__range_remove(start_at, mem_size, from, 1); 967 else 968 e820__range_remove(start_at, mem_size, 0, 0); 969 } else { 970 e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); 971 } 972 973 return *p == '\0' ? 0 : -EINVAL; 974 } 975 976 static int __init parse_memmap_opt(char *str) 977 { 978 while (str) { 979 char *k = strchr(str, ','); 980 981 if (k) 982 *k++ = 0; 983 984 parse_memmap_one(str); 985 str = k; 986 } 987 988 return 0; 989 } 990 early_param("memmap", parse_memmap_opt); 991 992 /* 993 * Called after parse_early_param(), after early parameters (such as mem=) 994 * have been processed, in which case we already have an E820 table filled in 995 * via the parameter callback function(s), but it's not sorted and printed yet: 996 */ 997 void __init e820__finish_early_params(void) 998 { 999 if (userdef) { 1000 if (e820__update_table(e820_table) < 0) 1001 early_panic("Invalid user supplied memory map"); 1002 1003 pr_info("user-defined physical RAM map:\n"); 1004 e820__print_table("user"); 1005 } 1006 } 1007 1008 static const char *__init e820_type_to_string(struct e820_entry *entry) 1009 { 1010 switch (entry->type) { 1011 case E820_TYPE_RAM: return "System RAM"; 1012 case E820_TYPE_ACPI: return "ACPI Tables"; 1013 case E820_TYPE_NVS: return "ACPI Non-volatile Storage"; 1014 case E820_TYPE_UNUSABLE: return "Unusable memory"; 1015 case E820_TYPE_PRAM: return "Persistent Memory (legacy)"; 1016 case E820_TYPE_PMEM: return "Persistent Memory"; 1017 case E820_TYPE_RESERVED: return "Reserved"; 1018 case E820_TYPE_SOFT_RESERVED: return "Soft Reserved"; 1019 default: return "Unknown E820 type"; 1020 } 1021 } 1022 1023 static unsigned long __init e820_type_to_iomem_type(struct e820_entry *entry) 1024 { 1025 switch (entry->type) { 1026 case E820_TYPE_RAM: return IORESOURCE_SYSTEM_RAM; 1027 case E820_TYPE_ACPI: /* Fall-through: */ 1028 case E820_TYPE_NVS: /* Fall-through: */ 1029 case E820_TYPE_UNUSABLE: /* Fall-through: */ 1030 case E820_TYPE_PRAM: /* Fall-through: */ 1031 case E820_TYPE_PMEM: /* Fall-through: */ 1032 case E820_TYPE_RESERVED: /* Fall-through: */ 1033 case E820_TYPE_SOFT_RESERVED: /* Fall-through: */ 1034 default: return IORESOURCE_MEM; 1035 } 1036 } 1037 1038 static unsigned long __init e820_type_to_iores_desc(struct e820_entry *entry) 1039 { 1040 switch (entry->type) { 1041 case E820_TYPE_ACPI: return IORES_DESC_ACPI_TABLES; 1042 case E820_TYPE_NVS: return IORES_DESC_ACPI_NV_STORAGE; 1043 case E820_TYPE_PMEM: return IORES_DESC_PERSISTENT_MEMORY; 1044 case E820_TYPE_PRAM: return IORES_DESC_PERSISTENT_MEMORY_LEGACY; 1045 case E820_TYPE_RESERVED: return IORES_DESC_RESERVED; 1046 case E820_TYPE_SOFT_RESERVED: return IORES_DESC_SOFT_RESERVED; 1047 case E820_TYPE_RAM: /* Fall-through: */ 1048 case E820_TYPE_UNUSABLE: /* Fall-through: */ 1049 default: return IORES_DESC_NONE; 1050 } 1051 } 1052 1053 static bool __init do_mark_busy(enum e820_type type, struct resource *res) 1054 { 1055 /* this is the legacy bios/dos rom-shadow + mmio region */ 1056 if (res->start < (1ULL<<20)) 1057 return true; 1058 1059 /* 1060 * Treat persistent memory and other special memory ranges like 1061 * device memory, i.e. reserve it for exclusive use of a driver 1062 */ 1063 switch (type) { 1064 case E820_TYPE_RESERVED: 1065 case E820_TYPE_SOFT_RESERVED: 1066 case E820_TYPE_PRAM: 1067 case E820_TYPE_PMEM: 1068 return false; 1069 case E820_TYPE_RAM: 1070 case E820_TYPE_ACPI: 1071 case E820_TYPE_NVS: 1072 case E820_TYPE_UNUSABLE: 1073 default: 1074 return true; 1075 } 1076 } 1077 1078 /* 1079 * Mark E820 reserved areas as busy for the resource manager: 1080 */ 1081 1082 static struct resource __initdata *e820_res; 1083 1084 void __init e820__reserve_resources(void) 1085 { 1086 int i; 1087 struct resource *res; 1088 u64 end; 1089 1090 res = memblock_alloc_or_panic(sizeof(*res) * e820_table->nr_entries, 1091 SMP_CACHE_BYTES); 1092 e820_res = res; 1093 1094 for (i = 0; i < e820_table->nr_entries; i++) { 1095 struct e820_entry *entry = e820_table->entries + i; 1096 1097 end = entry->addr + entry->size - 1; 1098 if (end != (resource_size_t)end) { 1099 res++; 1100 continue; 1101 } 1102 res->start = entry->addr; 1103 res->end = end; 1104 res->name = e820_type_to_string(entry); 1105 res->flags = e820_type_to_iomem_type(entry); 1106 res->desc = e820_type_to_iores_desc(entry); 1107 1108 /* 1109 * Don't register the region that could be conflicted with 1110 * PCI device BAR resources and insert them later in 1111 * pcibios_resource_survey(): 1112 */ 1113 if (do_mark_busy(entry->type, res)) { 1114 res->flags |= IORESOURCE_BUSY; 1115 insert_resource(&iomem_resource, res); 1116 } 1117 res++; 1118 } 1119 1120 /* Expose the kexec e820 table to the sysfs. */ 1121 for (i = 0; i < e820_table_kexec->nr_entries; i++) { 1122 struct e820_entry *entry = e820_table_kexec->entries + i; 1123 1124 firmware_map_add_early(entry->addr, entry->addr + entry->size, e820_type_to_string(entry)); 1125 } 1126 } 1127 1128 /* 1129 * How much should we pad the end of RAM, depending on where it is? 1130 */ 1131 static unsigned long __init ram_alignment(resource_size_t pos) 1132 { 1133 unsigned long mb = pos >> 20; 1134 1135 /* To 64kB in the first megabyte */ 1136 if (!mb) 1137 return 64*1024; 1138 1139 /* To 1MB in the first 16MB */ 1140 if (mb < 16) 1141 return 1024*1024; 1142 1143 /* To 64MB for anything above that */ 1144 return 64*1024*1024; 1145 } 1146 1147 #define MAX_RESOURCE_SIZE ((resource_size_t)-1) 1148 1149 void __init e820__reserve_resources_late(void) 1150 { 1151 int i; 1152 struct resource *res; 1153 1154 res = e820_res; 1155 for (i = 0; i < e820_table->nr_entries; i++) { 1156 if (!res->parent && res->end) 1157 insert_resource_expand_to_fit(&iomem_resource, res); 1158 res++; 1159 } 1160 1161 /* 1162 * Try to bump up RAM regions to reasonable boundaries, to 1163 * avoid stolen RAM: 1164 */ 1165 for (i = 0; i < e820_table->nr_entries; i++) { 1166 struct e820_entry *entry = &e820_table->entries[i]; 1167 u64 start, end; 1168 1169 if (entry->type != E820_TYPE_RAM) 1170 continue; 1171 1172 start = entry->addr + entry->size; 1173 end = round_up(start, ram_alignment(start)) - 1; 1174 if (end > MAX_RESOURCE_SIZE) 1175 end = MAX_RESOURCE_SIZE; 1176 if (start >= end) 1177 continue; 1178 1179 printk(KERN_DEBUG "e820: reserve RAM buffer [mem %#010llx-%#010llx]\n", start, end); 1180 reserve_region_with_split(&iomem_resource, start, end, "RAM buffer"); 1181 } 1182 } 1183 1184 /* 1185 * Pass the firmware (bootloader) E820 map to the kernel and process it: 1186 */ 1187 char *__init e820__memory_setup_default(void) 1188 { 1189 char *who = "BIOS-e820"; 1190 1191 /* 1192 * Try to copy the BIOS-supplied E820-map. 1193 * 1194 * Otherwise fake a memory map; one section from 0k->640k, 1195 * the next section from 1mb->appropriate_mem_k 1196 */ 1197 if (append_e820_table(boot_params.e820_table, boot_params.e820_entries) < 0) { 1198 u64 mem_size; 1199 1200 /* Compare results from other methods and take the one that gives more RAM: */ 1201 if (boot_params.alt_mem_k < boot_params.screen_info.ext_mem_k) { 1202 mem_size = boot_params.screen_info.ext_mem_k; 1203 who = "BIOS-88"; 1204 } else { 1205 mem_size = boot_params.alt_mem_k; 1206 who = "BIOS-e801"; 1207 } 1208 1209 e820_table->nr_entries = 0; 1210 e820__range_add(0, LOWMEMSIZE(), E820_TYPE_RAM); 1211 e820__range_add(HIGH_MEMORY, mem_size << 10, E820_TYPE_RAM); 1212 } 1213 1214 /* We just appended a lot of ranges, sanitize the table: */ 1215 e820__update_table(e820_table); 1216 1217 return who; 1218 } 1219 1220 /* 1221 * Calls e820__memory_setup_default() in essence to pick up the firmware/bootloader 1222 * E820 map - with an optional platform quirk available for virtual platforms 1223 * to override this method of boot environment processing: 1224 */ 1225 void __init e820__memory_setup(void) 1226 { 1227 char *who; 1228 1229 /* This is a firmware interface ABI - make sure we don't break it: */ 1230 BUILD_BUG_ON(sizeof(struct boot_e820_entry) != 20); 1231 1232 who = x86_init.resources.memory_setup(); 1233 1234 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); 1235 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); 1236 1237 pr_info("BIOS-provided physical RAM map:\n"); 1238 e820__print_table(who); 1239 } 1240 1241 void __init e820__memblock_setup(void) 1242 { 1243 int i; 1244 u64 end; 1245 1246 #ifdef CONFIG_MEMORY_HOTPLUG 1247 /* 1248 * Memory used by the kernel cannot be hot-removed because Linux 1249 * cannot migrate the kernel pages. When memory hotplug is 1250 * enabled, we should prevent memblock from allocating memory 1251 * for the kernel. 1252 * 1253 * ACPI SRAT records all hotpluggable memory ranges. But before 1254 * SRAT is parsed, we don't know about it. 1255 * 1256 * The kernel image is loaded into memory at very early time. We 1257 * cannot prevent this anyway. So on NUMA system, we set any 1258 * node the kernel resides in as un-hotpluggable. 1259 * 1260 * Since on modern servers, one node could have double-digit 1261 * gigabytes memory, we can assume the memory around the kernel 1262 * image is also un-hotpluggable. So before SRAT is parsed, just 1263 * allocate memory near the kernel image to try the best to keep 1264 * the kernel away from hotpluggable memory. 1265 */ 1266 if (movable_node_is_enabled()) 1267 memblock_set_bottom_up(true); 1268 #endif 1269 1270 /* 1271 * At this point only the first megabyte is mapped for sure, the 1272 * rest of the memory cannot be used for memblock resizing 1273 */ 1274 memblock_set_current_limit(ISA_END_ADDRESS); 1275 1276 /* 1277 * The bootstrap memblock region count maximum is 128 entries 1278 * (INIT_MEMBLOCK_REGIONS), but EFI might pass us more E820 entries 1279 * than that - so allow memblock resizing. 1280 * 1281 * This is safe, because this call happens pretty late during x86 setup, 1282 * so we know about reserved memory regions already. (This is important 1283 * so that memblock resizing does no stomp over reserved areas.) 1284 */ 1285 memblock_allow_resize(); 1286 1287 for (i = 0; i < e820_table->nr_entries; i++) { 1288 struct e820_entry *entry = &e820_table->entries[i]; 1289 1290 end = entry->addr + entry->size; 1291 if (end != (resource_size_t)end) 1292 continue; 1293 1294 if (entry->type == E820_TYPE_SOFT_RESERVED) 1295 memblock_reserve(entry->addr, entry->size); 1296 1297 if (entry->type != E820_TYPE_RAM) 1298 continue; 1299 1300 memblock_add(entry->addr, entry->size); 1301 } 1302 1303 /* Throw away partial pages: */ 1304 memblock_trim_memory(PAGE_SIZE); 1305 1306 memblock_dump_all(); 1307 } 1308