1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Low level x86 E820 memory map handling functions. 4 * 5 * The firmware and bootloader passes us the "E820 table", which is the primary 6 * physical memory layout description available about x86 systems. 7 * 8 * The kernel takes the E820 memory layout and optionally modifies it with 9 * quirks and other tweaks, and feeds that into the generic Linux memory 10 * allocation code routines via a platform independent interface (memblock, etc.). 11 */ 12 #include <linux/memblock.h> 13 #include <linux/suspend.h> 14 #include <linux/acpi.h> 15 #include <linux/firmware-map.h> 16 #include <linux/sort.h> 17 #include <linux/kvm_types.h> 18 19 #include <asm/e820/api.h> 20 #include <asm/setup.h> 21 22 /* 23 * We organize the E820 table into three main data structures: 24 * 25 * - 'e820_table_firmware': the original firmware version passed to us by the 26 * bootloader - not modified by the kernel. It is composed of two parts: 27 * the first 128 E820 memory entries in boot_params.e820_table and the remaining 28 * (if any) entries of the SETUP_E820_EXT nodes. We use this to: 29 * 30 * - the hibernation code uses it to generate a kernel-independent CRC32 31 * checksum of the physical memory layout of a system. 32 * 33 * - 'e820_table_kexec': a slightly modified (by the kernel) firmware version 34 * passed to us by the bootloader - the major difference between 35 * e820_table_firmware[] and this one is that e820_table_kexec[] 36 * might be modified by the kexec itself to fake an mptable. 37 * We use this to: 38 * 39 * - kexec, which is a bootloader in disguise, uses the original E820 40 * layout to pass to the kexec-ed kernel. This way the original kernel 41 * can have a restricted E820 map while the kexec()-ed kexec-kernel 42 * can have access to full memory - etc. 43 * 44 * Export the memory layout via /sys/firmware/memmap. kexec-tools uses 45 * the entries to create an E820 table for the kexec kernel. 46 * 47 * kexec_file_load in-kernel code uses the table for the kexec kernel. 48 * 49 * - 'e820_table': this is the main E820 table that is massaged by the 50 * low level x86 platform code, or modified by boot parameters, before 51 * passed on to higher level MM layers. 52 * 53 * Once the E820 map has been converted to the standard Linux memory layout 54 * information its role stops - modifying it has no effect and does not get 55 * re-propagated. So its main role is a temporary bootstrap storage of firmware 56 * specific memory layout data during early bootup. 57 */ 58 static struct e820_table e820_table_init __initdata; 59 static struct e820_table e820_table_kexec_init __initdata; 60 static struct e820_table e820_table_firmware_init __initdata; 61 62 __refdata struct e820_table *e820_table = &e820_table_init; 63 __refdata struct e820_table *e820_table_kexec = &e820_table_kexec_init; 64 __refdata struct e820_table *e820_table_firmware = &e820_table_firmware_init; 65 66 /* For PCI or other memory-mapped resources */ 67 unsigned long pci_mem_start = 0xaeedbabe; 68 #ifdef CONFIG_PCI 69 EXPORT_SYMBOL(pci_mem_start); 70 #endif 71 72 /* 73 * This function checks if any part of the range <start,end> is mapped 74 * with type. 75 */ 76 static bool _e820__mapped_any(struct e820_table *table, 77 u64 start, u64 end, enum e820_type type) 78 { 79 u32 idx; 80 81 for (idx = 0; idx < table->nr_entries; idx++) { 82 struct e820_entry *entry = &table->entries[idx]; 83 84 if (type && entry->type != type) 85 continue; 86 if (entry->addr >= end || entry->addr + entry->size <= start) 87 continue; 88 return true; 89 } 90 return false; 91 } 92 93 bool e820__mapped_raw_any(u64 start, u64 end, enum e820_type type) 94 { 95 return _e820__mapped_any(e820_table_firmware, start, end, type); 96 } 97 EXPORT_SYMBOL_FOR_KVM(e820__mapped_raw_any); 98 99 bool e820__mapped_any(u64 start, u64 end, enum e820_type type) 100 { 101 return _e820__mapped_any(e820_table, start, end, type); 102 } 103 EXPORT_SYMBOL_GPL(e820__mapped_any); 104 105 /* 106 * This function checks if the entire <start,end> range is mapped with 'type'. 107 * 108 * Note: this function only works correctly once the E820 table is sorted and 109 * not-overlapping (at least for the range specified), which is the case normally. 110 */ 111 static struct e820_entry *__e820__mapped_all(u64 start, u64 end, 112 enum e820_type type) 113 { 114 u32 idx; 115 116 for (idx = 0; idx < e820_table->nr_entries; idx++) { 117 struct e820_entry *entry = &e820_table->entries[idx]; 118 119 if (type && entry->type != type) 120 continue; 121 122 /* Is the region (part) in overlap with the current region? */ 123 if (entry->addr >= end || entry->addr + entry->size <= start) 124 continue; 125 126 /* 127 * If the region is at the beginning of <start,end> we move 128 * 'start' to the end of the region since it's ok until there 129 */ 130 if (entry->addr <= start) 131 start = entry->addr + entry->size; 132 133 /* 134 * If 'start' is now at or beyond 'end', we're done, full 135 * coverage of the desired range exists: 136 */ 137 if (start >= end) 138 return entry; 139 } 140 141 return NULL; 142 } 143 144 /* 145 * This function checks if the entire range <start,end> is mapped with type. 146 */ 147 bool __init e820__mapped_all(u64 start, u64 end, enum e820_type type) 148 { 149 return __e820__mapped_all(start, end, type); 150 } 151 152 /* 153 * This function returns the type associated with the range <start,end>. 154 */ 155 int e820__get_entry_type(u64 start, u64 end) 156 { 157 struct e820_entry *entry = __e820__mapped_all(start, end, 0); 158 159 return entry ? entry->type : -EINVAL; 160 } 161 162 /* 163 * Add a memory region to the kernel E820 map. 164 */ 165 static void __init __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type) 166 { 167 u32 idx = table->nr_entries; 168 169 if (idx >= ARRAY_SIZE(table->entries)) { 170 pr_err("too many E820 table entries; ignoring [mem %#010llx-%#010llx]\n", 171 start, start + size-1); 172 return; 173 } 174 175 table->entries[idx].addr = start; 176 table->entries[idx].size = size; 177 table->entries[idx].type = type; 178 179 table->nr_entries++; 180 } 181 182 void __init e820__range_add(u64 start, u64 size, enum e820_type type) 183 { 184 __e820__range_add(e820_table, start, size, type); 185 } 186 187 static void __init e820_print_type(enum e820_type type) 188 { 189 switch (type) { 190 case E820_TYPE_RAM: pr_cont(" System RAM"); break; 191 case E820_TYPE_RESERVED: pr_cont(" device reserved"); break; 192 case E820_TYPE_SOFT_RESERVED: pr_cont(" soft reserved"); break; 193 case E820_TYPE_ACPI: pr_cont(" ACPI data"); break; 194 case E820_TYPE_NVS: pr_cont(" ACPI NVS"); break; 195 case E820_TYPE_UNUSABLE: pr_cont(" unusable"); break; 196 case E820_TYPE_PMEM: /* Fall through: */ 197 case E820_TYPE_PRAM: pr_cont(" persistent RAM (type %u)", type); break; 198 default: pr_cont(" type %u", type); break; 199 } 200 } 201 202 static void __init e820__print_table(const char *who) 203 { 204 u64 range_end_prev = 0; 205 u32 idx; 206 207 for (idx = 0; idx < e820_table->nr_entries; idx++) { 208 struct e820_entry *entry = e820_table->entries + idx; 209 u64 range_start, range_end; 210 211 range_start = entry->addr; 212 range_end = entry->addr + entry->size; 213 214 /* Out of order E820 maps should not happen: */ 215 if (range_start < range_end_prev) 216 pr_info(FW_BUG "out of order E820 entry!\n"); 217 218 if (range_start > range_end_prev) { 219 pr_info("%s: [gap %#018Lx-%#018Lx]\n", 220 who, 221 range_end_prev, 222 range_start-1); 223 } 224 225 pr_info("%s: [mem %#018Lx-%#018Lx] ", who, range_start, range_end-1); 226 e820_print_type(entry->type); 227 pr_cont("\n"); 228 229 range_end_prev = range_end; 230 } 231 } 232 233 /* 234 * Sanitize an E820 map. 235 * 236 * Some E820 layouts include overlapping entries. The following 237 * replaces the original E820 map with a new one, removing overlaps, 238 * and resolving conflicting memory types in favor of highest 239 * numbered type. 240 * 241 * The input parameter 'entries' points to an array of 'struct 242 * e820_entry' which on entry has elements in the range [0, *nr_entries) 243 * valid, and which has space for up to max_nr_entries entries. 244 * On return, the resulting sanitized E820 map entries will be in 245 * overwritten in the same location, starting at 'entries'. 246 * 247 * The integer pointed to by nr_entries must be valid on entry (the 248 * current number of valid entries located at 'entries'). If the 249 * sanitizing succeeds the *nr_entries will be updated with the new 250 * number of valid entries (something no more than max_nr_entries). 251 * 252 * The return value from e820__update_table() is zero if it 253 * successfully 'sanitized' the map entries passed in, and is -1 254 * if it did nothing, which can happen if either of (1) it was 255 * only passed one map entry, or (2) any of the input map entries 256 * were invalid (start + size < start, meaning that the size was 257 * so big the described memory range wrapped around through zero.) 258 * 259 * Visually we're performing the following 260 * (1,2,3,4 = memory types)... 261 * 262 * Sample memory map (w/overlaps): 263 * ____22__________________ 264 * ______________________4_ 265 * ____1111________________ 266 * _44_____________________ 267 * 11111111________________ 268 * ____________________33__ 269 * ___________44___________ 270 * __________33333_________ 271 * ______________22________ 272 * ___________________2222_ 273 * _________111111111______ 274 * _____________________11_ 275 * _________________4______ 276 * 277 * Sanitized equivalent (no overlap): 278 * 1_______________________ 279 * _44_____________________ 280 * ___1____________________ 281 * ____22__________________ 282 * ______11________________ 283 * _________1______________ 284 * __________3_____________ 285 * ___________44___________ 286 * _____________33_________ 287 * _______________2________ 288 * ________________1_______ 289 * _________________4______ 290 * ___________________2____ 291 * ____________________33__ 292 * ______________________4_ 293 */ 294 struct change_member { 295 /* Pointer to the original entry: */ 296 struct e820_entry *entry; 297 /* Address for this change point: */ 298 u64 addr; 299 }; 300 301 static struct change_member change_point_list[2*E820_MAX_ENTRIES] __initdata; 302 static struct change_member *change_point[2*E820_MAX_ENTRIES] __initdata; 303 static struct e820_entry *overlap_list[E820_MAX_ENTRIES] __initdata; 304 static struct e820_entry new_entries[E820_MAX_ENTRIES] __initdata; 305 306 static int __init cpcompare(const void *a, const void *b) 307 { 308 struct change_member * const *app = a, * const *bpp = b; 309 const struct change_member *ap = *app, *bp = *bpp; 310 311 /* 312 * Inputs are pointers to two elements of change_point[]. If their 313 * addresses are not equal, their difference dominates. If the addresses 314 * are equal, then consider one that represents the end of its region 315 * to be greater than one that does not. 316 */ 317 if (ap->addr != bp->addr) 318 return ap->addr > bp->addr ? 1 : -1; 319 320 return (ap->addr != ap->entry->addr) - (bp->addr != bp->entry->addr); 321 } 322 323 /* 324 * Can two consecutive E820 entries of this same E820 type be merged? 325 */ 326 static bool e820_type_mergeable(enum e820_type type) 327 { 328 /* 329 * These types may indicate distinct platform ranges aligned to 330 * NUMA node, protection domain, performance domain, or other 331 * boundaries. Do not merge them. 332 */ 333 if (type == E820_TYPE_PRAM) 334 return false; 335 if (type == E820_TYPE_SOFT_RESERVED) 336 return false; 337 338 return true; 339 } 340 341 int __init e820__update_table(struct e820_table *table) 342 { 343 struct e820_entry *entries = table->entries; 344 u32 max_nr_entries = ARRAY_SIZE(table->entries); 345 enum e820_type current_type, last_type; 346 u64 last_addr; 347 u32 new_nr_entries, overlap_entries; 348 u32 idx, chg_idx, chg_nr; 349 350 /* If there's only one memory region, don't bother: */ 351 if (table->nr_entries < 2) 352 return -1; 353 354 BUG_ON(table->nr_entries > max_nr_entries); 355 356 /* Bail out if we find any unreasonable addresses in the map: */ 357 for (idx = 0; idx < table->nr_entries; idx++) { 358 if (entries[idx].addr + entries[idx].size < entries[idx].addr) 359 return -1; 360 } 361 362 /* Create pointers for initial change-point information (for sorting): */ 363 for (idx = 0; idx < 2 * table->nr_entries; idx++) 364 change_point[idx] = &change_point_list[idx]; 365 366 /* 367 * Record all known change-points (starting and ending addresses), 368 * omitting empty memory regions: 369 */ 370 chg_idx = 0; 371 for (idx = 0; idx < table->nr_entries; idx++) { 372 if (entries[idx].size != 0) { 373 change_point[chg_idx]->addr = entries[idx].addr; 374 change_point[chg_idx++]->entry = &entries[idx]; 375 change_point[chg_idx]->addr = entries[idx].addr + entries[idx].size; 376 change_point[chg_idx++]->entry = &entries[idx]; 377 } 378 } 379 chg_nr = chg_idx; 380 381 /* Sort change-point list by memory addresses (low -> high): */ 382 sort(change_point, chg_nr, sizeof(*change_point), cpcompare, NULL); 383 384 /* Create a new memory map, removing overlaps: */ 385 overlap_entries = 0; /* Number of entries in the overlap table */ 386 new_nr_entries = 0; /* Index for creating new map entries */ 387 last_type = 0; /* Start with undefined memory type */ 388 last_addr = 0; /* Start with 0 as last starting address */ 389 390 /* Loop through change-points, determining effect on the new map: */ 391 for (chg_idx = 0; chg_idx < chg_nr; chg_idx++) { 392 /* Keep track of all overlapping entries */ 393 if (change_point[chg_idx]->addr == change_point[chg_idx]->entry->addr) { 394 /* Add map entry to overlap list (> 1 entry implies an overlap) */ 395 overlap_list[overlap_entries++] = change_point[chg_idx]->entry; 396 } else { 397 /* Remove entry from list (order independent, so swap with last): */ 398 for (idx = 0; idx < overlap_entries; idx++) { 399 if (overlap_list[idx] == change_point[chg_idx]->entry) 400 overlap_list[idx] = overlap_list[overlap_entries-1]; 401 } 402 overlap_entries--; 403 } 404 /* 405 * If there are overlapping entries, decide which 406 * "type" to use (larger value takes precedence -- 407 * 1=usable, 2,3,4,4+=unusable) 408 */ 409 current_type = 0; 410 for (idx = 0; idx < overlap_entries; idx++) { 411 if (overlap_list[idx]->type > current_type) 412 current_type = overlap_list[idx]->type; 413 } 414 415 /* Continue building up new map based on this information: */ 416 if (current_type != last_type || !e820_type_mergeable(current_type)) { 417 if (last_type) { 418 new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr; 419 /* Move forward only if the new size was non-zero: */ 420 if (new_entries[new_nr_entries].size != 0) 421 /* No more space left for new entries? */ 422 if (++new_nr_entries >= max_nr_entries) 423 break; 424 } 425 if (current_type) { 426 new_entries[new_nr_entries].addr = change_point[chg_idx]->addr; 427 new_entries[new_nr_entries].type = current_type; 428 last_addr = change_point[chg_idx]->addr; 429 } 430 last_type = current_type; 431 } 432 } 433 434 /* Copy the new entries into the original location: */ 435 memcpy(entries, new_entries, new_nr_entries*sizeof(*entries)); 436 table->nr_entries = new_nr_entries; 437 438 return 0; 439 } 440 441 static int __init __append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) 442 { 443 struct boot_e820_entry *entry = entries; 444 445 while (nr_entries) { 446 u64 start = entry->addr; 447 u64 size = entry->size; 448 u64 end = start + size - 1; 449 u32 type = entry->type; 450 451 /* Ignore the entry on 64-bit overflow: */ 452 if (start > end && likely(size)) 453 return -1; 454 455 e820__range_add(start, size, type); 456 457 entry++; 458 nr_entries--; 459 } 460 return 0; 461 } 462 463 /* 464 * Copy the BIOS E820 map into a safe place. 465 * 466 * Sanity-check it while we're at it.. 467 * 468 * If we're lucky and live on a modern system, the setup code 469 * will have given us a memory map that we can use to properly 470 * set up memory. If we aren't, we'll fake a memory map. 471 */ 472 static int __init append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) 473 { 474 /* Only one memory region (or negative)? Ignore it */ 475 if (nr_entries < 2) 476 return -1; 477 478 return __append_e820_table(entries, nr_entries); 479 } 480 481 static u64 __init 482 __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) 483 { 484 u64 end; 485 u32 idx; 486 u64 real_updated_size = 0; 487 488 BUG_ON(old_type == new_type); 489 490 if (size > (ULLONG_MAX - start)) 491 size = ULLONG_MAX - start; 492 493 end = start + size; 494 printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx]", start, end - 1); 495 e820_print_type(old_type); 496 pr_cont(" ==>"); 497 e820_print_type(new_type); 498 pr_cont("\n"); 499 500 for (idx = 0; idx < table->nr_entries; idx++) { 501 struct e820_entry *entry = &table->entries[idx]; 502 u64 final_start, final_end; 503 u64 entry_end; 504 505 if (entry->type != old_type) 506 continue; 507 508 entry_end = entry->addr + entry->size; 509 510 /* Completely covered by new range? */ 511 if (entry->addr >= start && entry_end <= end) { 512 entry->type = new_type; 513 real_updated_size += entry->size; 514 continue; 515 } 516 517 /* New range is completely covered? */ 518 if (entry->addr < start && entry_end > end) { 519 __e820__range_add(table, start, size, new_type); 520 __e820__range_add(table, end, entry_end - end, entry->type); 521 entry->size = start - entry->addr; 522 real_updated_size += size; 523 continue; 524 } 525 526 /* Partially covered: */ 527 final_start = max(start, entry->addr); 528 final_end = min(end, entry_end); 529 if (final_start >= final_end) 530 continue; 531 532 __e820__range_add(table, final_start, final_end - final_start, new_type); 533 534 real_updated_size += final_end - final_start; 535 536 /* 537 * Left range could be head or tail, so need to update 538 * its size first: 539 */ 540 entry->size -= final_end - final_start; 541 if (entry->addr < final_start) 542 continue; 543 544 entry->addr = final_end; 545 } 546 return real_updated_size; 547 } 548 549 u64 __init e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) 550 { 551 return __e820__range_update(e820_table, start, size, old_type, new_type); 552 } 553 554 u64 __init e820__range_update_table(struct e820_table *t, u64 start, u64 size, 555 enum e820_type old_type, enum e820_type new_type) 556 { 557 return __e820__range_update(t, start, size, old_type, new_type); 558 } 559 560 /* Remove a range of memory from the E820 table: */ 561 u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type) 562 { 563 u32 idx; 564 u64 end; 565 u64 real_removed_size = 0; 566 567 if (size > (ULLONG_MAX - start)) 568 size = ULLONG_MAX - start; 569 570 end = start + size; 571 printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx]", start, end - 1); 572 if (check_type) 573 e820_print_type(old_type); 574 pr_cont("\n"); 575 576 for (idx = 0; idx < e820_table->nr_entries; idx++) { 577 struct e820_entry *entry = &e820_table->entries[idx]; 578 u64 final_start, final_end; 579 u64 entry_end; 580 581 if (check_type && entry->type != old_type) 582 continue; 583 584 entry_end = entry->addr + entry->size; 585 586 /* Completely covered? */ 587 if (entry->addr >= start && entry_end <= end) { 588 real_removed_size += entry->size; 589 memset(entry, 0, sizeof(*entry)); 590 continue; 591 } 592 593 /* Is the new range completely covered? */ 594 if (entry->addr < start && entry_end > end) { 595 e820__range_add(end, entry_end - end, entry->type); 596 entry->size = start - entry->addr; 597 real_removed_size += size; 598 continue; 599 } 600 601 /* Partially covered: */ 602 final_start = max(start, entry->addr); 603 final_end = min(end, entry_end); 604 if (final_start >= final_end) 605 continue; 606 607 real_removed_size += final_end - final_start; 608 609 /* 610 * Left range could be head or tail, so need to update 611 * the size first: 612 */ 613 entry->size -= final_end - final_start; 614 if (entry->addr < final_start) 615 continue; 616 617 entry->addr = final_end; 618 } 619 return real_removed_size; 620 } 621 622 void __init e820__update_table_print(void) 623 { 624 if (e820__update_table(e820_table)) 625 return; 626 627 pr_info("modified physical RAM map:\n"); 628 e820__print_table("modified"); 629 } 630 631 static void __init e820__update_table_kexec(void) 632 { 633 e820__update_table(e820_table_kexec); 634 } 635 636 #define MAX_GAP_END 0x100000000ull 637 638 /* 639 * Search for a gap in the E820 memory space from 0 to MAX_GAP_END (4GB). 640 */ 641 static int __init e820_search_gap(unsigned long *gapstart, unsigned long *gapsize) 642 { 643 u64 last = MAX_GAP_END; 644 int idx = e820_table->nr_entries; 645 int found = 0; 646 647 while (--idx >= 0) { 648 u64 start = e820_table->entries[idx].addr; 649 u64 end = start + e820_table->entries[idx].size; 650 651 /* 652 * Since "last" is at most 4GB, we know we'll 653 * fit in 32 bits if this condition is true: 654 */ 655 if (last > end) { 656 unsigned long gap = last - end; 657 658 if (gap >= *gapsize) { 659 *gapsize = gap; 660 *gapstart = end; 661 found = 1; 662 } 663 } 664 if (start < last) 665 last = start; 666 } 667 return found; 668 } 669 670 /* 671 * Search for the biggest gap in the low 32 bits of the E820 672 * memory space. We pass this space to the PCI subsystem, so 673 * that it can assign MMIO resources for hotplug or 674 * unconfigured devices in. 675 * 676 * Hopefully the BIOS let enough space left. 677 */ 678 __init void e820__setup_pci_gap(void) 679 { 680 unsigned long gapstart, gapsize; 681 int found; 682 683 gapsize = 0x400000; 684 found = e820_search_gap(&gapstart, &gapsize); 685 686 if (!found) { 687 #ifdef CONFIG_X86_64 688 gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; 689 pr_err("Cannot find an available gap in the 32-bit address range\n"); 690 pr_err("PCI devices with unassigned 32-bit BARs may not work!\n"); 691 #else 692 gapstart = 0x10000000; 693 #endif 694 } 695 696 /* 697 * e820__reserve_resources_late() protects stolen RAM already: 698 */ 699 pci_mem_start = gapstart; 700 701 pr_info("[gap %#010lx-%#010lx] available for PCI devices\n", 702 gapstart, gapstart + gapsize - 1); 703 } 704 705 /* 706 * Called late during init, in free_initmem(). 707 * 708 * Initial e820_table and e820_table_kexec are largish __initdata arrays. 709 * 710 * Copy them to a (usually much smaller) dynamically allocated area that is 711 * sized precisely after the number of e820 entries. 712 * 713 * This is done after we've performed all the fixes and tweaks to the tables. 714 * All functions which modify them are __init functions, which won't exist 715 * after free_initmem(). 716 */ 717 __init void e820__reallocate_tables(void) 718 { 719 struct e820_table *n; 720 int size; 721 722 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table->nr_entries; 723 n = kmemdup(e820_table, size, GFP_KERNEL); 724 BUG_ON(!n); 725 e820_table = n; 726 727 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_kexec->nr_entries; 728 n = kmemdup(e820_table_kexec, size, GFP_KERNEL); 729 BUG_ON(!n); 730 e820_table_kexec = n; 731 732 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_firmware->nr_entries; 733 n = kmemdup(e820_table_firmware, size, GFP_KERNEL); 734 BUG_ON(!n); 735 e820_table_firmware = n; 736 } 737 738 /* 739 * Because of the small fixed size of struct boot_params, only the first 740 * 128 E820 memory entries are passed to the kernel via boot_params.e820_table, 741 * the remaining (if any) entries are passed via the SETUP_E820_EXT node of 742 * struct setup_data, which is parsed here. 743 */ 744 void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len) 745 { 746 int entries; 747 struct boot_e820_entry *extmap; 748 struct setup_data *sdata; 749 750 sdata = early_memremap(phys_addr, data_len); 751 entries = sdata->len / sizeof(*extmap); 752 extmap = (struct boot_e820_entry *)(sdata->data); 753 754 __append_e820_table(extmap, entries); 755 e820__update_table(e820_table); 756 757 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); 758 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); 759 760 early_memunmap(sdata, data_len); 761 pr_info("extended physical RAM map:\n"); 762 e820__print_table("extended"); 763 } 764 765 /* 766 * Find the ranges of physical addresses that do not correspond to 767 * E820 RAM areas and register the corresponding pages as 'nosave' for 768 * hibernation (32-bit) or software suspend and suspend to RAM (64-bit). 769 * 770 * This function requires the E820 map to be sorted and without any 771 * overlapping entries. 772 */ 773 void __init e820__register_nosave_regions(unsigned long limit_pfn) 774 { 775 u32 idx; 776 u64 last_addr = 0; 777 778 for (idx = 0; idx < e820_table->nr_entries; idx++) { 779 struct e820_entry *entry = &e820_table->entries[idx]; 780 781 if (entry->type != E820_TYPE_RAM) 782 continue; 783 784 if (last_addr < entry->addr) 785 register_nosave_region(PFN_DOWN(last_addr), PFN_UP(entry->addr)); 786 787 last_addr = entry->addr + entry->size; 788 } 789 790 register_nosave_region(PFN_DOWN(last_addr), limit_pfn); 791 } 792 793 #ifdef CONFIG_ACPI 794 /* 795 * Register ACPI NVS memory regions, so that we can save/restore them during 796 * hibernation and the subsequent resume: 797 */ 798 static int __init e820__register_nvs_regions(void) 799 { 800 u32 idx; 801 802 for (idx = 0; idx < e820_table->nr_entries; idx++) { 803 struct e820_entry *entry = &e820_table->entries[idx]; 804 805 if (entry->type == E820_TYPE_NVS) 806 acpi_nvs_register(entry->addr, entry->size); 807 } 808 809 return 0; 810 } 811 core_initcall(e820__register_nvs_regions); 812 #endif 813 814 /* 815 * Allocate the requested number of bytes with the requested alignment 816 * and return (the physical address) to the caller. Also register this 817 * range in the 'kexec' E820 table as a reserved range. 818 * 819 * This allows kexec to fake a new mptable, as if it came from the real 820 * system. 821 */ 822 u64 __init e820__memblock_alloc_reserved(u64 size, u64 align) 823 { 824 u64 addr; 825 826 addr = memblock_phys_alloc(size, align); 827 if (addr) { 828 e820__range_update_table(e820_table_kexec, addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); 829 pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); 830 e820__update_table_kexec(); 831 } 832 833 return addr; 834 } 835 836 #ifdef CONFIG_X86_32 837 # ifdef CONFIG_X86_PAE 838 # define MAX_ARCH_PFN (1ULL<<(36-PAGE_SHIFT)) 839 # else 840 # define MAX_ARCH_PFN (1ULL<<(32-PAGE_SHIFT)) 841 # endif 842 #else /* CONFIG_X86_32 */ 843 # define MAX_ARCH_PFN MAXMEM>>PAGE_SHIFT 844 #endif 845 846 /* 847 * Find the highest page frame number we have available 848 */ 849 static unsigned long __init e820__end_ram_pfn(unsigned long limit_pfn) 850 { 851 u32 idx; 852 unsigned long last_pfn = 0; 853 unsigned long max_arch_pfn = MAX_ARCH_PFN; 854 855 for (idx = 0; idx < e820_table->nr_entries; idx++) { 856 struct e820_entry *entry = &e820_table->entries[idx]; 857 unsigned long start_pfn; 858 unsigned long end_pfn; 859 860 if (entry->type != E820_TYPE_RAM && 861 entry->type != E820_TYPE_ACPI) 862 continue; 863 864 start_pfn = entry->addr >> PAGE_SHIFT; 865 end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT; 866 867 if (start_pfn >= limit_pfn) 868 continue; 869 if (end_pfn > limit_pfn) { 870 last_pfn = limit_pfn; 871 break; 872 } 873 if (end_pfn > last_pfn) 874 last_pfn = end_pfn; 875 } 876 877 if (last_pfn > max_arch_pfn) 878 last_pfn = max_arch_pfn; 879 880 pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n", 881 last_pfn, max_arch_pfn); 882 return last_pfn; 883 } 884 885 unsigned long __init e820__end_of_ram_pfn(void) 886 { 887 return e820__end_ram_pfn(MAX_ARCH_PFN); 888 } 889 890 unsigned long __init e820__end_of_low_ram_pfn(void) 891 { 892 return e820__end_ram_pfn(1UL << (32 - PAGE_SHIFT)); 893 } 894 895 static int userdef __initdata; 896 897 /* The "mem=nopentium" boot option disables 4MB page tables on 32-bit kernels: */ 898 static int __init parse_memopt(char *p) 899 { 900 u64 mem_size; 901 902 if (!p) 903 return -EINVAL; 904 905 if (!strcmp(p, "nopentium")) { 906 #ifdef CONFIG_X86_32 907 setup_clear_cpu_cap(X86_FEATURE_PSE); 908 return 0; 909 #else 910 pr_warn("mem=nopentium ignored! (only supported on x86_32)\n"); 911 return -EINVAL; 912 #endif 913 } 914 915 userdef = 1; 916 mem_size = memparse(p, &p); 917 918 /* Don't remove all memory when getting "mem={invalid}" parameter: */ 919 if (mem_size == 0) 920 return -EINVAL; 921 922 e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); 923 924 #ifdef CONFIG_MEMORY_HOTPLUG 925 max_mem_size = mem_size; 926 #endif 927 928 return 0; 929 } 930 early_param("mem", parse_memopt); 931 932 static int __init parse_memmap_one(char *p) 933 { 934 char *oldp; 935 u64 start_at, mem_size; 936 937 if (!p) 938 return -EINVAL; 939 940 if (!strncmp(p, "exactmap", 8)) { 941 e820_table->nr_entries = 0; 942 userdef = 1; 943 return 0; 944 } 945 946 oldp = p; 947 mem_size = memparse(p, &p); 948 if (p == oldp) 949 return -EINVAL; 950 951 userdef = 1; 952 if (*p == '@') { 953 start_at = memparse(p+1, &p); 954 e820__range_add(start_at, mem_size, E820_TYPE_RAM); 955 } else if (*p == '#') { 956 start_at = memparse(p+1, &p); 957 e820__range_add(start_at, mem_size, E820_TYPE_ACPI); 958 } else if (*p == '$') { 959 start_at = memparse(p+1, &p); 960 e820__range_add(start_at, mem_size, E820_TYPE_RESERVED); 961 } else if (*p == '!') { 962 start_at = memparse(p+1, &p); 963 e820__range_add(start_at, mem_size, E820_TYPE_PRAM); 964 } else if (*p == '%') { 965 enum e820_type from = 0, to = 0; 966 967 start_at = memparse(p + 1, &p); 968 if (*p == '-') 969 from = simple_strtoull(p + 1, &p, 0); 970 if (*p == '+') 971 to = simple_strtoull(p + 1, &p, 0); 972 if (*p != '\0') 973 return -EINVAL; 974 if (from && to) 975 e820__range_update(start_at, mem_size, from, to); 976 else if (to) 977 e820__range_add(start_at, mem_size, to); 978 else if (from) 979 e820__range_remove(start_at, mem_size, from, 1); 980 else 981 e820__range_remove(start_at, mem_size, 0, 0); 982 } else { 983 e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); 984 } 985 986 return *p == '\0' ? 0 : -EINVAL; 987 } 988 989 static int __init parse_memmap_opt(char *str) 990 { 991 while (str) { 992 char *k = strchr(str, ','); 993 994 if (k) 995 *k++ = 0; 996 997 parse_memmap_one(str); 998 str = k; 999 } 1000 1001 return 0; 1002 } 1003 early_param("memmap", parse_memmap_opt); 1004 1005 /* 1006 * Called after parse_early_param(), after early parameters (such as mem=) 1007 * have been processed, in which case we already have an E820 table filled in 1008 * via the parameter callback function(s), but it's not sorted and printed yet: 1009 */ 1010 void __init e820__finish_early_params(void) 1011 { 1012 if (userdef) { 1013 if (e820__update_table(e820_table) < 0) 1014 panic("Invalid user supplied memory map"); 1015 1016 pr_info("user-defined physical RAM map:\n"); 1017 e820__print_table("user"); 1018 } 1019 } 1020 1021 static const char *__init e820_type_to_string(struct e820_entry *entry) 1022 { 1023 switch (entry->type) { 1024 case E820_TYPE_RAM: return "System RAM"; 1025 case E820_TYPE_ACPI: return "ACPI Tables"; 1026 case E820_TYPE_NVS: return "ACPI Non-volatile Storage"; 1027 case E820_TYPE_UNUSABLE: return "Unusable memory"; 1028 case E820_TYPE_PRAM: return "Persistent Memory (legacy)"; 1029 case E820_TYPE_PMEM: return "Persistent Memory"; 1030 case E820_TYPE_RESERVED: return "Reserved"; 1031 case E820_TYPE_SOFT_RESERVED: return "Soft Reserved"; 1032 default: return "Unknown E820 type"; 1033 } 1034 } 1035 1036 static unsigned long __init e820_type_to_iomem_type(struct e820_entry *entry) 1037 { 1038 switch (entry->type) { 1039 case E820_TYPE_RAM: return IORESOURCE_SYSTEM_RAM; 1040 case E820_TYPE_ACPI: /* Fall-through: */ 1041 case E820_TYPE_NVS: /* Fall-through: */ 1042 case E820_TYPE_UNUSABLE: /* Fall-through: */ 1043 case E820_TYPE_PRAM: /* Fall-through: */ 1044 case E820_TYPE_PMEM: /* Fall-through: */ 1045 case E820_TYPE_RESERVED: /* Fall-through: */ 1046 case E820_TYPE_SOFT_RESERVED: /* Fall-through: */ 1047 default: return IORESOURCE_MEM; 1048 } 1049 } 1050 1051 static unsigned long __init e820_type_to_iores_desc(struct e820_entry *entry) 1052 { 1053 switch (entry->type) { 1054 case E820_TYPE_ACPI: return IORES_DESC_ACPI_TABLES; 1055 case E820_TYPE_NVS: return IORES_DESC_ACPI_NV_STORAGE; 1056 case E820_TYPE_PMEM: return IORES_DESC_PERSISTENT_MEMORY; 1057 case E820_TYPE_PRAM: return IORES_DESC_PERSISTENT_MEMORY_LEGACY; 1058 case E820_TYPE_RESERVED: return IORES_DESC_RESERVED; 1059 case E820_TYPE_SOFT_RESERVED: return IORES_DESC_SOFT_RESERVED; 1060 case E820_TYPE_RAM: /* Fall-through: */ 1061 case E820_TYPE_UNUSABLE: /* Fall-through: */ 1062 default: return IORES_DESC_NONE; 1063 } 1064 } 1065 1066 /* 1067 * We assign one resource entry for each E820 map entry: 1068 */ 1069 static struct resource __initdata *e820_res; 1070 1071 /* 1072 * Is this a device address region that should not be marked busy? 1073 * (Versus system address regions that we register & lock early.) 1074 */ 1075 static bool __init e820_device_region(enum e820_type type, struct resource *res) 1076 { 1077 /* This is the legacy BIOS/DOS ROM-shadow + MMIO region: */ 1078 if (res->start < (1ULL<<20)) 1079 return false; 1080 1081 /* 1082 * Treat persistent memory and other special memory ranges like 1083 * device memory, i.e. keep it available for exclusive use of a 1084 * driver: 1085 */ 1086 switch (type) { 1087 case E820_TYPE_RESERVED: 1088 case E820_TYPE_SOFT_RESERVED: 1089 case E820_TYPE_PRAM: 1090 case E820_TYPE_PMEM: 1091 return true; 1092 case E820_TYPE_RAM: 1093 case E820_TYPE_ACPI: 1094 case E820_TYPE_NVS: 1095 case E820_TYPE_UNUSABLE: 1096 default: 1097 return false; 1098 } 1099 } 1100 1101 /* 1102 * Mark E820 system regions as busy for the resource manager: 1103 */ 1104 void __init e820__reserve_resources(void) 1105 { 1106 u32 idx; 1107 struct resource *res; 1108 u64 end; 1109 1110 res = memblock_alloc_or_panic(sizeof(*res) * e820_table->nr_entries, 1111 SMP_CACHE_BYTES); 1112 e820_res = res; 1113 1114 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1115 struct e820_entry *entry = e820_table->entries + idx; 1116 1117 end = entry->addr + entry->size - 1; 1118 if (end != (resource_size_t)end) { 1119 res++; 1120 continue; 1121 } 1122 res->start = entry->addr; 1123 res->end = end; 1124 res->name = e820_type_to_string(entry); 1125 res->flags = e820_type_to_iomem_type(entry); 1126 res->desc = e820_type_to_iores_desc(entry); 1127 1128 /* 1129 * Skip and don't register device regions that could be conflicted 1130 * with PCI device BAR resources. They get inserted later in 1131 * pcibios_resource_survey() -> e820__reserve_resources_late(): 1132 */ 1133 if (!e820_device_region(entry->type, res)) { 1134 res->flags |= IORESOURCE_BUSY; 1135 insert_resource(&iomem_resource, res); 1136 } 1137 res++; 1138 } 1139 1140 /* Expose the kexec e820 table to sysfs: */ 1141 for (idx = 0; idx < e820_table_kexec->nr_entries; idx++) { 1142 struct e820_entry *entry = e820_table_kexec->entries + idx; 1143 1144 firmware_map_add_early(entry->addr, entry->addr + entry->size, e820_type_to_string(entry)); 1145 } 1146 } 1147 1148 /* 1149 * How much should we pad the end of RAM, depending on where it is? 1150 */ 1151 static unsigned long __init ram_alignment(resource_size_t pos) 1152 { 1153 unsigned long mb = pos >> 20; 1154 1155 /* To 64kB in the first megabyte */ 1156 if (!mb) 1157 return 64*1024; 1158 1159 /* To 1MB in the first 16MB */ 1160 if (mb < 16) 1161 return 1024*1024; 1162 1163 /* To 64MB for anything above that */ 1164 return 64*1024*1024; 1165 } 1166 1167 #define MAX_RESOURCE_SIZE ((resource_size_t)-1) 1168 1169 void __init e820__reserve_resources_late(void) 1170 { 1171 u32 idx; 1172 struct resource *res; 1173 1174 /* 1175 * Register device address regions listed in the E820 map, 1176 * these can be claimed by device drivers later on: 1177 */ 1178 res = e820_res; 1179 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1180 if (!res->parent && res->end) 1181 insert_resource_expand_to_fit(&iomem_resource, res); 1182 res++; 1183 } 1184 1185 /* 1186 * Create additional 'gaps' at the end of RAM regions, 1187 * rounding them up to 64k/1MB/64MB boundaries, should 1188 * they be weirdly sized, and register extra, locked 1189 * resource regions for them, to make sure drivers 1190 * won't claim those addresses. 1191 * 1192 * These are basically blind guesses and heuristics to 1193 * avoid resource conflicts with broken firmware that 1194 * doesn't properly list 'stolen RAM' as a system region 1195 * in the E820 map. 1196 */ 1197 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1198 struct e820_entry *entry = &e820_table->entries[idx]; 1199 u64 start, end; 1200 1201 if (entry->type != E820_TYPE_RAM) 1202 continue; 1203 1204 start = entry->addr + entry->size; 1205 end = round_up(start, ram_alignment(start)) - 1; 1206 if (end > MAX_RESOURCE_SIZE) 1207 end = MAX_RESOURCE_SIZE; 1208 if (start >= end) 1209 continue; 1210 1211 pr_info("e820: register RAM buffer resource [mem %#010llx-%#010llx]\n", start, end); 1212 reserve_region_with_split(&iomem_resource, start, end, "RAM buffer"); 1213 } 1214 } 1215 1216 /* 1217 * Pass the firmware (bootloader) E820 map to the kernel and process it: 1218 */ 1219 char *__init e820__memory_setup_default(void) 1220 { 1221 char *who = "BIOS-e820"; 1222 1223 /* 1224 * Try to copy the BIOS-supplied E820-map. 1225 * 1226 * Otherwise fake a memory map; one section from 0k->640k, 1227 * the next section from 1mb->appropriate_mem_k 1228 */ 1229 if (append_e820_table(boot_params.e820_table, boot_params.e820_entries) < 0) { 1230 u64 mem_size; 1231 1232 /* Compare results from other methods and take the one that gives more RAM: */ 1233 if (boot_params.alt_mem_k < boot_params.screen_info.ext_mem_k) { 1234 mem_size = boot_params.screen_info.ext_mem_k; 1235 who = "BIOS-88"; 1236 } else { 1237 mem_size = boot_params.alt_mem_k; 1238 who = "BIOS-e801"; 1239 } 1240 1241 e820_table->nr_entries = 0; 1242 e820__range_add(0, LOWMEMSIZE(), E820_TYPE_RAM); 1243 e820__range_add(HIGH_MEMORY, mem_size << 10, E820_TYPE_RAM); 1244 } 1245 1246 /* We just appended a lot of ranges, sanitize the table: */ 1247 e820__update_table(e820_table); 1248 1249 return who; 1250 } 1251 1252 /* 1253 * Calls e820__memory_setup_default() in essence to pick up the firmware/bootloader 1254 * E820 map - with an optional platform quirk available for virtual platforms 1255 * to override this method of boot environment processing: 1256 */ 1257 void __init e820__memory_setup(void) 1258 { 1259 char *who; 1260 1261 /* This is a firmware interface ABI - make sure we don't break it: */ 1262 BUILD_BUG_ON(sizeof(struct boot_e820_entry) != 20); 1263 1264 who = x86_init.resources.memory_setup(); 1265 1266 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); 1267 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); 1268 1269 pr_info("BIOS-provided physical RAM map:\n"); 1270 e820__print_table(who); 1271 } 1272 1273 void __init e820__memblock_setup(void) 1274 { 1275 u32 idx; 1276 u64 end; 1277 1278 #ifdef CONFIG_MEMORY_HOTPLUG 1279 /* 1280 * Memory used by the kernel cannot be hot-removed because Linux 1281 * cannot migrate the kernel pages. When memory hotplug is 1282 * enabled, we should prevent memblock from allocating memory 1283 * for the kernel. 1284 * 1285 * ACPI SRAT records all hotpluggable memory ranges. But before 1286 * SRAT is parsed, we don't know about it. 1287 * 1288 * The kernel image is loaded into memory at very early time. We 1289 * cannot prevent this anyway. So on NUMA system, we set any 1290 * node the kernel resides in as un-hotpluggable. 1291 * 1292 * Since on modern servers, one node could have double-digit 1293 * gigabytes memory, we can assume the memory around the kernel 1294 * image is also un-hotpluggable. So before SRAT is parsed, just 1295 * allocate memory near the kernel image to try the best to keep 1296 * the kernel away from hotpluggable memory. 1297 */ 1298 if (movable_node_is_enabled()) 1299 memblock_set_bottom_up(true); 1300 #endif 1301 1302 /* 1303 * At this point only the first megabyte is mapped for sure, the 1304 * rest of the memory cannot be used for memblock resizing 1305 */ 1306 memblock_set_current_limit(ISA_END_ADDRESS); 1307 1308 /* 1309 * The bootstrap memblock region count maximum is 128 entries 1310 * (INIT_MEMBLOCK_REGIONS), but EFI might pass us more E820 entries 1311 * than that - so allow memblock resizing. 1312 * 1313 * This is safe, because this call happens pretty late during x86 setup, 1314 * so we know about reserved memory regions already. (This is important 1315 * so that memblock resizing does no stomp over reserved areas.) 1316 */ 1317 memblock_allow_resize(); 1318 1319 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1320 struct e820_entry *entry = &e820_table->entries[idx]; 1321 1322 end = entry->addr + entry->size; 1323 if (end != (resource_size_t)end) 1324 continue; 1325 1326 if (entry->type == E820_TYPE_SOFT_RESERVED) 1327 memblock_reserve(entry->addr, entry->size); 1328 1329 if (entry->type != E820_TYPE_RAM) 1330 continue; 1331 1332 memblock_add(entry->addr, entry->size); 1333 } 1334 1335 /* 1336 * At this point memblock is only allowed to allocate from memory 1337 * below 1M (aka ISA_END_ADDRESS) up until direct map is completely set 1338 * up in init_mem_mapping(). 1339 * 1340 * KHO kernels are special and use only scratch memory for memblock 1341 * allocations, but memory below 1M is ignored by kernel after early 1342 * boot and cannot be naturally marked as scratch. 1343 * 1344 * To allow allocation of the real-mode trampoline and a few (if any) 1345 * other very early allocations from below 1M forcibly mark the memory 1346 * below 1M as scratch. 1347 * 1348 * After real mode trampoline is allocated, we clear that scratch 1349 * marking. 1350 */ 1351 memblock_mark_kho_scratch(0, SZ_1M); 1352 1353 /* 1354 * 32-bit systems are limited to 4BG of memory even with HIGHMEM and 1355 * to even less without it. 1356 * Discard memory after max_pfn - the actual limit detected at runtime. 1357 */ 1358 if (IS_ENABLED(CONFIG_X86_32)) 1359 memblock_remove(PFN_PHYS(max_pfn), -1); 1360 1361 /* Throw away partial pages: */ 1362 memblock_trim_memory(PAGE_SIZE); 1363 1364 memblock_dump_all(); 1365 } 1366