1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Low level x86 E820 memory map handling functions. 4 * 5 * The firmware and bootloader passes us the "E820 table", which is the primary 6 * physical memory layout description available about x86 systems. 7 * 8 * The kernel takes the E820 memory layout and optionally modifies it with 9 * quirks and other tweaks, and feeds that into the generic Linux memory 10 * allocation code routines via a platform independent interface (memblock, etc.). 11 */ 12 #include <linux/memblock.h> 13 #include <linux/suspend.h> 14 #include <linux/acpi.h> 15 #include <linux/firmware-map.h> 16 #include <linux/sort.h> 17 #include <linux/kvm_types.h> 18 19 #include <asm/e820/api.h> 20 #include <asm/setup.h> 21 22 /* 23 * We organize the E820 table into three main data structures: 24 * 25 * - 'e820_table_firmware': the original firmware version passed to us by the 26 * bootloader - not modified by the kernel. It is composed of two parts: 27 * the first 128 E820 memory entries in boot_params.e820_table and the remaining 28 * (if any) entries of the SETUP_E820_EXT nodes. We use this to: 29 * 30 * - the hibernation code uses it to generate a kernel-independent CRC32 31 * checksum of the physical memory layout of a system. 32 * 33 * - 'e820_table_kexec': a slightly modified (by the kernel) firmware version 34 * passed to us by the bootloader - the major difference between 35 * e820_table_firmware[] and this one is that e820_table_kexec[] 36 * might be modified by the kexec itself to fake an mptable. 37 * We use this to: 38 * 39 * - kexec, which is a bootloader in disguise, uses the original E820 40 * layout to pass to the kexec-ed kernel. This way the original kernel 41 * can have a restricted E820 map while the kexec()-ed kexec-kernel 42 * can have access to full memory - etc. 43 * 44 * Export the memory layout via /sys/firmware/memmap. kexec-tools uses 45 * the entries to create an E820 table for the kexec kernel. 46 * 47 * kexec_file_load in-kernel code uses the table for the kexec kernel. 48 * 49 * - 'e820_table': this is the main E820 table that is massaged by the 50 * low level x86 platform code, or modified by boot parameters, before 51 * passed on to higher level MM layers. 52 * 53 * Once the E820 map has been converted to the standard Linux memory layout 54 * information its role stops - modifying it has no effect and does not get 55 * re-propagated. So its main role is a temporary bootstrap storage of firmware 56 * specific memory layout data during early bootup. 57 */ 58 __initdata static struct e820_table e820_table_init; 59 __initdata static struct e820_table e820_table_kexec_init; 60 __initdata static struct e820_table e820_table_firmware_init; 61 62 __refdata struct e820_table *e820_table = &e820_table_init; 63 __refdata struct e820_table *e820_table_kexec = &e820_table_kexec_init; 64 __refdata struct e820_table *e820_table_firmware = &e820_table_firmware_init; 65 66 /* For PCI or other memory-mapped resources */ 67 unsigned long pci_mem_start = 0xaeedbabe; 68 #ifdef CONFIG_PCI 69 EXPORT_SYMBOL(pci_mem_start); 70 #endif 71 72 /* 73 * This function checks if any part of the range <start,end> is mapped 74 * with type. 75 */ 76 static bool _e820__mapped_any(struct e820_table *table, 77 u64 start, u64 end, enum e820_type type) 78 { 79 u32 idx; 80 81 for (idx = 0; idx < table->nr_entries; idx++) { 82 struct e820_entry *entry = &table->entries[idx]; 83 84 if (type && entry->type != type) 85 continue; 86 if (entry->addr >= end || entry->addr + entry->size <= start) 87 continue; 88 return true; 89 } 90 return false; 91 } 92 93 bool e820__mapped_raw_any(u64 start, u64 end, enum e820_type type) 94 { 95 return _e820__mapped_any(e820_table_firmware, start, end, type); 96 } 97 EXPORT_SYMBOL_FOR_KVM(e820__mapped_raw_any); 98 99 bool e820__mapped_any(u64 start, u64 end, enum e820_type type) 100 { 101 return _e820__mapped_any(e820_table, start, end, type); 102 } 103 EXPORT_SYMBOL_GPL(e820__mapped_any); 104 105 /* 106 * This function checks if the entire <start,end> range is mapped with 'type'. 107 * 108 * Note: this function only works correctly once the E820 table is sorted and 109 * not-overlapping (at least for the range specified), which is the case normally. 110 */ 111 static struct e820_entry *__e820__mapped_all(u64 start, u64 end, 112 enum e820_type type) 113 { 114 u32 idx; 115 116 for (idx = 0; idx < e820_table->nr_entries; idx++) { 117 struct e820_entry *entry = &e820_table->entries[idx]; 118 119 if (type && entry->type != type) 120 continue; 121 122 /* Is the region (part) in overlap with the current region? */ 123 if (entry->addr >= end || entry->addr + entry->size <= start) 124 continue; 125 126 /* 127 * If the region is at the beginning of <start,end> we move 128 * 'start' to the end of the region since it's ok until there 129 */ 130 if (entry->addr <= start) 131 start = entry->addr + entry->size; 132 133 /* 134 * If 'start' is now at or beyond 'end', we're done, full 135 * coverage of the desired range exists: 136 */ 137 if (start >= end) 138 return entry; 139 } 140 141 return NULL; 142 } 143 144 /* 145 * This function checks if the entire range <start,end> is mapped with type. 146 */ 147 __init bool e820__mapped_all(u64 start, u64 end, enum e820_type type) 148 { 149 return __e820__mapped_all(start, end, type); 150 } 151 152 /* 153 * This function returns the type associated with the range <start,end>. 154 */ 155 int e820__get_entry_type(u64 start, u64 end) 156 { 157 struct e820_entry *entry = __e820__mapped_all(start, end, 0); 158 159 return entry ? entry->type : -EINVAL; 160 } 161 162 /* 163 * Add a memory region to the kernel E820 map. 164 */ 165 __init static void __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type) 166 { 167 u32 idx = table->nr_entries; 168 struct e820_entry *entry_new; 169 170 if (idx >= ARRAY_SIZE(table->entries)) { 171 pr_err("E820 table full; ignoring [mem %#010llx-%#010llx]\n", 172 start, start + size-1); 173 return; 174 } 175 176 entry_new = table->entries + idx; 177 178 entry_new->addr = start; 179 entry_new->size = size; 180 entry_new->type = type; 181 182 table->nr_entries++; 183 } 184 185 __init void e820__range_add(u64 start, u64 size, enum e820_type type) 186 { 187 __e820__range_add(e820_table, start, size, type); 188 } 189 190 __init static void e820_print_type(enum e820_type type) 191 { 192 switch (type) { 193 case E820_TYPE_RAM: pr_cont(" System RAM"); break; 194 case E820_TYPE_RESERVED: pr_cont(" device reserved"); break; 195 case E820_TYPE_SOFT_RESERVED: pr_cont(" soft reserved"); break; 196 case E820_TYPE_ACPI: pr_cont(" ACPI data"); break; 197 case E820_TYPE_NVS: pr_cont(" ACPI NVS"); break; 198 case E820_TYPE_UNUSABLE: pr_cont(" unusable"); break; 199 case E820_TYPE_PMEM: /* Fall through: */ 200 case E820_TYPE_PRAM: pr_cont(" persistent RAM (type %u)", type); break; 201 default: pr_cont(" type %u", type); break; 202 } 203 } 204 205 __init static void e820__print_table(const char *who) 206 { 207 u64 range_end_prev = 0; 208 u32 idx; 209 210 for (idx = 0; idx < e820_table->nr_entries; idx++) { 211 struct e820_entry *entry = e820_table->entries + idx; 212 u64 range_start, range_end; 213 214 range_start = entry->addr; 215 range_end = entry->addr + entry->size; 216 217 /* Out of order E820 maps should not happen: */ 218 if (range_start < range_end_prev) 219 pr_info(FW_BUG "out of order E820 entry!\n"); 220 221 if (range_start > range_end_prev) { 222 pr_info("%s: [gap %#018Lx-%#018Lx]\n", 223 who, 224 range_end_prev, 225 range_start-1); 226 } 227 228 pr_info("%s: [mem %#018Lx-%#018Lx] ", who, range_start, range_end-1); 229 e820_print_type(entry->type); 230 pr_cont("\n"); 231 232 range_end_prev = range_end; 233 } 234 } 235 236 /* 237 * Sanitize an E820 map. 238 * 239 * Some E820 layouts include overlapping entries. The following 240 * replaces the original E820 map with a new one, removing overlaps, 241 * and resolving conflicting memory types in favor of highest 242 * numbered type. 243 * 244 * The input parameter 'entries' points to an array of 'struct 245 * e820_entry' which on entry has elements in the range [0, *nr_entries) 246 * valid, and which has space for up to max_nr_entries entries. 247 * On return, the resulting sanitized E820 map entries will be in 248 * overwritten in the same location, starting at 'entries'. 249 * 250 * The integer pointed to by nr_entries must be valid on entry (the 251 * current number of valid entries located at 'entries'). If the 252 * sanitizing succeeds the *nr_entries will be updated with the new 253 * number of valid entries (something no more than max_nr_entries). 254 * 255 * The return value from e820__update_table() is zero if it 256 * successfully 'sanitized' the map entries passed in, and is -1 257 * if it did nothing, which can happen if either of (1) it was 258 * only passed one map entry, or (2) any of the input map entries 259 * were invalid (start + size < start, meaning that the size was 260 * so big the described memory range wrapped around through zero.) 261 * 262 * Visually we're performing the following 263 * (1,2,3,4 = memory types)... 264 * 265 * Sample memory map (w/overlaps): 266 * ____22__________________ 267 * ______________________4_ 268 * ____1111________________ 269 * _44_____________________ 270 * 11111111________________ 271 * ____________________33__ 272 * ___________44___________ 273 * __________33333_________ 274 * ______________22________ 275 * ___________________2222_ 276 * _________111111111______ 277 * _____________________11_ 278 * _________________4______ 279 * 280 * Sanitized equivalent (no overlap): 281 * 1_______________________ 282 * _44_____________________ 283 * ___1____________________ 284 * ____22__________________ 285 * ______11________________ 286 * _________1______________ 287 * __________3_____________ 288 * ___________44___________ 289 * _____________33_________ 290 * _______________2________ 291 * ________________1_______ 292 * _________________4______ 293 * ___________________2____ 294 * ____________________33__ 295 * ______________________4_ 296 */ 297 struct change_member { 298 /* Pointer to the original entry: */ 299 struct e820_entry *entry; 300 /* Address for this change point: */ 301 u64 addr; 302 }; 303 304 __initdata static struct change_member change_point_list[2*E820_MAX_ENTRIES]; 305 __initdata static struct change_member *change_point[2*E820_MAX_ENTRIES]; 306 __initdata static struct e820_entry *overlap_list[E820_MAX_ENTRIES]; 307 __initdata static struct e820_entry new_entries[E820_MAX_ENTRIES]; 308 309 __init static int cpcompare(const void *a, const void *b) 310 { 311 struct change_member * const *app = a, * const *bpp = b; 312 const struct change_member *ap = *app, *bp = *bpp; 313 314 /* 315 * Inputs are pointers to two elements of change_point[]. If their 316 * addresses are not equal, their difference dominates. If the addresses 317 * are equal, then consider one that represents the end of its region 318 * to be greater than one that does not. 319 */ 320 if (ap->addr != bp->addr) 321 return ap->addr > bp->addr ? 1 : -1; 322 323 return (ap->addr != ap->entry->addr) - (bp->addr != bp->entry->addr); 324 } 325 326 /* 327 * Can two consecutive E820 entries of this same E820 type be merged? 328 */ 329 static bool e820_type_mergeable(enum e820_type type) 330 { 331 /* 332 * These types may indicate distinct platform ranges aligned to 333 * NUMA node, protection domain, performance domain, or other 334 * boundaries. Do not merge them. 335 */ 336 if (type == E820_TYPE_PRAM) 337 return false; 338 if (type == E820_TYPE_SOFT_RESERVED) 339 return false; 340 341 return true; 342 } 343 344 __init int e820__update_table(struct e820_table *table) 345 { 346 struct e820_entry *entries = table->entries; 347 u32 max_nr_entries = ARRAY_SIZE(table->entries); 348 enum e820_type current_type, last_type; 349 u64 last_addr; 350 u32 new_nr_entries, overlap_entries; 351 u32 idx, chg_idx, chg_nr; 352 353 /* If there's only one memory region, don't bother: */ 354 if (table->nr_entries < 2) 355 return -1; 356 357 BUG_ON(table->nr_entries > max_nr_entries); 358 359 /* Bail out if we find any unreasonable addresses in the map: */ 360 for (idx = 0; idx < table->nr_entries; idx++) { 361 if (entries[idx].addr + entries[idx].size < entries[idx].addr) 362 return -1; 363 } 364 365 /* Create pointers for initial change-point information (for sorting): */ 366 for (idx = 0; idx < 2 * table->nr_entries; idx++) 367 change_point[idx] = &change_point_list[idx]; 368 369 /* 370 * Record all known change-points (starting and ending addresses), 371 * omitting empty memory regions: 372 */ 373 chg_idx = 0; 374 for (idx = 0; idx < table->nr_entries; idx++) { 375 if (entries[idx].size != 0) { 376 change_point[chg_idx]->addr = entries[idx].addr; 377 change_point[chg_idx++]->entry = &entries[idx]; 378 change_point[chg_idx]->addr = entries[idx].addr + entries[idx].size; 379 change_point[chg_idx++]->entry = &entries[idx]; 380 } 381 } 382 chg_nr = chg_idx; 383 384 /* Sort change-point list by memory addresses (low -> high): */ 385 sort(change_point, chg_nr, sizeof(*change_point), cpcompare, NULL); 386 387 /* Create a new memory map, removing overlaps: */ 388 overlap_entries = 0; /* Number of entries in the overlap table */ 389 new_nr_entries = 0; /* Index for creating new map entries */ 390 last_type = 0; /* Start with undefined memory type */ 391 last_addr = 0; /* Start with 0 as last starting address */ 392 393 /* Loop through change-points, determining effect on the new map: */ 394 for (chg_idx = 0; chg_idx < chg_nr; chg_idx++) { 395 /* Keep track of all overlapping entries */ 396 if (change_point[chg_idx]->addr == change_point[chg_idx]->entry->addr) { 397 /* Add map entry to overlap list (> 1 entry implies an overlap) */ 398 overlap_list[overlap_entries++] = change_point[chg_idx]->entry; 399 } else { 400 /* Remove entry from list (order independent, so swap with last): */ 401 for (idx = 0; idx < overlap_entries; idx++) { 402 if (overlap_list[idx] == change_point[chg_idx]->entry) 403 overlap_list[idx] = overlap_list[overlap_entries-1]; 404 } 405 overlap_entries--; 406 } 407 /* 408 * If there are overlapping entries, decide which 409 * "type" to use (larger value takes precedence -- 410 * 1=usable, 2,3,4,4+=unusable) 411 */ 412 current_type = 0; 413 for (idx = 0; idx < overlap_entries; idx++) { 414 if (overlap_list[idx]->type > current_type) 415 current_type = overlap_list[idx]->type; 416 } 417 418 /* Continue building up new map based on this information: */ 419 if (current_type != last_type || !e820_type_mergeable(current_type)) { 420 if (last_type) { 421 new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr; 422 /* Move forward only if the new size was non-zero: */ 423 if (new_entries[new_nr_entries].size != 0) 424 /* No more space left for new entries? */ 425 if (++new_nr_entries >= max_nr_entries) 426 break; 427 } 428 if (current_type) { 429 new_entries[new_nr_entries].addr = change_point[chg_idx]->addr; 430 new_entries[new_nr_entries].type = current_type; 431 last_addr = change_point[chg_idx]->addr; 432 } 433 last_type = current_type; 434 } 435 } 436 437 /* Copy the new entries into the original location: */ 438 memcpy(entries, new_entries, new_nr_entries*sizeof(*entries)); 439 table->nr_entries = new_nr_entries; 440 441 return 0; 442 } 443 444 /* 445 * Copy the BIOS E820 map into the kernel's e820_table. 446 * 447 * Sanity-check it while we're at it.. 448 */ 449 __init static int append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) 450 { 451 struct boot_e820_entry *entry = entries; 452 453 while (nr_entries) { 454 u64 start = entry->addr; 455 u64 size = entry->size; 456 u64 end = start + size-1; 457 u32 type = entry->type; 458 459 /* Ignore the remaining entries on 64-bit overflow: */ 460 if (start > end && likely(size)) 461 return -1; 462 463 e820__range_add(start, size, type); 464 465 entry++; 466 nr_entries--; 467 } 468 return 0; 469 } 470 471 __init static u64 472 __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) 473 { 474 u64 end; 475 u32 idx; 476 u64 real_updated_size = 0; 477 478 BUG_ON(old_type == new_type); 479 480 if (size > (ULLONG_MAX - start)) 481 size = ULLONG_MAX - start; 482 483 end = start + size; 484 printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx]", start, end - 1); 485 e820_print_type(old_type); 486 pr_cont(" ==>"); 487 e820_print_type(new_type); 488 pr_cont("\n"); 489 490 for (idx = 0; idx < table->nr_entries; idx++) { 491 struct e820_entry *entry = &table->entries[idx]; 492 u64 final_start, final_end; 493 u64 entry_end; 494 495 if (entry->type != old_type) 496 continue; 497 498 entry_end = entry->addr + entry->size; 499 500 /* Completely covered by new range? */ 501 if (entry->addr >= start && entry_end <= end) { 502 entry->type = new_type; 503 real_updated_size += entry->size; 504 continue; 505 } 506 507 /* New range is completely covered? */ 508 if (entry->addr < start && entry_end > end) { 509 __e820__range_add(table, start, size, new_type); 510 __e820__range_add(table, end, entry_end - end, entry->type); 511 entry->size = start - entry->addr; 512 real_updated_size += size; 513 continue; 514 } 515 516 /* Partially covered: */ 517 final_start = max(start, entry->addr); 518 final_end = min(end, entry_end); 519 if (final_start >= final_end) 520 continue; 521 522 __e820__range_add(table, final_start, final_end - final_start, new_type); 523 524 real_updated_size += final_end - final_start; 525 526 /* 527 * Left range could be head or tail, so need to update 528 * its size first: 529 */ 530 entry->size -= final_end - final_start; 531 if (entry->addr < final_start) 532 continue; 533 534 entry->addr = final_end; 535 } 536 return real_updated_size; 537 } 538 539 __init u64 e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) 540 { 541 return __e820__range_update(e820_table, start, size, old_type, new_type); 542 } 543 544 __init u64 e820__range_update_table(struct e820_table *t, u64 start, u64 size, 545 enum e820_type old_type, enum e820_type new_type) 546 { 547 return __e820__range_update(t, start, size, old_type, new_type); 548 } 549 550 /* Remove a range of memory from the E820 table: */ 551 __init void e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type) 552 { 553 u32 idx; 554 u64 end; 555 556 if (size > (ULLONG_MAX - start)) 557 size = ULLONG_MAX - start; 558 559 end = start + size; 560 printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx]", start, end - 1); 561 if (check_type) 562 e820_print_type(old_type); 563 pr_cont("\n"); 564 565 for (idx = 0; idx < e820_table->nr_entries; idx++) { 566 struct e820_entry *entry = &e820_table->entries[idx]; 567 u64 final_start, final_end; 568 u64 entry_end; 569 570 if (check_type && entry->type != old_type) 571 continue; 572 573 entry_end = entry->addr + entry->size; 574 575 /* Completely covered? */ 576 if (entry->addr >= start && entry_end <= end) { 577 memset(entry, 0, sizeof(*entry)); 578 continue; 579 } 580 581 /* Is the new range completely covered? */ 582 if (entry->addr < start && entry_end > end) { 583 e820__range_add(end, entry_end - end, entry->type); 584 entry->size = start - entry->addr; 585 continue; 586 } 587 588 /* Partially covered: */ 589 final_start = max(start, entry->addr); 590 final_end = min(end, entry_end); 591 if (final_start >= final_end) 592 continue; 593 594 /* 595 * Left range could be head or tail, so need to update 596 * the size first: 597 */ 598 entry->size -= final_end - final_start; 599 if (entry->addr < final_start) 600 continue; 601 602 entry->addr = final_end; 603 } 604 } 605 606 __init void e820__update_table_print(void) 607 { 608 if (e820__update_table(e820_table)) 609 return; 610 611 pr_info("modified physical RAM map:\n"); 612 e820__print_table("modified"); 613 } 614 615 __init static void e820__update_table_kexec(void) 616 { 617 e820__update_table(e820_table_kexec); 618 } 619 620 #define MAX_GAP_END 0x100000000ull 621 622 /* 623 * Search for a gap in the E820 memory space from 0 to MAX_GAP_END (4GB). 624 */ 625 __init static int e820_search_gap(unsigned long *max_gap_start, unsigned long *max_gap_size) 626 { 627 u64 last = MAX_GAP_END; 628 int idx = e820_table->nr_entries; 629 int found = 0; 630 631 while (--idx >= 0) { 632 u64 start = e820_table->entries[idx].addr; 633 u64 end = start + e820_table->entries[idx].size; 634 635 /* 636 * Since "last" is at most 4GB, we know we'll 637 * fit in 32 bits if this condition is true: 638 */ 639 if (last > end) { 640 unsigned long gap = last - end; 641 642 if (gap > *max_gap_size) { 643 *max_gap_size = gap; 644 *max_gap_start = end; 645 found = 1; 646 } 647 } 648 if (start < last) 649 last = start; 650 } 651 return found; 652 } 653 654 /* 655 * Search for the biggest gap in the low 32 bits of the E820 656 * memory space. We pass this space to the PCI subsystem, so 657 * that it can assign MMIO resources for hotplug or 658 * unconfigured devices in. 659 * 660 * Hopefully the BIOS let enough space left. 661 */ 662 __init void e820__setup_pci_gap(void) 663 { 664 unsigned long max_gap_start, max_gap_size; 665 int found; 666 667 max_gap_size = SZ_4M; 668 found = e820_search_gap(&max_gap_start, &max_gap_size); 669 670 if (!found) { 671 #ifdef CONFIG_X86_64 672 max_gap_start = (max_pfn << PAGE_SHIFT) + SZ_1M; 673 pr_err("Cannot find an available gap in the 32-bit address range\n"); 674 pr_err("PCI devices with unassigned 32-bit BARs may not work!\n"); 675 #else 676 max_gap_start = 0x10000000; 677 #endif 678 } 679 680 /* 681 * e820__reserve_resources_late() protects stolen RAM already: 682 */ 683 pci_mem_start = max_gap_start; 684 685 pr_info("[gap %#010lx-%#010lx] available for PCI devices\n", 686 max_gap_start, max_gap_start + max_gap_size - 1); 687 } 688 689 /* 690 * Called late during init, in free_initmem(). 691 * 692 * Initial e820_table and e820_table_kexec are largish __initdata arrays. 693 * 694 * Copy them to a (usually much smaller) dynamically allocated area that is 695 * sized precisely after the number of e820 entries. 696 * 697 * This is done after we've performed all the fixes and tweaks to the tables. 698 * All functions which modify them are __init functions, which won't exist 699 * after free_initmem(). 700 */ 701 __init void e820__reallocate_tables(void) 702 { 703 struct e820_table *n; 704 int size; 705 706 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table->nr_entries; 707 n = kmemdup(e820_table, size, GFP_KERNEL); 708 BUG_ON(!n); 709 e820_table = n; 710 711 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_kexec->nr_entries; 712 n = kmemdup(e820_table_kexec, size, GFP_KERNEL); 713 BUG_ON(!n); 714 e820_table_kexec = n; 715 716 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_firmware->nr_entries; 717 n = kmemdup(e820_table_firmware, size, GFP_KERNEL); 718 BUG_ON(!n); 719 e820_table_firmware = n; 720 } 721 722 /* 723 * Because of the small fixed size of struct boot_params, only the first 724 * 128 E820 memory entries are passed to the kernel via boot_params.e820_table, 725 * the remaining (if any) entries are passed via the SETUP_E820_EXT node of 726 * struct setup_data, which is parsed here. 727 */ 728 __init void e820__memory_setup_extended(u64 phys_addr, u32 data_len) 729 { 730 int entries; 731 struct boot_e820_entry *extmap; 732 struct setup_data *sdata; 733 734 sdata = early_memremap(phys_addr, data_len); 735 entries = sdata->len / sizeof(*extmap); 736 extmap = (struct boot_e820_entry *)(sdata->data); 737 738 append_e820_table(extmap, entries); 739 e820__update_table(e820_table); 740 741 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); 742 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); 743 744 early_memunmap(sdata, data_len); 745 pr_info("extended physical RAM map:\n"); 746 e820__print_table("extended"); 747 } 748 749 /* 750 * Find the ranges of physical addresses that do not correspond to 751 * E820 RAM areas and register the corresponding pages as 'nosave' for 752 * hibernation (32-bit) or software suspend and suspend to RAM (64-bit). 753 * 754 * This function requires the E820 map to be sorted and without any 755 * overlapping entries. 756 */ 757 __init void e820__register_nosave_regions(unsigned long limit_pfn) 758 { 759 u32 idx; 760 u64 last_addr = 0; 761 762 for (idx = 0; idx < e820_table->nr_entries; idx++) { 763 struct e820_entry *entry = &e820_table->entries[idx]; 764 765 if (entry->type != E820_TYPE_RAM) 766 continue; 767 768 if (last_addr < entry->addr) 769 register_nosave_region(PFN_DOWN(last_addr), PFN_UP(entry->addr)); 770 771 last_addr = entry->addr + entry->size; 772 } 773 774 register_nosave_region(PFN_DOWN(last_addr), limit_pfn); 775 } 776 777 #ifdef CONFIG_ACPI 778 /* 779 * Register ACPI NVS memory regions, so that we can save/restore them during 780 * hibernation and the subsequent resume: 781 */ 782 __init static int e820__register_nvs_regions(void) 783 { 784 u32 idx; 785 786 for (idx = 0; idx < e820_table->nr_entries; idx++) { 787 struct e820_entry *entry = &e820_table->entries[idx]; 788 789 if (entry->type == E820_TYPE_NVS) 790 acpi_nvs_register(entry->addr, entry->size); 791 } 792 793 return 0; 794 } 795 core_initcall(e820__register_nvs_regions); 796 #endif 797 798 /* 799 * Allocate the requested number of bytes with the requested alignment 800 * and return (the physical address) to the caller. Also register this 801 * range in the 'kexec' E820 table as a reserved range. 802 * 803 * This allows kexec to fake a new mptable, as if it came from the real 804 * system. 805 */ 806 __init u64 e820__memblock_alloc_reserved(u64 size, u64 align) 807 { 808 u64 addr; 809 810 addr = memblock_phys_alloc(size, align); 811 if (addr) { 812 e820__range_update_table(e820_table_kexec, addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); 813 pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); 814 e820__update_table_kexec(); 815 } 816 817 return addr; 818 } 819 820 #ifdef CONFIG_X86_32 821 # ifdef CONFIG_X86_PAE 822 # define MAX_ARCH_PFN (1ULL<<(36-PAGE_SHIFT)) 823 # else 824 # define MAX_ARCH_PFN (1ULL<<(32-PAGE_SHIFT)) 825 # endif 826 #else /* CONFIG_X86_32 */ 827 # define MAX_ARCH_PFN MAXMEM>>PAGE_SHIFT 828 #endif 829 830 /* 831 * Find the highest page frame number we have available 832 */ 833 __init static unsigned long e820__end_ram_pfn(unsigned long limit_pfn) 834 { 835 u32 idx; 836 unsigned long last_pfn = 0; 837 unsigned long max_arch_pfn = MAX_ARCH_PFN; 838 839 for (idx = 0; idx < e820_table->nr_entries; idx++) { 840 struct e820_entry *entry = &e820_table->entries[idx]; 841 unsigned long start_pfn; 842 unsigned long end_pfn; 843 844 if (entry->type != E820_TYPE_RAM && 845 entry->type != E820_TYPE_ACPI) 846 continue; 847 848 start_pfn = entry->addr >> PAGE_SHIFT; 849 end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT; 850 851 if (start_pfn >= limit_pfn) 852 continue; 853 if (end_pfn > limit_pfn) { 854 last_pfn = limit_pfn; 855 break; 856 } 857 if (end_pfn > last_pfn) 858 last_pfn = end_pfn; 859 } 860 861 if (last_pfn > max_arch_pfn) 862 last_pfn = max_arch_pfn; 863 864 pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n", 865 last_pfn, max_arch_pfn); 866 return last_pfn; 867 } 868 869 __init unsigned long e820__end_of_ram_pfn(void) 870 { 871 return e820__end_ram_pfn(MAX_ARCH_PFN); 872 } 873 874 __init unsigned long e820__end_of_low_ram_pfn(void) 875 { 876 return e820__end_ram_pfn(1UL << (32 - PAGE_SHIFT)); 877 } 878 879 __initdata static int userdef; 880 881 /* The "mem=nopentium" boot option disables 4MB page tables on 32-bit kernels: */ 882 __init static int parse_memopt(char *p) 883 { 884 u64 mem_size; 885 886 if (!p) 887 return -EINVAL; 888 889 if (!strcmp(p, "nopentium")) { 890 #ifdef CONFIG_X86_32 891 setup_clear_cpu_cap(X86_FEATURE_PSE); 892 return 0; 893 #else 894 pr_warn("mem=nopentium ignored! (only supported on x86_32)\n"); 895 return -EINVAL; 896 #endif 897 } 898 899 userdef = 1; 900 mem_size = memparse(p, &p); 901 902 /* Don't remove all memory when getting "mem={invalid}" parameter: */ 903 if (mem_size == 0) 904 return -EINVAL; 905 906 e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); 907 908 #ifdef CONFIG_MEMORY_HOTPLUG 909 max_mem_size = mem_size; 910 #endif 911 912 return 0; 913 } 914 early_param("mem", parse_memopt); 915 916 __init static int parse_memmap_one(char *p) 917 { 918 char *oldp; 919 u64 start_at, mem_size; 920 921 if (!p) 922 return -EINVAL; 923 924 if (!strncmp(p, "exactmap", 8)) { 925 e820_table->nr_entries = 0; 926 userdef = 1; 927 return 0; 928 } 929 930 oldp = p; 931 mem_size = memparse(p, &p); 932 if (p == oldp) 933 return -EINVAL; 934 935 userdef = 1; 936 if (*p == '@') { 937 start_at = memparse(p+1, &p); 938 e820__range_add(start_at, mem_size, E820_TYPE_RAM); 939 } else if (*p == '#') { 940 start_at = memparse(p+1, &p); 941 e820__range_add(start_at, mem_size, E820_TYPE_ACPI); 942 } else if (*p == '$') { 943 start_at = memparse(p+1, &p); 944 e820__range_add(start_at, mem_size, E820_TYPE_RESERVED); 945 } else if (*p == '!') { 946 start_at = memparse(p+1, &p); 947 e820__range_add(start_at, mem_size, E820_TYPE_PRAM); 948 } else if (*p == '%') { 949 enum e820_type from = 0, to = 0; 950 951 start_at = memparse(p + 1, &p); 952 if (*p == '-') 953 from = simple_strtoull(p + 1, &p, 0); 954 if (*p == '+') 955 to = simple_strtoull(p + 1, &p, 0); 956 if (*p != '\0') 957 return -EINVAL; 958 if (from && to) 959 e820__range_update(start_at, mem_size, from, to); 960 else if (to) 961 e820__range_add(start_at, mem_size, to); 962 else if (from) 963 e820__range_remove(start_at, mem_size, from, 1); 964 else 965 e820__range_remove(start_at, mem_size, 0, 0); 966 } else { 967 e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); 968 } 969 970 return *p == '\0' ? 0 : -EINVAL; 971 } 972 973 __init static int parse_memmap_opt(char *str) 974 { 975 while (str) { 976 char *k = strchr(str, ','); 977 978 if (k) 979 *k++ = 0; 980 981 parse_memmap_one(str); 982 str = k; 983 } 984 985 return 0; 986 } 987 early_param("memmap", parse_memmap_opt); 988 989 /* 990 * Called after parse_early_param(), after early parameters (such as mem=) 991 * have been processed, in which case we already have an E820 table filled in 992 * via the parameter callback function(s), but it's not sorted and printed yet: 993 */ 994 __init void e820__finish_early_params(void) 995 { 996 if (userdef) { 997 if (e820__update_table(e820_table) < 0) 998 panic("Invalid user supplied memory map"); 999 1000 pr_info("user-defined physical RAM map:\n"); 1001 e820__print_table("user"); 1002 } 1003 } 1004 1005 __init static const char * e820_type_to_string(struct e820_entry *entry) 1006 { 1007 switch (entry->type) { 1008 case E820_TYPE_RAM: return "System RAM"; 1009 case E820_TYPE_ACPI: return "ACPI Tables"; 1010 case E820_TYPE_NVS: return "ACPI Non-volatile Storage"; 1011 case E820_TYPE_UNUSABLE: return "Unusable memory"; 1012 case E820_TYPE_PRAM: return "Persistent Memory (legacy)"; 1013 case E820_TYPE_PMEM: return "Persistent Memory"; 1014 case E820_TYPE_RESERVED: return "Reserved"; 1015 case E820_TYPE_SOFT_RESERVED: return "Soft Reserved"; 1016 default: return "Unknown E820 type"; 1017 } 1018 } 1019 1020 __init static unsigned long e820_type_to_iomem_type(struct e820_entry *entry) 1021 { 1022 switch (entry->type) { 1023 case E820_TYPE_RAM: return IORESOURCE_SYSTEM_RAM; 1024 case E820_TYPE_ACPI: /* Fall-through: */ 1025 case E820_TYPE_NVS: /* Fall-through: */ 1026 case E820_TYPE_UNUSABLE: /* Fall-through: */ 1027 case E820_TYPE_PRAM: /* Fall-through: */ 1028 case E820_TYPE_PMEM: /* Fall-through: */ 1029 case E820_TYPE_RESERVED: /* Fall-through: */ 1030 case E820_TYPE_SOFT_RESERVED: /* Fall-through: */ 1031 default: return IORESOURCE_MEM; 1032 } 1033 } 1034 1035 __init static unsigned long e820_type_to_iores_desc(struct e820_entry *entry) 1036 { 1037 switch (entry->type) { 1038 case E820_TYPE_ACPI: return IORES_DESC_ACPI_TABLES; 1039 case E820_TYPE_NVS: return IORES_DESC_ACPI_NV_STORAGE; 1040 case E820_TYPE_PMEM: return IORES_DESC_PERSISTENT_MEMORY; 1041 case E820_TYPE_PRAM: return IORES_DESC_PERSISTENT_MEMORY_LEGACY; 1042 case E820_TYPE_RESERVED: return IORES_DESC_RESERVED; 1043 case E820_TYPE_SOFT_RESERVED: return IORES_DESC_SOFT_RESERVED; 1044 case E820_TYPE_RAM: /* Fall-through: */ 1045 case E820_TYPE_UNUSABLE: /* Fall-through: */ 1046 default: return IORES_DESC_NONE; 1047 } 1048 } 1049 1050 /* 1051 * We assign one resource entry for each E820 map entry: 1052 */ 1053 __initdata static struct resource *e820_res; 1054 1055 /* 1056 * Is this a device address region that should not be marked busy? 1057 * (Versus system address regions that we register & lock early.) 1058 */ 1059 __init static bool e820_device_region(enum e820_type type, struct resource *res) 1060 { 1061 /* This is the legacy BIOS/DOS ROM-shadow + MMIO region: */ 1062 if (res->start < (1ULL<<20)) 1063 return false; 1064 1065 /* 1066 * Treat persistent memory and other special memory ranges like 1067 * device memory, i.e. keep it available for exclusive use of a 1068 * driver: 1069 */ 1070 switch (type) { 1071 case E820_TYPE_RESERVED: 1072 case E820_TYPE_SOFT_RESERVED: 1073 case E820_TYPE_PRAM: 1074 case E820_TYPE_PMEM: 1075 return true; 1076 case E820_TYPE_RAM: 1077 case E820_TYPE_ACPI: 1078 case E820_TYPE_NVS: 1079 case E820_TYPE_UNUSABLE: 1080 default: 1081 return false; 1082 } 1083 } 1084 1085 /* 1086 * Mark E820 system regions as busy for the resource manager: 1087 */ 1088 __init void e820__reserve_resources(void) 1089 { 1090 u32 idx; 1091 struct resource *res; 1092 u64 end; 1093 1094 res = memblock_alloc_or_panic(sizeof(*res) * e820_table->nr_entries, 1095 SMP_CACHE_BYTES); 1096 e820_res = res; 1097 1098 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1099 struct e820_entry *entry = e820_table->entries + idx; 1100 1101 end = entry->addr + entry->size - 1; 1102 if (end != (resource_size_t)end) { 1103 res++; 1104 continue; 1105 } 1106 res->start = entry->addr; 1107 res->end = end; 1108 res->name = e820_type_to_string(entry); 1109 res->flags = e820_type_to_iomem_type(entry); 1110 res->desc = e820_type_to_iores_desc(entry); 1111 1112 /* 1113 * Skip and don't register device regions that could be conflicted 1114 * with PCI device BAR resources. They get inserted later in 1115 * pcibios_resource_survey() -> e820__reserve_resources_late(): 1116 */ 1117 if (!e820_device_region(entry->type, res)) { 1118 res->flags |= IORESOURCE_BUSY; 1119 insert_resource(&iomem_resource, res); 1120 } 1121 res++; 1122 } 1123 1124 /* Expose the kexec e820 table to sysfs: */ 1125 for (idx = 0; idx < e820_table_kexec->nr_entries; idx++) { 1126 struct e820_entry *entry = e820_table_kexec->entries + idx; 1127 1128 firmware_map_add_early(entry->addr, entry->addr + entry->size, e820_type_to_string(entry)); 1129 } 1130 } 1131 1132 /* 1133 * How much should we pad the end of RAM, depending on where it is? 1134 */ 1135 __init static unsigned long ram_alignment(resource_size_t pos) 1136 { 1137 unsigned long mb = pos >> 20; 1138 1139 /* To 64kB in the first megabyte */ 1140 if (!mb) 1141 return 64*1024; 1142 1143 /* To 1MB in the first 16MB */ 1144 if (mb < 16) 1145 return 1024*1024; 1146 1147 /* To 64MB for anything above that */ 1148 return 64*1024*1024; 1149 } 1150 1151 #define MAX_RESOURCE_SIZE ((resource_size_t)-1) 1152 1153 __init void e820__reserve_resources_late(void) 1154 { 1155 u32 idx; 1156 struct resource *res; 1157 1158 /* 1159 * Register device address regions listed in the E820 map, 1160 * these can be claimed by device drivers later on: 1161 */ 1162 res = e820_res; 1163 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1164 if (!res->parent && res->end) 1165 insert_resource_expand_to_fit(&iomem_resource, res); 1166 res++; 1167 } 1168 1169 /* 1170 * Create additional 'gaps' at the end of RAM regions, 1171 * rounding them up to 64k/1MB/64MB boundaries, should 1172 * they be weirdly sized, and register extra, locked 1173 * resource regions for them, to make sure drivers 1174 * won't claim those addresses. 1175 * 1176 * These are basically blind guesses and heuristics to 1177 * avoid resource conflicts with broken firmware that 1178 * doesn't properly list 'stolen RAM' as a system region 1179 * in the E820 map. 1180 */ 1181 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1182 struct e820_entry *entry = &e820_table->entries[idx]; 1183 u64 start, end; 1184 1185 if (entry->type != E820_TYPE_RAM) 1186 continue; 1187 1188 start = entry->addr + entry->size; 1189 end = round_up(start, ram_alignment(start)) - 1; 1190 if (end > MAX_RESOURCE_SIZE) 1191 end = MAX_RESOURCE_SIZE; 1192 if (start >= end) 1193 continue; 1194 1195 pr_info("e820: register RAM buffer resource [mem %#010llx-%#010llx]\n", start, end); 1196 reserve_region_with_split(&iomem_resource, start, end, "RAM buffer"); 1197 } 1198 } 1199 1200 /* 1201 * Pass the firmware (bootloader) E820 map to the kernel and process it: 1202 */ 1203 __init char * e820__memory_setup_default(void) 1204 { 1205 char *who = "BIOS-e820"; 1206 1207 /* 1208 * Try to copy the BIOS-supplied E820-map. 1209 * 1210 * Otherwise fake a memory map; one section from 0k->640k, 1211 * the next section from 1mb->appropriate_mem_k 1212 */ 1213 if (append_e820_table(boot_params.e820_table, boot_params.e820_entries) < 0) { 1214 u64 mem_size; 1215 1216 /* Compare results from other methods and take the one that gives more RAM: */ 1217 if (boot_params.alt_mem_k < boot_params.screen_info.ext_mem_k) { 1218 mem_size = boot_params.screen_info.ext_mem_k; 1219 who = "BIOS-88"; 1220 } else { 1221 mem_size = boot_params.alt_mem_k; 1222 who = "BIOS-e801"; 1223 } 1224 1225 e820_table->nr_entries = 0; 1226 e820__range_add(0, LOWMEMSIZE(), E820_TYPE_RAM); 1227 e820__range_add(HIGH_MEMORY, mem_size << 10, E820_TYPE_RAM); 1228 } 1229 1230 /* We just appended a lot of ranges, sanitize the table: */ 1231 e820__update_table(e820_table); 1232 1233 return who; 1234 } 1235 1236 /* 1237 * Calls e820__memory_setup_default() in essence to pick up the firmware/bootloader 1238 * E820 map - with an optional platform quirk available for virtual platforms 1239 * to override this method of boot environment processing: 1240 */ 1241 __init void e820__memory_setup(void) 1242 { 1243 char *who; 1244 1245 /* This is a firmware interface ABI - make sure we don't break it: */ 1246 BUILD_BUG_ON(sizeof(struct boot_e820_entry) != 20); 1247 1248 who = x86_init.resources.memory_setup(); 1249 1250 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); 1251 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); 1252 1253 pr_info("BIOS-provided physical RAM map:\n"); 1254 e820__print_table(who); 1255 } 1256 1257 __init void e820__memblock_setup(void) 1258 { 1259 u32 idx; 1260 u64 end; 1261 1262 #ifdef CONFIG_MEMORY_HOTPLUG 1263 /* 1264 * Memory used by the kernel cannot be hot-removed because Linux 1265 * cannot migrate the kernel pages. When memory hotplug is 1266 * enabled, we should prevent memblock from allocating memory 1267 * for the kernel. 1268 * 1269 * ACPI SRAT records all hotpluggable memory ranges. But before 1270 * SRAT is parsed, we don't know about it. 1271 * 1272 * The kernel image is loaded into memory at very early time. We 1273 * cannot prevent this anyway. So on NUMA system, we set any 1274 * node the kernel resides in as un-hotpluggable. 1275 * 1276 * Since on modern servers, one node could have double-digit 1277 * gigabytes memory, we can assume the memory around the kernel 1278 * image is also un-hotpluggable. So before SRAT is parsed, just 1279 * allocate memory near the kernel image to try the best to keep 1280 * the kernel away from hotpluggable memory. 1281 */ 1282 if (movable_node_is_enabled()) 1283 memblock_set_bottom_up(true); 1284 #endif 1285 1286 /* 1287 * At this point only the first megabyte is mapped for sure, the 1288 * rest of the memory cannot be used for memblock resizing 1289 */ 1290 memblock_set_current_limit(ISA_END_ADDRESS); 1291 1292 /* 1293 * The bootstrap memblock region count maximum is 128 entries 1294 * (INIT_MEMBLOCK_REGIONS), but EFI might pass us more E820 entries 1295 * than that - so allow memblock resizing. 1296 * 1297 * This is safe, because this call happens pretty late during x86 setup, 1298 * so we know about reserved memory regions already. (This is important 1299 * so that memblock resizing does no stomp over reserved areas.) 1300 */ 1301 memblock_allow_resize(); 1302 1303 for (idx = 0; idx < e820_table->nr_entries; idx++) { 1304 struct e820_entry *entry = &e820_table->entries[idx]; 1305 1306 end = entry->addr + entry->size; 1307 if (end != (resource_size_t)end) 1308 continue; 1309 1310 if (entry->type == E820_TYPE_SOFT_RESERVED) 1311 memblock_reserve(entry->addr, entry->size); 1312 1313 if (entry->type != E820_TYPE_RAM) 1314 continue; 1315 1316 memblock_add(entry->addr, entry->size); 1317 } 1318 1319 /* 1320 * At this point memblock is only allowed to allocate from memory 1321 * below 1M (aka ISA_END_ADDRESS) up until direct map is completely set 1322 * up in init_mem_mapping(). 1323 * 1324 * KHO kernels are special and use only scratch memory for memblock 1325 * allocations, but memory below 1M is ignored by kernel after early 1326 * boot and cannot be naturally marked as scratch. 1327 * 1328 * To allow allocation of the real-mode trampoline and a few (if any) 1329 * other very early allocations from below 1M forcibly mark the memory 1330 * below 1M as scratch. 1331 * 1332 * After real mode trampoline is allocated, we clear that scratch 1333 * marking. 1334 */ 1335 memblock_mark_kho_scratch(0, SZ_1M); 1336 1337 /* 1338 * 32-bit systems are limited to 4BG of memory even with HIGHMEM and 1339 * to even less without it. 1340 * Discard memory after max_pfn - the actual limit detected at runtime. 1341 */ 1342 if (IS_ENABLED(CONFIG_X86_32)) 1343 memblock_remove(PFN_PHYS(max_pfn), -1); 1344 1345 /* Throw away partial pages: */ 1346 memblock_trim_memory(PAGE_SIZE); 1347 1348 memblock_dump_all(); 1349 } 1350