1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Machine specific setup for xen 4 * 5 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 6 */ 7 8 #include <linux/init.h> 9 #include <linux/sched.h> 10 #include <linux/mm.h> 11 #include <linux/pm.h> 12 #include <linux/memblock.h> 13 #include <linux/cpuidle.h> 14 #include <linux/cpufreq.h> 15 #include <linux/memory_hotplug.h> 16 17 #include <asm/elf.h> 18 #include <asm/vdso.h> 19 #include <asm/e820/api.h> 20 #include <asm/setup.h> 21 #include <asm/acpi.h> 22 #include <asm/numa.h> 23 #include <asm/idtentry.h> 24 #include <asm/xen/hypervisor.h> 25 #include <asm/xen/hypercall.h> 26 27 #include <xen/xen.h> 28 #include <xen/page.h> 29 #include <xen/interface/callback.h> 30 #include <xen/interface/memory.h> 31 #include <xen/interface/physdev.h> 32 #include <xen/features.h> 33 #include <xen/hvc-console.h> 34 #include "xen-ops.h" 35 #include "mmu.h" 36 37 #define GB(x) ((uint64_t)(x) * 1024 * 1024 * 1024) 38 39 /* Amount of extra memory space we add to the e820 ranges */ 40 struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; 41 42 /* Number of pages released from the initial allocation. */ 43 unsigned long xen_released_pages; 44 45 /* E820 map used during setting up memory. */ 46 static struct e820_table xen_e820_table __initdata; 47 48 /* 49 * Buffer used to remap identity mapped pages. We only need the virtual space. 50 * The physical page behind this address is remapped as needed to different 51 * buffer pages. 52 */ 53 #define REMAP_SIZE (P2M_PER_PAGE - 3) 54 static struct { 55 unsigned long next_area_mfn; 56 unsigned long target_pfn; 57 unsigned long size; 58 unsigned long mfns[REMAP_SIZE]; 59 } xen_remap_buf __initdata __aligned(PAGE_SIZE); 60 static unsigned long xen_remap_mfn __initdata = INVALID_P2M_ENTRY; 61 62 /* 63 * The maximum amount of extra memory compared to the base size. The 64 * main scaling factor is the size of struct page. At extreme ratios 65 * of base:extra, all the base memory can be filled with page 66 * structures for the extra memory, leaving no space for anything 67 * else. 68 * 69 * 10x seems like a reasonable balance between scaling flexibility and 70 * leaving a practically usable system. 71 */ 72 #define EXTRA_MEM_RATIO (10) 73 74 static bool xen_512gb_limit __initdata = IS_ENABLED(CONFIG_XEN_512GB); 75 76 static void __init xen_parse_512gb(void) 77 { 78 bool val = false; 79 char *arg; 80 81 arg = strstr(xen_start_info->cmd_line, "xen_512gb_limit"); 82 if (!arg) 83 return; 84 85 arg = strstr(xen_start_info->cmd_line, "xen_512gb_limit="); 86 if (!arg) 87 val = true; 88 else if (strtobool(arg + strlen("xen_512gb_limit="), &val)) 89 return; 90 91 xen_512gb_limit = val; 92 } 93 94 static void __init xen_add_extra_mem(unsigned long start_pfn, 95 unsigned long n_pfns) 96 { 97 int i; 98 99 /* 100 * No need to check for zero size, should happen rarely and will only 101 * write a new entry regarded to be unused due to zero size. 102 */ 103 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { 104 /* Add new region. */ 105 if (xen_extra_mem[i].n_pfns == 0) { 106 xen_extra_mem[i].start_pfn = start_pfn; 107 xen_extra_mem[i].n_pfns = n_pfns; 108 break; 109 } 110 /* Append to existing region. */ 111 if (xen_extra_mem[i].start_pfn + xen_extra_mem[i].n_pfns == 112 start_pfn) { 113 xen_extra_mem[i].n_pfns += n_pfns; 114 break; 115 } 116 } 117 if (i == XEN_EXTRA_MEM_MAX_REGIONS) 118 printk(KERN_WARNING "Warning: not enough extra memory regions\n"); 119 120 memblock_reserve(PFN_PHYS(start_pfn), PFN_PHYS(n_pfns)); 121 } 122 123 static void __init xen_del_extra_mem(unsigned long start_pfn, 124 unsigned long n_pfns) 125 { 126 int i; 127 unsigned long start_r, size_r; 128 129 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { 130 start_r = xen_extra_mem[i].start_pfn; 131 size_r = xen_extra_mem[i].n_pfns; 132 133 /* Start of region. */ 134 if (start_r == start_pfn) { 135 BUG_ON(n_pfns > size_r); 136 xen_extra_mem[i].start_pfn += n_pfns; 137 xen_extra_mem[i].n_pfns -= n_pfns; 138 break; 139 } 140 /* End of region. */ 141 if (start_r + size_r == start_pfn + n_pfns) { 142 BUG_ON(n_pfns > size_r); 143 xen_extra_mem[i].n_pfns -= n_pfns; 144 break; 145 } 146 /* Mid of region. */ 147 if (start_pfn > start_r && start_pfn < start_r + size_r) { 148 BUG_ON(start_pfn + n_pfns > start_r + size_r); 149 xen_extra_mem[i].n_pfns = start_pfn - start_r; 150 /* Calling memblock_reserve() again is okay. */ 151 xen_add_extra_mem(start_pfn + n_pfns, start_r + size_r - 152 (start_pfn + n_pfns)); 153 break; 154 } 155 } 156 memblock_phys_free(PFN_PHYS(start_pfn), PFN_PHYS(n_pfns)); 157 } 158 159 /* 160 * Called during boot before the p2m list can take entries beyond the 161 * hypervisor supplied p2m list. Entries in extra mem are to be regarded as 162 * invalid. 163 */ 164 unsigned long __ref xen_chk_extra_mem(unsigned long pfn) 165 { 166 int i; 167 168 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { 169 if (pfn >= xen_extra_mem[i].start_pfn && 170 pfn < xen_extra_mem[i].start_pfn + xen_extra_mem[i].n_pfns) 171 return INVALID_P2M_ENTRY; 172 } 173 174 return IDENTITY_FRAME(pfn); 175 } 176 177 /* 178 * Mark all pfns of extra mem as invalid in p2m list. 179 */ 180 void __init xen_inv_extra_mem(void) 181 { 182 unsigned long pfn, pfn_s, pfn_e; 183 int i; 184 185 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { 186 if (!xen_extra_mem[i].n_pfns) 187 continue; 188 pfn_s = xen_extra_mem[i].start_pfn; 189 pfn_e = pfn_s + xen_extra_mem[i].n_pfns; 190 for (pfn = pfn_s; pfn < pfn_e; pfn++) 191 set_phys_to_machine(pfn, INVALID_P2M_ENTRY); 192 } 193 } 194 195 /* 196 * Finds the next RAM pfn available in the E820 map after min_pfn. 197 * This function updates min_pfn with the pfn found and returns 198 * the size of that range or zero if not found. 199 */ 200 static unsigned long __init xen_find_pfn_range(unsigned long *min_pfn) 201 { 202 const struct e820_entry *entry = xen_e820_table.entries; 203 unsigned int i; 204 unsigned long done = 0; 205 206 for (i = 0; i < xen_e820_table.nr_entries; i++, entry++) { 207 unsigned long s_pfn; 208 unsigned long e_pfn; 209 210 if (entry->type != E820_TYPE_RAM) 211 continue; 212 213 e_pfn = PFN_DOWN(entry->addr + entry->size); 214 215 /* We only care about E820 after this */ 216 if (e_pfn <= *min_pfn) 217 continue; 218 219 s_pfn = PFN_UP(entry->addr); 220 221 /* If min_pfn falls within the E820 entry, we want to start 222 * at the min_pfn PFN. 223 */ 224 if (s_pfn <= *min_pfn) { 225 done = e_pfn - *min_pfn; 226 } else { 227 done = e_pfn - s_pfn; 228 *min_pfn = s_pfn; 229 } 230 break; 231 } 232 233 return done; 234 } 235 236 static int __init xen_free_mfn(unsigned long mfn) 237 { 238 struct xen_memory_reservation reservation = { 239 .address_bits = 0, 240 .extent_order = 0, 241 .domid = DOMID_SELF 242 }; 243 244 set_xen_guest_handle(reservation.extent_start, &mfn); 245 reservation.nr_extents = 1; 246 247 return HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation); 248 } 249 250 /* 251 * This releases a chunk of memory and then does the identity map. It's used 252 * as a fallback if the remapping fails. 253 */ 254 static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn, 255 unsigned long end_pfn, unsigned long nr_pages) 256 { 257 unsigned long pfn, end; 258 int ret; 259 260 WARN_ON(start_pfn > end_pfn); 261 262 /* Release pages first. */ 263 end = min(end_pfn, nr_pages); 264 for (pfn = start_pfn; pfn < end; pfn++) { 265 unsigned long mfn = pfn_to_mfn(pfn); 266 267 /* Make sure pfn exists to start with */ 268 if (mfn == INVALID_P2M_ENTRY || mfn_to_pfn(mfn) != pfn) 269 continue; 270 271 ret = xen_free_mfn(mfn); 272 WARN(ret != 1, "Failed to release pfn %lx err=%d\n", pfn, ret); 273 274 if (ret == 1) { 275 xen_released_pages++; 276 if (!__set_phys_to_machine(pfn, INVALID_P2M_ENTRY)) 277 break; 278 } else 279 break; 280 } 281 282 set_phys_range_identity(start_pfn, end_pfn); 283 } 284 285 /* 286 * Helper function to update the p2m and m2p tables and kernel mapping. 287 */ 288 static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn) 289 { 290 struct mmu_update update = { 291 .ptr = ((uint64_t)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE, 292 .val = pfn 293 }; 294 295 /* Update p2m */ 296 if (!set_phys_to_machine(pfn, mfn)) { 297 WARN(1, "Failed to set p2m mapping for pfn=%ld mfn=%ld\n", 298 pfn, mfn); 299 BUG(); 300 } 301 302 /* Update m2p */ 303 if (HYPERVISOR_mmu_update(&update, 1, NULL, DOMID_SELF) < 0) { 304 WARN(1, "Failed to set m2p mapping for mfn=%ld pfn=%ld\n", 305 mfn, pfn); 306 BUG(); 307 } 308 309 if (HYPERVISOR_update_va_mapping((unsigned long)__va(pfn << PAGE_SHIFT), 310 mfn_pte(mfn, PAGE_KERNEL), 0)) { 311 WARN(1, "Failed to update kernel mapping for mfn=%ld pfn=%ld\n", 312 mfn, pfn); 313 BUG(); 314 } 315 } 316 317 /* 318 * This function updates the p2m and m2p tables with an identity map from 319 * start_pfn to start_pfn+size and prepares remapping the underlying RAM of the 320 * original allocation at remap_pfn. The information needed for remapping is 321 * saved in the memory itself to avoid the need for allocating buffers. The 322 * complete remap information is contained in a list of MFNs each containing 323 * up to REMAP_SIZE MFNs and the start target PFN for doing the remap. 324 * This enables us to preserve the original mfn sequence while doing the 325 * remapping at a time when the memory management is capable of allocating 326 * virtual and physical memory in arbitrary amounts, see 'xen_remap_memory' and 327 * its callers. 328 */ 329 static void __init xen_do_set_identity_and_remap_chunk( 330 unsigned long start_pfn, unsigned long size, unsigned long remap_pfn) 331 { 332 unsigned long buf = (unsigned long)&xen_remap_buf; 333 unsigned long mfn_save, mfn; 334 unsigned long ident_pfn_iter, remap_pfn_iter; 335 unsigned long ident_end_pfn = start_pfn + size; 336 unsigned long left = size; 337 unsigned int i, chunk; 338 339 WARN_ON(size == 0); 340 341 mfn_save = virt_to_mfn(buf); 342 343 for (ident_pfn_iter = start_pfn, remap_pfn_iter = remap_pfn; 344 ident_pfn_iter < ident_end_pfn; 345 ident_pfn_iter += REMAP_SIZE, remap_pfn_iter += REMAP_SIZE) { 346 chunk = (left < REMAP_SIZE) ? left : REMAP_SIZE; 347 348 /* Map first pfn to xen_remap_buf */ 349 mfn = pfn_to_mfn(ident_pfn_iter); 350 set_pte_mfn(buf, mfn, PAGE_KERNEL); 351 352 /* Save mapping information in page */ 353 xen_remap_buf.next_area_mfn = xen_remap_mfn; 354 xen_remap_buf.target_pfn = remap_pfn_iter; 355 xen_remap_buf.size = chunk; 356 for (i = 0; i < chunk; i++) 357 xen_remap_buf.mfns[i] = pfn_to_mfn(ident_pfn_iter + i); 358 359 /* Put remap buf into list. */ 360 xen_remap_mfn = mfn; 361 362 /* Set identity map */ 363 set_phys_range_identity(ident_pfn_iter, ident_pfn_iter + chunk); 364 365 left -= chunk; 366 } 367 368 /* Restore old xen_remap_buf mapping */ 369 set_pte_mfn(buf, mfn_save, PAGE_KERNEL); 370 } 371 372 /* 373 * This function takes a contiguous pfn range that needs to be identity mapped 374 * and: 375 * 376 * 1) Finds a new range of pfns to use to remap based on E820 and remap_pfn. 377 * 2) Calls the do_ function to actually do the mapping/remapping work. 378 * 379 * The goal is to not allocate additional memory but to remap the existing 380 * pages. In the case of an error the underlying memory is simply released back 381 * to Xen and not remapped. 382 */ 383 static unsigned long __init xen_set_identity_and_remap_chunk( 384 unsigned long start_pfn, unsigned long end_pfn, unsigned long nr_pages, 385 unsigned long remap_pfn) 386 { 387 unsigned long pfn; 388 unsigned long i = 0; 389 unsigned long n = end_pfn - start_pfn; 390 391 if (remap_pfn == 0) 392 remap_pfn = nr_pages; 393 394 while (i < n) { 395 unsigned long cur_pfn = start_pfn + i; 396 unsigned long left = n - i; 397 unsigned long size = left; 398 unsigned long remap_range_size; 399 400 /* Do not remap pages beyond the current allocation */ 401 if (cur_pfn >= nr_pages) { 402 /* Identity map remaining pages */ 403 set_phys_range_identity(cur_pfn, cur_pfn + size); 404 break; 405 } 406 if (cur_pfn + size > nr_pages) 407 size = nr_pages - cur_pfn; 408 409 remap_range_size = xen_find_pfn_range(&remap_pfn); 410 if (!remap_range_size) { 411 pr_warn("Unable to find available pfn range, not remapping identity pages\n"); 412 xen_set_identity_and_release_chunk(cur_pfn, 413 cur_pfn + left, nr_pages); 414 break; 415 } 416 /* Adjust size to fit in current e820 RAM region */ 417 if (size > remap_range_size) 418 size = remap_range_size; 419 420 xen_do_set_identity_and_remap_chunk(cur_pfn, size, remap_pfn); 421 422 /* Update variables to reflect new mappings. */ 423 i += size; 424 remap_pfn += size; 425 } 426 427 /* 428 * If the PFNs are currently mapped, their VA mappings need to be 429 * zapped. 430 */ 431 for (pfn = start_pfn; pfn <= max_pfn_mapped && pfn < end_pfn; pfn++) 432 (void)HYPERVISOR_update_va_mapping( 433 (unsigned long)__va(pfn << PAGE_SHIFT), 434 native_make_pte(0), 0); 435 436 return remap_pfn; 437 } 438 439 static unsigned long __init xen_count_remap_pages( 440 unsigned long start_pfn, unsigned long end_pfn, unsigned long nr_pages, 441 unsigned long remap_pages) 442 { 443 if (start_pfn >= nr_pages) 444 return remap_pages; 445 446 return remap_pages + min(end_pfn, nr_pages) - start_pfn; 447 } 448 449 static unsigned long __init xen_foreach_remap_area(unsigned long nr_pages, 450 unsigned long (*func)(unsigned long start_pfn, unsigned long end_pfn, 451 unsigned long nr_pages, unsigned long last_val)) 452 { 453 phys_addr_t start = 0; 454 unsigned long ret_val = 0; 455 const struct e820_entry *entry = xen_e820_table.entries; 456 int i; 457 458 /* 459 * Combine non-RAM regions and gaps until a RAM region (or the 460 * end of the map) is reached, then call the provided function 461 * to perform its duty on the non-RAM region. 462 * 463 * The combined non-RAM regions are rounded to a whole number 464 * of pages so any partial pages are accessible via the 1:1 465 * mapping. This is needed for some BIOSes that put (for 466 * example) the DMI tables in a reserved region that begins on 467 * a non-page boundary. 468 */ 469 for (i = 0; i < xen_e820_table.nr_entries; i++, entry++) { 470 phys_addr_t end = entry->addr + entry->size; 471 if (entry->type == E820_TYPE_RAM || i == xen_e820_table.nr_entries - 1) { 472 unsigned long start_pfn = PFN_DOWN(start); 473 unsigned long end_pfn = PFN_UP(end); 474 475 if (entry->type == E820_TYPE_RAM) 476 end_pfn = PFN_UP(entry->addr); 477 478 if (start_pfn < end_pfn) 479 ret_val = func(start_pfn, end_pfn, nr_pages, 480 ret_val); 481 start = end; 482 } 483 } 484 485 return ret_val; 486 } 487 488 /* 489 * Remap the memory prepared in xen_do_set_identity_and_remap_chunk(). 490 * The remap information (which mfn remap to which pfn) is contained in the 491 * to be remapped memory itself in a linked list anchored at xen_remap_mfn. 492 * This scheme allows to remap the different chunks in arbitrary order while 493 * the resulting mapping will be independent from the order. 494 */ 495 void __init xen_remap_memory(void) 496 { 497 unsigned long buf = (unsigned long)&xen_remap_buf; 498 unsigned long mfn_save, pfn; 499 unsigned long remapped = 0; 500 unsigned int i; 501 unsigned long pfn_s = ~0UL; 502 unsigned long len = 0; 503 504 mfn_save = virt_to_mfn(buf); 505 506 while (xen_remap_mfn != INVALID_P2M_ENTRY) { 507 /* Map the remap information */ 508 set_pte_mfn(buf, xen_remap_mfn, PAGE_KERNEL); 509 510 BUG_ON(xen_remap_mfn != xen_remap_buf.mfns[0]); 511 512 pfn = xen_remap_buf.target_pfn; 513 for (i = 0; i < xen_remap_buf.size; i++) { 514 xen_update_mem_tables(pfn, xen_remap_buf.mfns[i]); 515 remapped++; 516 pfn++; 517 } 518 if (pfn_s == ~0UL || pfn == pfn_s) { 519 pfn_s = xen_remap_buf.target_pfn; 520 len += xen_remap_buf.size; 521 } else if (pfn_s + len == xen_remap_buf.target_pfn) { 522 len += xen_remap_buf.size; 523 } else { 524 xen_del_extra_mem(pfn_s, len); 525 pfn_s = xen_remap_buf.target_pfn; 526 len = xen_remap_buf.size; 527 } 528 xen_remap_mfn = xen_remap_buf.next_area_mfn; 529 } 530 531 if (pfn_s != ~0UL && len) 532 xen_del_extra_mem(pfn_s, len); 533 534 set_pte_mfn(buf, mfn_save, PAGE_KERNEL); 535 536 pr_info("Remapped %ld page(s)\n", remapped); 537 } 538 539 static unsigned long __init xen_get_pages_limit(void) 540 { 541 unsigned long limit; 542 543 limit = MAXMEM / PAGE_SIZE; 544 if (!xen_initial_domain() && xen_512gb_limit) 545 limit = GB(512) / PAGE_SIZE; 546 547 return limit; 548 } 549 550 static unsigned long __init xen_get_max_pages(void) 551 { 552 unsigned long max_pages, limit; 553 domid_t domid = DOMID_SELF; 554 long ret; 555 556 limit = xen_get_pages_limit(); 557 max_pages = limit; 558 559 /* 560 * For the initial domain we use the maximum reservation as 561 * the maximum page. 562 * 563 * For guest domains the current maximum reservation reflects 564 * the current maximum rather than the static maximum. In this 565 * case the e820 map provided to us will cover the static 566 * maximum region. 567 */ 568 if (xen_initial_domain()) { 569 ret = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid); 570 if (ret > 0) 571 max_pages = ret; 572 } 573 574 return min(max_pages, limit); 575 } 576 577 static void __init xen_align_and_add_e820_region(phys_addr_t start, 578 phys_addr_t size, int type) 579 { 580 phys_addr_t end = start + size; 581 582 /* Align RAM regions to page boundaries. */ 583 if (type == E820_TYPE_RAM) { 584 start = PAGE_ALIGN(start); 585 end &= ~((phys_addr_t)PAGE_SIZE - 1); 586 #ifdef CONFIG_MEMORY_HOTPLUG 587 /* 588 * Don't allow adding memory not in E820 map while booting the 589 * system. Once the balloon driver is up it will remove that 590 * restriction again. 591 */ 592 max_mem_size = end; 593 #endif 594 } 595 596 e820__range_add(start, end - start, type); 597 } 598 599 static void __init xen_ignore_unusable(void) 600 { 601 struct e820_entry *entry = xen_e820_table.entries; 602 unsigned int i; 603 604 for (i = 0; i < xen_e820_table.nr_entries; i++, entry++) { 605 if (entry->type == E820_TYPE_UNUSABLE) 606 entry->type = E820_TYPE_RAM; 607 } 608 } 609 610 bool __init xen_is_e820_reserved(phys_addr_t start, phys_addr_t size) 611 { 612 struct e820_entry *entry; 613 unsigned mapcnt; 614 phys_addr_t end; 615 616 if (!size) 617 return false; 618 619 end = start + size; 620 entry = xen_e820_table.entries; 621 622 for (mapcnt = 0; mapcnt < xen_e820_table.nr_entries; mapcnt++) { 623 if (entry->type == E820_TYPE_RAM && entry->addr <= start && 624 (entry->addr + entry->size) >= end) 625 return false; 626 627 entry++; 628 } 629 630 return true; 631 } 632 633 /* 634 * Find a free area in physical memory not yet reserved and compliant with 635 * E820 map. 636 * Used to relocate pre-allocated areas like initrd or p2m list which are in 637 * conflict with the to be used E820 map. 638 * In case no area is found, return 0. Otherwise return the physical address 639 * of the area which is already reserved for convenience. 640 */ 641 phys_addr_t __init xen_find_free_area(phys_addr_t size) 642 { 643 unsigned mapcnt; 644 phys_addr_t addr, start; 645 struct e820_entry *entry = xen_e820_table.entries; 646 647 for (mapcnt = 0; mapcnt < xen_e820_table.nr_entries; mapcnt++, entry++) { 648 if (entry->type != E820_TYPE_RAM || entry->size < size) 649 continue; 650 start = entry->addr; 651 for (addr = start; addr < start + size; addr += PAGE_SIZE) { 652 if (!memblock_is_reserved(addr)) 653 continue; 654 start = addr + PAGE_SIZE; 655 if (start + size > entry->addr + entry->size) 656 break; 657 } 658 if (addr >= start + size) { 659 memblock_reserve(start, size); 660 return start; 661 } 662 } 663 664 return 0; 665 } 666 667 /* 668 * Like memcpy, but with physical addresses for dest and src. 669 */ 670 static void __init xen_phys_memcpy(phys_addr_t dest, phys_addr_t src, 671 phys_addr_t n) 672 { 673 phys_addr_t dest_off, src_off, dest_len, src_len, len; 674 void *from, *to; 675 676 while (n) { 677 dest_off = dest & ~PAGE_MASK; 678 src_off = src & ~PAGE_MASK; 679 dest_len = n; 680 if (dest_len > (NR_FIX_BTMAPS << PAGE_SHIFT) - dest_off) 681 dest_len = (NR_FIX_BTMAPS << PAGE_SHIFT) - dest_off; 682 src_len = n; 683 if (src_len > (NR_FIX_BTMAPS << PAGE_SHIFT) - src_off) 684 src_len = (NR_FIX_BTMAPS << PAGE_SHIFT) - src_off; 685 len = min(dest_len, src_len); 686 to = early_memremap(dest - dest_off, dest_len + dest_off); 687 from = early_memremap(src - src_off, src_len + src_off); 688 memcpy(to, from, len); 689 early_memunmap(to, dest_len + dest_off); 690 early_memunmap(from, src_len + src_off); 691 n -= len; 692 dest += len; 693 src += len; 694 } 695 } 696 697 /* 698 * Reserve Xen mfn_list. 699 */ 700 static void __init xen_reserve_xen_mfnlist(void) 701 { 702 phys_addr_t start, size; 703 704 if (xen_start_info->mfn_list >= __START_KERNEL_map) { 705 start = __pa(xen_start_info->mfn_list); 706 size = PFN_ALIGN(xen_start_info->nr_pages * 707 sizeof(unsigned long)); 708 } else { 709 start = PFN_PHYS(xen_start_info->first_p2m_pfn); 710 size = PFN_PHYS(xen_start_info->nr_p2m_frames); 711 } 712 713 memblock_reserve(start, size); 714 if (!xen_is_e820_reserved(start, size)) 715 return; 716 717 xen_relocate_p2m(); 718 memblock_phys_free(start, size); 719 } 720 721 /** 722 * xen_memory_setup - Hook for machine specific memory setup. 723 **/ 724 char * __init xen_memory_setup(void) 725 { 726 unsigned long max_pfn, pfn_s, n_pfns; 727 phys_addr_t mem_end, addr, size, chunk_size; 728 u32 type; 729 int rc; 730 struct xen_memory_map memmap; 731 unsigned long max_pages; 732 unsigned long extra_pages = 0; 733 int i; 734 int op; 735 736 xen_parse_512gb(); 737 max_pfn = xen_get_pages_limit(); 738 max_pfn = min(max_pfn, xen_start_info->nr_pages); 739 mem_end = PFN_PHYS(max_pfn); 740 741 memmap.nr_entries = ARRAY_SIZE(xen_e820_table.entries); 742 set_xen_guest_handle(memmap.buffer, xen_e820_table.entries); 743 744 #if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_XEN_BALLOON) 745 xen_saved_max_mem_size = max_mem_size; 746 #endif 747 748 op = xen_initial_domain() ? 749 XENMEM_machine_memory_map : 750 XENMEM_memory_map; 751 rc = HYPERVISOR_memory_op(op, &memmap); 752 if (rc == -ENOSYS) { 753 BUG_ON(xen_initial_domain()); 754 memmap.nr_entries = 1; 755 xen_e820_table.entries[0].addr = 0ULL; 756 xen_e820_table.entries[0].size = mem_end; 757 /* 8MB slack (to balance backend allocations). */ 758 xen_e820_table.entries[0].size += 8ULL << 20; 759 xen_e820_table.entries[0].type = E820_TYPE_RAM; 760 rc = 0; 761 } 762 BUG_ON(rc); 763 BUG_ON(memmap.nr_entries == 0); 764 xen_e820_table.nr_entries = memmap.nr_entries; 765 766 /* 767 * Xen won't allow a 1:1 mapping to be created to UNUSABLE 768 * regions, so if we're using the machine memory map leave the 769 * region as RAM as it is in the pseudo-physical map. 770 * 771 * UNUSABLE regions in domUs are not handled and will need 772 * a patch in the future. 773 */ 774 if (xen_initial_domain()) 775 xen_ignore_unusable(); 776 777 /* Make sure the Xen-supplied memory map is well-ordered. */ 778 e820__update_table(&xen_e820_table); 779 780 max_pages = xen_get_max_pages(); 781 782 /* How many extra pages do we need due to remapping? */ 783 max_pages += xen_foreach_remap_area(max_pfn, xen_count_remap_pages); 784 785 if (max_pages > max_pfn) 786 extra_pages += max_pages - max_pfn; 787 788 /* 789 * Clamp the amount of extra memory to a EXTRA_MEM_RATIO 790 * factor the base size. 791 * 792 * Make sure we have no memory above max_pages, as this area 793 * isn't handled by the p2m management. 794 */ 795 extra_pages = min3(EXTRA_MEM_RATIO * min(max_pfn, PFN_DOWN(MAXMEM)), 796 extra_pages, max_pages - max_pfn); 797 i = 0; 798 addr = xen_e820_table.entries[0].addr; 799 size = xen_e820_table.entries[0].size; 800 while (i < xen_e820_table.nr_entries) { 801 bool discard = false; 802 803 chunk_size = size; 804 type = xen_e820_table.entries[i].type; 805 806 if (type == E820_TYPE_RAM) { 807 if (addr < mem_end) { 808 chunk_size = min(size, mem_end - addr); 809 } else if (extra_pages) { 810 chunk_size = min(size, PFN_PHYS(extra_pages)); 811 pfn_s = PFN_UP(addr); 812 n_pfns = PFN_DOWN(addr + chunk_size) - pfn_s; 813 extra_pages -= n_pfns; 814 xen_add_extra_mem(pfn_s, n_pfns); 815 xen_max_p2m_pfn = pfn_s + n_pfns; 816 } else 817 discard = true; 818 } 819 820 if (!discard) 821 xen_align_and_add_e820_region(addr, chunk_size, type); 822 823 addr += chunk_size; 824 size -= chunk_size; 825 if (size == 0) { 826 i++; 827 if (i < xen_e820_table.nr_entries) { 828 addr = xen_e820_table.entries[i].addr; 829 size = xen_e820_table.entries[i].size; 830 } 831 } 832 } 833 834 /* 835 * Set the rest as identity mapped, in case PCI BARs are 836 * located here. 837 */ 838 set_phys_range_identity(addr / PAGE_SIZE, ~0ul); 839 840 /* 841 * In domU, the ISA region is normal, usable memory, but we 842 * reserve ISA memory anyway because too many things poke 843 * about in there. 844 */ 845 e820__range_add(ISA_START_ADDRESS, ISA_END_ADDRESS - ISA_START_ADDRESS, E820_TYPE_RESERVED); 846 847 e820__update_table(e820_table); 848 849 /* 850 * Check whether the kernel itself conflicts with the target E820 map. 851 * Failing now is better than running into weird problems later due 852 * to relocating (and even reusing) pages with kernel text or data. 853 */ 854 if (xen_is_e820_reserved(__pa_symbol(_text), 855 __pa_symbol(__bss_stop) - __pa_symbol(_text))) { 856 xen_raw_console_write("Xen hypervisor allocated kernel memory conflicts with E820 map\n"); 857 BUG(); 858 } 859 860 /* 861 * Check for a conflict of the hypervisor supplied page tables with 862 * the target E820 map. 863 */ 864 xen_pt_check_e820(); 865 866 xen_reserve_xen_mfnlist(); 867 868 /* Check for a conflict of the initrd with the target E820 map. */ 869 if (xen_is_e820_reserved(boot_params.hdr.ramdisk_image, 870 boot_params.hdr.ramdisk_size)) { 871 phys_addr_t new_area, start, size; 872 873 new_area = xen_find_free_area(boot_params.hdr.ramdisk_size); 874 if (!new_area) { 875 xen_raw_console_write("Can't find new memory area for initrd needed due to E820 map conflict\n"); 876 BUG(); 877 } 878 879 start = boot_params.hdr.ramdisk_image; 880 size = boot_params.hdr.ramdisk_size; 881 xen_phys_memcpy(new_area, start, size); 882 pr_info("initrd moved from [mem %#010llx-%#010llx] to [mem %#010llx-%#010llx]\n", 883 start, start + size, new_area, new_area + size); 884 memblock_phys_free(start, size); 885 boot_params.hdr.ramdisk_image = new_area; 886 boot_params.ext_ramdisk_image = new_area >> 32; 887 } 888 889 /* 890 * Set identity map on non-RAM pages and prepare remapping the 891 * underlying RAM. 892 */ 893 xen_foreach_remap_area(max_pfn, xen_set_identity_and_remap_chunk); 894 895 pr_info("Released %ld page(s)\n", xen_released_pages); 896 897 return "Xen"; 898 } 899 900 static int register_callback(unsigned type, const void *func) 901 { 902 struct callback_register callback = { 903 .type = type, 904 .address = XEN_CALLBACK(__KERNEL_CS, func), 905 .flags = CALLBACKF_mask_events, 906 }; 907 908 return HYPERVISOR_callback_op(CALLBACKOP_register, &callback); 909 } 910 911 void xen_enable_sysenter(void) 912 { 913 int ret; 914 unsigned sysenter_feature; 915 916 sysenter_feature = X86_FEATURE_SYSENTER32; 917 918 if (!boot_cpu_has(sysenter_feature)) 919 return; 920 921 ret = register_callback(CALLBACKTYPE_sysenter, xen_entry_SYSENTER_compat); 922 if(ret != 0) 923 setup_clear_cpu_cap(sysenter_feature); 924 } 925 926 void xen_enable_syscall(void) 927 { 928 int ret; 929 930 ret = register_callback(CALLBACKTYPE_syscall, xen_entry_SYSCALL_64); 931 if (ret != 0) { 932 printk(KERN_ERR "Failed to set syscall callback: %d\n", ret); 933 /* Pretty fatal; 64-bit userspace has no other 934 mechanism for syscalls. */ 935 } 936 937 if (boot_cpu_has(X86_FEATURE_SYSCALL32)) { 938 ret = register_callback(CALLBACKTYPE_syscall32, 939 xen_entry_SYSCALL_compat); 940 if (ret != 0) 941 setup_clear_cpu_cap(X86_FEATURE_SYSCALL32); 942 } 943 } 944 945 static void __init xen_pvmmu_arch_setup(void) 946 { 947 HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments); 948 HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_writable_pagetables); 949 950 HYPERVISOR_vm_assist(VMASST_CMD_enable, 951 VMASST_TYPE_pae_extended_cr3); 952 953 if (register_callback(CALLBACKTYPE_event, 954 xen_asm_exc_xen_hypervisor_callback) || 955 register_callback(CALLBACKTYPE_failsafe, xen_failsafe_callback)) 956 BUG(); 957 958 xen_enable_sysenter(); 959 xen_enable_syscall(); 960 } 961 962 /* This function is not called for HVM domains */ 963 void __init xen_arch_setup(void) 964 { 965 xen_panic_handler_init(); 966 xen_pvmmu_arch_setup(); 967 968 #ifdef CONFIG_ACPI 969 if (!(xen_start_info->flags & SIF_INITDOMAIN)) { 970 printk(KERN_INFO "ACPI in unprivileged domain disabled\n"); 971 disable_acpi(); 972 } 973 #endif 974 975 memcpy(boot_command_line, xen_start_info->cmd_line, 976 MAX_GUEST_CMDLINE > COMMAND_LINE_SIZE ? 977 COMMAND_LINE_SIZE : MAX_GUEST_CMDLINE); 978 979 /* Set up idle, making sure it calls safe_halt() pvop */ 980 disable_cpuidle(); 981 disable_cpufreq(); 982 WARN_ON(xen_set_default_idle()); 983 #ifdef CONFIG_NUMA 984 numa_off = 1; 985 #endif 986 } 987