1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Core of Xen paravirt_ops implementation. 4 * 5 * This file contains the xen_paravirt_ops structure itself, and the 6 * implementations for: 7 * - privileged instructions 8 * - interrupt flags 9 * - segment operations 10 * - booting and setup 11 * 12 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 13 */ 14 15 #include <linux/cpu.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/smp.h> 19 #include <linux/preempt.h> 20 #include <linux/hardirq.h> 21 #include <linux/percpu.h> 22 #include <linux/delay.h> 23 #include <linux/start_kernel.h> 24 #include <linux/sched.h> 25 #include <linux/kprobes.h> 26 #include <linux/kstrtox.h> 27 #include <linux/memblock.h> 28 #include <linux/export.h> 29 #include <linux/mm.h> 30 #include <linux/page-flags.h> 31 #include <linux/pci.h> 32 #include <linux/gfp.h> 33 #include <linux/edd.h> 34 #include <linux/reboot.h> 35 #include <linux/virtio_anchor.h> 36 #include <linux/stackprotector.h> 37 38 #include <xen/xen.h> 39 #include <xen/events.h> 40 #include <xen/interface/xen.h> 41 #include <xen/interface/version.h> 42 #include <xen/interface/physdev.h> 43 #include <xen/interface/vcpu.h> 44 #include <xen/interface/memory.h> 45 #include <xen/interface/nmi.h> 46 #include <xen/interface/xen-mca.h> 47 #include <xen/features.h> 48 #include <xen/page.h> 49 #include <xen/hvc-console.h> 50 #include <xen/acpi.h> 51 52 #include <asm/paravirt.h> 53 #include <asm/apic.h> 54 #include <asm/page.h> 55 #include <asm/xen/pci.h> 56 #include <asm/xen/hypercall.h> 57 #include <asm/xen/hypervisor.h> 58 #include <asm/xen/cpuid.h> 59 #include <asm/fixmap.h> 60 #include <asm/processor.h> 61 #include <asm/proto.h> 62 #include <asm/msr-index.h> 63 #include <asm/traps.h> 64 #include <asm/setup.h> 65 #include <asm/desc.h> 66 #include <asm/pgalloc.h> 67 #include <asm/tlbflush.h> 68 #include <asm/reboot.h> 69 #include <asm/hypervisor.h> 70 #include <asm/mach_traps.h> 71 #include <asm/mtrr.h> 72 #include <asm/mwait.h> 73 #include <asm/pci_x86.h> 74 #include <asm/cpu.h> 75 #ifdef CONFIG_X86_IOPL_IOPERM 76 #include <asm/io_bitmap.h> 77 #endif 78 79 #ifdef CONFIG_ACPI 80 #include <linux/acpi.h> 81 #include <asm/acpi.h> 82 #include <acpi/proc_cap_intel.h> 83 #include <acpi/processor.h> 84 #include <xen/interface/platform.h> 85 #endif 86 87 #include "xen-ops.h" 88 #include "mmu.h" 89 #include "smp.h" 90 #include "multicalls.h" 91 #include "pmu.h" 92 93 #include "../kernel/cpu/cpu.h" /* get_cpu_cap() */ 94 95 void *xen_initial_gdt; 96 97 static int xen_cpu_up_prepare_pv(unsigned int cpu); 98 static int xen_cpu_dead_pv(unsigned int cpu); 99 100 struct tls_descs { 101 struct desc_struct desc[3]; 102 }; 103 104 DEFINE_PER_CPU(enum xen_lazy_mode, xen_lazy_mode) = XEN_LAZY_NONE; 105 DEFINE_PER_CPU(unsigned int, xen_lazy_nesting); 106 107 enum xen_lazy_mode xen_get_lazy_mode(void) 108 { 109 if (in_interrupt()) 110 return XEN_LAZY_NONE; 111 112 return this_cpu_read(xen_lazy_mode); 113 } 114 115 /* 116 * Updating the 3 TLS descriptors in the GDT on every task switch is 117 * surprisingly expensive so we avoid updating them if they haven't 118 * changed. Since Xen writes different descriptors than the one 119 * passed in the update_descriptor hypercall we keep shadow copies to 120 * compare against. 121 */ 122 static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc); 123 124 static __read_mostly bool xen_msr_safe = IS_ENABLED(CONFIG_XEN_PV_MSR_SAFE); 125 126 static int __init parse_xen_msr_safe(char *str) 127 { 128 if (str) 129 return kstrtobool(str, &xen_msr_safe); 130 return -EINVAL; 131 } 132 early_param("xen_msr_safe", parse_xen_msr_safe); 133 134 /* Get MTRR settings from Xen and put them into mtrr_state. */ 135 static void __init xen_set_mtrr_data(void) 136 { 137 #ifdef CONFIG_MTRR 138 struct xen_platform_op op = { 139 .cmd = XENPF_read_memtype, 140 .interface_version = XENPF_INTERFACE_VERSION, 141 }; 142 unsigned int reg; 143 unsigned long mask; 144 uint32_t eax, width; 145 static struct mtrr_var_range var[MTRR_MAX_VAR_RANGES] __initdata; 146 147 /* Get physical address width (only 64-bit cpus supported). */ 148 width = 36; 149 eax = cpuid_eax(0x80000000); 150 if ((eax >> 16) == 0x8000 && eax >= 0x80000008) { 151 eax = cpuid_eax(0x80000008); 152 width = eax & 0xff; 153 } 154 155 for (reg = 0; reg < MTRR_MAX_VAR_RANGES; reg++) { 156 op.u.read_memtype.reg = reg; 157 if (HYPERVISOR_platform_op(&op)) 158 break; 159 160 /* 161 * Only called in dom0, which has all RAM PFNs mapped at 162 * RAM MFNs, and all PCI space etc. is identity mapped. 163 * This means we can treat MFN == PFN regarding MTRR settings. 164 */ 165 var[reg].base_lo = op.u.read_memtype.type; 166 var[reg].base_lo |= op.u.read_memtype.mfn << PAGE_SHIFT; 167 var[reg].base_hi = op.u.read_memtype.mfn >> (32 - PAGE_SHIFT); 168 mask = ~((op.u.read_memtype.nr_mfns << PAGE_SHIFT) - 1); 169 mask &= (1UL << width) - 1; 170 if (mask) 171 mask |= MTRR_PHYSMASK_V; 172 var[reg].mask_lo = mask; 173 var[reg].mask_hi = mask >> 32; 174 } 175 176 /* Only overwrite MTRR state if any MTRR could be got from Xen. */ 177 if (reg) 178 mtrr_overwrite_state(var, reg, MTRR_TYPE_UNCACHABLE); 179 #endif 180 } 181 182 static void __init xen_pv_init_platform(void) 183 { 184 /* PV guests can't operate virtio devices without grants. */ 185 if (IS_ENABLED(CONFIG_XEN_VIRTIO)) 186 virtio_set_mem_acc_cb(xen_virtio_restricted_mem_acc); 187 188 populate_extra_pte(fix_to_virt(FIX_PARAVIRT_BOOTMAP)); 189 190 set_fixmap(FIX_PARAVIRT_BOOTMAP, xen_start_info->shared_info); 191 HYPERVISOR_shared_info = (void *)fix_to_virt(FIX_PARAVIRT_BOOTMAP); 192 193 /* xen clock uses per-cpu vcpu_info, need to init it for boot cpu */ 194 xen_vcpu_info_reset(0); 195 196 /* pvclock is in shared info area */ 197 xen_init_time_ops(); 198 199 if (xen_initial_domain()) 200 xen_set_mtrr_data(); 201 else 202 mtrr_overwrite_state(NULL, 0, MTRR_TYPE_WRBACK); 203 204 /* Adjust nr_cpu_ids before "enumeration" happens */ 205 xen_smp_count_cpus(); 206 } 207 208 static void __init xen_pv_guest_late_init(void) 209 { 210 #ifndef CONFIG_SMP 211 /* Setup shared vcpu info for non-smp configurations */ 212 xen_setup_vcpu_info_placement(); 213 #endif 214 } 215 216 static __read_mostly unsigned int cpuid_leaf5_ecx_val; 217 static __read_mostly unsigned int cpuid_leaf5_edx_val; 218 219 static void xen_cpuid(unsigned int *ax, unsigned int *bx, 220 unsigned int *cx, unsigned int *dx) 221 { 222 unsigned int maskebx = ~0; 223 unsigned int or_ebx = 0; 224 225 /* 226 * Mask out inconvenient features, to try and disable as many 227 * unsupported kernel subsystems as possible. 228 */ 229 switch (*ax) { 230 case 0x1: 231 /* Replace initial APIC ID in bits 24-31 of EBX. */ 232 /* See xen_pv_smp_config() for related topology preparations. */ 233 maskebx = 0x00ffffff; 234 or_ebx = smp_processor_id() << 24; 235 break; 236 237 case CPUID_MWAIT_LEAF: 238 /* Synthesize the values.. */ 239 *ax = 0; 240 *bx = 0; 241 *cx = cpuid_leaf5_ecx_val; 242 *dx = cpuid_leaf5_edx_val; 243 return; 244 245 case 0xb: 246 /* Suppress extended topology stuff */ 247 maskebx = 0; 248 break; 249 } 250 251 asm(XEN_EMULATE_PREFIX "cpuid" 252 : "=a" (*ax), 253 "=b" (*bx), 254 "=c" (*cx), 255 "=d" (*dx) 256 : "0" (*ax), "2" (*cx)); 257 258 *bx &= maskebx; 259 *bx |= or_ebx; 260 } 261 262 static bool __init xen_check_mwait(void) 263 { 264 #ifdef CONFIG_ACPI 265 struct xen_platform_op op = { 266 .cmd = XENPF_set_processor_pminfo, 267 .u.set_pminfo.id = -1, 268 .u.set_pminfo.type = XEN_PM_PDC, 269 }; 270 uint32_t buf[3]; 271 unsigned int ax, bx, cx, dx; 272 unsigned int mwait_mask; 273 274 /* We need to determine whether it is OK to expose the MWAIT 275 * capability to the kernel to harvest deeper than C3 states from ACPI 276 * _CST using the processor_harvest_xen.c module. For this to work, we 277 * need to gather the MWAIT_LEAF values (which the cstate.c code 278 * checks against). The hypervisor won't expose the MWAIT flag because 279 * it would break backwards compatibility; so we will find out directly 280 * from the hardware and hypercall. 281 */ 282 if (!xen_initial_domain()) 283 return false; 284 285 /* 286 * When running under platform earlier than Xen4.2, do not expose 287 * mwait, to avoid the risk of loading native acpi pad driver 288 */ 289 if (!xen_running_on_version_or_later(4, 2)) 290 return false; 291 292 ax = 1; 293 cx = 0; 294 295 native_cpuid(&ax, &bx, &cx, &dx); 296 297 mwait_mask = (1 << (X86_FEATURE_EST % 32)) | 298 (1 << (X86_FEATURE_MWAIT % 32)); 299 300 if ((cx & mwait_mask) != mwait_mask) 301 return false; 302 303 /* We need to emulate the MWAIT_LEAF and for that we need both 304 * ecx and edx. The hypercall provides only partial information. 305 */ 306 307 ax = CPUID_MWAIT_LEAF; 308 bx = 0; 309 cx = 0; 310 dx = 0; 311 312 native_cpuid(&ax, &bx, &cx, &dx); 313 314 /* Ask the Hypervisor whether to clear ACPI_PROC_CAP_C_C2C3_FFH. If so, 315 * don't expose MWAIT_LEAF and let ACPI pick the IOPORT version of C3. 316 */ 317 buf[0] = ACPI_PDC_REVISION_ID; 318 buf[1] = 1; 319 buf[2] = (ACPI_PROC_CAP_C_CAPABILITY_SMP | ACPI_PROC_CAP_EST_CAPABILITY_SWSMP); 320 321 set_xen_guest_handle(op.u.set_pminfo.pdc, buf); 322 323 if ((HYPERVISOR_platform_op(&op) == 0) && 324 (buf[2] & (ACPI_PROC_CAP_C_C1_FFH | ACPI_PROC_CAP_C_C2C3_FFH))) { 325 cpuid_leaf5_ecx_val = cx; 326 cpuid_leaf5_edx_val = dx; 327 } 328 return true; 329 #else 330 return false; 331 #endif 332 } 333 334 static bool __init xen_check_xsave(void) 335 { 336 unsigned int cx, xsave_mask; 337 338 cx = cpuid_ecx(1); 339 340 xsave_mask = (1 << (X86_FEATURE_XSAVE % 32)) | 341 (1 << (X86_FEATURE_OSXSAVE % 32)); 342 343 /* Xen will set CR4.OSXSAVE if supported and not disabled by force */ 344 return (cx & xsave_mask) == xsave_mask; 345 } 346 347 static void __init xen_init_capabilities(void) 348 { 349 setup_force_cpu_cap(X86_FEATURE_XENPV); 350 setup_clear_cpu_cap(X86_FEATURE_DCA); 351 setup_clear_cpu_cap(X86_FEATURE_APERFMPERF); 352 setup_clear_cpu_cap(X86_FEATURE_MTRR); 353 setup_clear_cpu_cap(X86_FEATURE_ACC); 354 setup_clear_cpu_cap(X86_FEATURE_X2APIC); 355 setup_clear_cpu_cap(X86_FEATURE_SME); 356 setup_clear_cpu_cap(X86_FEATURE_LKGS); 357 358 /* 359 * Xen PV would need some work to support PCID: CR3 handling as well 360 * as xen_flush_tlb_others() would need updating. 361 */ 362 setup_clear_cpu_cap(X86_FEATURE_PCID); 363 364 if (!xen_initial_domain()) 365 setup_clear_cpu_cap(X86_FEATURE_ACPI); 366 367 if (xen_check_mwait()) 368 setup_force_cpu_cap(X86_FEATURE_MWAIT); 369 else 370 setup_clear_cpu_cap(X86_FEATURE_MWAIT); 371 372 if (!xen_check_xsave()) { 373 setup_clear_cpu_cap(X86_FEATURE_XSAVE); 374 setup_clear_cpu_cap(X86_FEATURE_OSXSAVE); 375 } 376 } 377 378 static noinstr void xen_set_debugreg(int reg, unsigned long val) 379 { 380 HYPERVISOR_set_debugreg(reg, val); 381 } 382 383 static noinstr unsigned long xen_get_debugreg(int reg) 384 { 385 return HYPERVISOR_get_debugreg(reg); 386 } 387 388 static void xen_start_context_switch(struct task_struct *prev) 389 { 390 BUG_ON(preemptible()); 391 392 if (this_cpu_read(xen_lazy_mode) == XEN_LAZY_MMU) { 393 arch_leave_lazy_mmu_mode(); 394 set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES); 395 } 396 enter_lazy(XEN_LAZY_CPU); 397 } 398 399 static void xen_end_context_switch(struct task_struct *next) 400 { 401 BUG_ON(preemptible()); 402 403 xen_mc_flush(); 404 leave_lazy(XEN_LAZY_CPU); 405 if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES)) 406 arch_enter_lazy_mmu_mode(); 407 } 408 409 static unsigned long xen_store_tr(void) 410 { 411 return 0; 412 } 413 414 /* 415 * Set the page permissions for a particular virtual address. If the 416 * address is a vmalloc mapping (or other non-linear mapping), then 417 * find the linear mapping of the page and also set its protections to 418 * match. 419 */ 420 static void set_aliased_prot(void *v, pgprot_t prot) 421 { 422 int level; 423 pte_t *ptep; 424 pte_t pte; 425 unsigned long pfn; 426 unsigned char dummy; 427 void *va; 428 429 ptep = lookup_address((unsigned long)v, &level); 430 BUG_ON(ptep == NULL); 431 432 pfn = pte_pfn(*ptep); 433 pte = pfn_pte(pfn, prot); 434 435 /* 436 * Careful: update_va_mapping() will fail if the virtual address 437 * we're poking isn't populated in the page tables. We don't 438 * need to worry about the direct map (that's always in the page 439 * tables), but we need to be careful about vmap space. In 440 * particular, the top level page table can lazily propagate 441 * entries between processes, so if we've switched mms since we 442 * vmapped the target in the first place, we might not have the 443 * top-level page table entry populated. 444 * 445 * We disable preemption because we want the same mm active when 446 * we probe the target and when we issue the hypercall. We'll 447 * have the same nominal mm, but if we're a kernel thread, lazy 448 * mm dropping could change our pgd. 449 * 450 * Out of an abundance of caution, this uses __get_user() to fault 451 * in the target address just in case there's some obscure case 452 * in which the target address isn't readable. 453 */ 454 455 preempt_disable(); 456 457 copy_from_kernel_nofault(&dummy, v, 1); 458 459 if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0)) 460 BUG(); 461 462 va = __va(PFN_PHYS(pfn)); 463 464 if (va != v && HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0)) 465 BUG(); 466 467 preempt_enable(); 468 } 469 470 static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries) 471 { 472 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE; 473 int i; 474 475 /* 476 * We need to mark the all aliases of the LDT pages RO. We 477 * don't need to call vm_flush_aliases(), though, since that's 478 * only responsible for flushing aliases out the TLBs, not the 479 * page tables, and Xen will flush the TLB for us if needed. 480 * 481 * To avoid confusing future readers: none of this is necessary 482 * to load the LDT. The hypervisor only checks this when the 483 * LDT is faulted in due to subsequent descriptor access. 484 */ 485 486 for (i = 0; i < entries; i += entries_per_page) 487 set_aliased_prot(ldt + i, PAGE_KERNEL_RO); 488 } 489 490 static void xen_free_ldt(struct desc_struct *ldt, unsigned entries) 491 { 492 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE; 493 int i; 494 495 for (i = 0; i < entries; i += entries_per_page) 496 set_aliased_prot(ldt + i, PAGE_KERNEL); 497 } 498 499 static void xen_set_ldt(const void *addr, unsigned entries) 500 { 501 struct mmuext_op *op; 502 struct multicall_space mcs = xen_mc_entry(sizeof(*op)); 503 504 trace_xen_cpu_set_ldt(addr, entries); 505 506 op = mcs.args; 507 op->cmd = MMUEXT_SET_LDT; 508 op->arg1.linear_addr = (unsigned long)addr; 509 op->arg2.nr_ents = entries; 510 511 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 512 513 xen_mc_issue(XEN_LAZY_CPU); 514 } 515 516 static void xen_load_gdt(const struct desc_ptr *dtr) 517 { 518 unsigned long va = dtr->address; 519 unsigned int size = dtr->size + 1; 520 unsigned long pfn, mfn; 521 int level; 522 pte_t *ptep; 523 void *virt; 524 525 /* @size should be at most GDT_SIZE which is smaller than PAGE_SIZE. */ 526 BUG_ON(size > PAGE_SIZE); 527 BUG_ON(va & ~PAGE_MASK); 528 529 /* 530 * The GDT is per-cpu and is in the percpu data area. 531 * That can be virtually mapped, so we need to do a 532 * page-walk to get the underlying MFN for the 533 * hypercall. The page can also be in the kernel's 534 * linear range, so we need to RO that mapping too. 535 */ 536 ptep = lookup_address(va, &level); 537 BUG_ON(ptep == NULL); 538 539 pfn = pte_pfn(*ptep); 540 mfn = pfn_to_mfn(pfn); 541 virt = __va(PFN_PHYS(pfn)); 542 543 make_lowmem_page_readonly((void *)va); 544 make_lowmem_page_readonly(virt); 545 546 if (HYPERVISOR_set_gdt(&mfn, size / sizeof(struct desc_struct))) 547 BUG(); 548 } 549 550 /* 551 * load_gdt for early boot, when the gdt is only mapped once 552 */ 553 static void __init xen_load_gdt_boot(const struct desc_ptr *dtr) 554 { 555 unsigned long va = dtr->address; 556 unsigned int size = dtr->size + 1; 557 unsigned long pfn, mfn; 558 pte_t pte; 559 560 /* @size should be at most GDT_SIZE which is smaller than PAGE_SIZE. */ 561 BUG_ON(size > PAGE_SIZE); 562 BUG_ON(va & ~PAGE_MASK); 563 564 pfn = virt_to_pfn((void *)va); 565 mfn = pfn_to_mfn(pfn); 566 567 pte = pfn_pte(pfn, PAGE_KERNEL_RO); 568 569 if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0)) 570 BUG(); 571 572 if (HYPERVISOR_set_gdt(&mfn, size / sizeof(struct desc_struct))) 573 BUG(); 574 } 575 576 static inline bool desc_equal(const struct desc_struct *d1, 577 const struct desc_struct *d2) 578 { 579 return !memcmp(d1, d2, sizeof(*d1)); 580 } 581 582 static void load_TLS_descriptor(struct thread_struct *t, 583 unsigned int cpu, unsigned int i) 584 { 585 struct desc_struct *shadow = &per_cpu(shadow_tls_desc, cpu).desc[i]; 586 struct desc_struct *gdt; 587 xmaddr_t maddr; 588 struct multicall_space mc; 589 590 if (desc_equal(shadow, &t->tls_array[i])) 591 return; 592 593 *shadow = t->tls_array[i]; 594 595 gdt = get_cpu_gdt_rw(cpu); 596 maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]); 597 mc = __xen_mc_entry(0); 598 599 MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]); 600 } 601 602 static void xen_load_tls(struct thread_struct *t, unsigned int cpu) 603 { 604 /* 605 * In lazy mode we need to zero %fs, otherwise we may get an 606 * exception between the new %fs descriptor being loaded and 607 * %fs being effectively cleared at __switch_to(). 608 */ 609 if (xen_get_lazy_mode() == XEN_LAZY_CPU) 610 loadsegment(fs, 0); 611 612 xen_mc_batch(); 613 614 load_TLS_descriptor(t, cpu, 0); 615 load_TLS_descriptor(t, cpu, 1); 616 load_TLS_descriptor(t, cpu, 2); 617 618 xen_mc_issue(XEN_LAZY_CPU); 619 } 620 621 static void xen_load_gs_index(unsigned int idx) 622 { 623 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx)) 624 BUG(); 625 } 626 627 static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum, 628 const void *ptr) 629 { 630 xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]); 631 u64 entry = *(u64 *)ptr; 632 633 trace_xen_cpu_write_ldt_entry(dt, entrynum, entry); 634 635 preempt_disable(); 636 637 xen_mc_flush(); 638 if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry)) 639 BUG(); 640 641 preempt_enable(); 642 } 643 644 void noist_exc_debug(struct pt_regs *regs); 645 646 DEFINE_IDTENTRY_RAW(xenpv_exc_nmi) 647 { 648 /* On Xen PV, NMI doesn't use IST. The C part is the same as native. */ 649 exc_nmi(regs); 650 } 651 652 DEFINE_IDTENTRY_RAW_ERRORCODE(xenpv_exc_double_fault) 653 { 654 /* On Xen PV, DF doesn't use IST. The C part is the same as native. */ 655 exc_double_fault(regs, error_code); 656 } 657 658 DEFINE_IDTENTRY_RAW(xenpv_exc_debug) 659 { 660 /* 661 * There's no IST on Xen PV, but we still need to dispatch 662 * to the correct handler. 663 */ 664 if (user_mode(regs)) 665 noist_exc_debug(regs); 666 else 667 exc_debug(regs); 668 } 669 670 DEFINE_IDTENTRY_RAW(exc_xen_unknown_trap) 671 { 672 /* This should never happen and there is no way to handle it. */ 673 instrumentation_begin(); 674 pr_err("Unknown trap in Xen PV mode."); 675 BUG(); 676 instrumentation_end(); 677 } 678 679 #ifdef CONFIG_X86_MCE 680 DEFINE_IDTENTRY_RAW(xenpv_exc_machine_check) 681 { 682 /* 683 * There's no IST on Xen PV, but we still need to dispatch 684 * to the correct handler. 685 */ 686 if (user_mode(regs)) 687 noist_exc_machine_check(regs); 688 else 689 exc_machine_check(regs); 690 } 691 #endif 692 693 struct trap_array_entry { 694 void (*orig)(void); 695 void (*xen)(void); 696 bool ist_okay; 697 }; 698 699 #define TRAP_ENTRY(func, ist_ok) { \ 700 .orig = asm_##func, \ 701 .xen = xen_asm_##func, \ 702 .ist_okay = ist_ok } 703 704 #define TRAP_ENTRY_REDIR(func, ist_ok) { \ 705 .orig = asm_##func, \ 706 .xen = xen_asm_xenpv_##func, \ 707 .ist_okay = ist_ok } 708 709 static struct trap_array_entry trap_array[] = { 710 TRAP_ENTRY_REDIR(exc_debug, true ), 711 TRAP_ENTRY_REDIR(exc_double_fault, true ), 712 #ifdef CONFIG_X86_MCE 713 TRAP_ENTRY_REDIR(exc_machine_check, true ), 714 #endif 715 TRAP_ENTRY_REDIR(exc_nmi, true ), 716 TRAP_ENTRY(exc_int3, false ), 717 TRAP_ENTRY(exc_overflow, false ), 718 #ifdef CONFIG_IA32_EMULATION 719 TRAP_ENTRY(int80_emulation, false ), 720 #endif 721 TRAP_ENTRY(exc_page_fault, false ), 722 TRAP_ENTRY(exc_divide_error, false ), 723 TRAP_ENTRY(exc_bounds, false ), 724 TRAP_ENTRY(exc_invalid_op, false ), 725 TRAP_ENTRY(exc_device_not_available, false ), 726 TRAP_ENTRY(exc_coproc_segment_overrun, false ), 727 TRAP_ENTRY(exc_invalid_tss, false ), 728 TRAP_ENTRY(exc_segment_not_present, false ), 729 TRAP_ENTRY(exc_stack_segment, false ), 730 TRAP_ENTRY(exc_general_protection, false ), 731 TRAP_ENTRY(exc_spurious_interrupt_bug, false ), 732 TRAP_ENTRY(exc_coprocessor_error, false ), 733 TRAP_ENTRY(exc_alignment_check, false ), 734 TRAP_ENTRY(exc_simd_coprocessor_error, false ), 735 #ifdef CONFIG_X86_CET 736 TRAP_ENTRY(exc_control_protection, false ), 737 #endif 738 }; 739 740 static bool __ref get_trap_addr(void **addr, unsigned int ist) 741 { 742 unsigned int nr; 743 bool ist_okay = false; 744 bool found = false; 745 746 /* 747 * Replace trap handler addresses by Xen specific ones. 748 * Check for known traps using IST and whitelist them. 749 * The debugger ones are the only ones we care about. 750 * Xen will handle faults like double_fault, so we should never see 751 * them. Warn if there's an unexpected IST-using fault handler. 752 */ 753 for (nr = 0; nr < ARRAY_SIZE(trap_array); nr++) { 754 struct trap_array_entry *entry = trap_array + nr; 755 756 if (*addr == entry->orig) { 757 *addr = entry->xen; 758 ist_okay = entry->ist_okay; 759 found = true; 760 break; 761 } 762 } 763 764 if (nr == ARRAY_SIZE(trap_array) && 765 *addr >= (void *)early_idt_handler_array[0] && 766 *addr < (void *)early_idt_handler_array[NUM_EXCEPTION_VECTORS]) { 767 nr = (*addr - (void *)early_idt_handler_array[0]) / 768 EARLY_IDT_HANDLER_SIZE; 769 *addr = (void *)xen_early_idt_handler_array[nr]; 770 found = true; 771 } 772 773 if (!found) 774 *addr = (void *)xen_asm_exc_xen_unknown_trap; 775 776 if (WARN_ON(found && ist != 0 && !ist_okay)) 777 return false; 778 779 return true; 780 } 781 782 static int cvt_gate_to_trap(int vector, const gate_desc *val, 783 struct trap_info *info) 784 { 785 unsigned long addr; 786 787 if (val->bits.type != GATE_TRAP && val->bits.type != GATE_INTERRUPT) 788 return 0; 789 790 info->vector = vector; 791 792 addr = gate_offset(val); 793 if (!get_trap_addr((void **)&addr, val->bits.ist)) 794 return 0; 795 info->address = addr; 796 797 info->cs = gate_segment(val); 798 info->flags = val->bits.dpl; 799 /* interrupt gates clear IF */ 800 if (val->bits.type == GATE_INTERRUPT) 801 info->flags |= 1 << 2; 802 803 return 1; 804 } 805 806 /* Locations of each CPU's IDT */ 807 static DEFINE_PER_CPU(struct desc_ptr, idt_desc); 808 809 /* Set an IDT entry. If the entry is part of the current IDT, then 810 also update Xen. */ 811 static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g) 812 { 813 unsigned long p = (unsigned long)&dt[entrynum]; 814 unsigned long start, end; 815 816 trace_xen_cpu_write_idt_entry(dt, entrynum, g); 817 818 preempt_disable(); 819 820 start = __this_cpu_read(idt_desc.address); 821 end = start + __this_cpu_read(idt_desc.size) + 1; 822 823 xen_mc_flush(); 824 825 native_write_idt_entry(dt, entrynum, g); 826 827 if (p >= start && (p + 8) <= end) { 828 struct trap_info info[2]; 829 830 info[1].address = 0; 831 832 if (cvt_gate_to_trap(entrynum, g, &info[0])) 833 if (HYPERVISOR_set_trap_table(info)) 834 BUG(); 835 } 836 837 preempt_enable(); 838 } 839 840 static unsigned xen_convert_trap_info(const struct desc_ptr *desc, 841 struct trap_info *traps, bool full) 842 { 843 unsigned in, out, count; 844 845 count = (desc->size+1) / sizeof(gate_desc); 846 BUG_ON(count > 256); 847 848 for (in = out = 0; in < count; in++) { 849 gate_desc *entry = (gate_desc *)(desc->address) + in; 850 851 if (cvt_gate_to_trap(in, entry, &traps[out]) || full) 852 out++; 853 } 854 855 return out; 856 } 857 858 void xen_copy_trap_info(struct trap_info *traps) 859 { 860 const struct desc_ptr *desc = this_cpu_ptr(&idt_desc); 861 862 xen_convert_trap_info(desc, traps, true); 863 } 864 865 /* Load a new IDT into Xen. In principle this can be per-CPU, so we 866 hold a spinlock to protect the static traps[] array (static because 867 it avoids allocation, and saves stack space). */ 868 static void xen_load_idt(const struct desc_ptr *desc) 869 { 870 static DEFINE_SPINLOCK(lock); 871 static struct trap_info traps[257]; 872 static const struct trap_info zero = { }; 873 unsigned out; 874 875 trace_xen_cpu_load_idt(desc); 876 877 spin_lock(&lock); 878 879 memcpy(this_cpu_ptr(&idt_desc), desc, sizeof(idt_desc)); 880 881 out = xen_convert_trap_info(desc, traps, false); 882 traps[out] = zero; 883 884 xen_mc_flush(); 885 if (HYPERVISOR_set_trap_table(traps)) 886 BUG(); 887 888 spin_unlock(&lock); 889 } 890 891 /* Write a GDT descriptor entry. Ignore LDT descriptors, since 892 they're handled differently. */ 893 static void xen_write_gdt_entry(struct desc_struct *dt, int entry, 894 const void *desc, int type) 895 { 896 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type); 897 898 preempt_disable(); 899 900 switch (type) { 901 case DESC_LDT: 902 case DESC_TSS: 903 /* ignore */ 904 break; 905 906 default: { 907 xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]); 908 909 xen_mc_flush(); 910 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc)) 911 BUG(); 912 } 913 914 } 915 916 preempt_enable(); 917 } 918 919 /* 920 * Version of write_gdt_entry for use at early boot-time needed to 921 * update an entry as simply as possible. 922 */ 923 static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry, 924 const void *desc, int type) 925 { 926 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type); 927 928 switch (type) { 929 case DESC_LDT: 930 case DESC_TSS: 931 /* ignore */ 932 break; 933 934 default: { 935 xmaddr_t maddr = virt_to_machine(&dt[entry]); 936 937 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc)) 938 dt[entry] = *(struct desc_struct *)desc; 939 } 940 941 } 942 } 943 944 static void xen_load_sp0(unsigned long sp0) 945 { 946 struct multicall_space mcs; 947 948 mcs = xen_mc_entry(0); 949 MULTI_stack_switch(mcs.mc, __KERNEL_DS, sp0); 950 xen_mc_issue(XEN_LAZY_CPU); 951 this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0); 952 } 953 954 #ifdef CONFIG_X86_IOPL_IOPERM 955 static void xen_invalidate_io_bitmap(void) 956 { 957 struct physdev_set_iobitmap iobitmap = { 958 .bitmap = NULL, 959 .nr_ports = 0, 960 }; 961 962 native_tss_invalidate_io_bitmap(); 963 HYPERVISOR_physdev_op(PHYSDEVOP_set_iobitmap, &iobitmap); 964 } 965 966 static void xen_update_io_bitmap(void) 967 { 968 struct physdev_set_iobitmap iobitmap; 969 struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw); 970 971 native_tss_update_io_bitmap(); 972 973 iobitmap.bitmap = (uint8_t *)(&tss->x86_tss) + 974 tss->x86_tss.io_bitmap_base; 975 if (tss->x86_tss.io_bitmap_base == IO_BITMAP_OFFSET_INVALID) 976 iobitmap.nr_ports = 0; 977 else 978 iobitmap.nr_ports = IO_BITMAP_BITS; 979 980 HYPERVISOR_physdev_op(PHYSDEVOP_set_iobitmap, &iobitmap); 981 } 982 #endif 983 984 static void xen_io_delay(void) 985 { 986 } 987 988 static DEFINE_PER_CPU(unsigned long, xen_cr0_value); 989 990 static unsigned long xen_read_cr0(void) 991 { 992 unsigned long cr0 = this_cpu_read(xen_cr0_value); 993 994 if (unlikely(cr0 == 0)) { 995 cr0 = native_read_cr0(); 996 this_cpu_write(xen_cr0_value, cr0); 997 } 998 999 return cr0; 1000 } 1001 1002 static void xen_write_cr0(unsigned long cr0) 1003 { 1004 struct multicall_space mcs; 1005 1006 this_cpu_write(xen_cr0_value, cr0); 1007 1008 /* Only pay attention to cr0.TS; everything else is 1009 ignored. */ 1010 mcs = xen_mc_entry(0); 1011 1012 MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0); 1013 1014 xen_mc_issue(XEN_LAZY_CPU); 1015 } 1016 1017 static void xen_write_cr4(unsigned long cr4) 1018 { 1019 cr4 &= ~(X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PCE); 1020 1021 native_write_cr4(cr4); 1022 } 1023 1024 static u64 xen_do_read_msr(unsigned int msr, int *err) 1025 { 1026 u64 val = 0; /* Avoid uninitialized value for safe variant. */ 1027 1028 if (pmu_msr_read(msr, &val, err)) 1029 return val; 1030 1031 if (err) 1032 val = native_read_msr_safe(msr, err); 1033 else 1034 val = native_read_msr(msr); 1035 1036 switch (msr) { 1037 case MSR_IA32_APICBASE: 1038 val &= ~X2APIC_ENABLE; 1039 break; 1040 } 1041 return val; 1042 } 1043 1044 static void set_seg(unsigned int which, unsigned int low, unsigned int high, 1045 int *err) 1046 { 1047 u64 base = ((u64)high << 32) | low; 1048 1049 if (HYPERVISOR_set_segment_base(which, base) == 0) 1050 return; 1051 1052 if (err) 1053 *err = -EIO; 1054 else 1055 WARN(1, "Xen set_segment_base(%u, %llx) failed\n", which, base); 1056 } 1057 1058 /* 1059 * Support write_msr_safe() and write_msr() semantics. 1060 * With err == NULL write_msr() semantics are selected. 1061 * Supplying an err pointer requires err to be pre-initialized with 0. 1062 */ 1063 static void xen_do_write_msr(unsigned int msr, unsigned int low, 1064 unsigned int high, int *err) 1065 { 1066 switch (msr) { 1067 case MSR_FS_BASE: 1068 set_seg(SEGBASE_FS, low, high, err); 1069 break; 1070 1071 case MSR_KERNEL_GS_BASE: 1072 set_seg(SEGBASE_GS_USER, low, high, err); 1073 break; 1074 1075 case MSR_GS_BASE: 1076 set_seg(SEGBASE_GS_KERNEL, low, high, err); 1077 break; 1078 1079 case MSR_STAR: 1080 case MSR_CSTAR: 1081 case MSR_LSTAR: 1082 case MSR_SYSCALL_MASK: 1083 case MSR_IA32_SYSENTER_CS: 1084 case MSR_IA32_SYSENTER_ESP: 1085 case MSR_IA32_SYSENTER_EIP: 1086 /* Fast syscall setup is all done in hypercalls, so 1087 these are all ignored. Stub them out here to stop 1088 Xen console noise. */ 1089 break; 1090 1091 default: 1092 if (!pmu_msr_write(msr, low, high, err)) { 1093 if (err) 1094 *err = native_write_msr_safe(msr, low, high); 1095 else 1096 native_write_msr(msr, low, high); 1097 } 1098 } 1099 } 1100 1101 static u64 xen_read_msr_safe(unsigned int msr, int *err) 1102 { 1103 return xen_do_read_msr(msr, err); 1104 } 1105 1106 static int xen_write_msr_safe(unsigned int msr, unsigned int low, 1107 unsigned int high) 1108 { 1109 int err = 0; 1110 1111 xen_do_write_msr(msr, low, high, &err); 1112 1113 return err; 1114 } 1115 1116 static u64 xen_read_msr(unsigned int msr) 1117 { 1118 int err; 1119 1120 return xen_do_read_msr(msr, xen_msr_safe ? &err : NULL); 1121 } 1122 1123 static void xen_write_msr(unsigned int msr, unsigned low, unsigned high) 1124 { 1125 int err; 1126 1127 xen_do_write_msr(msr, low, high, xen_msr_safe ? &err : NULL); 1128 } 1129 1130 /* This is called once we have the cpu_possible_mask */ 1131 void __init xen_setup_vcpu_info_placement(void) 1132 { 1133 int cpu; 1134 1135 for_each_possible_cpu(cpu) { 1136 /* Set up direct vCPU id mapping for PV guests. */ 1137 per_cpu(xen_vcpu_id, cpu) = cpu; 1138 xen_vcpu_setup(cpu); 1139 } 1140 1141 pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct); 1142 pv_ops.irq.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); 1143 pv_ops.irq.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); 1144 pv_ops.mmu.read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2_direct); 1145 } 1146 1147 static const struct pv_info xen_info __initconst = { 1148 .extra_user_64bit_cs = FLAT_USER_CS64, 1149 .name = "Xen", 1150 }; 1151 1152 static const typeof(pv_ops) xen_cpu_ops __initconst = { 1153 .cpu = { 1154 .cpuid = xen_cpuid, 1155 1156 .set_debugreg = xen_set_debugreg, 1157 .get_debugreg = xen_get_debugreg, 1158 1159 .read_cr0 = xen_read_cr0, 1160 .write_cr0 = xen_write_cr0, 1161 1162 .write_cr4 = xen_write_cr4, 1163 1164 .wbinvd = pv_native_wbinvd, 1165 1166 .read_msr = xen_read_msr, 1167 .write_msr = xen_write_msr, 1168 1169 .read_msr_safe = xen_read_msr_safe, 1170 .write_msr_safe = xen_write_msr_safe, 1171 1172 .read_pmc = xen_read_pmc, 1173 1174 .load_tr_desc = paravirt_nop, 1175 .set_ldt = xen_set_ldt, 1176 .load_gdt = xen_load_gdt, 1177 .load_idt = xen_load_idt, 1178 .load_tls = xen_load_tls, 1179 .load_gs_index = xen_load_gs_index, 1180 1181 .alloc_ldt = xen_alloc_ldt, 1182 .free_ldt = xen_free_ldt, 1183 1184 .store_tr = xen_store_tr, 1185 1186 .write_ldt_entry = xen_write_ldt_entry, 1187 .write_gdt_entry = xen_write_gdt_entry, 1188 .write_idt_entry = xen_write_idt_entry, 1189 .load_sp0 = xen_load_sp0, 1190 1191 #ifdef CONFIG_X86_IOPL_IOPERM 1192 .invalidate_io_bitmap = xen_invalidate_io_bitmap, 1193 .update_io_bitmap = xen_update_io_bitmap, 1194 #endif 1195 .io_delay = xen_io_delay, 1196 1197 .start_context_switch = xen_start_context_switch, 1198 .end_context_switch = xen_end_context_switch, 1199 }, 1200 }; 1201 1202 static void xen_restart(char *msg) 1203 { 1204 xen_reboot(SHUTDOWN_reboot); 1205 } 1206 1207 static void xen_machine_halt(void) 1208 { 1209 xen_reboot(SHUTDOWN_poweroff); 1210 } 1211 1212 static void xen_machine_power_off(void) 1213 { 1214 do_kernel_power_off(); 1215 xen_reboot(SHUTDOWN_poweroff); 1216 } 1217 1218 static void xen_crash_shutdown(struct pt_regs *regs) 1219 { 1220 xen_reboot(SHUTDOWN_crash); 1221 } 1222 1223 static const struct machine_ops xen_machine_ops __initconst = { 1224 .restart = xen_restart, 1225 .halt = xen_machine_halt, 1226 .power_off = xen_machine_power_off, 1227 .shutdown = xen_machine_halt, 1228 .crash_shutdown = xen_crash_shutdown, 1229 .emergency_restart = xen_emergency_restart, 1230 }; 1231 1232 static unsigned char xen_get_nmi_reason(void) 1233 { 1234 unsigned char reason = 0; 1235 1236 /* Construct a value which looks like it came from port 0x61. */ 1237 if (test_bit(_XEN_NMIREASON_io_error, 1238 &HYPERVISOR_shared_info->arch.nmi_reason)) 1239 reason |= NMI_REASON_IOCHK; 1240 if (test_bit(_XEN_NMIREASON_pci_serr, 1241 &HYPERVISOR_shared_info->arch.nmi_reason)) 1242 reason |= NMI_REASON_SERR; 1243 1244 return reason; 1245 } 1246 1247 static void __init xen_boot_params_init_edd(void) 1248 { 1249 #if IS_ENABLED(CONFIG_EDD) 1250 struct xen_platform_op op; 1251 struct edd_info *edd_info; 1252 u32 *mbr_signature; 1253 unsigned nr; 1254 int ret; 1255 1256 edd_info = boot_params.eddbuf; 1257 mbr_signature = boot_params.edd_mbr_sig_buffer; 1258 1259 op.cmd = XENPF_firmware_info; 1260 1261 op.u.firmware_info.type = XEN_FW_DISK_INFO; 1262 for (nr = 0; nr < EDDMAXNR; nr++) { 1263 struct edd_info *info = edd_info + nr; 1264 1265 op.u.firmware_info.index = nr; 1266 info->params.length = sizeof(info->params); 1267 set_xen_guest_handle(op.u.firmware_info.u.disk_info.edd_params, 1268 &info->params); 1269 ret = HYPERVISOR_platform_op(&op); 1270 if (ret) 1271 break; 1272 1273 #define C(x) info->x = op.u.firmware_info.u.disk_info.x 1274 C(device); 1275 C(version); 1276 C(interface_support); 1277 C(legacy_max_cylinder); 1278 C(legacy_max_head); 1279 C(legacy_sectors_per_track); 1280 #undef C 1281 } 1282 boot_params.eddbuf_entries = nr; 1283 1284 op.u.firmware_info.type = XEN_FW_DISK_MBR_SIGNATURE; 1285 for (nr = 0; nr < EDD_MBR_SIG_MAX; nr++) { 1286 op.u.firmware_info.index = nr; 1287 ret = HYPERVISOR_platform_op(&op); 1288 if (ret) 1289 break; 1290 mbr_signature[nr] = op.u.firmware_info.u.disk_mbr_signature.mbr_signature; 1291 } 1292 boot_params.edd_mbr_sig_buf_entries = nr; 1293 #endif 1294 } 1295 1296 /* 1297 * Set up the GDT and segment registers for -fstack-protector. Until 1298 * we do this, we have to be careful not to call any stack-protected 1299 * function, which is most of the kernel. 1300 */ 1301 static void __init xen_setup_gdt(int cpu) 1302 { 1303 pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot; 1304 pv_ops.cpu.load_gdt = xen_load_gdt_boot; 1305 1306 switch_gdt_and_percpu_base(cpu); 1307 1308 pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry; 1309 pv_ops.cpu.load_gdt = xen_load_gdt; 1310 } 1311 1312 static void __init xen_dom0_set_legacy_features(void) 1313 { 1314 x86_platform.legacy.rtc = 1; 1315 } 1316 1317 static void __init xen_domu_set_legacy_features(void) 1318 { 1319 x86_platform.legacy.rtc = 0; 1320 } 1321 1322 extern void early_xen_iret_patch(void); 1323 1324 /* First C function to be called on Xen boot */ 1325 asmlinkage __visible void __init xen_start_kernel(struct start_info *si) 1326 { 1327 struct physdev_set_iopl set_iopl; 1328 unsigned long initrd_start = 0; 1329 int rc; 1330 1331 if (!si) 1332 return; 1333 1334 clear_bss(); 1335 1336 xen_start_info = si; 1337 1338 __text_gen_insn(&early_xen_iret_patch, 1339 JMP32_INSN_OPCODE, &early_xen_iret_patch, &xen_iret, 1340 JMP32_INSN_SIZE); 1341 1342 xen_domain_type = XEN_PV_DOMAIN; 1343 xen_start_flags = xen_start_info->flags; 1344 1345 xen_setup_features(); 1346 1347 /* Install Xen paravirt ops */ 1348 pv_info = xen_info; 1349 pv_ops.cpu = xen_cpu_ops.cpu; 1350 xen_init_irq_ops(); 1351 1352 /* 1353 * Setup xen_vcpu early because it is needed for 1354 * local_irq_disable(), irqs_disabled(), e.g. in printk(). 1355 * 1356 * Don't do the full vcpu_info placement stuff until we have 1357 * the cpu_possible_mask and a non-dummy shared_info. 1358 */ 1359 xen_vcpu_info_reset(0); 1360 1361 x86_platform.get_nmi_reason = xen_get_nmi_reason; 1362 x86_platform.realmode_reserve = x86_init_noop; 1363 x86_platform.realmode_init = x86_init_noop; 1364 1365 x86_init.resources.memory_setup = xen_memory_setup; 1366 x86_init.irqs.intr_mode_select = x86_init_noop; 1367 x86_init.irqs.intr_mode_init = x86_64_probe_apic; 1368 x86_init.oem.arch_setup = xen_arch_setup; 1369 x86_init.oem.banner = xen_banner; 1370 x86_init.hyper.init_platform = xen_pv_init_platform; 1371 x86_init.hyper.guest_late_init = xen_pv_guest_late_init; 1372 1373 /* 1374 * Set up some pagetable state before starting to set any ptes. 1375 */ 1376 1377 xen_setup_machphys_mapping(); 1378 xen_init_mmu_ops(); 1379 1380 /* Prevent unwanted bits from being set in PTEs. */ 1381 __supported_pte_mask &= ~_PAGE_GLOBAL; 1382 __default_kernel_pte_mask &= ~_PAGE_GLOBAL; 1383 1384 /* Get mfn list */ 1385 xen_build_dynamic_phys_to_machine(); 1386 1387 /* Work out if we support NX */ 1388 get_cpu_cap(&boot_cpu_data); 1389 x86_configure_nx(); 1390 1391 /* 1392 * Set up kernel GDT and segment registers, mainly so that 1393 * -fstack-protector code can be executed. 1394 */ 1395 xen_setup_gdt(0); 1396 1397 /* Determine virtual and physical address sizes */ 1398 get_cpu_address_sizes(&boot_cpu_data); 1399 1400 /* Let's presume PV guests always boot on vCPU with id 0. */ 1401 per_cpu(xen_vcpu_id, 0) = 0; 1402 1403 idt_setup_early_handler(); 1404 1405 xen_init_capabilities(); 1406 1407 /* 1408 * set up the basic apic ops. 1409 */ 1410 xen_init_apic(); 1411 1412 machine_ops = xen_machine_ops; 1413 1414 /* 1415 * The only reliable way to retain the initial address of the 1416 * percpu gdt_page is to remember it here, so we can go and 1417 * mark it RW later, when the initial percpu area is freed. 1418 */ 1419 xen_initial_gdt = &per_cpu(gdt_page, 0); 1420 1421 xen_smp_init(); 1422 1423 #ifdef CONFIG_ACPI_NUMA 1424 /* 1425 * The pages we from Xen are not related to machine pages, so 1426 * any NUMA information the kernel tries to get from ACPI will 1427 * be meaningless. Prevent it from trying. 1428 */ 1429 disable_srat(); 1430 #endif 1431 WARN_ON(xen_cpuhp_setup(xen_cpu_up_prepare_pv, xen_cpu_dead_pv)); 1432 1433 local_irq_disable(); 1434 early_boot_irqs_disabled = true; 1435 1436 xen_raw_console_write("mapping kernel into physical memory\n"); 1437 xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, 1438 xen_start_info->nr_pages); 1439 xen_reserve_special_pages(); 1440 1441 /* 1442 * We used to do this in xen_arch_setup, but that is too late 1443 * on AMD were early_cpu_init (run before ->arch_setup()) calls 1444 * early_amd_init which pokes 0xcf8 port. 1445 */ 1446 set_iopl.iopl = 1; 1447 rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 1448 if (rc != 0) 1449 xen_raw_printk("physdev_op failed %d\n", rc); 1450 1451 1452 if (xen_start_info->mod_start) { 1453 if (xen_start_info->flags & SIF_MOD_START_PFN) 1454 initrd_start = PFN_PHYS(xen_start_info->mod_start); 1455 else 1456 initrd_start = __pa(xen_start_info->mod_start); 1457 } 1458 1459 /* Poke various useful things into boot_params */ 1460 boot_params.hdr.type_of_loader = (9 << 4) | 0; 1461 boot_params.hdr.ramdisk_image = initrd_start; 1462 boot_params.hdr.ramdisk_size = xen_start_info->mod_len; 1463 boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line); 1464 boot_params.hdr.hardware_subarch = X86_SUBARCH_XEN; 1465 1466 if (!xen_initial_domain()) { 1467 if (pci_xen) 1468 x86_init.pci.arch_init = pci_xen_init; 1469 x86_platform.set_legacy_features = 1470 xen_domu_set_legacy_features; 1471 } else { 1472 const struct dom0_vga_console_info *info = 1473 (void *)((char *)xen_start_info + 1474 xen_start_info->console.dom0.info_off); 1475 struct xen_platform_op op = { 1476 .cmd = XENPF_firmware_info, 1477 .interface_version = XENPF_INTERFACE_VERSION, 1478 .u.firmware_info.type = XEN_FW_KBD_SHIFT_FLAGS, 1479 }; 1480 1481 x86_platform.set_legacy_features = 1482 xen_dom0_set_legacy_features; 1483 xen_init_vga(info, xen_start_info->console.dom0.info_size, 1484 &boot_params.screen_info); 1485 xen_start_info->console.domU.mfn = 0; 1486 xen_start_info->console.domU.evtchn = 0; 1487 1488 if (HYPERVISOR_platform_op(&op) == 0) 1489 boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags; 1490 1491 /* Make sure ACS will be enabled */ 1492 pci_request_acs(); 1493 1494 xen_acpi_sleep_register(); 1495 1496 xen_boot_params_init_edd(); 1497 1498 #ifdef CONFIG_ACPI 1499 /* 1500 * Disable selecting "Firmware First mode" for correctable 1501 * memory errors, as this is the duty of the hypervisor to 1502 * decide. 1503 */ 1504 acpi_disable_cmcff = 1; 1505 #endif 1506 } 1507 1508 xen_add_preferred_consoles(); 1509 1510 #ifdef CONFIG_PCI 1511 /* PCI BIOS service won't work from a PV guest. */ 1512 pci_probe &= ~PCI_PROBE_BIOS; 1513 #endif 1514 xen_raw_console_write("about to get started...\n"); 1515 1516 /* We need this for printk timestamps */ 1517 xen_setup_runstate_info(0); 1518 1519 xen_efi_init(&boot_params); 1520 1521 /* Start the world */ 1522 cr4_init_shadow(); /* 32b kernel does this in i386_start_kernel() */ 1523 x86_64_start_reservations((char *)__pa_symbol(&boot_params)); 1524 } 1525 1526 static int xen_cpu_up_prepare_pv(unsigned int cpu) 1527 { 1528 int rc; 1529 1530 if (per_cpu(xen_vcpu, cpu) == NULL) 1531 return -ENODEV; 1532 1533 xen_setup_timer(cpu); 1534 1535 rc = xen_smp_intr_init(cpu); 1536 if (rc) { 1537 WARN(1, "xen_smp_intr_init() for CPU %d failed: %d\n", 1538 cpu, rc); 1539 return rc; 1540 } 1541 1542 rc = xen_smp_intr_init_pv(cpu); 1543 if (rc) { 1544 WARN(1, "xen_smp_intr_init_pv() for CPU %d failed: %d\n", 1545 cpu, rc); 1546 return rc; 1547 } 1548 1549 return 0; 1550 } 1551 1552 static int xen_cpu_dead_pv(unsigned int cpu) 1553 { 1554 xen_smp_intr_free(cpu); 1555 xen_smp_intr_free_pv(cpu); 1556 1557 xen_teardown_timer(cpu); 1558 1559 return 0; 1560 } 1561 1562 static uint32_t __init xen_platform_pv(void) 1563 { 1564 if (xen_pv_domain()) 1565 return xen_cpuid_base(); 1566 1567 return 0; 1568 } 1569 1570 const __initconst struct hypervisor_x86 x86_hyper_xen_pv = { 1571 .name = "Xen PV", 1572 .detect = xen_platform_pv, 1573 .type = X86_HYPER_XEN_PV, 1574 .runtime.pin_vcpu = xen_pin_vcpu, 1575 .ignore_nopv = true, 1576 }; 1577