1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * X86 specific Hyper-V initialization code. 4 * 5 * Copyright (C) 2016, Microsoft, Inc. 6 * 7 * Author : K. Y. Srinivasan <kys@microsoft.com> 8 */ 9 10 #include <linux/efi.h> 11 #include <linux/types.h> 12 #include <linux/bitfield.h> 13 #include <linux/io.h> 14 #include <asm/apic.h> 15 #include <asm/desc.h> 16 #include <asm/hypervisor.h> 17 #include <asm/hyperv-tlfs.h> 18 #include <asm/mshyperv.h> 19 #include <asm/idtentry.h> 20 #include <linux/kexec.h> 21 #include <linux/version.h> 22 #include <linux/vmalloc.h> 23 #include <linux/mm.h> 24 #include <linux/hyperv.h> 25 #include <linux/slab.h> 26 #include <linux/kernel.h> 27 #include <linux/cpuhotplug.h> 28 #include <linux/syscore_ops.h> 29 #include <clocksource/hyperv_timer.h> 30 #include <linux/highmem.h> 31 #include <linux/swiotlb.h> 32 33 int hyperv_init_cpuhp; 34 u64 hv_current_partition_id = ~0ull; 35 EXPORT_SYMBOL_GPL(hv_current_partition_id); 36 37 void *hv_hypercall_pg; 38 EXPORT_SYMBOL_GPL(hv_hypercall_pg); 39 40 union hv_ghcb * __percpu *hv_ghcb_pg; 41 42 /* Storage to save the hypercall page temporarily for hibernation */ 43 static void *hv_hypercall_pg_saved; 44 45 struct hv_vp_assist_page **hv_vp_assist_page; 46 EXPORT_SYMBOL_GPL(hv_vp_assist_page); 47 48 static int hyperv_init_ghcb(void) 49 { 50 u64 ghcb_gpa; 51 void *ghcb_va; 52 void **ghcb_base; 53 54 if (!hv_isolation_type_snp()) 55 return 0; 56 57 if (!hv_ghcb_pg) 58 return -EINVAL; 59 60 /* 61 * GHCB page is allocated by paravisor. The address 62 * returned by MSR_AMD64_SEV_ES_GHCB is above shared 63 * memory boundary and map it here. 64 */ 65 rdmsrl(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa); 66 ghcb_va = memremap(ghcb_gpa, HV_HYP_PAGE_SIZE, MEMREMAP_WB); 67 if (!ghcb_va) 68 return -ENOMEM; 69 70 ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg); 71 *ghcb_base = ghcb_va; 72 73 return 0; 74 } 75 76 static int hv_cpu_init(unsigned int cpu) 77 { 78 union hv_vp_assist_msr_contents msr = { 0 }; 79 struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()]; 80 int ret; 81 82 ret = hv_common_cpu_init(cpu); 83 if (ret) 84 return ret; 85 86 if (!hv_vp_assist_page) 87 return 0; 88 89 if (!*hvp) { 90 if (hv_root_partition) { 91 /* 92 * For root partition we get the hypervisor provided VP assist 93 * page, instead of allocating a new page. 94 */ 95 rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); 96 *hvp = memremap(msr.pfn << 97 HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT, 98 PAGE_SIZE, MEMREMAP_WB); 99 } else { 100 /* 101 * The VP assist page is an "overlay" page (see Hyper-V TLFS's 102 * Section 5.2.1 "GPA Overlay Pages"). Here it must be zeroed 103 * out to make sure we always write the EOI MSR in 104 * hv_apic_eoi_write() *after* the EOI optimization is disabled 105 * in hv_cpu_die(), otherwise a CPU may not be stopped in the 106 * case of CPU offlining and the VM will hang. 107 */ 108 *hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); 109 if (*hvp) 110 msr.pfn = vmalloc_to_pfn(*hvp); 111 } 112 WARN_ON(!(*hvp)); 113 if (*hvp) { 114 msr.enable = 1; 115 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); 116 } 117 } 118 119 return hyperv_init_ghcb(); 120 } 121 122 static void (*hv_reenlightenment_cb)(void); 123 124 static void hv_reenlightenment_notify(struct work_struct *dummy) 125 { 126 struct hv_tsc_emulation_status emu_status; 127 128 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 129 130 /* Don't issue the callback if TSC accesses are not emulated */ 131 if (hv_reenlightenment_cb && emu_status.inprogress) 132 hv_reenlightenment_cb(); 133 } 134 static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify); 135 136 void hyperv_stop_tsc_emulation(void) 137 { 138 u64 freq; 139 struct hv_tsc_emulation_status emu_status; 140 141 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 142 emu_status.inprogress = 0; 143 wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 144 145 rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq); 146 tsc_khz = div64_u64(freq, 1000); 147 } 148 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation); 149 150 static inline bool hv_reenlightenment_available(void) 151 { 152 /* 153 * Check for required features and privileges to make TSC frequency 154 * change notifications work. 155 */ 156 return ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS && 157 ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE && 158 ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT; 159 } 160 161 DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment) 162 { 163 ack_APIC_irq(); 164 inc_irq_stat(irq_hv_reenlightenment_count); 165 schedule_delayed_work(&hv_reenlightenment_work, HZ/10); 166 } 167 168 void set_hv_tscchange_cb(void (*cb)(void)) 169 { 170 struct hv_reenlightenment_control re_ctrl = { 171 .vector = HYPERV_REENLIGHTENMENT_VECTOR, 172 .enabled = 1, 173 }; 174 struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1}; 175 176 if (!hv_reenlightenment_available()) { 177 pr_warn("Hyper-V: reenlightenment support is unavailable\n"); 178 return; 179 } 180 181 if (!hv_vp_index) 182 return; 183 184 hv_reenlightenment_cb = cb; 185 186 /* Make sure callback is registered before we write to MSRs */ 187 wmb(); 188 189 re_ctrl.target_vp = hv_vp_index[get_cpu()]; 190 191 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 192 wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl)); 193 194 put_cpu(); 195 } 196 EXPORT_SYMBOL_GPL(set_hv_tscchange_cb); 197 198 void clear_hv_tscchange_cb(void) 199 { 200 struct hv_reenlightenment_control re_ctrl; 201 202 if (!hv_reenlightenment_available()) 203 return; 204 205 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 206 re_ctrl.enabled = 0; 207 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 208 209 hv_reenlightenment_cb = NULL; 210 } 211 EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb); 212 213 static int hv_cpu_die(unsigned int cpu) 214 { 215 struct hv_reenlightenment_control re_ctrl; 216 unsigned int new_cpu; 217 void **ghcb_va; 218 219 if (hv_ghcb_pg) { 220 ghcb_va = (void **)this_cpu_ptr(hv_ghcb_pg); 221 if (*ghcb_va) 222 memunmap(*ghcb_va); 223 *ghcb_va = NULL; 224 } 225 226 hv_common_cpu_die(cpu); 227 228 if (hv_vp_assist_page && hv_vp_assist_page[cpu]) { 229 union hv_vp_assist_msr_contents msr = { 0 }; 230 if (hv_root_partition) { 231 /* 232 * For root partition the VP assist page is mapped to 233 * hypervisor provided page, and thus we unmap the 234 * page here and nullify it, so that in future we have 235 * correct page address mapped in hv_cpu_init. 236 */ 237 memunmap(hv_vp_assist_page[cpu]); 238 hv_vp_assist_page[cpu] = NULL; 239 rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); 240 msr.enable = 0; 241 } 242 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); 243 } 244 245 if (hv_reenlightenment_cb == NULL) 246 return 0; 247 248 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 249 if (re_ctrl.target_vp == hv_vp_index[cpu]) { 250 /* 251 * Reassign reenlightenment notifications to some other online 252 * CPU or just disable the feature if there are no online CPUs 253 * left (happens on hibernation). 254 */ 255 new_cpu = cpumask_any_but(cpu_online_mask, cpu); 256 257 if (new_cpu < nr_cpu_ids) 258 re_ctrl.target_vp = hv_vp_index[new_cpu]; 259 else 260 re_ctrl.enabled = 0; 261 262 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 263 } 264 265 return 0; 266 } 267 268 static int __init hv_pci_init(void) 269 { 270 int gen2vm = efi_enabled(EFI_BOOT); 271 272 /* 273 * For Generation-2 VM, we exit from pci_arch_init() by returning 0. 274 * The purpose is to suppress the harmless warning: 275 * "PCI: Fatal: No config space access function found" 276 */ 277 if (gen2vm) 278 return 0; 279 280 /* For Generation-1 VM, we'll proceed in pci_arch_init(). */ 281 return 1; 282 } 283 284 static int hv_suspend(void) 285 { 286 union hv_x64_msr_hypercall_contents hypercall_msr; 287 int ret; 288 289 if (hv_root_partition) 290 return -EPERM; 291 292 /* 293 * Reset the hypercall page as it is going to be invalidated 294 * across hibernation. Setting hv_hypercall_pg to NULL ensures 295 * that any subsequent hypercall operation fails safely instead of 296 * crashing due to an access of an invalid page. The hypercall page 297 * pointer is restored on resume. 298 */ 299 hv_hypercall_pg_saved = hv_hypercall_pg; 300 hv_hypercall_pg = NULL; 301 302 /* Disable the hypercall page in the hypervisor */ 303 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 304 hypercall_msr.enable = 0; 305 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 306 307 ret = hv_cpu_die(0); 308 return ret; 309 } 310 311 static void hv_resume(void) 312 { 313 union hv_x64_msr_hypercall_contents hypercall_msr; 314 int ret; 315 316 ret = hv_cpu_init(0); 317 WARN_ON(ret); 318 319 /* Re-enable the hypercall page */ 320 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 321 hypercall_msr.enable = 1; 322 hypercall_msr.guest_physical_address = 323 vmalloc_to_pfn(hv_hypercall_pg_saved); 324 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 325 326 hv_hypercall_pg = hv_hypercall_pg_saved; 327 hv_hypercall_pg_saved = NULL; 328 329 /* 330 * Reenlightenment notifications are disabled by hv_cpu_die(0), 331 * reenable them here if hv_reenlightenment_cb was previously set. 332 */ 333 if (hv_reenlightenment_cb) 334 set_hv_tscchange_cb(hv_reenlightenment_cb); 335 } 336 337 /* Note: when the ops are called, only CPU0 is online and IRQs are disabled. */ 338 static struct syscore_ops hv_syscore_ops = { 339 .suspend = hv_suspend, 340 .resume = hv_resume, 341 }; 342 343 static void (* __initdata old_setup_percpu_clockev)(void); 344 345 static void __init hv_stimer_setup_percpu_clockev(void) 346 { 347 /* 348 * Ignore any errors in setting up stimer clockevents 349 * as we can run with the LAPIC timer as a fallback. 350 */ 351 (void)hv_stimer_alloc(false); 352 353 /* 354 * Still register the LAPIC timer, because the direct-mode STIMER is 355 * not supported by old versions of Hyper-V. This also allows users 356 * to switch to LAPIC timer via /sys, if they want to. 357 */ 358 if (old_setup_percpu_clockev) 359 old_setup_percpu_clockev(); 360 } 361 362 static void __init hv_get_partition_id(void) 363 { 364 struct hv_get_partition_id *output_page; 365 u64 status; 366 unsigned long flags; 367 368 local_irq_save(flags); 369 output_page = *this_cpu_ptr(hyperv_pcpu_output_arg); 370 status = hv_do_hypercall(HVCALL_GET_PARTITION_ID, NULL, output_page); 371 if (!hv_result_success(status)) { 372 /* No point in proceeding if this failed */ 373 pr_err("Failed to get partition ID: %lld\n", status); 374 BUG(); 375 } 376 hv_current_partition_id = output_page->partition_id; 377 local_irq_restore(flags); 378 } 379 380 /* 381 * This function is to be invoked early in the boot sequence after the 382 * hypervisor has been detected. 383 * 384 * 1. Setup the hypercall page. 385 * 2. Register Hyper-V specific clocksource. 386 * 3. Setup Hyper-V specific APIC entry points. 387 */ 388 void __init hyperv_init(void) 389 { 390 u64 guest_id; 391 union hv_x64_msr_hypercall_contents hypercall_msr; 392 int cpuhp; 393 394 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 395 return; 396 397 if (hv_common_init()) 398 return; 399 400 hv_vp_assist_page = kcalloc(num_possible_cpus(), 401 sizeof(*hv_vp_assist_page), GFP_KERNEL); 402 if (!hv_vp_assist_page) { 403 ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 404 goto common_free; 405 } 406 407 if (hv_isolation_type_snp()) { 408 hv_ghcb_pg = alloc_percpu(union hv_ghcb *); 409 if (!hv_ghcb_pg) 410 goto free_vp_assist_page; 411 } 412 413 cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online", 414 hv_cpu_init, hv_cpu_die); 415 if (cpuhp < 0) 416 goto free_ghcb_page; 417 418 /* 419 * Setup the hypercall page and enable hypercalls. 420 * 1. Register the guest ID 421 * 2. Enable the hypercall and register the hypercall page 422 */ 423 guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); 424 wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 425 426 /* Hyper-V requires to write guest os id via ghcb in SNP IVM. */ 427 hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, guest_id); 428 429 hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, 430 VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX, 431 VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, 432 __builtin_return_address(0)); 433 if (hv_hypercall_pg == NULL) 434 goto clean_guest_os_id; 435 436 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 437 hypercall_msr.enable = 1; 438 439 if (hv_root_partition) { 440 struct page *pg; 441 void *src, *dst; 442 443 /* 444 * For the root partition, the hypervisor will set up its 445 * hypercall page. The hypervisor guarantees it will not show 446 * up in the root's address space. The root can't change the 447 * location of the hypercall page. 448 * 449 * Order is important here. We must enable the hypercall page 450 * so it is populated with code, then copy the code to an 451 * executable page. 452 */ 453 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 454 455 pg = vmalloc_to_page(hv_hypercall_pg); 456 dst = kmap(pg); 457 src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE, 458 MEMREMAP_WB); 459 BUG_ON(!(src && dst)); 460 memcpy(dst, src, HV_HYP_PAGE_SIZE); 461 memunmap(src); 462 kunmap(pg); 463 } else { 464 hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); 465 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 466 } 467 468 /* 469 * hyperv_init() is called before LAPIC is initialized: see 470 * apic_intr_mode_init() -> x86_platform.apic_post_init() and 471 * apic_bsp_setup() -> setup_local_APIC(). The direct-mode STIMER 472 * depends on LAPIC, so hv_stimer_alloc() should be called from 473 * x86_init.timers.setup_percpu_clockev. 474 */ 475 old_setup_percpu_clockev = x86_init.timers.setup_percpu_clockev; 476 x86_init.timers.setup_percpu_clockev = hv_stimer_setup_percpu_clockev; 477 478 hv_apic_init(); 479 480 x86_init.pci.arch_init = hv_pci_init; 481 482 register_syscore_ops(&hv_syscore_ops); 483 484 hyperv_init_cpuhp = cpuhp; 485 486 if (cpuid_ebx(HYPERV_CPUID_FEATURES) & HV_ACCESS_PARTITION_ID) 487 hv_get_partition_id(); 488 489 BUG_ON(hv_root_partition && hv_current_partition_id == ~0ull); 490 491 #ifdef CONFIG_PCI_MSI 492 /* 493 * If we're running as root, we want to create our own PCI MSI domain. 494 * We can't set this in hv_pci_init because that would be too late. 495 */ 496 if (hv_root_partition) 497 x86_init.irqs.create_pci_msi_domain = hv_create_pci_msi_domain; 498 #endif 499 500 /* Query the VMs extended capability once, so that it can be cached. */ 501 hv_query_ext_cap(0); 502 503 #ifdef CONFIG_SWIOTLB 504 /* 505 * Swiotlb bounce buffer needs to be mapped in extra address 506 * space. Map function doesn't work in the early place and so 507 * call swiotlb_update_mem_attributes() here. 508 */ 509 if (hv_is_isolation_supported()) 510 swiotlb_update_mem_attributes(); 511 #endif 512 513 return; 514 515 clean_guest_os_id: 516 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 517 hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, 0); 518 cpuhp_remove_state(cpuhp); 519 free_ghcb_page: 520 free_percpu(hv_ghcb_pg); 521 free_vp_assist_page: 522 kfree(hv_vp_assist_page); 523 hv_vp_assist_page = NULL; 524 common_free: 525 hv_common_free(); 526 } 527 528 /* 529 * This routine is called before kexec/kdump, it does the required cleanup. 530 */ 531 void hyperv_cleanup(void) 532 { 533 union hv_x64_msr_hypercall_contents hypercall_msr; 534 535 unregister_syscore_ops(&hv_syscore_ops); 536 537 /* Reset our OS id */ 538 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 539 hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, 0); 540 541 /* 542 * Reset hypercall page reference before reset the page, 543 * let hypercall operations fail safely rather than 544 * panic the kernel for using invalid hypercall page 545 */ 546 hv_hypercall_pg = NULL; 547 548 /* Reset the hypercall page */ 549 hypercall_msr.as_uint64 = 0; 550 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 551 552 /* Reset the TSC page */ 553 hypercall_msr.as_uint64 = 0; 554 wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64); 555 } 556 557 void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die) 558 { 559 static bool panic_reported; 560 u64 guest_id; 561 562 if (in_die && !panic_on_oops) 563 return; 564 565 /* 566 * We prefer to report panic on 'die' chain as we have proper 567 * registers to report, but if we miss it (e.g. on BUG()) we need 568 * to report it on 'panic'. 569 */ 570 if (panic_reported) 571 return; 572 panic_reported = true; 573 574 rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 575 576 wrmsrl(HV_X64_MSR_CRASH_P0, err); 577 wrmsrl(HV_X64_MSR_CRASH_P1, guest_id); 578 wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip); 579 wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax); 580 wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp); 581 582 /* 583 * Let Hyper-V know there is crash data available 584 */ 585 wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY); 586 } 587 EXPORT_SYMBOL_GPL(hyperv_report_panic); 588 589 bool hv_is_hyperv_initialized(void) 590 { 591 union hv_x64_msr_hypercall_contents hypercall_msr; 592 593 /* 594 * Ensure that we're really on Hyper-V, and not a KVM or Xen 595 * emulation of Hyper-V 596 */ 597 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 598 return false; 599 600 /* 601 * Verify that earlier initialization succeeded by checking 602 * that the hypercall page is setup 603 */ 604 hypercall_msr.as_uint64 = 0; 605 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 606 607 return hypercall_msr.enable; 608 } 609 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized); 610