1 /* 2 * X86 specific Hyper-V initialization code. 3 * 4 * Copyright (C) 2016, Microsoft, Inc. 5 * 6 * Author : K. Y. Srinivasan <kys@microsoft.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License version 2 as published 10 * by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, but 13 * WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 15 * NON INFRINGEMENT. See the GNU General Public License for more 16 * details. 17 * 18 */ 19 20 #include <linux/types.h> 21 #include <asm/apic.h> 22 #include <asm/desc.h> 23 #include <asm/hypervisor.h> 24 #include <asm/hyperv-tlfs.h> 25 #include <asm/mshyperv.h> 26 #include <linux/version.h> 27 #include <linux/vmalloc.h> 28 #include <linux/mm.h> 29 #include <linux/clockchips.h> 30 #include <linux/hyperv.h> 31 #include <linux/slab.h> 32 #include <linux/cpuhotplug.h> 33 34 #ifdef CONFIG_HYPERV_TSCPAGE 35 36 static struct ms_hyperv_tsc_page *tsc_pg; 37 38 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 39 { 40 return tsc_pg; 41 } 42 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 43 44 static u64 read_hv_clock_tsc(struct clocksource *arg) 45 { 46 u64 current_tick = hv_read_tsc_page(tsc_pg); 47 48 if (current_tick == U64_MAX) 49 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 50 51 return current_tick; 52 } 53 54 static struct clocksource hyperv_cs_tsc = { 55 .name = "hyperv_clocksource_tsc_page", 56 .rating = 400, 57 .read = read_hv_clock_tsc, 58 .mask = CLOCKSOURCE_MASK(64), 59 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 60 }; 61 #endif 62 63 static u64 read_hv_clock_msr(struct clocksource *arg) 64 { 65 u64 current_tick; 66 /* 67 * Read the partition counter to get the current tick count. This count 68 * is set to 0 when the partition is created and is incremented in 69 * 100 nanosecond units. 70 */ 71 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 72 return current_tick; 73 } 74 75 static struct clocksource hyperv_cs_msr = { 76 .name = "hyperv_clocksource_msr", 77 .rating = 400, 78 .read = read_hv_clock_msr, 79 .mask = CLOCKSOURCE_MASK(64), 80 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 81 }; 82 83 void *hv_hypercall_pg; 84 EXPORT_SYMBOL_GPL(hv_hypercall_pg); 85 struct clocksource *hyperv_cs; 86 EXPORT_SYMBOL_GPL(hyperv_cs); 87 88 u32 *hv_vp_index; 89 EXPORT_SYMBOL_GPL(hv_vp_index); 90 91 struct hv_vp_assist_page **hv_vp_assist_page; 92 EXPORT_SYMBOL_GPL(hv_vp_assist_page); 93 94 void __percpu **hyperv_pcpu_input_arg; 95 EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg); 96 97 u32 hv_max_vp_index; 98 99 static int hv_cpu_init(unsigned int cpu) 100 { 101 u64 msr_vp_index; 102 struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()]; 103 void **input_arg; 104 105 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 106 *input_arg = page_address(alloc_page(GFP_KERNEL)); 107 108 hv_get_vp_index(msr_vp_index); 109 110 hv_vp_index[smp_processor_id()] = msr_vp_index; 111 112 if (msr_vp_index > hv_max_vp_index) 113 hv_max_vp_index = msr_vp_index; 114 115 if (!hv_vp_assist_page) 116 return 0; 117 118 if (!*hvp) 119 *hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 120 121 if (*hvp) { 122 u64 val; 123 124 val = vmalloc_to_pfn(*hvp); 125 val = (val << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) | 126 HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; 127 128 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, val); 129 } 130 131 return 0; 132 } 133 134 static void (*hv_reenlightenment_cb)(void); 135 136 static void hv_reenlightenment_notify(struct work_struct *dummy) 137 { 138 struct hv_tsc_emulation_status emu_status; 139 140 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 141 142 /* Don't issue the callback if TSC accesses are not emulated */ 143 if (hv_reenlightenment_cb && emu_status.inprogress) 144 hv_reenlightenment_cb(); 145 } 146 static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify); 147 148 void hyperv_stop_tsc_emulation(void) 149 { 150 u64 freq; 151 struct hv_tsc_emulation_status emu_status; 152 153 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 154 emu_status.inprogress = 0; 155 wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 156 157 rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq); 158 tsc_khz = div64_u64(freq, 1000); 159 } 160 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation); 161 162 static inline bool hv_reenlightenment_available(void) 163 { 164 /* 165 * Check for required features and priviliges to make TSC frequency 166 * change notifications work. 167 */ 168 return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS && 169 ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE && 170 ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT; 171 } 172 173 __visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs) 174 { 175 entering_ack_irq(); 176 177 inc_irq_stat(irq_hv_reenlightenment_count); 178 179 schedule_delayed_work(&hv_reenlightenment_work, HZ/10); 180 181 exiting_irq(); 182 } 183 184 void set_hv_tscchange_cb(void (*cb)(void)) 185 { 186 struct hv_reenlightenment_control re_ctrl = { 187 .vector = HYPERV_REENLIGHTENMENT_VECTOR, 188 .enabled = 1, 189 .target_vp = hv_vp_index[smp_processor_id()] 190 }; 191 struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1}; 192 193 if (!hv_reenlightenment_available()) { 194 pr_warn("Hyper-V: reenlightenment support is unavailable\n"); 195 return; 196 } 197 198 hv_reenlightenment_cb = cb; 199 200 /* Make sure callback is registered before we write to MSRs */ 201 wmb(); 202 203 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 204 wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl)); 205 } 206 EXPORT_SYMBOL_GPL(set_hv_tscchange_cb); 207 208 void clear_hv_tscchange_cb(void) 209 { 210 struct hv_reenlightenment_control re_ctrl; 211 212 if (!hv_reenlightenment_available()) 213 return; 214 215 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 216 re_ctrl.enabled = 0; 217 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 218 219 hv_reenlightenment_cb = NULL; 220 } 221 EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb); 222 223 static int hv_cpu_die(unsigned int cpu) 224 { 225 struct hv_reenlightenment_control re_ctrl; 226 unsigned int new_cpu; 227 unsigned long flags; 228 void **input_arg; 229 void *input_pg = NULL; 230 231 local_irq_save(flags); 232 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 233 input_pg = *input_arg; 234 *input_arg = NULL; 235 local_irq_restore(flags); 236 free_page((unsigned long)input_pg); 237 238 if (hv_vp_assist_page && hv_vp_assist_page[cpu]) 239 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0); 240 241 if (hv_reenlightenment_cb == NULL) 242 return 0; 243 244 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 245 if (re_ctrl.target_vp == hv_vp_index[cpu]) { 246 /* Reassign to some other online CPU */ 247 new_cpu = cpumask_any_but(cpu_online_mask, cpu); 248 249 re_ctrl.target_vp = hv_vp_index[new_cpu]; 250 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 251 } 252 253 return 0; 254 } 255 256 /* 257 * This function is to be invoked early in the boot sequence after the 258 * hypervisor has been detected. 259 * 260 * 1. Setup the hypercall page. 261 * 2. Register Hyper-V specific clocksource. 262 * 3. Setup Hyper-V specific APIC entry points. 263 */ 264 void __init hyperv_init(void) 265 { 266 u64 guest_id, required_msrs; 267 union hv_x64_msr_hypercall_contents hypercall_msr; 268 int cpuhp, i; 269 270 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 271 return; 272 273 /* Absolutely required MSRs */ 274 required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE | 275 HV_X64_MSR_VP_INDEX_AVAILABLE; 276 277 if ((ms_hyperv.features & required_msrs) != required_msrs) 278 return; 279 280 /* 281 * Allocate the per-CPU state for the hypercall input arg. 282 * If this allocation fails, we will not be able to setup 283 * (per-CPU) hypercall input page and thus this failure is 284 * fatal on Hyper-V. 285 */ 286 hyperv_pcpu_input_arg = alloc_percpu(void *); 287 288 BUG_ON(hyperv_pcpu_input_arg == NULL); 289 290 /* Allocate percpu VP index */ 291 hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index), 292 GFP_KERNEL); 293 if (!hv_vp_index) 294 return; 295 296 for (i = 0; i < num_possible_cpus(); i++) 297 hv_vp_index[i] = VP_INVAL; 298 299 hv_vp_assist_page = kcalloc(num_possible_cpus(), 300 sizeof(*hv_vp_assist_page), GFP_KERNEL); 301 if (!hv_vp_assist_page) { 302 ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 303 goto free_vp_index; 304 } 305 306 cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online", 307 hv_cpu_init, hv_cpu_die); 308 if (cpuhp < 0) 309 goto free_vp_assist_page; 310 311 /* 312 * Setup the hypercall page and enable hypercalls. 313 * 1. Register the guest ID 314 * 2. Enable the hypercall and register the hypercall page 315 */ 316 guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); 317 wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 318 319 hv_hypercall_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX); 320 if (hv_hypercall_pg == NULL) { 321 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 322 goto remove_cpuhp_state; 323 } 324 325 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 326 hypercall_msr.enable = 1; 327 hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); 328 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 329 330 hv_apic_init(); 331 332 /* 333 * Register Hyper-V specific clocksource. 334 */ 335 #ifdef CONFIG_HYPERV_TSCPAGE 336 if (ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE) { 337 union hv_x64_msr_hypercall_contents tsc_msr; 338 339 tsc_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 340 if (!tsc_pg) 341 goto register_msr_cs; 342 343 hyperv_cs = &hyperv_cs_tsc; 344 345 rdmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 346 347 tsc_msr.enable = 1; 348 tsc_msr.guest_physical_address = vmalloc_to_pfn(tsc_pg); 349 350 wrmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 351 352 hyperv_cs_tsc.archdata.vclock_mode = VCLOCK_HVCLOCK; 353 354 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 355 return; 356 } 357 register_msr_cs: 358 #endif 359 /* 360 * For 32 bit guests just use the MSR based mechanism for reading 361 * the partition counter. 362 */ 363 364 hyperv_cs = &hyperv_cs_msr; 365 if (ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE) 366 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 367 368 return; 369 370 remove_cpuhp_state: 371 cpuhp_remove_state(cpuhp); 372 free_vp_assist_page: 373 kfree(hv_vp_assist_page); 374 hv_vp_assist_page = NULL; 375 free_vp_index: 376 kfree(hv_vp_index); 377 hv_vp_index = NULL; 378 } 379 380 /* 381 * This routine is called before kexec/kdump, it does the required cleanup. 382 */ 383 void hyperv_cleanup(void) 384 { 385 union hv_x64_msr_hypercall_contents hypercall_msr; 386 387 /* Reset our OS id */ 388 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 389 390 /* Reset the hypercall page */ 391 hypercall_msr.as_uint64 = 0; 392 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 393 394 /* Reset the TSC page */ 395 hypercall_msr.as_uint64 = 0; 396 wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64); 397 } 398 EXPORT_SYMBOL_GPL(hyperv_cleanup); 399 400 void hyperv_report_panic(struct pt_regs *regs, long err) 401 { 402 static bool panic_reported; 403 u64 guest_id; 404 405 /* 406 * We prefer to report panic on 'die' chain as we have proper 407 * registers to report, but if we miss it (e.g. on BUG()) we need 408 * to report it on 'panic'. 409 */ 410 if (panic_reported) 411 return; 412 panic_reported = true; 413 414 rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 415 416 wrmsrl(HV_X64_MSR_CRASH_P0, err); 417 wrmsrl(HV_X64_MSR_CRASH_P1, guest_id); 418 wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip); 419 wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax); 420 wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp); 421 422 /* 423 * Let Hyper-V know there is crash data available 424 */ 425 wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY); 426 } 427 EXPORT_SYMBOL_GPL(hyperv_report_panic); 428 429 /** 430 * hyperv_report_panic_msg - report panic message to Hyper-V 431 * @pa: physical address of the panic page containing the message 432 * @size: size of the message in the page 433 */ 434 void hyperv_report_panic_msg(phys_addr_t pa, size_t size) 435 { 436 /* 437 * P3 to contain the physical address of the panic page & P4 to 438 * contain the size of the panic data in that page. Rest of the 439 * registers are no-op when the NOTIFY_MSG flag is set. 440 */ 441 wrmsrl(HV_X64_MSR_CRASH_P0, 0); 442 wrmsrl(HV_X64_MSR_CRASH_P1, 0); 443 wrmsrl(HV_X64_MSR_CRASH_P2, 0); 444 wrmsrl(HV_X64_MSR_CRASH_P3, pa); 445 wrmsrl(HV_X64_MSR_CRASH_P4, size); 446 447 /* 448 * Let Hyper-V know there is crash data available along with 449 * the panic message. 450 */ 451 wrmsrl(HV_X64_MSR_CRASH_CTL, 452 (HV_CRASH_CTL_CRASH_NOTIFY | HV_CRASH_CTL_CRASH_NOTIFY_MSG)); 453 } 454 EXPORT_SYMBOL_GPL(hyperv_report_panic_msg); 455 456 bool hv_is_hyperv_initialized(void) 457 { 458 union hv_x64_msr_hypercall_contents hypercall_msr; 459 460 /* 461 * Ensure that we're really on Hyper-V, and not a KVM or Xen 462 * emulation of Hyper-V 463 */ 464 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 465 return false; 466 467 /* 468 * Verify that earlier initialization succeeded by checking 469 * that the hypercall page is setup 470 */ 471 hypercall_msr.as_uint64 = 0; 472 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 473 474 return hypercall_msr.enable; 475 } 476 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized); 477