1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM Microsoft Hyper-V emulation 4 * 5 * derived from arch/x86/kvm/x86.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com> 12 * 13 * Authors: 14 * Avi Kivity <avi@qumranet.com> 15 * Yaniv Kamay <yaniv@qumranet.com> 16 * Amit Shah <amit.shah@qumranet.com> 17 * Ben-Ami Yassour <benami@il.ibm.com> 18 * Andrey Smetanin <asmetanin@virtuozzo.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include "x86.h" 23 #include "lapic.h" 24 #include "ioapic.h" 25 #include "cpuid.h" 26 #include "hyperv.h" 27 #include "mmu.h" 28 #include "xen.h" 29 30 #include <linux/cpu.h> 31 #include <linux/kvm_host.h> 32 #include <linux/highmem.h> 33 #include <linux/sched/cputime.h> 34 #include <linux/spinlock.h> 35 #include <linux/eventfd.h> 36 37 #include <asm/apicdef.h> 38 #include <asm/mshyperv.h> 39 #include <trace/events/kvm.h> 40 41 #include "trace.h" 42 #include "irq.h" 43 #include "fpu.h" 44 45 #define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, HV_VCPUS_PER_SPARSE_BANK) 46 47 /* 48 * As per Hyper-V TLFS, extended hypercalls start from 0x8001 49 * (HvExtCallQueryCapabilities). Response of this hypercalls is a 64 bit value 50 * where each bit tells which extended hypercall is available besides 51 * HvExtCallQueryCapabilities. 52 * 53 * 0x8001 - First extended hypercall, HvExtCallQueryCapabilities, no bit 54 * assigned. 55 * 56 * 0x8002 - Bit 0 57 * 0x8003 - Bit 1 58 * .. 59 * 0x8041 - Bit 63 60 * 61 * Therefore, HV_EXT_CALL_MAX = 0x8001 + 64 62 */ 63 #define HV_EXT_CALL_MAX (HV_EXT_CALL_QUERY_CAPABILITIES + 64) 64 65 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 66 bool vcpu_kick); 67 68 static inline u64 synic_read_sint(struct kvm_vcpu_hv_synic *synic, int sint) 69 { 70 return atomic64_read(&synic->sint[sint]); 71 } 72 73 static inline int synic_get_sint_vector(u64 sint_value) 74 { 75 if (sint_value & HV_SYNIC_SINT_MASKED) 76 return -1; 77 return sint_value & HV_SYNIC_SINT_VECTOR_MASK; 78 } 79 80 static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic, 81 int vector) 82 { 83 int i; 84 85 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 86 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 87 return true; 88 } 89 return false; 90 } 91 92 static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic, 93 int vector) 94 { 95 int i; 96 u64 sint_value; 97 98 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 99 sint_value = synic_read_sint(synic, i); 100 if (synic_get_sint_vector(sint_value) == vector && 101 sint_value & HV_SYNIC_SINT_AUTO_EOI) 102 return true; 103 } 104 return false; 105 } 106 107 static void synic_update_vector(struct kvm_vcpu_hv_synic *synic, 108 int vector) 109 { 110 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 111 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 112 bool auto_eoi_old, auto_eoi_new; 113 114 if (vector < HV_SYNIC_FIRST_VALID_VECTOR) 115 return; 116 117 if (synic_has_vector_connected(synic, vector)) 118 __set_bit(vector, synic->vec_bitmap); 119 else 120 __clear_bit(vector, synic->vec_bitmap); 121 122 auto_eoi_old = !bitmap_empty(synic->auto_eoi_bitmap, 256); 123 124 if (synic_has_vector_auto_eoi(synic, vector)) 125 __set_bit(vector, synic->auto_eoi_bitmap); 126 else 127 __clear_bit(vector, synic->auto_eoi_bitmap); 128 129 auto_eoi_new = !bitmap_empty(synic->auto_eoi_bitmap, 256); 130 131 if (auto_eoi_old == auto_eoi_new) 132 return; 133 134 if (!enable_apicv) 135 return; 136 137 down_write(&vcpu->kvm->arch.apicv_update_lock); 138 139 if (auto_eoi_new) 140 hv->synic_auto_eoi_used++; 141 else 142 hv->synic_auto_eoi_used--; 143 144 /* 145 * Inhibit APICv if any vCPU is using SynIC's AutoEOI, which relies on 146 * the hypervisor to manually inject IRQs. 147 */ 148 __kvm_set_or_clear_apicv_inhibit(vcpu->kvm, 149 APICV_INHIBIT_REASON_HYPERV, 150 !!hv->synic_auto_eoi_used); 151 152 up_write(&vcpu->kvm->arch.apicv_update_lock); 153 } 154 155 static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, 156 u64 data, bool host) 157 { 158 int vector, old_vector; 159 bool masked; 160 161 vector = data & HV_SYNIC_SINT_VECTOR_MASK; 162 masked = data & HV_SYNIC_SINT_MASKED; 163 164 /* 165 * Valid vectors are 16-255, however, nested Hyper-V attempts to write 166 * default '0x10000' value on boot and this should not #GP. We need to 167 * allow zero-initing the register from host as well. 168 */ 169 if (vector < HV_SYNIC_FIRST_VALID_VECTOR && !host && !masked) 170 return 1; 171 /* 172 * Guest may configure multiple SINTs to use the same vector, so 173 * we maintain a bitmap of vectors handled by synic, and a 174 * bitmap of vectors with auto-eoi behavior. The bitmaps are 175 * updated here, and atomically queried on fast paths. 176 */ 177 old_vector = synic_read_sint(synic, sint) & HV_SYNIC_SINT_VECTOR_MASK; 178 179 atomic64_set(&synic->sint[sint], data); 180 181 synic_update_vector(synic, old_vector); 182 183 synic_update_vector(synic, vector); 184 185 /* Load SynIC vectors into EOI exit bitmap */ 186 kvm_make_request(KVM_REQ_SCAN_IOAPIC, hv_synic_to_vcpu(synic)); 187 return 0; 188 } 189 190 static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx) 191 { 192 struct kvm_vcpu *vcpu = NULL; 193 unsigned long i; 194 195 if (vpidx >= KVM_MAX_VCPUS) 196 return NULL; 197 198 vcpu = kvm_get_vcpu(kvm, vpidx); 199 if (vcpu && kvm_hv_get_vpindex(vcpu) == vpidx) 200 return vcpu; 201 kvm_for_each_vcpu(i, vcpu, kvm) 202 if (kvm_hv_get_vpindex(vcpu) == vpidx) 203 return vcpu; 204 return NULL; 205 } 206 207 static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx) 208 { 209 struct kvm_vcpu *vcpu; 210 struct kvm_vcpu_hv_synic *synic; 211 212 vcpu = get_vcpu_by_vpidx(kvm, vpidx); 213 if (!vcpu || !to_hv_vcpu(vcpu)) 214 return NULL; 215 synic = to_hv_synic(vcpu); 216 return (synic->active) ? synic : NULL; 217 } 218 219 static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint) 220 { 221 struct kvm *kvm = vcpu->kvm; 222 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 223 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 224 struct kvm_vcpu_hv_stimer *stimer; 225 int gsi, idx; 226 227 trace_kvm_hv_notify_acked_sint(vcpu->vcpu_id, sint); 228 229 /* Try to deliver pending Hyper-V SynIC timers messages */ 230 for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) { 231 stimer = &hv_vcpu->stimer[idx]; 232 if (stimer->msg_pending && stimer->config.enable && 233 !stimer->config.direct_mode && 234 stimer->config.sintx == sint) 235 stimer_mark_pending(stimer, false); 236 } 237 238 idx = srcu_read_lock(&kvm->irq_srcu); 239 gsi = atomic_read(&synic->sint_to_gsi[sint]); 240 if (gsi != -1) 241 kvm_notify_acked_gsi(kvm, gsi); 242 srcu_read_unlock(&kvm->irq_srcu, idx); 243 } 244 245 static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr) 246 { 247 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 248 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 249 250 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC; 251 hv_vcpu->exit.u.synic.msr = msr; 252 hv_vcpu->exit.u.synic.control = synic->control; 253 hv_vcpu->exit.u.synic.evt_page = synic->evt_page; 254 hv_vcpu->exit.u.synic.msg_page = synic->msg_page; 255 256 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 257 } 258 259 static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, 260 u32 msr, u64 data, bool host) 261 { 262 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 263 int ret; 264 265 if (!synic->active && (!host || data)) 266 return 1; 267 268 trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host); 269 270 ret = 0; 271 switch (msr) { 272 case HV_X64_MSR_SCONTROL: 273 synic->control = data; 274 if (!host) 275 synic_exit(synic, msr); 276 break; 277 case HV_X64_MSR_SVERSION: 278 if (!host) { 279 ret = 1; 280 break; 281 } 282 synic->version = data; 283 break; 284 case HV_X64_MSR_SIEFP: 285 if ((data & HV_SYNIC_SIEFP_ENABLE) && !host && 286 !synic->dont_zero_synic_pages) 287 if (kvm_clear_guest(vcpu->kvm, 288 data & PAGE_MASK, PAGE_SIZE)) { 289 ret = 1; 290 break; 291 } 292 synic->evt_page = data; 293 if (!host) 294 synic_exit(synic, msr); 295 break; 296 case HV_X64_MSR_SIMP: 297 if ((data & HV_SYNIC_SIMP_ENABLE) && !host && 298 !synic->dont_zero_synic_pages) 299 if (kvm_clear_guest(vcpu->kvm, 300 data & PAGE_MASK, PAGE_SIZE)) { 301 ret = 1; 302 break; 303 } 304 synic->msg_page = data; 305 if (!host) 306 synic_exit(synic, msr); 307 break; 308 case HV_X64_MSR_EOM: { 309 int i; 310 311 if (!synic->active) 312 break; 313 314 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 315 kvm_hv_notify_acked_sint(vcpu, i); 316 break; 317 } 318 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 319 ret = synic_set_sint(synic, msr - HV_X64_MSR_SINT0, data, host); 320 break; 321 default: 322 ret = 1; 323 break; 324 } 325 return ret; 326 } 327 328 static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) 329 { 330 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 331 332 return hv_vcpu->cpuid_cache.syndbg_cap_eax & 333 HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 334 } 335 336 static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) 337 { 338 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 339 340 if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL) 341 hv->hv_syndbg.control.status = 342 vcpu->run->hyperv.u.syndbg.status; 343 return 1; 344 } 345 346 static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr) 347 { 348 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 349 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 350 351 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG; 352 hv_vcpu->exit.u.syndbg.msr = msr; 353 hv_vcpu->exit.u.syndbg.control = syndbg->control.control; 354 hv_vcpu->exit.u.syndbg.send_page = syndbg->control.send_page; 355 hv_vcpu->exit.u.syndbg.recv_page = syndbg->control.recv_page; 356 hv_vcpu->exit.u.syndbg.pending_page = syndbg->control.pending_page; 357 vcpu->arch.complete_userspace_io = 358 kvm_hv_syndbg_complete_userspace; 359 360 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 361 } 362 363 static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 364 { 365 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 366 367 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 368 return 1; 369 370 trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id, 371 to_hv_vcpu(vcpu)->vp_index, msr, data); 372 switch (msr) { 373 case HV_X64_MSR_SYNDBG_CONTROL: 374 syndbg->control.control = data; 375 if (!host) 376 syndbg_exit(vcpu, msr); 377 break; 378 case HV_X64_MSR_SYNDBG_STATUS: 379 syndbg->control.status = data; 380 break; 381 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 382 syndbg->control.send_page = data; 383 break; 384 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 385 syndbg->control.recv_page = data; 386 break; 387 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 388 syndbg->control.pending_page = data; 389 if (!host) 390 syndbg_exit(vcpu, msr); 391 break; 392 case HV_X64_MSR_SYNDBG_OPTIONS: 393 syndbg->options = data; 394 break; 395 default: 396 break; 397 } 398 399 return 0; 400 } 401 402 static int syndbg_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 403 { 404 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 405 406 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 407 return 1; 408 409 switch (msr) { 410 case HV_X64_MSR_SYNDBG_CONTROL: 411 *pdata = syndbg->control.control; 412 break; 413 case HV_X64_MSR_SYNDBG_STATUS: 414 *pdata = syndbg->control.status; 415 break; 416 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 417 *pdata = syndbg->control.send_page; 418 break; 419 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 420 *pdata = syndbg->control.recv_page; 421 break; 422 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 423 *pdata = syndbg->control.pending_page; 424 break; 425 case HV_X64_MSR_SYNDBG_OPTIONS: 426 *pdata = syndbg->options; 427 break; 428 default: 429 break; 430 } 431 432 trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, kvm_hv_get_vpindex(vcpu), msr, *pdata); 433 434 return 0; 435 } 436 437 static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata, 438 bool host) 439 { 440 int ret; 441 442 if (!synic->active && !host) 443 return 1; 444 445 ret = 0; 446 switch (msr) { 447 case HV_X64_MSR_SCONTROL: 448 *pdata = synic->control; 449 break; 450 case HV_X64_MSR_SVERSION: 451 *pdata = synic->version; 452 break; 453 case HV_X64_MSR_SIEFP: 454 *pdata = synic->evt_page; 455 break; 456 case HV_X64_MSR_SIMP: 457 *pdata = synic->msg_page; 458 break; 459 case HV_X64_MSR_EOM: 460 *pdata = 0; 461 break; 462 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 463 *pdata = atomic64_read(&synic->sint[msr - HV_X64_MSR_SINT0]); 464 break; 465 default: 466 ret = 1; 467 break; 468 } 469 return ret; 470 } 471 472 static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint) 473 { 474 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 475 struct kvm_lapic_irq irq; 476 int ret, vector; 477 478 if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 479 return -EINVAL; 480 481 if (sint >= ARRAY_SIZE(synic->sint)) 482 return -EINVAL; 483 484 vector = synic_get_sint_vector(synic_read_sint(synic, sint)); 485 if (vector < 0) 486 return -ENOENT; 487 488 memset(&irq, 0, sizeof(irq)); 489 irq.shorthand = APIC_DEST_SELF; 490 irq.dest_mode = APIC_DEST_PHYSICAL; 491 irq.delivery_mode = APIC_DM_FIXED; 492 irq.vector = vector; 493 irq.level = 1; 494 495 ret = kvm_irq_delivery_to_apic(vcpu->kvm, vcpu->arch.apic, &irq); 496 trace_kvm_hv_synic_set_irq(vcpu->vcpu_id, sint, irq.vector, ret); 497 return ret; 498 } 499 500 int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 501 int irq_source_id, int level, bool line_status) 502 { 503 struct kvm_vcpu_hv_synic *synic; 504 505 if (!level) 506 return -1; 507 508 synic = synic_get(kvm, e->hv_sint.vcpu); 509 if (!synic) 510 return -EINVAL; 511 512 return synic_set_irq(synic, e->hv_sint.sint); 513 } 514 515 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector) 516 { 517 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 518 int i; 519 520 trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector); 521 522 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 523 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 524 kvm_hv_notify_acked_sint(vcpu, i); 525 } 526 527 static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi) 528 { 529 struct kvm_vcpu_hv_synic *synic; 530 531 synic = synic_get(kvm, vpidx); 532 if (!synic) 533 return -EINVAL; 534 535 if (sint >= ARRAY_SIZE(synic->sint_to_gsi)) 536 return -EINVAL; 537 538 atomic_set(&synic->sint_to_gsi[sint], gsi); 539 return 0; 540 } 541 542 void kvm_hv_irq_routing_update(struct kvm *kvm) 543 { 544 struct kvm_irq_routing_table *irq_rt; 545 struct kvm_kernel_irq_routing_entry *e; 546 u32 gsi; 547 548 irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu, 549 lockdep_is_held(&kvm->irq_lock)); 550 551 for (gsi = 0; gsi < irq_rt->nr_rt_entries; gsi++) { 552 hlist_for_each_entry(e, &irq_rt->map[gsi], link) { 553 if (e->type == KVM_IRQ_ROUTING_HV_SINT) 554 kvm_hv_set_sint_gsi(kvm, e->hv_sint.vcpu, 555 e->hv_sint.sint, gsi); 556 } 557 } 558 } 559 560 static void synic_init(struct kvm_vcpu_hv_synic *synic) 561 { 562 int i; 563 564 memset(synic, 0, sizeof(*synic)); 565 synic->version = HV_SYNIC_VERSION_1; 566 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 567 atomic64_set(&synic->sint[i], HV_SYNIC_SINT_MASKED); 568 atomic_set(&synic->sint_to_gsi[i], -1); 569 } 570 } 571 572 static u64 get_time_ref_counter(struct kvm *kvm) 573 { 574 struct kvm_hv *hv = to_kvm_hv(kvm); 575 struct kvm_vcpu *vcpu; 576 u64 tsc; 577 578 /* 579 * Fall back to get_kvmclock_ns() when TSC page hasn't been set up, 580 * is broken, disabled or being updated. 581 */ 582 if (hv->hv_tsc_page_status != HV_TSC_PAGE_SET) 583 return div_u64(get_kvmclock_ns(kvm), 100); 584 585 vcpu = kvm_get_vcpu(kvm, 0); 586 tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 587 return mul_u64_u64_shr(tsc, hv->tsc_ref.tsc_scale, 64) 588 + hv->tsc_ref.tsc_offset; 589 } 590 591 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 592 bool vcpu_kick) 593 { 594 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 595 596 set_bit(stimer->index, 597 to_hv_vcpu(vcpu)->stimer_pending_bitmap); 598 kvm_make_request(KVM_REQ_HV_STIMER, vcpu); 599 if (vcpu_kick) 600 kvm_vcpu_kick(vcpu); 601 } 602 603 static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer) 604 { 605 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 606 607 trace_kvm_hv_stimer_cleanup(hv_stimer_to_vcpu(stimer)->vcpu_id, 608 stimer->index); 609 610 hrtimer_cancel(&stimer->timer); 611 clear_bit(stimer->index, 612 to_hv_vcpu(vcpu)->stimer_pending_bitmap); 613 stimer->msg_pending = false; 614 stimer->exp_time = 0; 615 } 616 617 static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) 618 { 619 struct kvm_vcpu_hv_stimer *stimer; 620 621 stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer); 622 trace_kvm_hv_stimer_callback(hv_stimer_to_vcpu(stimer)->vcpu_id, 623 stimer->index); 624 stimer_mark_pending(stimer, true); 625 626 return HRTIMER_NORESTART; 627 } 628 629 /* 630 * stimer_start() assumptions: 631 * a) stimer->count is not equal to 0 632 * b) stimer->config has HV_STIMER_ENABLE flag 633 */ 634 static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) 635 { 636 u64 time_now; 637 ktime_t ktime_now; 638 639 time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm); 640 ktime_now = ktime_get(); 641 642 if (stimer->config.periodic) { 643 if (stimer->exp_time) { 644 if (time_now >= stimer->exp_time) { 645 u64 remainder; 646 647 div64_u64_rem(time_now - stimer->exp_time, 648 stimer->count, &remainder); 649 stimer->exp_time = 650 time_now + (stimer->count - remainder); 651 } 652 } else 653 stimer->exp_time = time_now + stimer->count; 654 655 trace_kvm_hv_stimer_start_periodic( 656 hv_stimer_to_vcpu(stimer)->vcpu_id, 657 stimer->index, 658 time_now, stimer->exp_time); 659 660 hrtimer_start(&stimer->timer, 661 ktime_add_ns(ktime_now, 662 100 * (stimer->exp_time - time_now)), 663 HRTIMER_MODE_ABS); 664 return 0; 665 } 666 stimer->exp_time = stimer->count; 667 if (time_now >= stimer->count) { 668 /* 669 * Expire timer according to Hypervisor Top-Level Functional 670 * specification v4(15.3.1): 671 * "If a one shot is enabled and the specified count is in 672 * the past, it will expire immediately." 673 */ 674 stimer_mark_pending(stimer, false); 675 return 0; 676 } 677 678 trace_kvm_hv_stimer_start_one_shot(hv_stimer_to_vcpu(stimer)->vcpu_id, 679 stimer->index, 680 time_now, stimer->count); 681 682 hrtimer_start(&stimer->timer, 683 ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)), 684 HRTIMER_MODE_ABS); 685 return 0; 686 } 687 688 static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, 689 bool host) 690 { 691 union hv_stimer_config new_config = {.as_uint64 = config}, 692 old_config = {.as_uint64 = stimer->config.as_uint64}; 693 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 694 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 695 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 696 697 if (!synic->active && (!host || config)) 698 return 1; 699 700 if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode && 701 !(hv_vcpu->cpuid_cache.features_edx & 702 HV_STIMER_DIRECT_MODE_AVAILABLE))) 703 return 1; 704 705 trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id, 706 stimer->index, config, host); 707 708 stimer_cleanup(stimer); 709 if (old_config.enable && 710 !new_config.direct_mode && new_config.sintx == 0) 711 new_config.enable = 0; 712 stimer->config.as_uint64 = new_config.as_uint64; 713 714 if (stimer->config.enable) 715 stimer_mark_pending(stimer, false); 716 717 return 0; 718 } 719 720 static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count, 721 bool host) 722 { 723 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 724 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 725 726 if (!synic->active && (!host || count)) 727 return 1; 728 729 trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id, 730 stimer->index, count, host); 731 732 stimer_cleanup(stimer); 733 stimer->count = count; 734 if (!host) { 735 if (stimer->count == 0) 736 stimer->config.enable = 0; 737 else if (stimer->config.auto_enable) 738 stimer->config.enable = 1; 739 } 740 741 if (stimer->config.enable) 742 stimer_mark_pending(stimer, false); 743 744 return 0; 745 } 746 747 static int stimer_get_config(struct kvm_vcpu_hv_stimer *stimer, u64 *pconfig) 748 { 749 *pconfig = stimer->config.as_uint64; 750 return 0; 751 } 752 753 static int stimer_get_count(struct kvm_vcpu_hv_stimer *stimer, u64 *pcount) 754 { 755 *pcount = stimer->count; 756 return 0; 757 } 758 759 static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, 760 struct hv_message *src_msg, bool no_retry) 761 { 762 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 763 int msg_off = offsetof(struct hv_message_page, sint_message[sint]); 764 gfn_t msg_page_gfn; 765 struct hv_message_header hv_hdr; 766 int r; 767 768 if (!(synic->msg_page & HV_SYNIC_SIMP_ENABLE)) 769 return -ENOENT; 770 771 msg_page_gfn = synic->msg_page >> PAGE_SHIFT; 772 773 /* 774 * Strictly following the spec-mandated ordering would assume setting 775 * .msg_pending before checking .message_type. However, this function 776 * is only called in vcpu context so the entire update is atomic from 777 * guest POV and thus the exact order here doesn't matter. 778 */ 779 r = kvm_vcpu_read_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_type, 780 msg_off + offsetof(struct hv_message, 781 header.message_type), 782 sizeof(hv_hdr.message_type)); 783 if (r < 0) 784 return r; 785 786 if (hv_hdr.message_type != HVMSG_NONE) { 787 if (no_retry) 788 return 0; 789 790 hv_hdr.message_flags.msg_pending = 1; 791 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, 792 &hv_hdr.message_flags, 793 msg_off + 794 offsetof(struct hv_message, 795 header.message_flags), 796 sizeof(hv_hdr.message_flags)); 797 if (r < 0) 798 return r; 799 return -EAGAIN; 800 } 801 802 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, src_msg, msg_off, 803 sizeof(src_msg->header) + 804 src_msg->header.payload_size); 805 if (r < 0) 806 return r; 807 808 r = synic_set_irq(synic, sint); 809 if (r < 0) 810 return r; 811 if (r == 0) 812 return -EFAULT; 813 return 0; 814 } 815 816 static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer) 817 { 818 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 819 struct hv_message *msg = &stimer->msg; 820 struct hv_timer_message_payload *payload = 821 (struct hv_timer_message_payload *)&msg->u.payload; 822 823 /* 824 * To avoid piling up periodic ticks, don't retry message 825 * delivery for them (within "lazy" lost ticks policy). 826 */ 827 bool no_retry = stimer->config.periodic; 828 829 payload->expiration_time = stimer->exp_time; 830 payload->delivery_time = get_time_ref_counter(vcpu->kvm); 831 return synic_deliver_msg(to_hv_synic(vcpu), 832 stimer->config.sintx, msg, 833 no_retry); 834 } 835 836 static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer) 837 { 838 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 839 struct kvm_lapic_irq irq = { 840 .delivery_mode = APIC_DM_FIXED, 841 .vector = stimer->config.apic_vector 842 }; 843 844 if (lapic_in_kernel(vcpu)) 845 return !kvm_apic_set_irq(vcpu, &irq, NULL); 846 return 0; 847 } 848 849 static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) 850 { 851 int r, direct = stimer->config.direct_mode; 852 853 stimer->msg_pending = true; 854 if (!direct) 855 r = stimer_send_msg(stimer); 856 else 857 r = stimer_notify_direct(stimer); 858 trace_kvm_hv_stimer_expiration(hv_stimer_to_vcpu(stimer)->vcpu_id, 859 stimer->index, direct, r); 860 if (!r) { 861 stimer->msg_pending = false; 862 if (!(stimer->config.periodic)) 863 stimer->config.enable = 0; 864 } 865 } 866 867 void kvm_hv_process_stimers(struct kvm_vcpu *vcpu) 868 { 869 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 870 struct kvm_vcpu_hv_stimer *stimer; 871 u64 time_now, exp_time; 872 int i; 873 874 if (!hv_vcpu) 875 return; 876 877 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 878 if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) { 879 stimer = &hv_vcpu->stimer[i]; 880 if (stimer->config.enable) { 881 exp_time = stimer->exp_time; 882 883 if (exp_time) { 884 time_now = 885 get_time_ref_counter(vcpu->kvm); 886 if (time_now >= exp_time) 887 stimer_expiration(stimer); 888 } 889 890 if ((stimer->config.enable) && 891 stimer->count) { 892 if (!stimer->msg_pending) 893 stimer_start(stimer); 894 } else 895 stimer_cleanup(stimer); 896 } 897 } 898 } 899 900 void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu) 901 { 902 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 903 int i; 904 905 if (!hv_vcpu) 906 return; 907 908 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 909 stimer_cleanup(&hv_vcpu->stimer[i]); 910 911 kfree(hv_vcpu); 912 vcpu->arch.hyperv = NULL; 913 } 914 915 bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu) 916 { 917 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 918 919 if (!hv_vcpu) 920 return false; 921 922 if (!(hv_vcpu->hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) 923 return false; 924 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 925 } 926 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_hv_assist_page_enabled); 927 928 int kvm_hv_get_assist_page(struct kvm_vcpu *vcpu) 929 { 930 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 931 932 if (!hv_vcpu || !kvm_hv_assist_page_enabled(vcpu)) 933 return -EFAULT; 934 935 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, 936 &hv_vcpu->vp_assist_page, sizeof(struct hv_vp_assist_page)); 937 } 938 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_hv_get_assist_page); 939 940 static void stimer_prepare_msg(struct kvm_vcpu_hv_stimer *stimer) 941 { 942 struct hv_message *msg = &stimer->msg; 943 struct hv_timer_message_payload *payload = 944 (struct hv_timer_message_payload *)&msg->u.payload; 945 946 memset(&msg->header, 0, sizeof(msg->header)); 947 msg->header.message_type = HVMSG_TIMER_EXPIRED; 948 msg->header.payload_size = sizeof(*payload); 949 950 payload->timer_index = stimer->index; 951 payload->expiration_time = 0; 952 payload->delivery_time = 0; 953 } 954 955 static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index) 956 { 957 memset(stimer, 0, sizeof(*stimer)); 958 stimer->index = timer_index; 959 hrtimer_setup(&stimer->timer, stimer_timer_callback, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 960 stimer_prepare_msg(stimer); 961 } 962 963 int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) 964 { 965 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 966 int i; 967 968 if (hv_vcpu) 969 return 0; 970 971 hv_vcpu = kzalloc_obj(struct kvm_vcpu_hv, GFP_KERNEL_ACCOUNT); 972 if (!hv_vcpu) 973 return -ENOMEM; 974 975 vcpu->arch.hyperv = hv_vcpu; 976 hv_vcpu->vcpu = vcpu; 977 978 synic_init(&hv_vcpu->synic); 979 980 bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); 981 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 982 stimer_init(&hv_vcpu->stimer[i], i); 983 984 hv_vcpu->vp_index = vcpu->vcpu_idx; 985 986 for (i = 0; i < HV_NR_TLB_FLUSH_FIFOS; i++) { 987 INIT_KFIFO(hv_vcpu->tlb_flush_fifo[i].entries); 988 spin_lock_init(&hv_vcpu->tlb_flush_fifo[i].write_lock); 989 } 990 991 return 0; 992 } 993 994 int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages) 995 { 996 struct kvm_vcpu_hv_synic *synic; 997 int r; 998 999 r = kvm_hv_vcpu_init(vcpu); 1000 if (r) 1001 return r; 1002 1003 synic = to_hv_synic(vcpu); 1004 1005 synic->active = true; 1006 synic->dont_zero_synic_pages = dont_zero_synic_pages; 1007 synic->control = HV_SYNIC_CONTROL_ENABLE; 1008 return 0; 1009 } 1010 1011 static bool kvm_hv_msr_partition_wide(u32 msr) 1012 { 1013 bool r = false; 1014 1015 switch (msr) { 1016 case HV_X64_MSR_GUEST_OS_ID: 1017 case HV_X64_MSR_HYPERCALL: 1018 case HV_X64_MSR_REFERENCE_TSC: 1019 case HV_X64_MSR_TIME_REF_COUNT: 1020 case HV_X64_MSR_CRASH_CTL: 1021 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1022 case HV_X64_MSR_RESET: 1023 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1024 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1025 case HV_X64_MSR_TSC_EMULATION_STATUS: 1026 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1027 case HV_X64_MSR_SYNDBG_OPTIONS: 1028 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1029 r = true; 1030 break; 1031 } 1032 1033 return r; 1034 } 1035 1036 static int kvm_hv_msr_get_crash_data(struct kvm *kvm, u32 index, u64 *pdata) 1037 { 1038 struct kvm_hv *hv = to_kvm_hv(kvm); 1039 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1040 1041 if (WARN_ON_ONCE(index >= size)) 1042 return -EINVAL; 1043 1044 *pdata = hv->hv_crash_param[array_index_nospec(index, size)]; 1045 return 0; 1046 } 1047 1048 static int kvm_hv_msr_get_crash_ctl(struct kvm *kvm, u64 *pdata) 1049 { 1050 struct kvm_hv *hv = to_kvm_hv(kvm); 1051 1052 *pdata = hv->hv_crash_ctl; 1053 return 0; 1054 } 1055 1056 static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data) 1057 { 1058 struct kvm_hv *hv = to_kvm_hv(kvm); 1059 1060 hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY; 1061 1062 return 0; 1063 } 1064 1065 static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data) 1066 { 1067 struct kvm_hv *hv = to_kvm_hv(kvm); 1068 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1069 1070 if (WARN_ON_ONCE(index >= size)) 1071 return -EINVAL; 1072 1073 hv->hv_crash_param[array_index_nospec(index, size)] = data; 1074 return 0; 1075 } 1076 1077 /* 1078 * The kvmclock and Hyper-V TSC page use similar formulas, and converting 1079 * between them is possible: 1080 * 1081 * kvmclock formula: 1082 * nsec = (ticks - tsc_timestamp) * tsc_to_system_mul * 2^(tsc_shift-32) 1083 * + system_time 1084 * 1085 * Hyper-V formula: 1086 * nsec/100 = ticks * scale / 2^64 + offset 1087 * 1088 * When tsc_timestamp = system_time = 0, offset is zero in the Hyper-V formula. 1089 * By dividing the kvmclock formula by 100 and equating what's left we get: 1090 * ticks * scale / 2^64 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1091 * scale / 2^64 = tsc_to_system_mul * 2^(tsc_shift-32) / 100 1092 * scale = tsc_to_system_mul * 2^(32+tsc_shift) / 100 1093 * 1094 * Now expand the kvmclock formula and divide by 100: 1095 * nsec = ticks * tsc_to_system_mul * 2^(tsc_shift-32) 1096 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) 1097 * + system_time 1098 * nsec/100 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1099 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1100 * + system_time / 100 1101 * 1102 * Replace tsc_to_system_mul * 2^(tsc_shift-32) / 100 by scale / 2^64: 1103 * nsec/100 = ticks * scale / 2^64 1104 * - tsc_timestamp * scale / 2^64 1105 * + system_time / 100 1106 * 1107 * Equate with the Hyper-V formula so that ticks * scale / 2^64 cancels out: 1108 * offset = system_time / 100 - tsc_timestamp * scale / 2^64 1109 * 1110 * These two equivalencies are implemented in this function. 1111 */ 1112 static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock, 1113 struct ms_hyperv_tsc_page *tsc_ref) 1114 { 1115 u64 max_mul; 1116 1117 if (!(hv_clock->flags & PVCLOCK_TSC_STABLE_BIT)) 1118 return false; 1119 1120 /* 1121 * check if scale would overflow, if so we use the time ref counter 1122 * tsc_to_system_mul * 2^(tsc_shift+32) / 100 >= 2^64 1123 * tsc_to_system_mul / 100 >= 2^(32-tsc_shift) 1124 * tsc_to_system_mul >= 100 * 2^(32-tsc_shift) 1125 */ 1126 max_mul = 100ull << (32 - hv_clock->tsc_shift); 1127 if (hv_clock->tsc_to_system_mul >= max_mul) 1128 return false; 1129 1130 /* 1131 * Otherwise compute the scale and offset according to the formulas 1132 * derived above. 1133 */ 1134 tsc_ref->tsc_scale = 1135 mul_u64_u32_div(1ULL << (32 + hv_clock->tsc_shift), 1136 hv_clock->tsc_to_system_mul, 1137 100); 1138 1139 tsc_ref->tsc_offset = hv_clock->system_time; 1140 do_div(tsc_ref->tsc_offset, 100); 1141 tsc_ref->tsc_offset -= 1142 mul_u64_u64_shr(hv_clock->tsc_timestamp, tsc_ref->tsc_scale, 64); 1143 return true; 1144 } 1145 1146 /* 1147 * Don't touch TSC page values if the guest has opted for TSC emulation after 1148 * migration. KVM doesn't fully support reenlightenment notifications and TSC 1149 * access emulation and Hyper-V is known to expect the values in TSC page to 1150 * stay constant before TSC access emulation is disabled from guest side 1151 * (HV_X64_MSR_TSC_EMULATION_STATUS). KVM userspace is expected to preserve TSC 1152 * frequency and guest visible TSC value across migration (and prevent it when 1153 * TSC scaling is unsupported). 1154 */ 1155 static inline bool tsc_page_update_unsafe(struct kvm_hv *hv) 1156 { 1157 return (hv->hv_tsc_page_status != HV_TSC_PAGE_GUEST_CHANGED) && 1158 hv->hv_tsc_emulation_control; 1159 } 1160 1161 void kvm_hv_setup_tsc_page(struct kvm *kvm, 1162 struct pvclock_vcpu_time_info *hv_clock) 1163 { 1164 struct kvm_hv *hv = to_kvm_hv(kvm); 1165 u32 tsc_seq; 1166 u64 gfn; 1167 1168 BUILD_BUG_ON(sizeof(tsc_seq) != sizeof(hv->tsc_ref.tsc_sequence)); 1169 BUILD_BUG_ON(offsetof(struct ms_hyperv_tsc_page, tsc_sequence) != 0); 1170 1171 guard(mutex)(&hv->hv_lock); 1172 1173 if (hv->hv_tsc_page_status == HV_TSC_PAGE_BROKEN || 1174 hv->hv_tsc_page_status == HV_TSC_PAGE_SET || 1175 hv->hv_tsc_page_status == HV_TSC_PAGE_UNSET) 1176 return; 1177 1178 if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE)) 1179 return; 1180 1181 gfn = hv->hv_tsc_page >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT; 1182 /* 1183 * Because the TSC parameters only vary when there is a 1184 * change in the master clock, do not bother with caching. 1185 */ 1186 if (unlikely(kvm_read_guest(kvm, gfn_to_gpa(gfn), 1187 &tsc_seq, sizeof(tsc_seq)))) 1188 goto out_err; 1189 1190 if (tsc_seq && tsc_page_update_unsafe(hv)) { 1191 if (kvm_read_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1192 goto out_err; 1193 1194 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1195 return; 1196 } 1197 1198 /* 1199 * While we're computing and writing the parameters, force the 1200 * guest to use the time reference count MSR. 1201 */ 1202 hv->tsc_ref.tsc_sequence = 0; 1203 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1204 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1205 goto out_err; 1206 1207 if (!compute_tsc_page_parameters(hv_clock, &hv->tsc_ref)) 1208 goto out_err; 1209 1210 /* Ensure sequence is zero before writing the rest of the struct. */ 1211 smp_wmb(); 1212 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1213 goto out_err; 1214 1215 /* 1216 * Now switch to the TSC page mechanism by writing the sequence. 1217 */ 1218 tsc_seq++; 1219 if (tsc_seq == 0xFFFFFFFF || tsc_seq == 0) 1220 tsc_seq = 1; 1221 1222 /* Write the struct entirely before the non-zero sequence. */ 1223 smp_wmb(); 1224 1225 hv->tsc_ref.tsc_sequence = tsc_seq; 1226 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1227 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1228 goto out_err; 1229 1230 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1231 return; 1232 1233 out_err: 1234 hv->hv_tsc_page_status = HV_TSC_PAGE_BROKEN; 1235 } 1236 1237 void kvm_hv_request_tsc_page_update(struct kvm *kvm) 1238 { 1239 struct kvm_hv *hv = to_kvm_hv(kvm); 1240 1241 mutex_lock(&hv->hv_lock); 1242 1243 if (hv->hv_tsc_page_status == HV_TSC_PAGE_SET && 1244 !tsc_page_update_unsafe(hv)) 1245 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1246 1247 mutex_unlock(&hv->hv_lock); 1248 } 1249 1250 static bool hv_check_msr_access(struct kvm_vcpu_hv *hv_vcpu, u32 msr) 1251 { 1252 if (!hv_vcpu->enforce_cpuid) 1253 return true; 1254 1255 switch (msr) { 1256 case HV_X64_MSR_GUEST_OS_ID: 1257 case HV_X64_MSR_HYPERCALL: 1258 return hv_vcpu->cpuid_cache.features_eax & 1259 HV_MSR_HYPERCALL_AVAILABLE; 1260 case HV_X64_MSR_VP_RUNTIME: 1261 return hv_vcpu->cpuid_cache.features_eax & 1262 HV_MSR_VP_RUNTIME_AVAILABLE; 1263 case HV_X64_MSR_TIME_REF_COUNT: 1264 return hv_vcpu->cpuid_cache.features_eax & 1265 HV_MSR_TIME_REF_COUNT_AVAILABLE; 1266 case HV_X64_MSR_VP_INDEX: 1267 return hv_vcpu->cpuid_cache.features_eax & 1268 HV_MSR_VP_INDEX_AVAILABLE; 1269 case HV_X64_MSR_RESET: 1270 return hv_vcpu->cpuid_cache.features_eax & 1271 HV_MSR_RESET_AVAILABLE; 1272 case HV_X64_MSR_REFERENCE_TSC: 1273 return hv_vcpu->cpuid_cache.features_eax & 1274 HV_MSR_REFERENCE_TSC_AVAILABLE; 1275 case HV_X64_MSR_SCONTROL: 1276 case HV_X64_MSR_SVERSION: 1277 case HV_X64_MSR_SIEFP: 1278 case HV_X64_MSR_SIMP: 1279 case HV_X64_MSR_EOM: 1280 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1281 return hv_vcpu->cpuid_cache.features_eax & 1282 HV_MSR_SYNIC_AVAILABLE; 1283 case HV_X64_MSR_STIMER0_CONFIG: 1284 case HV_X64_MSR_STIMER1_CONFIG: 1285 case HV_X64_MSR_STIMER2_CONFIG: 1286 case HV_X64_MSR_STIMER3_CONFIG: 1287 case HV_X64_MSR_STIMER0_COUNT: 1288 case HV_X64_MSR_STIMER1_COUNT: 1289 case HV_X64_MSR_STIMER2_COUNT: 1290 case HV_X64_MSR_STIMER3_COUNT: 1291 return hv_vcpu->cpuid_cache.features_eax & 1292 HV_MSR_SYNTIMER_AVAILABLE; 1293 case HV_X64_MSR_EOI: 1294 case HV_X64_MSR_ICR: 1295 case HV_X64_MSR_TPR: 1296 case HV_X64_MSR_VP_ASSIST_PAGE: 1297 return hv_vcpu->cpuid_cache.features_eax & 1298 HV_MSR_APIC_ACCESS_AVAILABLE; 1299 case HV_X64_MSR_TSC_FREQUENCY: 1300 case HV_X64_MSR_APIC_FREQUENCY: 1301 return hv_vcpu->cpuid_cache.features_eax & 1302 HV_ACCESS_FREQUENCY_MSRS; 1303 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1304 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1305 case HV_X64_MSR_TSC_EMULATION_STATUS: 1306 return hv_vcpu->cpuid_cache.features_eax & 1307 HV_ACCESS_REENLIGHTENMENT; 1308 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1309 return hv_vcpu->cpuid_cache.features_eax & 1310 HV_ACCESS_TSC_INVARIANT; 1311 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1312 case HV_X64_MSR_CRASH_CTL: 1313 return hv_vcpu->cpuid_cache.features_edx & 1314 HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 1315 case HV_X64_MSR_SYNDBG_OPTIONS: 1316 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1317 return hv_vcpu->cpuid_cache.features_edx & 1318 HV_FEATURE_DEBUG_MSRS_AVAILABLE; 1319 default: 1320 break; 1321 } 1322 1323 return false; 1324 } 1325 1326 #define KVM_HV_WIN2016_GUEST_ID 0x1040a00003839 1327 #define KVM_HV_WIN2016_GUEST_ID_MASK (~GENMASK_ULL(23, 16)) /* mask out the service version */ 1328 1329 /* 1330 * Hyper-V enabled Windows Server 2016 SMP VMs fail to boot in !XSAVES && XSAVEC 1331 * configuration. 1332 * Such configuration can result from, for example, AMD Erratum 1386 workaround. 1333 * 1334 * Print a notice so users aren't left wondering what's suddenly gone wrong. 1335 */ 1336 static void __kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu) 1337 { 1338 struct kvm *kvm = vcpu->kvm; 1339 struct kvm_hv *hv = to_kvm_hv(kvm); 1340 1341 /* Check again under the hv_lock. */ 1342 if (hv->xsaves_xsavec_checked) 1343 return; 1344 1345 if ((hv->hv_guest_os_id & KVM_HV_WIN2016_GUEST_ID_MASK) != 1346 KVM_HV_WIN2016_GUEST_ID) 1347 return; 1348 1349 hv->xsaves_xsavec_checked = true; 1350 1351 /* UP configurations aren't affected */ 1352 if (atomic_read(&kvm->online_vcpus) < 2) 1353 return; 1354 1355 if (guest_cpuid_has(vcpu, X86_FEATURE_XSAVES) || 1356 !guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVEC)) 1357 return; 1358 1359 pr_notice_ratelimited("Booting SMP Windows KVM VM with !XSAVES && XSAVEC. " 1360 "If it fails to boot try disabling XSAVEC in the VM config.\n"); 1361 } 1362 1363 void kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu) 1364 { 1365 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1366 1367 if (!vcpu->arch.hyperv_enabled || 1368 hv->xsaves_xsavec_checked) 1369 return; 1370 1371 mutex_lock(&hv->hv_lock); 1372 __kvm_hv_xsaves_xsavec_maybe_warn(vcpu); 1373 mutex_unlock(&hv->hv_lock); 1374 } 1375 1376 static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, 1377 bool host) 1378 { 1379 struct kvm *kvm = vcpu->kvm; 1380 struct kvm_hv *hv = to_kvm_hv(kvm); 1381 1382 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1383 return 1; 1384 1385 switch (msr) { 1386 case HV_X64_MSR_GUEST_OS_ID: 1387 hv->hv_guest_os_id = data; 1388 /* setting guest os id to zero disables hypercall page */ 1389 if (!hv->hv_guest_os_id) 1390 hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE; 1391 break; 1392 case HV_X64_MSR_HYPERCALL: { 1393 u8 instructions[9]; 1394 int i = 0; 1395 u64 addr; 1396 1397 /* if guest os id is not set hypercall should remain disabled */ 1398 if (!hv->hv_guest_os_id) 1399 break; 1400 if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) { 1401 hv->hv_hypercall = data; 1402 break; 1403 } 1404 1405 /* 1406 * If Xen and Hyper-V hypercalls are both enabled, disambiguate 1407 * the same way Xen itself does, by setting the bit 31 of EAX 1408 * which is RsvdZ in the 32-bit Hyper-V hypercall ABI and just 1409 * going to be clobbered on 64-bit. 1410 */ 1411 if (kvm_xen_hypercall_enabled(kvm)) { 1412 /* orl $0x80000000, %eax */ 1413 instructions[i++] = 0x0d; 1414 instructions[i++] = 0x00; 1415 instructions[i++] = 0x00; 1416 instructions[i++] = 0x00; 1417 instructions[i++] = 0x80; 1418 } 1419 1420 /* vmcall/vmmcall */ 1421 kvm_x86_call(patch_hypercall)(vcpu, instructions + i); 1422 i += 3; 1423 1424 /* ret */ 1425 ((unsigned char *)instructions)[i++] = 0xc3; 1426 1427 addr = data & HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK; 1428 if (kvm_vcpu_write_guest(vcpu, addr, instructions, i)) 1429 return 1; 1430 hv->hv_hypercall = data; 1431 break; 1432 } 1433 case HV_X64_MSR_REFERENCE_TSC: 1434 hv->hv_tsc_page = data; 1435 if (hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) { 1436 if (!host) 1437 hv->hv_tsc_page_status = HV_TSC_PAGE_GUEST_CHANGED; 1438 else 1439 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1440 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1441 } else { 1442 hv->hv_tsc_page_status = HV_TSC_PAGE_UNSET; 1443 } 1444 break; 1445 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1446 return kvm_hv_msr_set_crash_data(kvm, 1447 msr - HV_X64_MSR_CRASH_P0, 1448 data); 1449 case HV_X64_MSR_CRASH_CTL: 1450 if (host) 1451 return kvm_hv_msr_set_crash_ctl(kvm, data); 1452 1453 if (data & HV_CRASH_CTL_CRASH_NOTIFY) { 1454 vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n", 1455 hv->hv_crash_param[0], 1456 hv->hv_crash_param[1], 1457 hv->hv_crash_param[2], 1458 hv->hv_crash_param[3], 1459 hv->hv_crash_param[4]); 1460 1461 /* Send notification about crash to user space */ 1462 kvm_make_request(KVM_REQ_HV_CRASH, vcpu); 1463 } 1464 break; 1465 case HV_X64_MSR_RESET: 1466 if (data == 1) { 1467 vcpu_debug(vcpu, "hyper-v reset requested\n"); 1468 kvm_make_request(KVM_REQ_HV_RESET, vcpu); 1469 } 1470 break; 1471 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1472 hv->hv_reenlightenment_control = data; 1473 break; 1474 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1475 hv->hv_tsc_emulation_control = data; 1476 break; 1477 case HV_X64_MSR_TSC_EMULATION_STATUS: 1478 if (data && !host) 1479 return 1; 1480 1481 hv->hv_tsc_emulation_status = data; 1482 break; 1483 case HV_X64_MSR_TIME_REF_COUNT: 1484 /* read-only, but still ignore it if host-initiated */ 1485 if (!host) 1486 return 1; 1487 break; 1488 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1489 /* Only bit 0 is supported */ 1490 if (data & ~HV_EXPOSE_INVARIANT_TSC) 1491 return 1; 1492 1493 /* The feature can't be disabled from the guest */ 1494 if (!host && hv->hv_invtsc_control && !data) 1495 return 1; 1496 1497 hv->hv_invtsc_control = data; 1498 break; 1499 case HV_X64_MSR_SYNDBG_OPTIONS: 1500 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1501 return syndbg_set_msr(vcpu, msr, data, host); 1502 default: 1503 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1504 return 1; 1505 } 1506 return 0; 1507 } 1508 1509 /* Calculate cpu time spent by current task in 100ns units */ 1510 static u64 current_task_runtime_100ns(void) 1511 { 1512 u64 utime, stime; 1513 1514 task_cputime_adjusted(current, &utime, &stime); 1515 1516 return div_u64(utime + stime, 100); 1517 } 1518 1519 static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1520 { 1521 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1522 1523 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1524 return 1; 1525 1526 switch (msr) { 1527 case HV_X64_MSR_VP_INDEX: { 1528 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1529 u32 new_vp_index = (u32)data; 1530 1531 if (!host || new_vp_index >= KVM_MAX_VCPUS) 1532 return 1; 1533 1534 if (new_vp_index == hv_vcpu->vp_index) 1535 return 0; 1536 1537 /* 1538 * The VP index is initialized to vcpu_index by 1539 * kvm_hv_vcpu_postcreate so they initially match. Now the 1540 * VP index is changing, adjust num_mismatched_vp_indexes if 1541 * it now matches or no longer matches vcpu_idx. 1542 */ 1543 if (hv_vcpu->vp_index == vcpu->vcpu_idx) 1544 atomic_inc(&hv->num_mismatched_vp_indexes); 1545 else if (new_vp_index == vcpu->vcpu_idx) 1546 atomic_dec(&hv->num_mismatched_vp_indexes); 1547 1548 hv_vcpu->vp_index = new_vp_index; 1549 break; 1550 } 1551 case HV_X64_MSR_VP_ASSIST_PAGE: { 1552 u64 gfn; 1553 unsigned long addr; 1554 1555 if (!(data & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) { 1556 hv_vcpu->hv_vapic = data; 1557 if (kvm_lapic_set_pv_eoi(vcpu, 0, 0)) 1558 return 1; 1559 break; 1560 } 1561 gfn = data >> HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT; 1562 addr = kvm_vcpu_gfn_to_hva(vcpu, gfn); 1563 if (kvm_is_error_hva(addr)) 1564 return 1; 1565 1566 /* 1567 * Clear apic_assist portion of struct hv_vp_assist_page 1568 * only, there can be valuable data in the rest which needs 1569 * to be preserved e.g. on migration. 1570 */ 1571 if (put_user(0, (u32 __user *)addr)) 1572 return 1; 1573 hv_vcpu->hv_vapic = data; 1574 kvm_vcpu_mark_page_dirty(vcpu, gfn); 1575 if (kvm_lapic_set_pv_eoi(vcpu, 1576 gfn_to_gpa(gfn) | KVM_MSR_ENABLED, 1577 sizeof(struct hv_vp_assist_page))) 1578 return 1; 1579 break; 1580 } 1581 case HV_X64_MSR_EOI: 1582 return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data); 1583 case HV_X64_MSR_ICR: 1584 return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data); 1585 case HV_X64_MSR_TPR: 1586 return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data); 1587 case HV_X64_MSR_VP_RUNTIME: 1588 if (!host) 1589 return 1; 1590 hv_vcpu->runtime_offset = data - current_task_runtime_100ns(); 1591 break; 1592 case HV_X64_MSR_SCONTROL: 1593 case HV_X64_MSR_SVERSION: 1594 case HV_X64_MSR_SIEFP: 1595 case HV_X64_MSR_SIMP: 1596 case HV_X64_MSR_EOM: 1597 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1598 return synic_set_msr(to_hv_synic(vcpu), msr, data, host); 1599 case HV_X64_MSR_STIMER0_CONFIG: 1600 case HV_X64_MSR_STIMER1_CONFIG: 1601 case HV_X64_MSR_STIMER2_CONFIG: 1602 case HV_X64_MSR_STIMER3_CONFIG: { 1603 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1604 1605 return stimer_set_config(to_hv_stimer(vcpu, timer_index), 1606 data, host); 1607 } 1608 case HV_X64_MSR_STIMER0_COUNT: 1609 case HV_X64_MSR_STIMER1_COUNT: 1610 case HV_X64_MSR_STIMER2_COUNT: 1611 case HV_X64_MSR_STIMER3_COUNT: { 1612 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1613 1614 return stimer_set_count(to_hv_stimer(vcpu, timer_index), 1615 data, host); 1616 } 1617 case HV_X64_MSR_TSC_FREQUENCY: 1618 case HV_X64_MSR_APIC_FREQUENCY: 1619 /* read-only, but still ignore it if host-initiated */ 1620 if (!host) 1621 return 1; 1622 break; 1623 default: 1624 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1625 return 1; 1626 } 1627 1628 return 0; 1629 } 1630 1631 static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1632 bool host) 1633 { 1634 u64 data = 0; 1635 struct kvm *kvm = vcpu->kvm; 1636 struct kvm_hv *hv = to_kvm_hv(kvm); 1637 1638 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1639 return 1; 1640 1641 switch (msr) { 1642 case HV_X64_MSR_GUEST_OS_ID: 1643 data = hv->hv_guest_os_id; 1644 break; 1645 case HV_X64_MSR_HYPERCALL: 1646 data = hv->hv_hypercall; 1647 break; 1648 case HV_X64_MSR_TIME_REF_COUNT: 1649 data = get_time_ref_counter(kvm); 1650 break; 1651 case HV_X64_MSR_REFERENCE_TSC: 1652 data = hv->hv_tsc_page; 1653 break; 1654 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1655 return kvm_hv_msr_get_crash_data(kvm, 1656 msr - HV_X64_MSR_CRASH_P0, 1657 pdata); 1658 case HV_X64_MSR_CRASH_CTL: 1659 return kvm_hv_msr_get_crash_ctl(kvm, pdata); 1660 case HV_X64_MSR_RESET: 1661 data = 0; 1662 break; 1663 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1664 data = hv->hv_reenlightenment_control; 1665 break; 1666 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1667 data = hv->hv_tsc_emulation_control; 1668 break; 1669 case HV_X64_MSR_TSC_EMULATION_STATUS: 1670 data = hv->hv_tsc_emulation_status; 1671 break; 1672 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1673 data = hv->hv_invtsc_control; 1674 break; 1675 case HV_X64_MSR_SYNDBG_OPTIONS: 1676 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1677 return syndbg_get_msr(vcpu, msr, pdata, host); 1678 default: 1679 kvm_pr_unimpl_rdmsr(vcpu, msr); 1680 return 1; 1681 } 1682 1683 *pdata = data; 1684 return 0; 1685 } 1686 1687 static int kvm_hv_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1688 bool host) 1689 { 1690 u64 data = 0; 1691 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1692 1693 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1694 return 1; 1695 1696 switch (msr) { 1697 case HV_X64_MSR_VP_INDEX: 1698 data = hv_vcpu->vp_index; 1699 break; 1700 case HV_X64_MSR_EOI: 1701 return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata); 1702 case HV_X64_MSR_ICR: 1703 return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata); 1704 case HV_X64_MSR_TPR: 1705 return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata); 1706 case HV_X64_MSR_VP_ASSIST_PAGE: 1707 data = hv_vcpu->hv_vapic; 1708 break; 1709 case HV_X64_MSR_VP_RUNTIME: 1710 data = current_task_runtime_100ns() + hv_vcpu->runtime_offset; 1711 break; 1712 case HV_X64_MSR_SCONTROL: 1713 case HV_X64_MSR_SVERSION: 1714 case HV_X64_MSR_SIEFP: 1715 case HV_X64_MSR_SIMP: 1716 case HV_X64_MSR_EOM: 1717 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1718 return synic_get_msr(to_hv_synic(vcpu), msr, pdata, host); 1719 case HV_X64_MSR_STIMER0_CONFIG: 1720 case HV_X64_MSR_STIMER1_CONFIG: 1721 case HV_X64_MSR_STIMER2_CONFIG: 1722 case HV_X64_MSR_STIMER3_CONFIG: { 1723 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1724 1725 return stimer_get_config(to_hv_stimer(vcpu, timer_index), 1726 pdata); 1727 } 1728 case HV_X64_MSR_STIMER0_COUNT: 1729 case HV_X64_MSR_STIMER1_COUNT: 1730 case HV_X64_MSR_STIMER2_COUNT: 1731 case HV_X64_MSR_STIMER3_COUNT: { 1732 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1733 1734 return stimer_get_count(to_hv_stimer(vcpu, timer_index), 1735 pdata); 1736 } 1737 case HV_X64_MSR_TSC_FREQUENCY: 1738 data = (u64)vcpu->arch.virtual_tsc_khz * 1000; 1739 break; 1740 case HV_X64_MSR_APIC_FREQUENCY: 1741 data = div64_u64(1000000000ULL, 1742 vcpu->kvm->arch.apic_bus_cycle_ns); 1743 break; 1744 default: 1745 kvm_pr_unimpl_rdmsr(vcpu, msr); 1746 return 1; 1747 } 1748 *pdata = data; 1749 return 0; 1750 } 1751 1752 int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1753 { 1754 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1755 1756 if (!host && !vcpu->arch.hyperv_enabled) 1757 return 1; 1758 1759 if (kvm_hv_vcpu_init(vcpu)) 1760 return 1; 1761 1762 if (kvm_hv_msr_partition_wide(msr)) { 1763 int r; 1764 1765 mutex_lock(&hv->hv_lock); 1766 r = kvm_hv_set_msr_pw(vcpu, msr, data, host); 1767 mutex_unlock(&hv->hv_lock); 1768 return r; 1769 } else 1770 return kvm_hv_set_msr(vcpu, msr, data, host); 1771 } 1772 1773 int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 1774 { 1775 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1776 1777 if (!host && !vcpu->arch.hyperv_enabled) 1778 return 1; 1779 1780 if (kvm_hv_vcpu_init(vcpu)) 1781 return 1; 1782 1783 if (kvm_hv_msr_partition_wide(msr)) { 1784 int r; 1785 1786 mutex_lock(&hv->hv_lock); 1787 r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host); 1788 mutex_unlock(&hv->hv_lock); 1789 return r; 1790 } else 1791 return kvm_hv_get_msr(vcpu, msr, pdata, host); 1792 } 1793 1794 static void sparse_set_to_vcpu_mask(struct kvm *kvm, u64 *sparse_banks, 1795 u64 valid_bank_mask, unsigned long *vcpu_mask) 1796 { 1797 struct kvm_hv *hv = to_kvm_hv(kvm); 1798 bool has_mismatch = atomic_read(&hv->num_mismatched_vp_indexes); 1799 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS]; 1800 struct kvm_vcpu *vcpu; 1801 int bank, sbank = 0; 1802 unsigned long i; 1803 u64 *bitmap; 1804 1805 BUILD_BUG_ON(sizeof(vp_bitmap) > 1806 sizeof(*vcpu_mask) * BITS_TO_LONGS(KVM_MAX_VCPUS)); 1807 1808 /* 1809 * If vp_index == vcpu_idx for all vCPUs, fill vcpu_mask directly, else 1810 * fill a temporary buffer and manually test each vCPU's VP index. 1811 */ 1812 if (likely(!has_mismatch)) 1813 bitmap = (u64 *)vcpu_mask; 1814 else 1815 bitmap = vp_bitmap; 1816 1817 /* 1818 * Each set of 64 VPs is packed into sparse_banks, with valid_bank_mask 1819 * having a '1' for each bank that exists in sparse_banks. Sets must 1820 * be in ascending order, i.e. bank0..bankN. 1821 */ 1822 memset(bitmap, 0, sizeof(vp_bitmap)); 1823 for_each_set_bit(bank, (unsigned long *)&valid_bank_mask, 1824 KVM_HV_MAX_SPARSE_VCPU_SET_BITS) 1825 bitmap[bank] = sparse_banks[sbank++]; 1826 1827 if (likely(!has_mismatch)) 1828 return; 1829 1830 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 1831 kvm_for_each_vcpu(i, vcpu, kvm) { 1832 if (test_bit(kvm_hv_get_vpindex(vcpu), (unsigned long *)vp_bitmap)) 1833 __set_bit(i, vcpu_mask); 1834 } 1835 } 1836 1837 static bool hv_is_vp_in_sparse_set(u32 vp_id, u64 valid_bank_mask, u64 sparse_banks[]) 1838 { 1839 int valid_bit_nr = vp_id / HV_VCPUS_PER_SPARSE_BANK; 1840 unsigned long sbank; 1841 1842 BUILD_BUG_ON(BITS_PER_TYPE(valid_bank_mask) != HV_MAX_SPARSE_VCPU_BANKS); 1843 1844 if (valid_bit_nr >= HV_MAX_SPARSE_VCPU_BANKS) 1845 return false; 1846 1847 if (!test_bit(valid_bit_nr, (unsigned long *)&valid_bank_mask)) 1848 return false; 1849 1850 /* 1851 * The index into the sparse bank is the number of preceding bits in 1852 * the valid mask. Optimize for VMs with <64 vCPUs by skipping the 1853 * fancy math if there can't possibly be preceding bits. 1854 */ 1855 if (valid_bit_nr) 1856 sbank = hweight64(valid_bank_mask & GENMASK_ULL(valid_bit_nr - 1, 0)); 1857 else 1858 sbank = 0; 1859 1860 return test_bit(vp_id % HV_VCPUS_PER_SPARSE_BANK, 1861 (unsigned long *)&sparse_banks[sbank]); 1862 } 1863 1864 struct kvm_hv_hcall { 1865 /* Hypercall input data */ 1866 u64 param; 1867 u64 ingpa; 1868 u64 outgpa; 1869 u16 code; 1870 u16 var_cnt; 1871 u16 rep_cnt; 1872 u16 rep_idx; 1873 bool fast; 1874 bool rep; 1875 sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS]; 1876 1877 /* 1878 * Current read offset when KVM reads hypercall input data gradually, 1879 * either offset in bytes from 'ingpa' for regular hypercalls or the 1880 * number of already consumed 'XMM halves' for 'fast' hypercalls. 1881 */ 1882 union { 1883 gpa_t data_offset; 1884 int consumed_xmm_halves; 1885 }; 1886 }; 1887 1888 1889 static int kvm_hv_get_hc_data(struct kvm *kvm, struct kvm_hv_hcall *hc, 1890 u16 orig_cnt, u16 cnt_cap, u64 *data) 1891 { 1892 /* 1893 * Preserve the original count when ignoring entries via a "cap", KVM 1894 * still needs to validate the guest input (though the non-XMM path 1895 * punts on the checks). 1896 */ 1897 u16 cnt = min(orig_cnt, cnt_cap); 1898 int i, j; 1899 1900 if (hc->fast) { 1901 /* 1902 * Each XMM holds two sparse banks, but do not count halves that 1903 * have already been consumed for hypercall parameters. 1904 */ 1905 if (orig_cnt > 2 * HV_HYPERCALL_MAX_XMM_REGISTERS - hc->consumed_xmm_halves) 1906 return HV_STATUS_INVALID_HYPERCALL_INPUT; 1907 1908 for (i = 0; i < cnt; i++) { 1909 j = i + hc->consumed_xmm_halves; 1910 if (j % 2) 1911 data[i] = sse128_hi(hc->xmm[j / 2]); 1912 else 1913 data[i] = sse128_lo(hc->xmm[j / 2]); 1914 } 1915 return 0; 1916 } 1917 1918 return kvm_read_guest(kvm, hc->ingpa + hc->data_offset, data, 1919 cnt * sizeof(*data)); 1920 } 1921 1922 static u64 kvm_get_sparse_vp_set(struct kvm *kvm, struct kvm_hv_hcall *hc, 1923 u64 *sparse_banks) 1924 { 1925 if (hc->var_cnt > HV_MAX_SPARSE_VCPU_BANKS) 1926 return -EINVAL; 1927 1928 /* Cap var_cnt to ignore banks that cannot contain a legal VP index. */ 1929 return kvm_hv_get_hc_data(kvm, hc, hc->var_cnt, KVM_HV_MAX_SPARSE_VCPU_SET_BITS, 1930 sparse_banks); 1931 } 1932 1933 static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc, u64 entries[]) 1934 { 1935 return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries); 1936 } 1937 1938 static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, 1939 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo, 1940 u64 *entries, int count) 1941 { 1942 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1943 u64 flush_all_entry = KVM_HV_TLB_FLUSHALL_ENTRY; 1944 1945 if (!hv_vcpu) 1946 return; 1947 1948 spin_lock(&tlb_flush_fifo->write_lock); 1949 1950 /* 1951 * All entries should fit on the fifo leaving one free for 'flush all' 1952 * entry in case another request comes in. In case there's not enough 1953 * space, just put 'flush all' entry there. 1954 */ 1955 if (count && entries && count < kfifo_avail(&tlb_flush_fifo->entries)) { 1956 WARN_ON(kfifo_in(&tlb_flush_fifo->entries, entries, count) != count); 1957 goto out_unlock; 1958 } 1959 1960 /* 1961 * Note: full fifo always contains 'flush all' entry, no need to check the 1962 * return value. 1963 */ 1964 kfifo_in(&tlb_flush_fifo->entries, &flush_all_entry, 1); 1965 1966 out_unlock: 1967 spin_unlock(&tlb_flush_fifo->write_lock); 1968 } 1969 1970 int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) 1971 { 1972 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 1973 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1974 u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; 1975 int i, j, count; 1976 gva_t gva; 1977 1978 if (!tdp_enabled || !hv_vcpu) 1979 return -EINVAL; 1980 1981 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu)); 1982 1983 count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); 1984 1985 for (i = 0; i < count; i++) { 1986 if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) 1987 goto out_flush_all; 1988 1989 /* 1990 * Lower 12 bits of 'address' encode the number of additional 1991 * pages to flush. 1992 */ 1993 gva = entries[i] & PAGE_MASK; 1994 for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { 1995 if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) 1996 continue; 1997 1998 kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); 1999 } 2000 2001 ++vcpu->stat.tlb_flush; 2002 } 2003 return 0; 2004 2005 out_flush_all: 2006 kfifo_reset_out(&tlb_flush_fifo->entries); 2007 2008 /* Fall back to full flush. */ 2009 return -ENOSPC; 2010 } 2011 2012 static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2013 { 2014 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2015 unsigned long *vcpu_mask = hv_vcpu->vcpu_mask; 2016 u64 *sparse_banks = hv_vcpu->sparse_banks; 2017 struct kvm *kvm = vcpu->kvm; 2018 struct hv_tlb_flush_ex flush_ex; 2019 struct hv_tlb_flush flush; 2020 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 2021 /* 2022 * Normally, there can be no more than 'KVM_HV_TLB_FLUSH_FIFO_SIZE' 2023 * entries on the TLB flush fifo. The last entry, however, needs to be 2024 * always left free for 'flush all' entry which gets placed when 2025 * there is not enough space to put all the requested entries. 2026 */ 2027 u64 __tlb_flush_entries[KVM_HV_TLB_FLUSH_FIFO_SIZE - 1]; 2028 u64 *tlb_flush_entries; 2029 u64 valid_bank_mask; 2030 struct kvm_vcpu *v; 2031 unsigned long i; 2032 bool all_cpus; 2033 2034 /* 2035 * The Hyper-V TLFS doesn't allow more than HV_MAX_SPARSE_VCPU_BANKS 2036 * sparse banks. Fail the build if KVM's max allowed number of 2037 * vCPUs (>4096) exceeds this limit. 2038 */ 2039 BUILD_BUG_ON(KVM_HV_MAX_SPARSE_VCPU_SET_BITS > HV_MAX_SPARSE_VCPU_BANKS); 2040 2041 /* 2042 * 'Slow' hypercall's first parameter is the address in guest's memory 2043 * where hypercall parameters are placed. This is either a GPA or a 2044 * nested GPA when KVM is handling the call from L2 ('direct' TLB 2045 * flush). Translate the address here so the memory can be uniformly 2046 * read with kvm_read_guest(). 2047 */ 2048 if (!hc->fast && mmu_is_nested(vcpu)) { 2049 hc->ingpa = kvm_x86_ops.nested_ops->translate_nested_gpa( 2050 vcpu, hc->ingpa, 2051 PFERR_GUEST_FINAL_MASK, NULL, 0); 2052 if (unlikely(hc->ingpa == INVALID_GPA)) 2053 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2054 } 2055 2056 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST || 2057 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE) { 2058 if (hc->fast) { 2059 flush.address_space = hc->ingpa; 2060 flush.flags = hc->outgpa; 2061 flush.processor_mask = sse128_lo(hc->xmm[0]); 2062 hc->consumed_xmm_halves = 1; 2063 } else { 2064 if (unlikely(kvm_read_guest(kvm, hc->ingpa, 2065 &flush, sizeof(flush)))) 2066 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2067 hc->data_offset = sizeof(flush); 2068 } 2069 2070 trace_kvm_hv_flush_tlb(flush.processor_mask, 2071 flush.address_space, flush.flags, 2072 is_guest_mode(vcpu)); 2073 2074 valid_bank_mask = BIT_ULL(0); 2075 sparse_banks[0] = flush.processor_mask; 2076 2077 /* 2078 * Work around possible WS2012 bug: it sends hypercalls 2079 * with processor_mask = 0x0 and HV_FLUSH_ALL_PROCESSORS clear, 2080 * while also expecting us to flush something and crashing if 2081 * we don't. Let's treat processor_mask == 0 same as 2082 * HV_FLUSH_ALL_PROCESSORS. 2083 */ 2084 all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) || 2085 flush.processor_mask == 0; 2086 } else { 2087 if (hc->fast) { 2088 flush_ex.address_space = hc->ingpa; 2089 flush_ex.flags = hc->outgpa; 2090 memcpy(&flush_ex.hv_vp_set, 2091 &hc->xmm[0], sizeof(hc->xmm[0])); 2092 hc->consumed_xmm_halves = 2; 2093 } else { 2094 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex, 2095 sizeof(flush_ex)))) 2096 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2097 hc->data_offset = sizeof(flush_ex); 2098 } 2099 2100 trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask, 2101 flush_ex.hv_vp_set.format, 2102 flush_ex.address_space, 2103 flush_ex.flags, is_guest_mode(vcpu)); 2104 2105 valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask; 2106 all_cpus = flush_ex.hv_vp_set.format != 2107 HV_GENERIC_SET_SPARSE_4K; 2108 2109 if (hc->var_cnt != hweight64(valid_bank_mask)) 2110 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2111 2112 if (!all_cpus) { 2113 if (!hc->var_cnt) 2114 goto ret_success; 2115 2116 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2117 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2118 } 2119 2120 /* 2121 * Hyper-V TLFS doesn't explicitly forbid non-empty sparse vCPU 2122 * banks (and, thus, non-zero 'var_cnt') for the 'all vCPUs' 2123 * case (HV_GENERIC_SET_ALL). Always adjust data_offset and 2124 * consumed_xmm_halves to make sure TLB flush entries are read 2125 * from the correct offset. 2126 */ 2127 if (hc->fast) 2128 hc->consumed_xmm_halves += hc->var_cnt; 2129 else 2130 hc->data_offset += hc->var_cnt * sizeof(sparse_banks[0]); 2131 } 2132 2133 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE || 2134 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX || 2135 hc->rep_cnt > ARRAY_SIZE(__tlb_flush_entries)) { 2136 tlb_flush_entries = NULL; 2137 } else { 2138 if (kvm_hv_get_tlb_flush_entries(kvm, hc, __tlb_flush_entries)) 2139 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2140 tlb_flush_entries = __tlb_flush_entries; 2141 } 2142 2143 /* 2144 * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't 2145 * analyze it here, flush TLB regardless of the specified address space. 2146 */ 2147 if (all_cpus && !is_guest_mode(vcpu)) { 2148 kvm_for_each_vcpu(i, v, kvm) { 2149 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false); 2150 hv_tlb_flush_enqueue(v, tlb_flush_fifo, 2151 tlb_flush_entries, hc->rep_cnt); 2152 } 2153 2154 kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH); 2155 } else if (!is_guest_mode(vcpu)) { 2156 sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, vcpu_mask); 2157 2158 for_each_set_bit(i, vcpu_mask, KVM_MAX_VCPUS) { 2159 v = kvm_get_vcpu(kvm, i); 2160 if (!v) 2161 continue; 2162 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false); 2163 hv_tlb_flush_enqueue(v, tlb_flush_fifo, 2164 tlb_flush_entries, hc->rep_cnt); 2165 } 2166 2167 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2168 } else { 2169 struct kvm_vcpu_hv *hv_v; 2170 2171 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 2172 2173 kvm_for_each_vcpu(i, v, kvm) { 2174 hv_v = to_hv_vcpu(v); 2175 2176 /* 2177 * The following check races with nested vCPUs entering/exiting 2178 * and/or migrating between L1's vCPUs, however the only case when 2179 * KVM *must* flush the TLB is when the target L2 vCPU keeps 2180 * running on the same L1 vCPU from the moment of the request until 2181 * kvm_hv_flush_tlb() returns. TLB is fully flushed in all other 2182 * cases, e.g. when the target L2 vCPU migrates to a different L1 2183 * vCPU or when the corresponding L1 vCPU temporary switches to a 2184 * different L2 vCPU while the request is being processed. 2185 */ 2186 if (!hv_v || hv_v->nested.vm_id != hv_vcpu->nested.vm_id) 2187 continue; 2188 2189 if (!all_cpus && 2190 !hv_is_vp_in_sparse_set(hv_v->nested.vp_id, valid_bank_mask, 2191 sparse_banks)) 2192 continue; 2193 2194 __set_bit(i, vcpu_mask); 2195 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, true); 2196 hv_tlb_flush_enqueue(v, tlb_flush_fifo, 2197 tlb_flush_entries, hc->rep_cnt); 2198 } 2199 2200 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2201 } 2202 2203 ret_success: 2204 /* We always do full TLB flush, set 'Reps completed' = 'Rep Count' */ 2205 return (u64)HV_STATUS_SUCCESS | 2206 ((u64)hc->rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET); 2207 } 2208 2209 static void kvm_hv_send_ipi_to_many(struct kvm *kvm, u32 vector, 2210 u64 *sparse_banks, u64 valid_bank_mask) 2211 { 2212 struct kvm_lapic_irq irq = { 2213 .delivery_mode = APIC_DM_FIXED, 2214 .vector = vector 2215 }; 2216 struct kvm_vcpu *vcpu; 2217 unsigned long i; 2218 2219 kvm_for_each_vcpu(i, vcpu, kvm) { 2220 if (sparse_banks && 2221 !hv_is_vp_in_sparse_set(kvm_hv_get_vpindex(vcpu), 2222 valid_bank_mask, sparse_banks)) 2223 continue; 2224 2225 /* We fail only when APIC is disabled */ 2226 kvm_apic_set_irq(vcpu, &irq, NULL); 2227 } 2228 } 2229 2230 static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2231 { 2232 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2233 u64 *sparse_banks = hv_vcpu->sparse_banks; 2234 struct kvm *kvm = vcpu->kvm; 2235 struct hv_send_ipi_ex send_ipi_ex; 2236 struct hv_send_ipi send_ipi; 2237 u64 valid_bank_mask; 2238 u32 vector; 2239 bool all_cpus; 2240 2241 if (!lapic_in_kernel(vcpu)) 2242 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2243 2244 if (hc->code == HVCALL_SEND_IPI) { 2245 if (!hc->fast) { 2246 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi, 2247 sizeof(send_ipi)))) 2248 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2249 sparse_banks[0] = send_ipi.cpu_mask; 2250 vector = send_ipi.vector; 2251 } else { 2252 /* 'reserved' part of hv_send_ipi should be 0 */ 2253 if (unlikely(hc->ingpa >> 32 != 0)) 2254 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2255 sparse_banks[0] = hc->outgpa; 2256 vector = (u32)hc->ingpa; 2257 } 2258 all_cpus = false; 2259 valid_bank_mask = BIT_ULL(0); 2260 2261 trace_kvm_hv_send_ipi(vector, sparse_banks[0]); 2262 } else { 2263 if (!hc->fast) { 2264 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex, 2265 sizeof(send_ipi_ex)))) 2266 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2267 } else { 2268 send_ipi_ex.vector = (u32)hc->ingpa; 2269 send_ipi_ex.vp_set.format = hc->outgpa; 2270 send_ipi_ex.vp_set.valid_bank_mask = sse128_lo(hc->xmm[0]); 2271 } 2272 2273 trace_kvm_hv_send_ipi_ex(send_ipi_ex.vector, 2274 send_ipi_ex.vp_set.format, 2275 send_ipi_ex.vp_set.valid_bank_mask); 2276 2277 vector = send_ipi_ex.vector; 2278 valid_bank_mask = send_ipi_ex.vp_set.valid_bank_mask; 2279 all_cpus = send_ipi_ex.vp_set.format == HV_GENERIC_SET_ALL; 2280 2281 if (hc->var_cnt != hweight64(valid_bank_mask)) 2282 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2283 2284 if (all_cpus) 2285 goto check_and_send_ipi; 2286 2287 if (!hc->var_cnt) 2288 goto ret_success; 2289 2290 if (!hc->fast) 2291 hc->data_offset = offsetof(struct hv_send_ipi_ex, 2292 vp_set.bank_contents); 2293 else 2294 hc->consumed_xmm_halves = 1; 2295 2296 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2297 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2298 } 2299 2300 check_and_send_ipi: 2301 if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) 2302 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2303 2304 if (all_cpus) 2305 kvm_hv_send_ipi_to_many(kvm, vector, NULL, 0); 2306 else 2307 kvm_hv_send_ipi_to_many(kvm, vector, sparse_banks, valid_bank_mask); 2308 2309 ret_success: 2310 return HV_STATUS_SUCCESS; 2311 } 2312 2313 void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled) 2314 { 2315 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2316 struct kvm_cpuid_entry2 *entry; 2317 2318 vcpu->arch.hyperv_enabled = hyperv_enabled; 2319 2320 if (!hv_vcpu) { 2321 /* 2322 * KVM should have already allocated kvm_vcpu_hv if Hyper-V is 2323 * enabled in CPUID. 2324 */ 2325 WARN_ON_ONCE(vcpu->arch.hyperv_enabled); 2326 return; 2327 } 2328 2329 memset(&hv_vcpu->cpuid_cache, 0, sizeof(hv_vcpu->cpuid_cache)); 2330 2331 if (!vcpu->arch.hyperv_enabled) 2332 return; 2333 2334 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_FEATURES); 2335 if (entry) { 2336 hv_vcpu->cpuid_cache.features_eax = entry->eax; 2337 hv_vcpu->cpuid_cache.features_ebx = entry->ebx; 2338 hv_vcpu->cpuid_cache.features_edx = entry->edx; 2339 } 2340 2341 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_ENLIGHTMENT_INFO); 2342 if (entry) { 2343 hv_vcpu->cpuid_cache.enlightenments_eax = entry->eax; 2344 hv_vcpu->cpuid_cache.enlightenments_ebx = entry->ebx; 2345 } 2346 2347 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES); 2348 if (entry) 2349 hv_vcpu->cpuid_cache.syndbg_cap_eax = entry->eax; 2350 2351 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_NESTED_FEATURES); 2352 if (entry) { 2353 hv_vcpu->cpuid_cache.nested_eax = entry->eax; 2354 hv_vcpu->cpuid_cache.nested_ebx = entry->ebx; 2355 } 2356 } 2357 2358 int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce) 2359 { 2360 struct kvm_vcpu_hv *hv_vcpu; 2361 int ret = 0; 2362 2363 if (!to_hv_vcpu(vcpu)) { 2364 if (enforce) { 2365 ret = kvm_hv_vcpu_init(vcpu); 2366 if (ret) 2367 return ret; 2368 } else { 2369 return 0; 2370 } 2371 } 2372 2373 hv_vcpu = to_hv_vcpu(vcpu); 2374 hv_vcpu->enforce_cpuid = enforce; 2375 2376 return ret; 2377 } 2378 2379 static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) 2380 { 2381 bool longmode; 2382 2383 longmode = is_64_bit_hypercall(vcpu); 2384 if (longmode) 2385 kvm_rax_write_raw(vcpu, result); 2386 else { 2387 kvm_edx_write(vcpu, result >> 32); 2388 kvm_eax_write(vcpu, result); 2389 } 2390 } 2391 2392 static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result) 2393 { 2394 u32 tlb_lock_count = 0; 2395 int ret; 2396 2397 if (hv_result_success(result) && is_guest_mode(vcpu) && 2398 kvm_hv_is_tlb_flush_hcall(vcpu) && 2399 kvm_read_guest(vcpu->kvm, to_hv_vcpu(vcpu)->nested.pa_page_gpa, 2400 &tlb_lock_count, sizeof(tlb_lock_count))) 2401 result = HV_STATUS_INVALID_HYPERCALL_INPUT; 2402 2403 trace_kvm_hv_hypercall_done(result); 2404 kvm_hv_hypercall_set_result(vcpu, result); 2405 ++vcpu->stat.hypercalls; 2406 2407 ret = kvm_skip_emulated_instruction(vcpu); 2408 2409 if (tlb_lock_count) 2410 kvm_x86_ops.nested_ops->hv_inject_synthetic_vmexit_post_tlb_flush(vcpu); 2411 2412 return ret; 2413 } 2414 2415 static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu) 2416 { 2417 return kvm_hv_hypercall_complete(vcpu, vcpu->run->hyperv.u.hcall.result); 2418 } 2419 2420 static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2421 { 2422 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 2423 struct eventfd_ctx *eventfd; 2424 2425 if (unlikely(!hc->fast)) { 2426 int ret; 2427 gpa_t gpa = hc->ingpa; 2428 2429 if ((gpa & (__alignof__(hc->ingpa) - 1)) || 2430 offset_in_page(gpa) + sizeof(hc->ingpa) > PAGE_SIZE) 2431 return HV_STATUS_INVALID_ALIGNMENT; 2432 2433 ret = kvm_vcpu_read_guest(vcpu, gpa, 2434 &hc->ingpa, sizeof(hc->ingpa)); 2435 if (ret < 0) 2436 return HV_STATUS_INVALID_ALIGNMENT; 2437 } 2438 2439 /* 2440 * Per spec, bits 32-47 contain the extra "flag number". However, we 2441 * have no use for it, and in all known usecases it is zero, so just 2442 * report lookup failure if it isn't. 2443 */ 2444 if (hc->ingpa & 0xffff00000000ULL) 2445 return HV_STATUS_INVALID_PORT_ID; 2446 /* remaining bits are reserved-zero */ 2447 if (hc->ingpa & ~KVM_HYPERV_CONN_ID_MASK) 2448 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2449 2450 /* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */ 2451 rcu_read_lock(); 2452 eventfd = idr_find(&hv->conn_to_evt, hc->ingpa); 2453 rcu_read_unlock(); 2454 if (!eventfd) 2455 return HV_STATUS_INVALID_PORT_ID; 2456 2457 eventfd_signal(eventfd); 2458 return HV_STATUS_SUCCESS; 2459 } 2460 2461 static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc) 2462 { 2463 switch (hc->code) { 2464 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2465 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2466 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2467 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2468 case HVCALL_SEND_IPI_EX: 2469 return true; 2470 } 2471 2472 return false; 2473 } 2474 2475 static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc) 2476 { 2477 int reg; 2478 2479 kvm_fpu_get(); 2480 for (reg = 0; reg < HV_HYPERCALL_MAX_XMM_REGISTERS; reg++) 2481 _kvm_read_sse_reg(reg, &hc->xmm[reg]); 2482 kvm_fpu_put(); 2483 } 2484 2485 static bool hv_check_hypercall_access(struct kvm_vcpu_hv *hv_vcpu, u16 code) 2486 { 2487 if (!hv_vcpu->enforce_cpuid) 2488 return true; 2489 2490 switch (code) { 2491 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2492 return hv_vcpu->cpuid_cache.enlightenments_ebx && 2493 hv_vcpu->cpuid_cache.enlightenments_ebx != U32_MAX; 2494 case HVCALL_POST_MESSAGE: 2495 return hv_vcpu->cpuid_cache.features_ebx & HV_POST_MESSAGES; 2496 case HVCALL_SIGNAL_EVENT: 2497 return hv_vcpu->cpuid_cache.features_ebx & HV_SIGNAL_EVENTS; 2498 case HVCALL_POST_DEBUG_DATA: 2499 case HVCALL_RETRIEVE_DEBUG_DATA: 2500 case HVCALL_RESET_DEBUG_SESSION: 2501 /* 2502 * Return 'true' when SynDBG is disabled so the resulting code 2503 * will be HV_STATUS_INVALID_HYPERCALL_CODE. 2504 */ 2505 return !kvm_hv_is_syndbg_enabled(hv_vcpu->vcpu) || 2506 hv_vcpu->cpuid_cache.features_ebx & HV_DEBUGGING; 2507 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2508 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2509 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2510 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2511 return false; 2512 fallthrough; 2513 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2514 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2515 return hv_vcpu->cpuid_cache.enlightenments_eax & 2516 HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2517 case HVCALL_SEND_IPI_EX: 2518 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2519 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2520 return false; 2521 fallthrough; 2522 case HVCALL_SEND_IPI: 2523 return hv_vcpu->cpuid_cache.enlightenments_eax & 2524 HV_X64_CLUSTER_IPI_RECOMMENDED; 2525 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2526 return hv_vcpu->cpuid_cache.features_ebx & 2527 HV_ENABLE_EXTENDED_HYPERCALLS; 2528 default: 2529 break; 2530 } 2531 2532 return true; 2533 } 2534 2535 int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 2536 { 2537 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2538 struct kvm_hv_hcall hc; 2539 u64 ret = HV_STATUS_SUCCESS; 2540 2541 /* 2542 * hypercall generates UD from non zero cpl and real mode 2543 * per HYPER-V spec 2544 */ 2545 if (kvm_x86_call(get_cpl)(vcpu) != 0 || !is_protmode(vcpu)) { 2546 kvm_queue_exception(vcpu, UD_VECTOR); 2547 return 1; 2548 } 2549 2550 #ifdef CONFIG_X86_64 2551 if (is_64_bit_hypercall(vcpu)) { 2552 hc.param = kvm_rcx_read_raw(vcpu); 2553 hc.ingpa = kvm_rdx_read_raw(vcpu); 2554 hc.outgpa = kvm_r8_read_raw(vcpu); 2555 } else 2556 #endif 2557 { 2558 hc.param = ((u64)kvm_edx_read(vcpu) << 32) | kvm_eax_read(vcpu); 2559 hc.ingpa = ((u64)kvm_ebx_read(vcpu) << 32) | kvm_ecx_read(vcpu); 2560 hc.outgpa = ((u64)kvm_edi_read(vcpu) << 32) | kvm_esi_read(vcpu); 2561 } 2562 2563 hc.code = hc.param & 0xffff; 2564 hc.var_cnt = (hc.param & HV_HYPERCALL_VARHEAD_MASK) >> HV_HYPERCALL_VARHEAD_OFFSET; 2565 hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT); 2566 hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff; 2567 hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff; 2568 hc.rep = !!(hc.rep_cnt || hc.rep_idx); 2569 2570 trace_kvm_hv_hypercall(hc.code, hc.fast, hc.var_cnt, hc.rep_cnt, 2571 hc.rep_idx, hc.ingpa, hc.outgpa); 2572 2573 if (unlikely(!hv_check_hypercall_access(hv_vcpu, hc.code))) { 2574 ret = HV_STATUS_ACCESS_DENIED; 2575 goto hypercall_complete; 2576 } 2577 2578 if (unlikely(hc.param & HV_HYPERCALL_RSVD_MASK)) { 2579 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2580 goto hypercall_complete; 2581 } 2582 2583 if (hc.fast && is_xmm_fast_hypercall(&hc)) { 2584 if (unlikely(hv_vcpu->enforce_cpuid && 2585 !(hv_vcpu->cpuid_cache.features_edx & 2586 HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE))) { 2587 kvm_queue_exception(vcpu, UD_VECTOR); 2588 return 1; 2589 } 2590 2591 kvm_hv_hypercall_read_xmm(&hc); 2592 } 2593 2594 switch (hc.code) { 2595 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2596 if (unlikely(hc.rep || hc.var_cnt)) { 2597 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2598 break; 2599 } 2600 kvm_vcpu_on_spin(vcpu, true); 2601 break; 2602 case HVCALL_SIGNAL_EVENT: 2603 if (unlikely(hc.rep || hc.var_cnt)) { 2604 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2605 break; 2606 } 2607 ret = kvm_hvcall_signal_event(vcpu, &hc); 2608 if (ret != HV_STATUS_INVALID_PORT_ID) 2609 break; 2610 fallthrough; /* maybe userspace knows this conn_id */ 2611 case HVCALL_POST_MESSAGE: 2612 /* don't bother userspace if it has no way to handle it */ 2613 if (unlikely(hc.rep || hc.var_cnt || !to_hv_synic(vcpu)->active)) { 2614 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2615 break; 2616 } 2617 goto hypercall_userspace_exit; 2618 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2619 if (unlikely(hc.var_cnt)) { 2620 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2621 break; 2622 } 2623 fallthrough; 2624 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2625 if (unlikely(!hc.rep_cnt || hc.rep_idx)) { 2626 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2627 break; 2628 } 2629 ret = kvm_hv_flush_tlb(vcpu, &hc); 2630 break; 2631 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2632 if (unlikely(hc.var_cnt)) { 2633 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2634 break; 2635 } 2636 fallthrough; 2637 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2638 if (unlikely(hc.rep)) { 2639 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2640 break; 2641 } 2642 ret = kvm_hv_flush_tlb(vcpu, &hc); 2643 break; 2644 case HVCALL_SEND_IPI: 2645 if (unlikely(hc.var_cnt)) { 2646 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2647 break; 2648 } 2649 fallthrough; 2650 case HVCALL_SEND_IPI_EX: 2651 if (unlikely(hc.rep)) { 2652 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2653 break; 2654 } 2655 ret = kvm_hv_send_ipi(vcpu, &hc); 2656 break; 2657 case HVCALL_POST_DEBUG_DATA: 2658 case HVCALL_RETRIEVE_DEBUG_DATA: 2659 if (unlikely(hc.fast)) { 2660 ret = HV_STATUS_INVALID_PARAMETER; 2661 break; 2662 } 2663 fallthrough; 2664 case HVCALL_RESET_DEBUG_SESSION: { 2665 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 2666 2667 if (!kvm_hv_is_syndbg_enabled(vcpu)) { 2668 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2669 break; 2670 } 2671 2672 if (!(syndbg->options & HV_X64_SYNDBG_OPTION_USE_HCALLS)) { 2673 ret = HV_STATUS_OPERATION_DENIED; 2674 break; 2675 } 2676 goto hypercall_userspace_exit; 2677 } 2678 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2679 if (unlikely(hc.fast)) { 2680 ret = HV_STATUS_INVALID_PARAMETER; 2681 break; 2682 } 2683 goto hypercall_userspace_exit; 2684 default: 2685 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2686 break; 2687 } 2688 2689 hypercall_complete: 2690 return kvm_hv_hypercall_complete(vcpu, ret); 2691 2692 hypercall_userspace_exit: 2693 vcpu->run->exit_reason = KVM_EXIT_HYPERV; 2694 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL; 2695 vcpu->run->hyperv.u.hcall.input = hc.param; 2696 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa; 2697 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa; 2698 vcpu->arch.complete_userspace_io = kvm_hv_hypercall_complete_userspace; 2699 return 0; 2700 } 2701 2702 void kvm_hv_init_vm(struct kvm *kvm) 2703 { 2704 struct kvm_hv *hv = to_kvm_hv(kvm); 2705 2706 mutex_init(&hv->hv_lock); 2707 idr_init(&hv->conn_to_evt); 2708 } 2709 2710 void kvm_hv_destroy_vm(struct kvm *kvm) 2711 { 2712 struct kvm_hv *hv = to_kvm_hv(kvm); 2713 struct eventfd_ctx *eventfd; 2714 int i; 2715 2716 idr_for_each_entry(&hv->conn_to_evt, eventfd, i) 2717 eventfd_ctx_put(eventfd); 2718 idr_destroy(&hv->conn_to_evt); 2719 } 2720 2721 static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd) 2722 { 2723 struct kvm_hv *hv = to_kvm_hv(kvm); 2724 struct eventfd_ctx *eventfd; 2725 int ret; 2726 2727 eventfd = eventfd_ctx_fdget(fd); 2728 if (IS_ERR(eventfd)) 2729 return PTR_ERR(eventfd); 2730 2731 mutex_lock(&hv->hv_lock); 2732 ret = idr_alloc(&hv->conn_to_evt, eventfd, conn_id, conn_id + 1, 2733 GFP_KERNEL_ACCOUNT); 2734 mutex_unlock(&hv->hv_lock); 2735 2736 if (ret >= 0) 2737 return 0; 2738 2739 if (ret == -ENOSPC) 2740 ret = -EEXIST; 2741 eventfd_ctx_put(eventfd); 2742 return ret; 2743 } 2744 2745 static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id) 2746 { 2747 struct kvm_hv *hv = to_kvm_hv(kvm); 2748 struct eventfd_ctx *eventfd; 2749 2750 mutex_lock(&hv->hv_lock); 2751 eventfd = idr_remove(&hv->conn_to_evt, conn_id); 2752 mutex_unlock(&hv->hv_lock); 2753 2754 if (!eventfd) 2755 return -ENOENT; 2756 2757 synchronize_srcu(&kvm->srcu); 2758 eventfd_ctx_put(eventfd); 2759 return 0; 2760 } 2761 2762 int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args) 2763 { 2764 if ((args->flags & ~KVM_HYPERV_EVENTFD_DEASSIGN) || 2765 (args->conn_id & ~KVM_HYPERV_CONN_ID_MASK)) 2766 return -EINVAL; 2767 2768 if (args->flags == KVM_HYPERV_EVENTFD_DEASSIGN) 2769 return kvm_hv_eventfd_deassign(kvm, args->conn_id); 2770 return kvm_hv_eventfd_assign(kvm, args->conn_id, args->fd); 2771 } 2772 2773 int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, 2774 struct kvm_cpuid_entry2 __user *entries) 2775 { 2776 uint16_t evmcs_ver = 0; 2777 struct kvm_cpuid_entry2 cpuid_entries[] = { 2778 { .function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS }, 2779 { .function = HYPERV_CPUID_INTERFACE }, 2780 { .function = HYPERV_CPUID_VERSION }, 2781 { .function = HYPERV_CPUID_FEATURES }, 2782 { .function = HYPERV_CPUID_ENLIGHTMENT_INFO }, 2783 { .function = HYPERV_CPUID_IMPLEMENT_LIMITS }, 2784 { .function = HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS }, 2785 { .function = HYPERV_CPUID_SYNDBG_INTERFACE }, 2786 { .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES }, 2787 { .function = HYPERV_CPUID_NESTED_FEATURES }, 2788 }; 2789 int i, nent = ARRAY_SIZE(cpuid_entries); 2790 2791 if (kvm_x86_ops.nested_ops->get_evmcs_version) 2792 evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu); 2793 2794 if (cpuid->nent < nent) 2795 return -E2BIG; 2796 2797 if (cpuid->nent > nent) 2798 cpuid->nent = nent; 2799 2800 for (i = 0; i < nent; i++) { 2801 struct kvm_cpuid_entry2 *ent = &cpuid_entries[i]; 2802 u32 signature[3]; 2803 2804 switch (ent->function) { 2805 case HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS: 2806 memcpy(signature, "Linux KVM Hv", 12); 2807 2808 ent->eax = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES; 2809 ent->ebx = signature[0]; 2810 ent->ecx = signature[1]; 2811 ent->edx = signature[2]; 2812 break; 2813 2814 case HYPERV_CPUID_INTERFACE: 2815 ent->eax = HYPERV_CPUID_SIGNATURE_EAX; 2816 break; 2817 2818 case HYPERV_CPUID_VERSION: 2819 /* 2820 * We implement some Hyper-V 2016 functions so let's use 2821 * this version. 2822 */ 2823 ent->eax = 0x00003839; 2824 ent->ebx = 0x000A0000; 2825 break; 2826 2827 case HYPERV_CPUID_FEATURES: 2828 ent->eax |= HV_MSR_VP_RUNTIME_AVAILABLE; 2829 ent->eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE; 2830 ent->eax |= HV_MSR_SYNIC_AVAILABLE; 2831 ent->eax |= HV_MSR_SYNTIMER_AVAILABLE; 2832 ent->eax |= HV_MSR_APIC_ACCESS_AVAILABLE; 2833 ent->eax |= HV_MSR_HYPERCALL_AVAILABLE; 2834 ent->eax |= HV_MSR_VP_INDEX_AVAILABLE; 2835 ent->eax |= HV_MSR_RESET_AVAILABLE; 2836 ent->eax |= HV_MSR_REFERENCE_TSC_AVAILABLE; 2837 ent->eax |= HV_ACCESS_FREQUENCY_MSRS; 2838 ent->eax |= HV_ACCESS_REENLIGHTENMENT; 2839 ent->eax |= HV_ACCESS_TSC_INVARIANT; 2840 2841 ent->ebx |= HV_POST_MESSAGES; 2842 ent->ebx |= HV_SIGNAL_EVENTS; 2843 ent->ebx |= HV_ENABLE_EXTENDED_HYPERCALLS; 2844 2845 ent->edx |= HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE; 2846 ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE; 2847 ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 2848 2849 ent->ebx |= HV_DEBUGGING; 2850 ent->edx |= HV_X64_GUEST_DEBUGGING_AVAILABLE; 2851 ent->edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE; 2852 ent->edx |= HV_FEATURE_EXT_GVA_RANGES_FLUSH; 2853 2854 /* 2855 * Direct Synthetic timers only make sense with in-kernel 2856 * LAPIC 2857 */ 2858 if (!vcpu || lapic_in_kernel(vcpu)) 2859 ent->edx |= HV_STIMER_DIRECT_MODE_AVAILABLE; 2860 2861 break; 2862 2863 case HYPERV_CPUID_ENLIGHTMENT_INFO: 2864 ent->eax |= HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2865 ent->eax |= HV_X64_APIC_ACCESS_RECOMMENDED; 2866 ent->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED; 2867 if (!vcpu || lapic_in_kernel(vcpu)) 2868 ent->eax |= HV_X64_CLUSTER_IPI_RECOMMENDED; 2869 ent->eax |= HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED; 2870 if (evmcs_ver) 2871 ent->eax |= HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 2872 if (!cpu_smt_possible()) 2873 ent->eax |= HV_X64_NO_NONARCH_CORESHARING; 2874 2875 ent->eax |= HV_DEPRECATING_AEOI_RECOMMENDED; 2876 /* 2877 * Default number of spinlock retry attempts, matches 2878 * HyperV 2016. 2879 */ 2880 ent->ebx = 0x00000FFF; 2881 2882 break; 2883 2884 case HYPERV_CPUID_IMPLEMENT_LIMITS: 2885 /* Maximum number of virtual processors */ 2886 ent->eax = KVM_MAX_VCPUS; 2887 /* 2888 * Maximum number of logical processors, matches 2889 * HyperV 2016. 2890 */ 2891 ent->ebx = 64; 2892 2893 break; 2894 2895 case HYPERV_CPUID_NESTED_FEATURES: 2896 ent->eax = evmcs_ver; 2897 ent->eax |= HV_X64_NESTED_DIRECT_FLUSH; 2898 ent->eax |= HV_X64_NESTED_MSR_BITMAP; 2899 ent->ebx |= HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; 2900 break; 2901 2902 case HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS: 2903 memcpy(signature, "Linux KVM Hv", 12); 2904 2905 ent->eax = 0; 2906 ent->ebx = signature[0]; 2907 ent->ecx = signature[1]; 2908 ent->edx = signature[2]; 2909 break; 2910 2911 case HYPERV_CPUID_SYNDBG_INTERFACE: 2912 memcpy(signature, "VS#1\0\0\0\0\0\0\0\0", 12); 2913 ent->eax = signature[0]; 2914 break; 2915 2916 case HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES: 2917 ent->eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 2918 break; 2919 2920 default: 2921 break; 2922 } 2923 } 2924 2925 if (copy_to_user(entries, cpuid_entries, 2926 nent * sizeof(struct kvm_cpuid_entry2))) 2927 return -EFAULT; 2928 2929 return 0; 2930 } 2931