1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM Microsoft Hyper-V emulation 4 * 5 * derived from arch/x86/kvm/x86.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com> 12 * 13 * Authors: 14 * Avi Kivity <avi@qumranet.com> 15 * Yaniv Kamay <yaniv@qumranet.com> 16 * Amit Shah <amit.shah@qumranet.com> 17 * Ben-Ami Yassour <benami@il.ibm.com> 18 * Andrey Smetanin <asmetanin@virtuozzo.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include "x86.h" 23 #include "lapic.h" 24 #include "ioapic.h" 25 #include "cpuid.h" 26 #include "hyperv.h" 27 #include "mmu.h" 28 #include "xen.h" 29 30 #include <linux/cpu.h> 31 #include <linux/kvm_host.h> 32 #include <linux/highmem.h> 33 #include <linux/sched/cputime.h> 34 #include <linux/spinlock.h> 35 #include <linux/eventfd.h> 36 37 #include <asm/apicdef.h> 38 #include <asm/mshyperv.h> 39 #include <trace/events/kvm.h> 40 41 #include "trace.h" 42 #include "irq.h" 43 #include "fpu.h" 44 45 #define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, HV_VCPUS_PER_SPARSE_BANK) 46 47 /* 48 * As per Hyper-V TLFS, extended hypercalls start from 0x8001 49 * (HvExtCallQueryCapabilities). Response of this hypercalls is a 64 bit value 50 * where each bit tells which extended hypercall is available besides 51 * HvExtCallQueryCapabilities. 52 * 53 * 0x8001 - First extended hypercall, HvExtCallQueryCapabilities, no bit 54 * assigned. 55 * 56 * 0x8002 - Bit 0 57 * 0x8003 - Bit 1 58 * .. 59 * 0x8041 - Bit 63 60 * 61 * Therefore, HV_EXT_CALL_MAX = 0x8001 + 64 62 */ 63 #define HV_EXT_CALL_MAX (HV_EXT_CALL_QUERY_CAPABILITIES + 64) 64 65 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 66 bool vcpu_kick); 67 68 static inline u64 synic_read_sint(struct kvm_vcpu_hv_synic *synic, int sint) 69 { 70 return atomic64_read(&synic->sint[sint]); 71 } 72 73 static inline int synic_get_sint_vector(u64 sint_value) 74 { 75 if (sint_value & HV_SYNIC_SINT_MASKED) 76 return -1; 77 return sint_value & HV_SYNIC_SINT_VECTOR_MASK; 78 } 79 80 static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic, 81 int vector) 82 { 83 int i; 84 85 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 86 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 87 return true; 88 } 89 return false; 90 } 91 92 static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic, 93 int vector) 94 { 95 int i; 96 u64 sint_value; 97 98 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 99 sint_value = synic_read_sint(synic, i); 100 if (synic_get_sint_vector(sint_value) == vector && 101 sint_value & HV_SYNIC_SINT_AUTO_EOI) 102 return true; 103 } 104 return false; 105 } 106 107 static void synic_update_vector(struct kvm_vcpu_hv_synic *synic, 108 int vector) 109 { 110 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 111 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 112 bool auto_eoi_old, auto_eoi_new; 113 114 if (vector < HV_SYNIC_FIRST_VALID_VECTOR) 115 return; 116 117 if (synic_has_vector_connected(synic, vector)) 118 __set_bit(vector, synic->vec_bitmap); 119 else 120 __clear_bit(vector, synic->vec_bitmap); 121 122 auto_eoi_old = !bitmap_empty(synic->auto_eoi_bitmap, 256); 123 124 if (synic_has_vector_auto_eoi(synic, vector)) 125 __set_bit(vector, synic->auto_eoi_bitmap); 126 else 127 __clear_bit(vector, synic->auto_eoi_bitmap); 128 129 auto_eoi_new = !bitmap_empty(synic->auto_eoi_bitmap, 256); 130 131 if (auto_eoi_old == auto_eoi_new) 132 return; 133 134 if (!enable_apicv) 135 return; 136 137 down_write(&vcpu->kvm->arch.apicv_update_lock); 138 139 if (auto_eoi_new) 140 hv->synic_auto_eoi_used++; 141 else 142 hv->synic_auto_eoi_used--; 143 144 /* 145 * Inhibit APICv if any vCPU is using SynIC's AutoEOI, which relies on 146 * the hypervisor to manually inject IRQs. 147 */ 148 __kvm_set_or_clear_apicv_inhibit(vcpu->kvm, 149 APICV_INHIBIT_REASON_HYPERV, 150 !!hv->synic_auto_eoi_used); 151 152 up_write(&vcpu->kvm->arch.apicv_update_lock); 153 } 154 155 static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, 156 u64 data, bool host) 157 { 158 int vector, old_vector; 159 bool masked; 160 161 vector = data & HV_SYNIC_SINT_VECTOR_MASK; 162 masked = data & HV_SYNIC_SINT_MASKED; 163 164 /* 165 * Valid vectors are 16-255, however, nested Hyper-V attempts to write 166 * default '0x10000' value on boot and this should not #GP. We need to 167 * allow zero-initing the register from host as well. 168 */ 169 if (vector < HV_SYNIC_FIRST_VALID_VECTOR && !host && !masked) 170 return 1; 171 /* 172 * Guest may configure multiple SINTs to use the same vector, so 173 * we maintain a bitmap of vectors handled by synic, and a 174 * bitmap of vectors with auto-eoi behavior. The bitmaps are 175 * updated here, and atomically queried on fast paths. 176 */ 177 old_vector = synic_read_sint(synic, sint) & HV_SYNIC_SINT_VECTOR_MASK; 178 179 atomic64_set(&synic->sint[sint], data); 180 181 synic_update_vector(synic, old_vector); 182 183 synic_update_vector(synic, vector); 184 185 /* Load SynIC vectors into EOI exit bitmap */ 186 kvm_make_request(KVM_REQ_SCAN_IOAPIC, hv_synic_to_vcpu(synic)); 187 return 0; 188 } 189 190 static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx) 191 { 192 struct kvm_vcpu *vcpu = NULL; 193 unsigned long i; 194 195 if (vpidx >= KVM_MAX_VCPUS) 196 return NULL; 197 198 vcpu = kvm_get_vcpu(kvm, vpidx); 199 if (vcpu && kvm_hv_get_vpindex(vcpu) == vpidx) 200 return vcpu; 201 kvm_for_each_vcpu(i, vcpu, kvm) 202 if (kvm_hv_get_vpindex(vcpu) == vpidx) 203 return vcpu; 204 return NULL; 205 } 206 207 static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx) 208 { 209 struct kvm_vcpu_hv_synic *synic; 210 struct kvm_vcpu_hv *hv_vcpu; 211 struct kvm_vcpu *vcpu; 212 213 vcpu = get_vcpu_by_vpidx(kvm, vpidx); 214 if (!vcpu) 215 return NULL; 216 217 hv_vcpu = to_hv_vcpu_safe(vcpu); 218 if (!hv_vcpu) 219 return NULL; 220 221 synic = &hv_vcpu->synic; 222 return READ_ONCE(synic->active) ? synic : NULL; 223 } 224 225 static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint) 226 { 227 struct kvm *kvm = vcpu->kvm; 228 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 229 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 230 struct kvm_vcpu_hv_stimer *stimer; 231 int gsi, idx; 232 233 trace_kvm_hv_notify_acked_sint(vcpu->vcpu_id, sint); 234 235 /* Try to deliver pending Hyper-V SynIC timers messages */ 236 for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) { 237 stimer = &hv_vcpu->stimer[idx]; 238 if (stimer->msg_pending && stimer->config.enable && 239 !stimer->config.direct_mode && 240 stimer->config.sintx == sint) 241 stimer_mark_pending(stimer, false); 242 } 243 244 idx = srcu_read_lock(&kvm->irq_srcu); 245 gsi = atomic_read(&synic->sint_to_gsi[sint]); 246 if (gsi != -1) 247 kvm_notify_acked_gsi(kvm, gsi); 248 srcu_read_unlock(&kvm->irq_srcu, idx); 249 } 250 251 static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr) 252 { 253 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 254 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 255 256 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC; 257 hv_vcpu->exit.u.synic.msr = msr; 258 hv_vcpu->exit.u.synic.control = synic->control; 259 hv_vcpu->exit.u.synic.evt_page = synic->evt_page; 260 hv_vcpu->exit.u.synic.msg_page = synic->msg_page; 261 262 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 263 } 264 265 static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, 266 u32 msr, u64 data, bool host) 267 { 268 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 269 int ret; 270 271 if (!synic->active && (!host || data)) 272 return 1; 273 274 trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host); 275 276 ret = 0; 277 switch (msr) { 278 case HV_X64_MSR_SCONTROL: 279 synic->control = data; 280 if (!host) 281 synic_exit(synic, msr); 282 break; 283 case HV_X64_MSR_SVERSION: 284 if (!host) { 285 ret = 1; 286 break; 287 } 288 synic->version = data; 289 break; 290 case HV_X64_MSR_SIEFP: 291 if ((data & HV_SYNIC_SIEFP_ENABLE) && !host && 292 !synic->dont_zero_synic_pages) 293 if (kvm_clear_guest(vcpu->kvm, 294 data & PAGE_MASK, PAGE_SIZE)) { 295 ret = 1; 296 break; 297 } 298 synic->evt_page = data; 299 if (!host) 300 synic_exit(synic, msr); 301 break; 302 case HV_X64_MSR_SIMP: 303 if ((data & HV_SYNIC_SIMP_ENABLE) && !host && 304 !synic->dont_zero_synic_pages) 305 if (kvm_clear_guest(vcpu->kvm, 306 data & PAGE_MASK, PAGE_SIZE)) { 307 ret = 1; 308 break; 309 } 310 synic->msg_page = data; 311 if (!host) 312 synic_exit(synic, msr); 313 break; 314 case HV_X64_MSR_EOM: { 315 int i; 316 317 if (!synic->active) 318 break; 319 320 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 321 kvm_hv_notify_acked_sint(vcpu, i); 322 break; 323 } 324 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 325 ret = synic_set_sint(synic, msr - HV_X64_MSR_SINT0, data, host); 326 break; 327 default: 328 ret = 1; 329 break; 330 } 331 return ret; 332 } 333 334 static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) 335 { 336 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 337 338 return hv_vcpu->cpuid_cache.syndbg_cap_eax & 339 HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 340 } 341 342 static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) 343 { 344 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 345 346 if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL) 347 hv->hv_syndbg.control.status = 348 vcpu->run->hyperv.u.syndbg.status; 349 return 1; 350 } 351 352 static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr) 353 { 354 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 355 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 356 357 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG; 358 hv_vcpu->exit.u.syndbg.msr = msr; 359 hv_vcpu->exit.u.syndbg.control = syndbg->control.control; 360 hv_vcpu->exit.u.syndbg.send_page = syndbg->control.send_page; 361 hv_vcpu->exit.u.syndbg.recv_page = syndbg->control.recv_page; 362 hv_vcpu->exit.u.syndbg.pending_page = syndbg->control.pending_page; 363 vcpu->arch.complete_userspace_io = 364 kvm_hv_syndbg_complete_userspace; 365 366 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 367 } 368 369 static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 370 { 371 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 372 373 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 374 return 1; 375 376 trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id, 377 to_hv_vcpu(vcpu)->vp_index, msr, data); 378 switch (msr) { 379 case HV_X64_MSR_SYNDBG_CONTROL: 380 syndbg->control.control = data; 381 if (!host) 382 syndbg_exit(vcpu, msr); 383 break; 384 case HV_X64_MSR_SYNDBG_STATUS: 385 syndbg->control.status = data; 386 break; 387 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 388 syndbg->control.send_page = data; 389 break; 390 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 391 syndbg->control.recv_page = data; 392 break; 393 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 394 syndbg->control.pending_page = data; 395 if (!host) 396 syndbg_exit(vcpu, msr); 397 break; 398 case HV_X64_MSR_SYNDBG_OPTIONS: 399 syndbg->options = data; 400 break; 401 default: 402 break; 403 } 404 405 return 0; 406 } 407 408 static int syndbg_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 409 { 410 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 411 412 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 413 return 1; 414 415 switch (msr) { 416 case HV_X64_MSR_SYNDBG_CONTROL: 417 *pdata = syndbg->control.control; 418 break; 419 case HV_X64_MSR_SYNDBG_STATUS: 420 *pdata = syndbg->control.status; 421 break; 422 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 423 *pdata = syndbg->control.send_page; 424 break; 425 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 426 *pdata = syndbg->control.recv_page; 427 break; 428 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 429 *pdata = syndbg->control.pending_page; 430 break; 431 case HV_X64_MSR_SYNDBG_OPTIONS: 432 *pdata = syndbg->options; 433 break; 434 default: 435 break; 436 } 437 438 trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, kvm_hv_get_vpindex(vcpu), msr, *pdata); 439 440 return 0; 441 } 442 443 static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata, 444 bool host) 445 { 446 int ret; 447 448 if (!synic->active && !host) 449 return 1; 450 451 ret = 0; 452 switch (msr) { 453 case HV_X64_MSR_SCONTROL: 454 *pdata = synic->control; 455 break; 456 case HV_X64_MSR_SVERSION: 457 *pdata = synic->version; 458 break; 459 case HV_X64_MSR_SIEFP: 460 *pdata = synic->evt_page; 461 break; 462 case HV_X64_MSR_SIMP: 463 *pdata = synic->msg_page; 464 break; 465 case HV_X64_MSR_EOM: 466 *pdata = 0; 467 break; 468 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 469 *pdata = atomic64_read(&synic->sint[msr - HV_X64_MSR_SINT0]); 470 break; 471 default: 472 ret = 1; 473 break; 474 } 475 return ret; 476 } 477 478 static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint) 479 { 480 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 481 struct kvm_lapic_irq irq; 482 int ret, vector; 483 484 if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 485 return -EINVAL; 486 487 if (sint >= ARRAY_SIZE(synic->sint)) 488 return -EINVAL; 489 490 vector = synic_get_sint_vector(synic_read_sint(synic, sint)); 491 if (vector < 0) 492 return -ENOENT; 493 494 memset(&irq, 0, sizeof(irq)); 495 irq.shorthand = APIC_DEST_SELF; 496 irq.dest_mode = APIC_DEST_PHYSICAL; 497 irq.delivery_mode = APIC_DM_FIXED; 498 irq.vector = vector; 499 irq.level = 1; 500 501 ret = kvm_irq_delivery_to_apic(vcpu->kvm, vcpu->arch.apic, &irq); 502 trace_kvm_hv_synic_set_irq(vcpu->vcpu_id, sint, irq.vector, ret); 503 return ret; 504 } 505 506 int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 507 int irq_source_id, int level, bool line_status) 508 { 509 struct kvm_vcpu_hv_synic *synic; 510 511 if (!level) 512 return -1; 513 514 synic = synic_get(kvm, e->hv_sint.vcpu); 515 if (!synic) 516 return -EINVAL; 517 518 return synic_set_irq(synic, e->hv_sint.sint); 519 } 520 521 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector) 522 { 523 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 524 int i; 525 526 trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector); 527 528 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 529 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 530 kvm_hv_notify_acked_sint(vcpu, i); 531 } 532 533 static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi) 534 { 535 struct kvm_vcpu_hv_synic *synic; 536 537 synic = synic_get(kvm, vpidx); 538 if (!synic) 539 return -EINVAL; 540 541 if (sint >= ARRAY_SIZE(synic->sint_to_gsi)) 542 return -EINVAL; 543 544 atomic_set(&synic->sint_to_gsi[sint], gsi); 545 return 0; 546 } 547 548 void kvm_hv_irq_routing_update(struct kvm *kvm) 549 { 550 struct kvm_irq_routing_table *irq_rt; 551 struct kvm_kernel_irq_routing_entry *e; 552 u32 gsi; 553 554 irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu, 555 lockdep_is_held(&kvm->irq_lock)); 556 557 for (gsi = 0; gsi < irq_rt->nr_rt_entries; gsi++) { 558 hlist_for_each_entry(e, &irq_rt->map[gsi], link) { 559 if (e->type == KVM_IRQ_ROUTING_HV_SINT) 560 kvm_hv_set_sint_gsi(kvm, e->hv_sint.vcpu, 561 e->hv_sint.sint, gsi); 562 } 563 } 564 } 565 566 static void synic_init(struct kvm_vcpu_hv_synic *synic) 567 { 568 int i; 569 570 memset(synic, 0, sizeof(*synic)); 571 synic->version = HV_SYNIC_VERSION_1; 572 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 573 atomic64_set(&synic->sint[i], HV_SYNIC_SINT_MASKED); 574 atomic_set(&synic->sint_to_gsi[i], -1); 575 } 576 } 577 578 static u64 get_time_ref_counter(struct kvm *kvm) 579 { 580 struct kvm_hv *hv = to_kvm_hv(kvm); 581 struct kvm_vcpu *vcpu; 582 u64 tsc; 583 584 /* 585 * Fall back to get_kvmclock_ns() when TSC page hasn't been set up, 586 * is broken, disabled or being updated. 587 */ 588 if (hv->hv_tsc_page_status != HV_TSC_PAGE_SET) 589 return div_u64(get_kvmclock_ns(kvm), 100); 590 591 vcpu = kvm_get_vcpu(kvm, 0); 592 tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 593 return mul_u64_u64_shr(tsc, hv->tsc_ref.tsc_scale, 64) 594 + hv->tsc_ref.tsc_offset; 595 } 596 597 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 598 bool vcpu_kick) 599 { 600 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 601 602 set_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap); 603 kvm_make_request(KVM_REQ_HV_STIMER, vcpu); 604 if (vcpu_kick) 605 kvm_vcpu_kick(vcpu); 606 } 607 608 static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer) 609 { 610 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 611 612 trace_kvm_hv_stimer_cleanup(hv_stimer_to_vcpu(stimer)->vcpu_id, 613 stimer->index); 614 615 hrtimer_cancel(&stimer->timer); 616 clear_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap); 617 stimer->msg_pending = false; 618 stimer->exp_time = 0; 619 } 620 621 static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) 622 { 623 struct kvm_vcpu_hv_stimer *stimer; 624 625 stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer); 626 trace_kvm_hv_stimer_callback(hv_stimer_to_vcpu(stimer)->vcpu_id, 627 stimer->index); 628 stimer_mark_pending(stimer, true); 629 630 return HRTIMER_NORESTART; 631 } 632 633 /* 634 * Translate a stimer expiry given in 100ns reference ticks into an 635 * an absolute deadline. Saturates on overflow. 636 */ 637 static ktime_t stimer_add_delta(ktime_t now, u64 delta_100ns) 638 { 639 if (delta_100ns >= KTIME_MAX / 100) 640 return KTIME_MAX; 641 642 return ktime_add_safe(now, 100 * delta_100ns); 643 } 644 645 /* 646 * stimer_start() assumptions: 647 * a) stimer->count is not equal to 0 648 * b) stimer->config has HV_STIMER_ENABLE flag 649 */ 650 static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) 651 { 652 u64 time_now; 653 ktime_t ktime_now; 654 ktime_t deadline; 655 656 time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm); 657 ktime_now = ktime_get(); 658 659 if (stimer->config.periodic) { 660 if (stimer->exp_time) { 661 if (time_now >= stimer->exp_time) { 662 u64 remainder; 663 664 div64_u64_rem(time_now - stimer->exp_time, 665 stimer->count, &remainder); 666 stimer->exp_time = 667 time_now + (stimer->count - remainder); 668 } 669 } else 670 stimer->exp_time = time_now + stimer->count; 671 672 trace_kvm_hv_stimer_start_periodic( 673 hv_stimer_to_vcpu(stimer)->vcpu_id, 674 stimer->index, 675 time_now, stimer->exp_time); 676 677 deadline = stimer_add_delta(ktime_now, stimer->exp_time - time_now); 678 hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); 679 return 0; 680 } 681 stimer->exp_time = stimer->count; 682 if (time_now >= stimer->count) { 683 /* 684 * Expire timer according to Hypervisor Top-Level Functional 685 * specification v4(15.3.1): 686 * "If a one shot is enabled and the specified count is in 687 * the past, it will expire immediately." 688 */ 689 stimer_mark_pending(stimer, false); 690 return 0; 691 } 692 693 trace_kvm_hv_stimer_start_one_shot(hv_stimer_to_vcpu(stimer)->vcpu_id, 694 stimer->index, 695 time_now, stimer->count); 696 697 deadline = stimer_add_delta(ktime_now, stimer->count - time_now); 698 hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); 699 700 return 0; 701 } 702 703 static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, 704 bool host) 705 { 706 union hv_stimer_config new_config = {.as_uint64 = config}, 707 old_config = {.as_uint64 = stimer->config.as_uint64}; 708 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 709 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 710 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 711 712 if (!synic->active && (!host || config)) 713 return 1; 714 715 if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode && 716 !(hv_vcpu->cpuid_cache.features_edx & 717 HV_STIMER_DIRECT_MODE_AVAILABLE))) 718 return 1; 719 720 trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id, 721 stimer->index, config, host); 722 723 stimer_cleanup(stimer); 724 if (old_config.enable && 725 !new_config.direct_mode && new_config.sintx == 0) 726 new_config.enable = 0; 727 stimer->config.as_uint64 = new_config.as_uint64; 728 729 if (stimer->config.enable) 730 stimer_mark_pending(stimer, false); 731 732 return 0; 733 } 734 735 static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count, 736 bool host) 737 { 738 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 739 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 740 741 if (!synic->active && (!host || count)) 742 return 1; 743 744 trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id, 745 stimer->index, count, host); 746 747 stimer_cleanup(stimer); 748 stimer->count = count; 749 if (!host) { 750 if (stimer->count == 0) 751 stimer->config.enable = 0; 752 else if (stimer->config.auto_enable) 753 stimer->config.enable = 1; 754 } 755 756 if (stimer->config.enable) 757 stimer_mark_pending(stimer, false); 758 759 return 0; 760 } 761 762 static int stimer_get_config(struct kvm_vcpu_hv_stimer *stimer, u64 *pconfig) 763 { 764 *pconfig = stimer->config.as_uint64; 765 return 0; 766 } 767 768 static int stimer_get_count(struct kvm_vcpu_hv_stimer *stimer, u64 *pcount) 769 { 770 *pcount = stimer->count; 771 return 0; 772 } 773 774 static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, 775 struct hv_message *src_msg, bool no_retry) 776 { 777 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 778 int msg_off = offsetof(struct hv_message_page, sint_message[sint]); 779 gfn_t msg_page_gfn; 780 struct hv_message_header hv_hdr; 781 int r; 782 783 if (!(synic->msg_page & HV_SYNIC_SIMP_ENABLE)) 784 return -ENOENT; 785 786 msg_page_gfn = synic->msg_page >> PAGE_SHIFT; 787 788 /* 789 * Strictly following the spec-mandated ordering would assume setting 790 * .msg_pending before checking .message_type. However, this function 791 * is only called in vcpu context so the entire update is atomic from 792 * guest POV and thus the exact order here doesn't matter. 793 */ 794 r = kvm_vcpu_read_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_type, 795 msg_off + offsetof(struct hv_message, 796 header.message_type), 797 sizeof(hv_hdr.message_type)); 798 if (r < 0) 799 return r; 800 801 if (hv_hdr.message_type != HVMSG_NONE) { 802 if (no_retry) 803 return 0; 804 805 hv_hdr.message_flags.msg_pending = 1; 806 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, 807 &hv_hdr.message_flags, 808 msg_off + 809 offsetof(struct hv_message, 810 header.message_flags), 811 sizeof(hv_hdr.message_flags)); 812 if (r < 0) 813 return r; 814 return -EAGAIN; 815 } 816 817 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, src_msg, msg_off, 818 sizeof(src_msg->header) + 819 src_msg->header.payload_size); 820 if (r < 0) 821 return r; 822 823 r = synic_set_irq(synic, sint); 824 if (r < 0) 825 return r; 826 if (r == 0) 827 return -EFAULT; 828 return 0; 829 } 830 831 static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer) 832 { 833 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 834 struct hv_message *msg = &stimer->msg; 835 struct hv_timer_message_payload *payload = 836 (struct hv_timer_message_payload *)&msg->u.payload; 837 838 /* 839 * To avoid piling up periodic ticks, don't retry message 840 * delivery for them (within "lazy" lost ticks policy). 841 */ 842 bool no_retry = stimer->config.periodic; 843 844 payload->expiration_time = stimer->exp_time; 845 payload->delivery_time = get_time_ref_counter(vcpu->kvm); 846 return synic_deliver_msg(to_hv_synic(vcpu), 847 stimer->config.sintx, msg, 848 no_retry); 849 } 850 851 static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer) 852 { 853 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 854 struct kvm_lapic_irq irq = { 855 .delivery_mode = APIC_DM_FIXED, 856 .vector = stimer->config.apic_vector 857 }; 858 859 if (lapic_in_kernel(vcpu)) 860 return !kvm_apic_set_irq(vcpu, &irq, NULL); 861 return 0; 862 } 863 864 static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) 865 { 866 int r, direct = stimer->config.direct_mode; 867 868 stimer->msg_pending = true; 869 if (!direct) 870 r = stimer_send_msg(stimer); 871 else 872 r = stimer_notify_direct(stimer); 873 trace_kvm_hv_stimer_expiration(hv_stimer_to_vcpu(stimer)->vcpu_id, 874 stimer->index, direct, r); 875 if (!r) { 876 stimer->msg_pending = false; 877 if (!(stimer->config.periodic)) 878 stimer->config.enable = 0; 879 } 880 } 881 882 void kvm_hv_process_stimers(struct kvm_vcpu *vcpu) 883 { 884 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 885 struct kvm_vcpu_hv_stimer *stimer; 886 u64 time_now, exp_time; 887 int i; 888 889 if (!hv_vcpu) 890 return; 891 892 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 893 if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) { 894 stimer = &hv_vcpu->stimer[i]; 895 if (stimer->config.enable) { 896 exp_time = stimer->exp_time; 897 898 if (exp_time) { 899 time_now = 900 get_time_ref_counter(vcpu->kvm); 901 if (time_now >= exp_time) 902 stimer_expiration(stimer); 903 } 904 905 if ((stimer->config.enable) && 906 stimer->count) { 907 if (!stimer->msg_pending) 908 stimer_start(stimer); 909 } else 910 stimer_cleanup(stimer); 911 } 912 } 913 } 914 915 void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu) 916 { 917 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 918 int i; 919 920 if (!hv_vcpu) 921 return; 922 923 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 924 stimer_cleanup(&hv_vcpu->stimer[i]); 925 926 kfree(hv_vcpu); 927 vcpu->arch.hyperv = NULL; 928 } 929 930 bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu) 931 { 932 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 933 934 if (!hv_vcpu) 935 return false; 936 937 if (!(hv_vcpu->hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) 938 return false; 939 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 940 } 941 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_hv_assist_page_enabled); 942 943 int kvm_hv_get_assist_page(struct kvm_vcpu *vcpu) 944 { 945 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 946 947 if (!hv_vcpu || !kvm_hv_assist_page_enabled(vcpu)) 948 return -EFAULT; 949 950 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, 951 &hv_vcpu->vp_assist_page, sizeof(struct hv_vp_assist_page)); 952 } 953 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_hv_get_assist_page); 954 955 static void stimer_prepare_msg(struct kvm_vcpu_hv_stimer *stimer) 956 { 957 struct hv_message *msg = &stimer->msg; 958 struct hv_timer_message_payload *payload = 959 (struct hv_timer_message_payload *)&msg->u.payload; 960 961 memset(&msg->header, 0, sizeof(msg->header)); 962 msg->header.message_type = HVMSG_TIMER_EXPIRED; 963 msg->header.payload_size = sizeof(*payload); 964 965 payload->timer_index = stimer->index; 966 payload->expiration_time = 0; 967 payload->delivery_time = 0; 968 } 969 970 static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index) 971 { 972 memset(stimer, 0, sizeof(*stimer)); 973 stimer->index = timer_index; 974 hrtimer_setup(&stimer->timer, stimer_timer_callback, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 975 stimer_prepare_msg(stimer); 976 } 977 978 int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) 979 { 980 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 981 int i; 982 983 if (hv_vcpu) 984 return 0; 985 986 hv_vcpu = kzalloc_obj(struct kvm_vcpu_hv, GFP_KERNEL_ACCOUNT); 987 if (!hv_vcpu) 988 return -ENOMEM; 989 990 hv_vcpu->vcpu = vcpu; 991 992 synic_init(&hv_vcpu->synic); 993 994 bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); 995 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 996 stimer_init(&hv_vcpu->stimer[i], i); 997 998 hv_vcpu->vp_index = vcpu->vcpu_idx; 999 1000 for (i = 0; i < HV_NR_TLB_FLUSH_FIFOS; i++) { 1001 INIT_KFIFO(hv_vcpu->tlb_flush_fifo[i].entries); 1002 spin_lock_init(&hv_vcpu->tlb_flush_fifo[i].write_lock); 1003 } 1004 1005 /* 1006 * Ensure the structure is fully initialized before it's visible to 1007 * other tasks, as much of the state can be legally accessed without 1008 * holding vcpu->mutex. 1009 * 1010 * Pairs with the smp_load_acquire() in to_hv_vcpu_safe(). 1011 */ 1012 smp_store_release(&vcpu->arch.hyperv, hv_vcpu); 1013 return 0; 1014 } 1015 1016 int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages) 1017 { 1018 struct kvm_vcpu_hv_synic *synic; 1019 int r; 1020 1021 r = kvm_hv_vcpu_init(vcpu); 1022 if (r) 1023 return r; 1024 1025 synic = to_hv_synic(vcpu); 1026 1027 WRITE_ONCE(synic->active, true); 1028 synic->dont_zero_synic_pages = dont_zero_synic_pages; 1029 synic->control = HV_SYNIC_CONTROL_ENABLE; 1030 return 0; 1031 } 1032 1033 static bool kvm_hv_msr_partition_wide(u32 msr) 1034 { 1035 bool r = false; 1036 1037 switch (msr) { 1038 case HV_X64_MSR_GUEST_OS_ID: 1039 case HV_X64_MSR_HYPERCALL: 1040 case HV_X64_MSR_REFERENCE_TSC: 1041 case HV_X64_MSR_TIME_REF_COUNT: 1042 case HV_X64_MSR_CRASH_CTL: 1043 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1044 case HV_X64_MSR_RESET: 1045 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1046 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1047 case HV_X64_MSR_TSC_EMULATION_STATUS: 1048 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1049 case HV_X64_MSR_SYNDBG_OPTIONS: 1050 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1051 r = true; 1052 break; 1053 } 1054 1055 return r; 1056 } 1057 1058 static int kvm_hv_msr_get_crash_data(struct kvm *kvm, u32 index, u64 *pdata) 1059 { 1060 struct kvm_hv *hv = to_kvm_hv(kvm); 1061 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1062 1063 if (WARN_ON_ONCE(index >= size)) 1064 return -EINVAL; 1065 1066 *pdata = hv->hv_crash_param[array_index_nospec(index, size)]; 1067 return 0; 1068 } 1069 1070 static int kvm_hv_msr_get_crash_ctl(struct kvm *kvm, u64 *pdata) 1071 { 1072 struct kvm_hv *hv = to_kvm_hv(kvm); 1073 1074 *pdata = hv->hv_crash_ctl; 1075 return 0; 1076 } 1077 1078 static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data) 1079 { 1080 struct kvm_hv *hv = to_kvm_hv(kvm); 1081 1082 hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY; 1083 1084 return 0; 1085 } 1086 1087 static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data) 1088 { 1089 struct kvm_hv *hv = to_kvm_hv(kvm); 1090 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1091 1092 if (WARN_ON_ONCE(index >= size)) 1093 return -EINVAL; 1094 1095 hv->hv_crash_param[array_index_nospec(index, size)] = data; 1096 return 0; 1097 } 1098 1099 /* 1100 * The kvmclock and Hyper-V TSC page use similar formulas, and converting 1101 * between them is possible: 1102 * 1103 * kvmclock formula: 1104 * nsec = (ticks - tsc_timestamp) * tsc_to_system_mul * 2^(tsc_shift-32) 1105 * + system_time 1106 * 1107 * Hyper-V formula: 1108 * nsec/100 = ticks * scale / 2^64 + offset 1109 * 1110 * When tsc_timestamp = system_time = 0, offset is zero in the Hyper-V formula. 1111 * By dividing the kvmclock formula by 100 and equating what's left we get: 1112 * ticks * scale / 2^64 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1113 * scale / 2^64 = tsc_to_system_mul * 2^(tsc_shift-32) / 100 1114 * scale = tsc_to_system_mul * 2^(32+tsc_shift) / 100 1115 * 1116 * Now expand the kvmclock formula and divide by 100: 1117 * nsec = ticks * tsc_to_system_mul * 2^(tsc_shift-32) 1118 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) 1119 * + system_time 1120 * nsec/100 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1121 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1122 * + system_time / 100 1123 * 1124 * Replace tsc_to_system_mul * 2^(tsc_shift-32) / 100 by scale / 2^64: 1125 * nsec/100 = ticks * scale / 2^64 1126 * - tsc_timestamp * scale / 2^64 1127 * + system_time / 100 1128 * 1129 * Equate with the Hyper-V formula so that ticks * scale / 2^64 cancels out: 1130 * offset = system_time / 100 - tsc_timestamp * scale / 2^64 1131 * 1132 * These two equivalencies are implemented in this function. 1133 */ 1134 static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock, 1135 struct ms_hyperv_tsc_page *tsc_ref) 1136 { 1137 u64 max_mul; 1138 1139 if (!(hv_clock->flags & PVCLOCK_TSC_STABLE_BIT)) 1140 return false; 1141 1142 /* 1143 * check if scale would overflow, if so we use the time ref counter 1144 * tsc_to_system_mul * 2^(tsc_shift+32) / 100 >= 2^64 1145 * tsc_to_system_mul / 100 >= 2^(32-tsc_shift) 1146 * tsc_to_system_mul >= 100 * 2^(32-tsc_shift) 1147 */ 1148 max_mul = 100ull << (32 - hv_clock->tsc_shift); 1149 if (hv_clock->tsc_to_system_mul >= max_mul) 1150 return false; 1151 1152 /* 1153 * Otherwise compute the scale and offset according to the formulas 1154 * derived above. 1155 */ 1156 tsc_ref->tsc_scale = 1157 mul_u64_u32_div(1ULL << (32 + hv_clock->tsc_shift), 1158 hv_clock->tsc_to_system_mul, 1159 100); 1160 1161 tsc_ref->tsc_offset = hv_clock->system_time; 1162 do_div(tsc_ref->tsc_offset, 100); 1163 tsc_ref->tsc_offset -= 1164 mul_u64_u64_shr(hv_clock->tsc_timestamp, tsc_ref->tsc_scale, 64); 1165 return true; 1166 } 1167 1168 /* 1169 * Don't touch TSC page values if the guest has opted for TSC emulation after 1170 * migration. KVM doesn't fully support reenlightenment notifications and TSC 1171 * access emulation and Hyper-V is known to expect the values in TSC page to 1172 * stay constant before TSC access emulation is disabled from guest side 1173 * (HV_X64_MSR_TSC_EMULATION_STATUS). KVM userspace is expected to preserve TSC 1174 * frequency and guest visible TSC value across migration (and prevent it when 1175 * TSC scaling is unsupported). 1176 */ 1177 static inline bool tsc_page_update_unsafe(struct kvm_hv *hv) 1178 { 1179 return (hv->hv_tsc_page_status != HV_TSC_PAGE_GUEST_CHANGED) && 1180 hv->hv_tsc_emulation_control; 1181 } 1182 1183 void kvm_hv_setup_tsc_page(struct kvm *kvm, 1184 struct pvclock_vcpu_time_info *hv_clock) 1185 { 1186 struct kvm_hv *hv = to_kvm_hv(kvm); 1187 u32 tsc_seq; 1188 u64 gfn; 1189 1190 BUILD_BUG_ON(sizeof(tsc_seq) != sizeof(hv->tsc_ref.tsc_sequence)); 1191 BUILD_BUG_ON(offsetof(struct ms_hyperv_tsc_page, tsc_sequence) != 0); 1192 1193 guard(mutex)(&hv->hv_lock); 1194 1195 if (hv->hv_tsc_page_status == HV_TSC_PAGE_BROKEN || 1196 hv->hv_tsc_page_status == HV_TSC_PAGE_SET || 1197 hv->hv_tsc_page_status == HV_TSC_PAGE_UNSET) 1198 return; 1199 1200 if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE)) 1201 return; 1202 1203 gfn = hv->hv_tsc_page >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT; 1204 /* 1205 * Because the TSC parameters only vary when there is a 1206 * change in the master clock, do not bother with caching. 1207 */ 1208 if (unlikely(kvm_read_guest(kvm, gfn_to_gpa(gfn), 1209 &tsc_seq, sizeof(tsc_seq)))) 1210 goto out_err; 1211 1212 if (tsc_seq && tsc_page_update_unsafe(hv)) { 1213 if (kvm_read_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1214 goto out_err; 1215 1216 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1217 return; 1218 } 1219 1220 /* 1221 * While we're computing and writing the parameters, force the 1222 * guest to use the time reference count MSR. 1223 */ 1224 hv->tsc_ref.tsc_sequence = 0; 1225 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1226 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1227 goto out_err; 1228 1229 if (!compute_tsc_page_parameters(hv_clock, &hv->tsc_ref)) 1230 goto out_err; 1231 1232 /* Ensure sequence is zero before writing the rest of the struct. */ 1233 smp_wmb(); 1234 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1235 goto out_err; 1236 1237 /* 1238 * Now switch to the TSC page mechanism by writing the sequence. 1239 */ 1240 tsc_seq++; 1241 if (tsc_seq == 0xFFFFFFFF || tsc_seq == 0) 1242 tsc_seq = 1; 1243 1244 /* Write the struct entirely before the non-zero sequence. */ 1245 smp_wmb(); 1246 1247 hv->tsc_ref.tsc_sequence = tsc_seq; 1248 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1249 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1250 goto out_err; 1251 1252 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1253 return; 1254 1255 out_err: 1256 hv->hv_tsc_page_status = HV_TSC_PAGE_BROKEN; 1257 } 1258 1259 void kvm_hv_request_tsc_page_update(struct kvm *kvm) 1260 { 1261 struct kvm_hv *hv = to_kvm_hv(kvm); 1262 1263 mutex_lock(&hv->hv_lock); 1264 1265 if (hv->hv_tsc_page_status == HV_TSC_PAGE_SET && 1266 !tsc_page_update_unsafe(hv)) 1267 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1268 1269 mutex_unlock(&hv->hv_lock); 1270 } 1271 1272 static bool hv_check_msr_access(struct kvm_vcpu_hv *hv_vcpu, u32 msr) 1273 { 1274 if (!hv_vcpu->enforce_cpuid) 1275 return true; 1276 1277 switch (msr) { 1278 case HV_X64_MSR_GUEST_OS_ID: 1279 case HV_X64_MSR_HYPERCALL: 1280 return hv_vcpu->cpuid_cache.features_eax & 1281 HV_MSR_HYPERCALL_AVAILABLE; 1282 case HV_X64_MSR_VP_RUNTIME: 1283 return hv_vcpu->cpuid_cache.features_eax & 1284 HV_MSR_VP_RUNTIME_AVAILABLE; 1285 case HV_X64_MSR_TIME_REF_COUNT: 1286 return hv_vcpu->cpuid_cache.features_eax & 1287 HV_MSR_TIME_REF_COUNT_AVAILABLE; 1288 case HV_X64_MSR_VP_INDEX: 1289 return hv_vcpu->cpuid_cache.features_eax & 1290 HV_MSR_VP_INDEX_AVAILABLE; 1291 case HV_X64_MSR_RESET: 1292 return hv_vcpu->cpuid_cache.features_eax & 1293 HV_MSR_RESET_AVAILABLE; 1294 case HV_X64_MSR_REFERENCE_TSC: 1295 return hv_vcpu->cpuid_cache.features_eax & 1296 HV_MSR_REFERENCE_TSC_AVAILABLE; 1297 case HV_X64_MSR_SCONTROL: 1298 case HV_X64_MSR_SVERSION: 1299 case HV_X64_MSR_SIEFP: 1300 case HV_X64_MSR_SIMP: 1301 case HV_X64_MSR_EOM: 1302 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1303 return hv_vcpu->cpuid_cache.features_eax & 1304 HV_MSR_SYNIC_AVAILABLE; 1305 case HV_X64_MSR_STIMER0_CONFIG: 1306 case HV_X64_MSR_STIMER1_CONFIG: 1307 case HV_X64_MSR_STIMER2_CONFIG: 1308 case HV_X64_MSR_STIMER3_CONFIG: 1309 case HV_X64_MSR_STIMER0_COUNT: 1310 case HV_X64_MSR_STIMER1_COUNT: 1311 case HV_X64_MSR_STIMER2_COUNT: 1312 case HV_X64_MSR_STIMER3_COUNT: 1313 return hv_vcpu->cpuid_cache.features_eax & 1314 HV_MSR_SYNTIMER_AVAILABLE; 1315 case HV_X64_MSR_EOI: 1316 case HV_X64_MSR_ICR: 1317 case HV_X64_MSR_TPR: 1318 case HV_X64_MSR_VP_ASSIST_PAGE: 1319 return hv_vcpu->cpuid_cache.features_eax & 1320 HV_MSR_APIC_ACCESS_AVAILABLE; 1321 case HV_X64_MSR_TSC_FREQUENCY: 1322 case HV_X64_MSR_APIC_FREQUENCY: 1323 return hv_vcpu->cpuid_cache.features_eax & 1324 HV_ACCESS_FREQUENCY_MSRS; 1325 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1326 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1327 case HV_X64_MSR_TSC_EMULATION_STATUS: 1328 return hv_vcpu->cpuid_cache.features_eax & 1329 HV_ACCESS_REENLIGHTENMENT; 1330 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1331 return hv_vcpu->cpuid_cache.features_eax & 1332 HV_ACCESS_TSC_INVARIANT; 1333 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1334 case HV_X64_MSR_CRASH_CTL: 1335 return hv_vcpu->cpuid_cache.features_edx & 1336 HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 1337 case HV_X64_MSR_SYNDBG_OPTIONS: 1338 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1339 return hv_vcpu->cpuid_cache.features_edx & 1340 HV_FEATURE_DEBUG_MSRS_AVAILABLE; 1341 default: 1342 break; 1343 } 1344 1345 return false; 1346 } 1347 1348 #define KVM_HV_WIN2016_GUEST_ID 0x1040a00003839 1349 #define KVM_HV_WIN2016_GUEST_ID_MASK (~GENMASK_ULL(23, 16)) /* mask out the service version */ 1350 1351 /* 1352 * Hyper-V enabled Windows Server 2016 SMP VMs fail to boot in !XSAVES && XSAVEC 1353 * configuration. 1354 * Such configuration can result from, for example, AMD Erratum 1386 workaround. 1355 * 1356 * Print a notice so users aren't left wondering what's suddenly gone wrong. 1357 */ 1358 static void __kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu) 1359 { 1360 struct kvm *kvm = vcpu->kvm; 1361 struct kvm_hv *hv = to_kvm_hv(kvm); 1362 1363 /* Check again under the hv_lock. */ 1364 if (hv->xsaves_xsavec_checked) 1365 return; 1366 1367 if ((hv->hv_guest_os_id & KVM_HV_WIN2016_GUEST_ID_MASK) != 1368 KVM_HV_WIN2016_GUEST_ID) 1369 return; 1370 1371 hv->xsaves_xsavec_checked = true; 1372 1373 /* UP configurations aren't affected */ 1374 if (atomic_read(&kvm->online_vcpus) < 2) 1375 return; 1376 1377 if (guest_cpuid_has(vcpu, X86_FEATURE_XSAVES) || 1378 !guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVEC)) 1379 return; 1380 1381 pr_notice_ratelimited("Booting SMP Windows KVM VM with !XSAVES && XSAVEC. " 1382 "If it fails to boot try disabling XSAVEC in the VM config.\n"); 1383 } 1384 1385 void kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu) 1386 { 1387 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1388 1389 if (!vcpu->arch.hyperv_enabled || 1390 hv->xsaves_xsavec_checked) 1391 return; 1392 1393 mutex_lock(&hv->hv_lock); 1394 __kvm_hv_xsaves_xsavec_maybe_warn(vcpu); 1395 mutex_unlock(&hv->hv_lock); 1396 } 1397 1398 static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, 1399 bool host) 1400 { 1401 struct kvm *kvm = vcpu->kvm; 1402 struct kvm_hv *hv = to_kvm_hv(kvm); 1403 1404 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1405 return 1; 1406 1407 switch (msr) { 1408 case HV_X64_MSR_GUEST_OS_ID: 1409 hv->hv_guest_os_id = data; 1410 /* setting guest os id to zero disables hypercall page */ 1411 if (!hv->hv_guest_os_id) 1412 hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE; 1413 break; 1414 case HV_X64_MSR_HYPERCALL: { 1415 u8 instructions[9]; 1416 int i = 0; 1417 u64 addr; 1418 1419 /* if guest os id is not set hypercall should remain disabled */ 1420 if (!hv->hv_guest_os_id) 1421 break; 1422 if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) { 1423 hv->hv_hypercall = data; 1424 break; 1425 } 1426 1427 /* 1428 * If Xen and Hyper-V hypercalls are both enabled, disambiguate 1429 * the same way Xen itself does, by setting the bit 31 of EAX 1430 * which is RsvdZ in the 32-bit Hyper-V hypercall ABI and just 1431 * going to be clobbered on 64-bit. 1432 */ 1433 if (kvm_xen_hypercall_enabled(kvm)) { 1434 /* orl $0x80000000, %eax */ 1435 instructions[i++] = 0x0d; 1436 instructions[i++] = 0x00; 1437 instructions[i++] = 0x00; 1438 instructions[i++] = 0x00; 1439 instructions[i++] = 0x80; 1440 } 1441 1442 /* vmcall/vmmcall */ 1443 kvm_x86_call(patch_hypercall)(vcpu, instructions + i); 1444 i += 3; 1445 1446 /* ret */ 1447 ((unsigned char *)instructions)[i++] = 0xc3; 1448 1449 addr = data & HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK; 1450 if (kvm_vcpu_write_guest(vcpu, addr, instructions, i)) 1451 return 1; 1452 hv->hv_hypercall = data; 1453 break; 1454 } 1455 case HV_X64_MSR_REFERENCE_TSC: 1456 hv->hv_tsc_page = data; 1457 if (hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) { 1458 if (!host) 1459 hv->hv_tsc_page_status = HV_TSC_PAGE_GUEST_CHANGED; 1460 else 1461 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1462 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1463 } else { 1464 hv->hv_tsc_page_status = HV_TSC_PAGE_UNSET; 1465 } 1466 break; 1467 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1468 return kvm_hv_msr_set_crash_data(kvm, 1469 msr - HV_X64_MSR_CRASH_P0, 1470 data); 1471 case HV_X64_MSR_CRASH_CTL: 1472 if (host) 1473 return kvm_hv_msr_set_crash_ctl(kvm, data); 1474 1475 if (data & HV_CRASH_CTL_CRASH_NOTIFY) { 1476 vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n", 1477 hv->hv_crash_param[0], 1478 hv->hv_crash_param[1], 1479 hv->hv_crash_param[2], 1480 hv->hv_crash_param[3], 1481 hv->hv_crash_param[4]); 1482 1483 /* Send notification about crash to user space */ 1484 kvm_make_request(KVM_REQ_HV_CRASH, vcpu); 1485 } 1486 break; 1487 case HV_X64_MSR_RESET: 1488 if (data == 1) { 1489 vcpu_debug(vcpu, "hyper-v reset requested\n"); 1490 kvm_make_request(KVM_REQ_HV_RESET, vcpu); 1491 } 1492 break; 1493 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1494 hv->hv_reenlightenment_control = data; 1495 break; 1496 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1497 hv->hv_tsc_emulation_control = data; 1498 break; 1499 case HV_X64_MSR_TSC_EMULATION_STATUS: 1500 if (data && !host) 1501 return 1; 1502 1503 hv->hv_tsc_emulation_status = data; 1504 break; 1505 case HV_X64_MSR_TIME_REF_COUNT: 1506 /* read-only, but still ignore it if host-initiated */ 1507 if (!host) 1508 return 1; 1509 break; 1510 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1511 /* Only bit 0 is supported */ 1512 if (data & ~HV_EXPOSE_INVARIANT_TSC) 1513 return 1; 1514 1515 /* The feature can't be disabled from the guest */ 1516 if (!host && hv->hv_invtsc_control && !data) 1517 return 1; 1518 1519 hv->hv_invtsc_control = data; 1520 break; 1521 case HV_X64_MSR_SYNDBG_OPTIONS: 1522 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1523 return syndbg_set_msr(vcpu, msr, data, host); 1524 default: 1525 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1526 return 1; 1527 } 1528 return 0; 1529 } 1530 1531 /* Calculate cpu time spent by current task in 100ns units */ 1532 static u64 current_task_runtime_100ns(void) 1533 { 1534 u64 utime, stime; 1535 1536 task_cputime_adjusted(current, &utime, &stime); 1537 1538 return div_u64(utime + stime, 100); 1539 } 1540 1541 static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1542 { 1543 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1544 1545 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1546 return 1; 1547 1548 switch (msr) { 1549 case HV_X64_MSR_VP_INDEX: { 1550 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1551 u32 new_vp_index = (u32)data; 1552 1553 if (!host || new_vp_index >= KVM_MAX_VCPUS) 1554 return 1; 1555 1556 if (new_vp_index == hv_vcpu->vp_index) 1557 return 0; 1558 1559 /* 1560 * The VP index is initialized to vcpu_index by 1561 * kvm_hv_vcpu_postcreate so they initially match. Now the 1562 * VP index is changing, adjust num_mismatched_vp_indexes if 1563 * it now matches or no longer matches vcpu_idx. 1564 */ 1565 if (hv_vcpu->vp_index == vcpu->vcpu_idx) 1566 atomic_inc(&hv->num_mismatched_vp_indexes); 1567 else if (new_vp_index == vcpu->vcpu_idx) 1568 atomic_dec(&hv->num_mismatched_vp_indexes); 1569 1570 hv_vcpu->vp_index = new_vp_index; 1571 break; 1572 } 1573 case HV_X64_MSR_VP_ASSIST_PAGE: { 1574 u64 gfn; 1575 unsigned long addr; 1576 1577 if (!(data & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) { 1578 hv_vcpu->hv_vapic = data; 1579 if (kvm_lapic_set_pv_eoi(vcpu, 0, 0)) 1580 return 1; 1581 break; 1582 } 1583 gfn = data >> HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT; 1584 addr = kvm_vcpu_gfn_to_hva(vcpu, gfn); 1585 if (kvm_is_error_hva(addr)) 1586 return 1; 1587 1588 /* 1589 * Clear apic_assist portion of struct hv_vp_assist_page 1590 * only, there can be valuable data in the rest which needs 1591 * to be preserved e.g. on migration. 1592 */ 1593 if (put_user(0, (u32 __user *)addr)) 1594 return 1; 1595 hv_vcpu->hv_vapic = data; 1596 kvm_vcpu_mark_page_dirty(vcpu, gfn); 1597 if (kvm_lapic_set_pv_eoi(vcpu, 1598 gfn_to_gpa(gfn) | KVM_MSR_ENABLED, 1599 sizeof(struct hv_vp_assist_page))) 1600 return 1; 1601 break; 1602 } 1603 case HV_X64_MSR_EOI: 1604 return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data); 1605 case HV_X64_MSR_ICR: 1606 return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data); 1607 case HV_X64_MSR_TPR: 1608 return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data); 1609 case HV_X64_MSR_VP_RUNTIME: 1610 if (!host) 1611 return 1; 1612 hv_vcpu->runtime_offset = data - current_task_runtime_100ns(); 1613 break; 1614 case HV_X64_MSR_SCONTROL: 1615 case HV_X64_MSR_SVERSION: 1616 case HV_X64_MSR_SIEFP: 1617 case HV_X64_MSR_SIMP: 1618 case HV_X64_MSR_EOM: 1619 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1620 return synic_set_msr(to_hv_synic(vcpu), msr, data, host); 1621 case HV_X64_MSR_STIMER0_CONFIG: 1622 case HV_X64_MSR_STIMER1_CONFIG: 1623 case HV_X64_MSR_STIMER2_CONFIG: 1624 case HV_X64_MSR_STIMER3_CONFIG: { 1625 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1626 1627 return stimer_set_config(to_hv_stimer(vcpu, timer_index), 1628 data, host); 1629 } 1630 case HV_X64_MSR_STIMER0_COUNT: 1631 case HV_X64_MSR_STIMER1_COUNT: 1632 case HV_X64_MSR_STIMER2_COUNT: 1633 case HV_X64_MSR_STIMER3_COUNT: { 1634 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1635 1636 return stimer_set_count(to_hv_stimer(vcpu, timer_index), 1637 data, host); 1638 } 1639 case HV_X64_MSR_TSC_FREQUENCY: 1640 case HV_X64_MSR_APIC_FREQUENCY: 1641 /* read-only, but still ignore it if host-initiated */ 1642 if (!host) 1643 return 1; 1644 break; 1645 default: 1646 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1647 return 1; 1648 } 1649 1650 return 0; 1651 } 1652 1653 static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1654 bool host) 1655 { 1656 u64 data = 0; 1657 struct kvm *kvm = vcpu->kvm; 1658 struct kvm_hv *hv = to_kvm_hv(kvm); 1659 1660 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1661 return 1; 1662 1663 switch (msr) { 1664 case HV_X64_MSR_GUEST_OS_ID: 1665 data = hv->hv_guest_os_id; 1666 break; 1667 case HV_X64_MSR_HYPERCALL: 1668 data = hv->hv_hypercall; 1669 break; 1670 case HV_X64_MSR_TIME_REF_COUNT: 1671 data = get_time_ref_counter(kvm); 1672 break; 1673 case HV_X64_MSR_REFERENCE_TSC: 1674 data = hv->hv_tsc_page; 1675 break; 1676 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1677 return kvm_hv_msr_get_crash_data(kvm, 1678 msr - HV_X64_MSR_CRASH_P0, 1679 pdata); 1680 case HV_X64_MSR_CRASH_CTL: 1681 return kvm_hv_msr_get_crash_ctl(kvm, pdata); 1682 case HV_X64_MSR_RESET: 1683 data = 0; 1684 break; 1685 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1686 data = hv->hv_reenlightenment_control; 1687 break; 1688 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1689 data = hv->hv_tsc_emulation_control; 1690 break; 1691 case HV_X64_MSR_TSC_EMULATION_STATUS: 1692 data = hv->hv_tsc_emulation_status; 1693 break; 1694 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1695 data = hv->hv_invtsc_control; 1696 break; 1697 case HV_X64_MSR_SYNDBG_OPTIONS: 1698 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1699 return syndbg_get_msr(vcpu, msr, pdata, host); 1700 default: 1701 kvm_pr_unimpl_rdmsr(vcpu, msr); 1702 return 1; 1703 } 1704 1705 *pdata = data; 1706 return 0; 1707 } 1708 1709 static int kvm_hv_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1710 bool host) 1711 { 1712 u64 data = 0; 1713 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1714 1715 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1716 return 1; 1717 1718 switch (msr) { 1719 case HV_X64_MSR_VP_INDEX: 1720 data = hv_vcpu->vp_index; 1721 break; 1722 case HV_X64_MSR_EOI: 1723 return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata); 1724 case HV_X64_MSR_ICR: 1725 return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata); 1726 case HV_X64_MSR_TPR: 1727 return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata); 1728 case HV_X64_MSR_VP_ASSIST_PAGE: 1729 data = hv_vcpu->hv_vapic; 1730 break; 1731 case HV_X64_MSR_VP_RUNTIME: 1732 data = current_task_runtime_100ns() + hv_vcpu->runtime_offset; 1733 break; 1734 case HV_X64_MSR_SCONTROL: 1735 case HV_X64_MSR_SVERSION: 1736 case HV_X64_MSR_SIEFP: 1737 case HV_X64_MSR_SIMP: 1738 case HV_X64_MSR_EOM: 1739 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1740 return synic_get_msr(to_hv_synic(vcpu), msr, pdata, host); 1741 case HV_X64_MSR_STIMER0_CONFIG: 1742 case HV_X64_MSR_STIMER1_CONFIG: 1743 case HV_X64_MSR_STIMER2_CONFIG: 1744 case HV_X64_MSR_STIMER3_CONFIG: { 1745 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1746 1747 return stimer_get_config(to_hv_stimer(vcpu, timer_index), 1748 pdata); 1749 } 1750 case HV_X64_MSR_STIMER0_COUNT: 1751 case HV_X64_MSR_STIMER1_COUNT: 1752 case HV_X64_MSR_STIMER2_COUNT: 1753 case HV_X64_MSR_STIMER3_COUNT: { 1754 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1755 1756 return stimer_get_count(to_hv_stimer(vcpu, timer_index), 1757 pdata); 1758 } 1759 case HV_X64_MSR_TSC_FREQUENCY: 1760 data = (u64)vcpu->arch.virtual_tsc_khz * 1000; 1761 break; 1762 case HV_X64_MSR_APIC_FREQUENCY: 1763 data = div64_u64(1000000000ULL, 1764 vcpu->kvm->arch.apic_bus_cycle_ns); 1765 break; 1766 default: 1767 kvm_pr_unimpl_rdmsr(vcpu, msr); 1768 return 1; 1769 } 1770 *pdata = data; 1771 return 0; 1772 } 1773 1774 int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1775 { 1776 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1777 1778 if (!host && !vcpu->arch.hyperv_enabled) 1779 return 1; 1780 1781 if (kvm_hv_vcpu_init(vcpu)) 1782 return 1; 1783 1784 if (kvm_hv_msr_partition_wide(msr)) { 1785 int r; 1786 1787 mutex_lock(&hv->hv_lock); 1788 r = kvm_hv_set_msr_pw(vcpu, msr, data, host); 1789 mutex_unlock(&hv->hv_lock); 1790 return r; 1791 } else 1792 return kvm_hv_set_msr(vcpu, msr, data, host); 1793 } 1794 1795 int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 1796 { 1797 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1798 1799 if (!host && !vcpu->arch.hyperv_enabled) 1800 return 1; 1801 1802 if (kvm_hv_vcpu_init(vcpu)) 1803 return 1; 1804 1805 if (kvm_hv_msr_partition_wide(msr)) { 1806 int r; 1807 1808 mutex_lock(&hv->hv_lock); 1809 r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host); 1810 mutex_unlock(&hv->hv_lock); 1811 return r; 1812 } else 1813 return kvm_hv_get_msr(vcpu, msr, pdata, host); 1814 } 1815 1816 static void sparse_set_to_vcpu_mask(struct kvm *kvm, u64 *sparse_banks, 1817 u64 valid_bank_mask, unsigned long *vcpu_mask) 1818 { 1819 struct kvm_hv *hv = to_kvm_hv(kvm); 1820 bool has_mismatch = atomic_read(&hv->num_mismatched_vp_indexes); 1821 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS]; 1822 struct kvm_vcpu *vcpu; 1823 int bank, sbank = 0; 1824 unsigned long i; 1825 u64 *bitmap; 1826 1827 BUILD_BUG_ON(sizeof(vp_bitmap) > 1828 sizeof(*vcpu_mask) * BITS_TO_LONGS(KVM_MAX_VCPUS)); 1829 1830 /* 1831 * If vp_index == vcpu_idx for all vCPUs, fill vcpu_mask directly, else 1832 * fill a temporary buffer and manually test each vCPU's VP index. 1833 */ 1834 if (likely(!has_mismatch)) 1835 bitmap = (u64 *)vcpu_mask; 1836 else 1837 bitmap = vp_bitmap; 1838 1839 /* 1840 * Each set of 64 VPs is packed into sparse_banks, with valid_bank_mask 1841 * having a '1' for each bank that exists in sparse_banks. Sets must 1842 * be in ascending order, i.e. bank0..bankN. 1843 */ 1844 memset(bitmap, 0, sizeof(vp_bitmap)); 1845 for_each_set_bit(bank, (unsigned long *)&valid_bank_mask, 1846 KVM_HV_MAX_SPARSE_VCPU_SET_BITS) 1847 bitmap[bank] = sparse_banks[sbank++]; 1848 1849 if (likely(!has_mismatch)) 1850 return; 1851 1852 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 1853 kvm_for_each_vcpu(i, vcpu, kvm) { 1854 if (test_bit(kvm_hv_get_vpindex(vcpu), (unsigned long *)vp_bitmap)) 1855 __set_bit(i, vcpu_mask); 1856 } 1857 } 1858 1859 static bool hv_is_vp_in_sparse_set(u32 vp_id, u64 valid_bank_mask, u64 sparse_banks[]) 1860 { 1861 int valid_bit_nr = vp_id / HV_VCPUS_PER_SPARSE_BANK; 1862 unsigned long sbank; 1863 1864 BUILD_BUG_ON(BITS_PER_TYPE(valid_bank_mask) != HV_MAX_SPARSE_VCPU_BANKS); 1865 1866 if (valid_bit_nr >= HV_MAX_SPARSE_VCPU_BANKS) 1867 return false; 1868 1869 if (!test_bit(valid_bit_nr, (unsigned long *)&valid_bank_mask)) 1870 return false; 1871 1872 /* 1873 * The index into the sparse bank is the number of preceding bits in 1874 * the valid mask. Optimize for VMs with <64 vCPUs by skipping the 1875 * fancy math if there can't possibly be preceding bits. 1876 */ 1877 if (valid_bit_nr) 1878 sbank = hweight64(valid_bank_mask & GENMASK_ULL(valid_bit_nr - 1, 0)); 1879 else 1880 sbank = 0; 1881 1882 return test_bit(vp_id % HV_VCPUS_PER_SPARSE_BANK, 1883 (unsigned long *)&sparse_banks[sbank]); 1884 } 1885 1886 struct kvm_hv_hcall { 1887 /* Hypercall input data */ 1888 u64 param; 1889 u64 ingpa; 1890 u64 outgpa; 1891 u16 code; 1892 u16 var_cnt; 1893 u16 rep_cnt; 1894 u16 rep_idx; 1895 bool fast; 1896 bool rep; 1897 sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS]; 1898 1899 /* 1900 * Current read offset when KVM reads hypercall input data gradually, 1901 * either offset in bytes from 'ingpa' for regular hypercalls or the 1902 * number of already consumed 'XMM halves' for 'fast' hypercalls. 1903 */ 1904 union { 1905 gpa_t data_offset; 1906 int consumed_xmm_halves; 1907 }; 1908 }; 1909 1910 1911 static int kvm_hv_get_hc_data(struct kvm *kvm, struct kvm_hv_hcall *hc, 1912 u16 orig_cnt, u16 cnt_cap, u64 *data) 1913 { 1914 /* 1915 * Preserve the original count when ignoring entries via a "cap", KVM 1916 * still needs to validate the guest input (though the non-XMM path 1917 * punts on the checks). 1918 */ 1919 u16 cnt = min(orig_cnt, cnt_cap); 1920 int i, j; 1921 1922 if (hc->fast) { 1923 /* 1924 * Each XMM holds two sparse banks, but do not count halves that 1925 * have already been consumed for hypercall parameters. 1926 */ 1927 if (orig_cnt > 2 * HV_HYPERCALL_MAX_XMM_REGISTERS - hc->consumed_xmm_halves) 1928 return HV_STATUS_INVALID_HYPERCALL_INPUT; 1929 1930 for (i = 0; i < cnt; i++) { 1931 j = i + hc->consumed_xmm_halves; 1932 if (j % 2) 1933 data[i] = sse128_hi(hc->xmm[j / 2]); 1934 else 1935 data[i] = sse128_lo(hc->xmm[j / 2]); 1936 } 1937 return 0; 1938 } 1939 1940 return kvm_read_guest(kvm, hc->ingpa + hc->data_offset, data, 1941 cnt * sizeof(*data)); 1942 } 1943 1944 static u64 kvm_get_sparse_vp_set(struct kvm *kvm, struct kvm_hv_hcall *hc, 1945 u64 *sparse_banks) 1946 { 1947 if (hc->var_cnt > HV_MAX_SPARSE_VCPU_BANKS) 1948 return -EINVAL; 1949 1950 /* Cap var_cnt to ignore banks that cannot contain a legal VP index. */ 1951 return kvm_hv_get_hc_data(kvm, hc, hc->var_cnt, KVM_HV_MAX_SPARSE_VCPU_SET_BITS, 1952 sparse_banks); 1953 } 1954 1955 static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc, u64 entries[]) 1956 { 1957 return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries); 1958 } 1959 1960 static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, u64 *entries, int count, 1961 bool is_guest_mode) 1962 { 1963 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 1964 u64 flush_all_entry = KVM_HV_TLB_FLUSHALL_ENTRY; 1965 1966 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode); 1967 if (!tlb_flush_fifo) 1968 return; 1969 1970 spin_lock(&tlb_flush_fifo->write_lock); 1971 1972 /* 1973 * All entries should fit on the fifo leaving one free for 'flush all' 1974 * entry in case another request comes in. In case there's not enough 1975 * space, just put 'flush all' entry there. 1976 */ 1977 if (count && entries && count < kfifo_avail(&tlb_flush_fifo->entries)) { 1978 WARN_ON(kfifo_in(&tlb_flush_fifo->entries, entries, count) != count); 1979 goto out_unlock; 1980 } 1981 1982 /* 1983 * Note: full fifo always contains 'flush all' entry, no need to check the 1984 * return value. 1985 */ 1986 kfifo_in(&tlb_flush_fifo->entries, &flush_all_entry, 1); 1987 1988 out_unlock: 1989 spin_unlock(&tlb_flush_fifo->write_lock); 1990 } 1991 1992 int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) 1993 { 1994 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 1995 u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; 1996 int i, j, count; 1997 gva_t gva; 1998 bool full = false; 1999 2000 if (!tdp_enabled) 2001 return -EINVAL; 2002 2003 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu)); 2004 if (!tlb_flush_fifo) 2005 return -EINVAL; 2006 2007 count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); 2008 2009 for (i = 0; i < count && !full; i++) { 2010 if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) 2011 goto out_flush_all; 2012 2013 /* 2014 * Lower 12 bits of 'address' encode the number of additional 2015 * pages to flush. 2016 */ 2017 gva = entries[i] & PAGE_MASK; 2018 for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) { 2019 if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) 2020 continue; 2021 2022 kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full); 2023 } 2024 2025 ++vcpu->stat.tlb_flush; 2026 } 2027 return 0; 2028 2029 out_flush_all: 2030 kfifo_reset_out(&tlb_flush_fifo->entries); 2031 2032 /* Fall back to full flush. */ 2033 return -ENOSPC; 2034 } 2035 2036 static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2037 { 2038 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2039 unsigned long *vcpu_mask = hv_vcpu->vcpu_mask; 2040 u64 *sparse_banks = hv_vcpu->sparse_banks; 2041 struct kvm *kvm = vcpu->kvm; 2042 struct hv_tlb_flush_ex flush_ex; 2043 struct hv_tlb_flush flush; 2044 /* 2045 * Normally, there can be no more than 'KVM_HV_TLB_FLUSH_FIFO_SIZE' 2046 * entries on the TLB flush fifo. The last entry, however, needs to be 2047 * always left free for 'flush all' entry which gets placed when 2048 * there is not enough space to put all the requested entries. 2049 */ 2050 u64 __tlb_flush_entries[KVM_HV_TLB_FLUSH_FIFO_SIZE - 1]; 2051 u64 *tlb_flush_entries; 2052 u64 valid_bank_mask; 2053 struct kvm_vcpu *v; 2054 unsigned long i; 2055 bool all_cpus; 2056 2057 /* 2058 * The Hyper-V TLFS doesn't allow more than HV_MAX_SPARSE_VCPU_BANKS 2059 * sparse banks. Fail the build if KVM's max allowed number of 2060 * vCPUs (>4096) exceeds this limit. 2061 */ 2062 BUILD_BUG_ON(KVM_HV_MAX_SPARSE_VCPU_SET_BITS > HV_MAX_SPARSE_VCPU_BANKS); 2063 2064 /* 2065 * 'Slow' hypercall's first parameter is the address in guest's memory 2066 * where hypercall parameters are placed. This is either a GPA or a 2067 * nested GPA when KVM is handling the call from L2 ('direct' TLB 2068 * flush). Translate the address here so the memory can be uniformly 2069 * read with kvm_read_guest(). 2070 */ 2071 if (!hc->fast) { 2072 hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.gva_walk, hc->ingpa, 2073 PFERR_GUEST_FINAL_MASK, NULL, 0); 2074 if (unlikely(hc->ingpa == INVALID_GPA)) 2075 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2076 } 2077 2078 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST || 2079 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE) { 2080 if (hc->fast) { 2081 flush.address_space = hc->ingpa; 2082 flush.flags = hc->outgpa; 2083 flush.processor_mask = sse128_lo(hc->xmm[0]); 2084 hc->consumed_xmm_halves = 1; 2085 } else { 2086 if (unlikely(kvm_read_guest(kvm, hc->ingpa, 2087 &flush, sizeof(flush)))) 2088 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2089 hc->data_offset = sizeof(flush); 2090 } 2091 2092 trace_kvm_hv_flush_tlb(flush.processor_mask, 2093 flush.address_space, flush.flags, 2094 is_guest_mode(vcpu)); 2095 2096 valid_bank_mask = BIT_ULL(0); 2097 sparse_banks[0] = flush.processor_mask; 2098 2099 /* 2100 * Work around possible WS2012 bug: it sends hypercalls 2101 * with processor_mask = 0x0 and HV_FLUSH_ALL_PROCESSORS clear, 2102 * while also expecting us to flush something and crashing if 2103 * we don't. Let's treat processor_mask == 0 same as 2104 * HV_FLUSH_ALL_PROCESSORS. 2105 */ 2106 all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) || 2107 flush.processor_mask == 0; 2108 } else { 2109 if (hc->fast) { 2110 flush_ex.address_space = hc->ingpa; 2111 flush_ex.flags = hc->outgpa; 2112 memcpy(&flush_ex.hv_vp_set, 2113 &hc->xmm[0], sizeof(hc->xmm[0])); 2114 hc->consumed_xmm_halves = 2; 2115 } else { 2116 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex, 2117 sizeof(flush_ex)))) 2118 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2119 hc->data_offset = sizeof(flush_ex); 2120 } 2121 2122 trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask, 2123 flush_ex.hv_vp_set.format, 2124 flush_ex.address_space, 2125 flush_ex.flags, is_guest_mode(vcpu)); 2126 2127 valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask; 2128 all_cpus = flush_ex.hv_vp_set.format != 2129 HV_GENERIC_SET_SPARSE_4K; 2130 2131 if (hc->var_cnt != hweight64(valid_bank_mask)) 2132 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2133 2134 if (!all_cpus) { 2135 if (!hc->var_cnt) 2136 goto ret_success; 2137 2138 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2139 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2140 } 2141 2142 /* 2143 * Hyper-V TLFS doesn't explicitly forbid non-empty sparse vCPU 2144 * banks (and, thus, non-zero 'var_cnt') for the 'all vCPUs' 2145 * case (HV_GENERIC_SET_ALL). Always adjust data_offset and 2146 * consumed_xmm_halves to make sure TLB flush entries are read 2147 * from the correct offset. 2148 */ 2149 if (hc->fast) 2150 hc->consumed_xmm_halves += hc->var_cnt; 2151 else 2152 hc->data_offset += hc->var_cnt * sizeof(sparse_banks[0]); 2153 } 2154 2155 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE || 2156 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX || 2157 hc->rep_cnt > ARRAY_SIZE(__tlb_flush_entries)) { 2158 tlb_flush_entries = NULL; 2159 } else { 2160 if (kvm_hv_get_tlb_flush_entries(kvm, hc, __tlb_flush_entries)) 2161 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2162 tlb_flush_entries = __tlb_flush_entries; 2163 } 2164 2165 /* 2166 * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't 2167 * analyze it here, flush TLB regardless of the specified address space. 2168 */ 2169 if (all_cpus && !is_guest_mode(vcpu)) { 2170 kvm_for_each_vcpu(i, v, kvm) 2171 hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false); 2172 2173 kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH); 2174 } else if (!is_guest_mode(vcpu)) { 2175 sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, vcpu_mask); 2176 2177 for_each_set_bit(i, vcpu_mask, KVM_MAX_VCPUS) { 2178 v = kvm_get_vcpu(kvm, i); 2179 if (!v) 2180 continue; 2181 hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false); 2182 } 2183 2184 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2185 } else { 2186 struct kvm_vcpu_hv *hv_v; 2187 2188 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 2189 2190 kvm_for_each_vcpu(i, v, kvm) { 2191 hv_v = to_hv_vcpu_safe(v); 2192 2193 /* 2194 * The following check races with nested vCPUs entering/exiting 2195 * and/or migrating between L1's vCPUs, however the only case when 2196 * KVM *must* flush the TLB is when the target L2 vCPU keeps 2197 * running on the same L1 vCPU from the moment of the request until 2198 * kvm_hv_flush_tlb() returns. TLB is fully flushed in all other 2199 * cases, e.g. when the target L2 vCPU migrates to a different L1 2200 * vCPU or when the corresponding L1 vCPU temporary switches to a 2201 * different L2 vCPU while the request is being processed. 2202 */ 2203 if (!hv_v || hv_v->nested.vm_id != hv_vcpu->nested.vm_id) 2204 continue; 2205 2206 if (!all_cpus && 2207 !hv_is_vp_in_sparse_set(hv_v->nested.vp_id, valid_bank_mask, 2208 sparse_banks)) 2209 continue; 2210 2211 __set_bit(i, vcpu_mask); 2212 hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, true); 2213 } 2214 2215 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2216 } 2217 2218 ret_success: 2219 /* We always do full TLB flush, set 'Reps completed' = 'Rep Count' */ 2220 return (u64)HV_STATUS_SUCCESS | 2221 ((u64)hc->rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET); 2222 } 2223 2224 static void kvm_hv_send_ipi_to_many(struct kvm *kvm, u32 vector, 2225 u64 *sparse_banks, u64 valid_bank_mask) 2226 { 2227 struct kvm_lapic_irq irq = { 2228 .delivery_mode = APIC_DM_FIXED, 2229 .vector = vector 2230 }; 2231 struct kvm_vcpu *vcpu; 2232 unsigned long i; 2233 2234 kvm_for_each_vcpu(i, vcpu, kvm) { 2235 if (sparse_banks && 2236 !hv_is_vp_in_sparse_set(kvm_hv_get_vpindex(vcpu), 2237 valid_bank_mask, sparse_banks)) 2238 continue; 2239 2240 /* We fail only when APIC is disabled */ 2241 kvm_apic_set_irq(vcpu, &irq, NULL); 2242 } 2243 } 2244 2245 static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2246 { 2247 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2248 u64 *sparse_banks = hv_vcpu->sparse_banks; 2249 struct kvm *kvm = vcpu->kvm; 2250 struct hv_send_ipi_ex send_ipi_ex; 2251 struct hv_send_ipi send_ipi; 2252 u64 valid_bank_mask; 2253 u32 vector; 2254 bool all_cpus; 2255 2256 if (!lapic_in_kernel(vcpu)) 2257 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2258 2259 if (hc->code == HVCALL_SEND_IPI) { 2260 if (!hc->fast) { 2261 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi, 2262 sizeof(send_ipi)))) 2263 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2264 sparse_banks[0] = send_ipi.cpu_mask; 2265 vector = send_ipi.vector; 2266 } else { 2267 /* 'reserved' part of hv_send_ipi should be 0 */ 2268 if (unlikely(hc->ingpa >> 32 != 0)) 2269 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2270 sparse_banks[0] = hc->outgpa; 2271 vector = (u32)hc->ingpa; 2272 } 2273 all_cpus = false; 2274 valid_bank_mask = BIT_ULL(0); 2275 2276 trace_kvm_hv_send_ipi(vector, sparse_banks[0]); 2277 } else { 2278 if (!hc->fast) { 2279 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex, 2280 sizeof(send_ipi_ex)))) 2281 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2282 } else { 2283 send_ipi_ex.vector = (u32)hc->ingpa; 2284 send_ipi_ex.vp_set.format = hc->outgpa; 2285 send_ipi_ex.vp_set.valid_bank_mask = sse128_lo(hc->xmm[0]); 2286 } 2287 2288 trace_kvm_hv_send_ipi_ex(send_ipi_ex.vector, 2289 send_ipi_ex.vp_set.format, 2290 send_ipi_ex.vp_set.valid_bank_mask); 2291 2292 vector = send_ipi_ex.vector; 2293 valid_bank_mask = send_ipi_ex.vp_set.valid_bank_mask; 2294 all_cpus = send_ipi_ex.vp_set.format == HV_GENERIC_SET_ALL; 2295 2296 if (hc->var_cnt != hweight64(valid_bank_mask)) 2297 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2298 2299 if (all_cpus) 2300 goto check_and_send_ipi; 2301 2302 if (!hc->var_cnt) 2303 goto ret_success; 2304 2305 if (!hc->fast) 2306 hc->data_offset = offsetof(struct hv_send_ipi_ex, 2307 vp_set.bank_contents); 2308 else 2309 hc->consumed_xmm_halves = 1; 2310 2311 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2312 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2313 } 2314 2315 check_and_send_ipi: 2316 if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) 2317 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2318 2319 if (all_cpus) 2320 kvm_hv_send_ipi_to_many(kvm, vector, NULL, 0); 2321 else 2322 kvm_hv_send_ipi_to_many(kvm, vector, sparse_banks, valid_bank_mask); 2323 2324 ret_success: 2325 return HV_STATUS_SUCCESS; 2326 } 2327 2328 void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled) 2329 { 2330 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2331 struct kvm_cpuid_entry2 *entry; 2332 2333 vcpu->arch.hyperv_enabled = hyperv_enabled; 2334 2335 if (!hv_vcpu) { 2336 /* 2337 * KVM should have already allocated kvm_vcpu_hv if Hyper-V is 2338 * enabled in CPUID. 2339 */ 2340 WARN_ON_ONCE(vcpu->arch.hyperv_enabled); 2341 return; 2342 } 2343 2344 memset(&hv_vcpu->cpuid_cache, 0, sizeof(hv_vcpu->cpuid_cache)); 2345 2346 if (!vcpu->arch.hyperv_enabled) 2347 return; 2348 2349 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_FEATURES); 2350 if (entry) { 2351 hv_vcpu->cpuid_cache.features_eax = entry->eax; 2352 hv_vcpu->cpuid_cache.features_ebx = entry->ebx; 2353 hv_vcpu->cpuid_cache.features_edx = entry->edx; 2354 } 2355 2356 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_ENLIGHTMENT_INFO); 2357 if (entry) { 2358 hv_vcpu->cpuid_cache.enlightenments_eax = entry->eax; 2359 hv_vcpu->cpuid_cache.enlightenments_ebx = entry->ebx; 2360 } 2361 2362 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES); 2363 if (entry) 2364 hv_vcpu->cpuid_cache.syndbg_cap_eax = entry->eax; 2365 2366 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_NESTED_FEATURES); 2367 if (entry) { 2368 hv_vcpu->cpuid_cache.nested_eax = entry->eax; 2369 hv_vcpu->cpuid_cache.nested_ebx = entry->ebx; 2370 } 2371 } 2372 2373 int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce) 2374 { 2375 struct kvm_vcpu_hv *hv_vcpu; 2376 int ret = 0; 2377 2378 if (!to_hv_vcpu(vcpu)) { 2379 if (enforce) { 2380 ret = kvm_hv_vcpu_init(vcpu); 2381 if (ret) 2382 return ret; 2383 } else { 2384 return 0; 2385 } 2386 } 2387 2388 hv_vcpu = to_hv_vcpu(vcpu); 2389 hv_vcpu->enforce_cpuid = enforce; 2390 2391 return ret; 2392 } 2393 2394 static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) 2395 { 2396 bool longmode; 2397 2398 longmode = is_64_bit_hypercall(vcpu); 2399 if (longmode) 2400 kvm_rax_write_raw(vcpu, result); 2401 else { 2402 kvm_edx_write(vcpu, result >> 32); 2403 kvm_eax_write(vcpu, result); 2404 } 2405 } 2406 2407 static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result) 2408 { 2409 u32 tlb_lock_count = 0; 2410 int ret; 2411 2412 if (hv_result_success(result) && is_guest_mode(vcpu) && 2413 kvm_hv_is_tlb_flush_hcall(vcpu) && 2414 kvm_read_guest(vcpu->kvm, to_hv_vcpu(vcpu)->nested.pa_page_gpa, 2415 &tlb_lock_count, sizeof(tlb_lock_count))) 2416 result = HV_STATUS_INVALID_HYPERCALL_INPUT; 2417 2418 trace_kvm_hv_hypercall_done(result); 2419 kvm_hv_hypercall_set_result(vcpu, result); 2420 ++vcpu->stat.hypercalls; 2421 2422 ret = kvm_skip_emulated_instruction(vcpu); 2423 2424 if (tlb_lock_count) 2425 kvm_nested_call(hv_inject_synthetic_vmexit_post_tlb_flush)(vcpu); 2426 2427 return ret; 2428 } 2429 2430 static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu) 2431 { 2432 return kvm_hv_hypercall_complete(vcpu, vcpu->run->hyperv.u.hcall.result); 2433 } 2434 2435 static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2436 { 2437 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 2438 struct eventfd_ctx *eventfd; 2439 2440 if (unlikely(!hc->fast)) { 2441 int ret; 2442 gpa_t gpa = hc->ingpa; 2443 2444 if ((gpa & (__alignof__(hc->ingpa) - 1)) || 2445 offset_in_page(gpa) + sizeof(hc->ingpa) > PAGE_SIZE) 2446 return HV_STATUS_INVALID_ALIGNMENT; 2447 2448 ret = kvm_vcpu_read_guest(vcpu, gpa, 2449 &hc->ingpa, sizeof(hc->ingpa)); 2450 if (ret < 0) 2451 return HV_STATUS_INVALID_ALIGNMENT; 2452 } 2453 2454 /* 2455 * Per spec, bits 32-47 contain the extra "flag number". However, we 2456 * have no use for it, and in all known usecases it is zero, so just 2457 * report lookup failure if it isn't. 2458 */ 2459 if (hc->ingpa & 0xffff00000000ULL) 2460 return HV_STATUS_INVALID_PORT_ID; 2461 /* remaining bits are reserved-zero */ 2462 if (hc->ingpa & ~KVM_HYPERV_CONN_ID_MASK) 2463 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2464 2465 /* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */ 2466 rcu_read_lock(); 2467 eventfd = idr_find(&hv->conn_to_evt, hc->ingpa); 2468 rcu_read_unlock(); 2469 if (!eventfd) 2470 return HV_STATUS_INVALID_PORT_ID; 2471 2472 eventfd_signal(eventfd); 2473 return HV_STATUS_SUCCESS; 2474 } 2475 2476 static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc) 2477 { 2478 switch (hc->code) { 2479 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2480 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2481 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2482 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2483 case HVCALL_SEND_IPI_EX: 2484 return true; 2485 } 2486 2487 return false; 2488 } 2489 2490 static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc) 2491 { 2492 int reg; 2493 2494 kvm_fpu_get(); 2495 for (reg = 0; reg < HV_HYPERCALL_MAX_XMM_REGISTERS; reg++) 2496 _kvm_read_sse_reg(reg, &hc->xmm[reg]); 2497 kvm_fpu_put(); 2498 } 2499 2500 static bool hv_check_hypercall_access(struct kvm_vcpu_hv *hv_vcpu, u16 code) 2501 { 2502 if (!hv_vcpu->enforce_cpuid) 2503 return true; 2504 2505 switch (code) { 2506 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2507 return hv_vcpu->cpuid_cache.enlightenments_ebx && 2508 hv_vcpu->cpuid_cache.enlightenments_ebx != U32_MAX; 2509 case HVCALL_POST_MESSAGE: 2510 return hv_vcpu->cpuid_cache.features_ebx & HV_POST_MESSAGES; 2511 case HVCALL_SIGNAL_EVENT: 2512 return hv_vcpu->cpuid_cache.features_ebx & HV_SIGNAL_EVENTS; 2513 case HVCALL_POST_DEBUG_DATA: 2514 case HVCALL_RETRIEVE_DEBUG_DATA: 2515 case HVCALL_RESET_DEBUG_SESSION: 2516 /* 2517 * Return 'true' when SynDBG is disabled so the resulting code 2518 * will be HV_STATUS_INVALID_HYPERCALL_CODE. 2519 */ 2520 return !kvm_hv_is_syndbg_enabled(hv_vcpu->vcpu) || 2521 hv_vcpu->cpuid_cache.features_ebx & HV_DEBUGGING; 2522 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2523 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2524 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2525 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2526 return false; 2527 fallthrough; 2528 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2529 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2530 return hv_vcpu->cpuid_cache.enlightenments_eax & 2531 HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2532 case HVCALL_SEND_IPI_EX: 2533 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2534 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2535 return false; 2536 fallthrough; 2537 case HVCALL_SEND_IPI: 2538 return hv_vcpu->cpuid_cache.enlightenments_eax & 2539 HV_X64_CLUSTER_IPI_RECOMMENDED; 2540 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2541 return hv_vcpu->cpuid_cache.features_ebx & 2542 HV_ENABLE_EXTENDED_HYPERCALLS; 2543 default: 2544 break; 2545 } 2546 2547 return true; 2548 } 2549 2550 int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 2551 { 2552 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2553 struct kvm_hv_hcall hc; 2554 u64 ret = HV_STATUS_SUCCESS; 2555 2556 /* 2557 * hypercall generates UD from non zero cpl and real mode 2558 * per HYPER-V spec 2559 */ 2560 if (kvm_x86_call(get_cpl)(vcpu) != 0 || !is_protmode(vcpu)) { 2561 kvm_queue_exception(vcpu, UD_VECTOR); 2562 return 1; 2563 } 2564 2565 #ifdef CONFIG_X86_64 2566 if (is_64_bit_hypercall(vcpu)) { 2567 hc.param = kvm_rcx_read_raw(vcpu); 2568 hc.ingpa = kvm_rdx_read_raw(vcpu); 2569 hc.outgpa = kvm_r8_read_raw(vcpu); 2570 } else 2571 #endif 2572 { 2573 hc.param = ((u64)kvm_edx_read(vcpu) << 32) | kvm_eax_read(vcpu); 2574 hc.ingpa = ((u64)kvm_ebx_read(vcpu) << 32) | kvm_ecx_read(vcpu); 2575 hc.outgpa = ((u64)kvm_edi_read(vcpu) << 32) | kvm_esi_read(vcpu); 2576 } 2577 2578 hc.code = hc.param & 0xffff; 2579 hc.var_cnt = (hc.param & HV_HYPERCALL_VARHEAD_MASK) >> HV_HYPERCALL_VARHEAD_OFFSET; 2580 hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT); 2581 hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff; 2582 hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff; 2583 hc.rep = !!(hc.rep_cnt || hc.rep_idx); 2584 2585 trace_kvm_hv_hypercall(hc.code, hc.fast, hc.var_cnt, hc.rep_cnt, 2586 hc.rep_idx, hc.ingpa, hc.outgpa); 2587 2588 if (unlikely(!hv_check_hypercall_access(hv_vcpu, hc.code))) { 2589 ret = HV_STATUS_ACCESS_DENIED; 2590 goto hypercall_complete; 2591 } 2592 2593 if (unlikely(hc.param & HV_HYPERCALL_RSVD_MASK)) { 2594 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2595 goto hypercall_complete; 2596 } 2597 2598 if (hc.fast && is_xmm_fast_hypercall(&hc)) { 2599 if (unlikely(hv_vcpu->enforce_cpuid && 2600 !(hv_vcpu->cpuid_cache.features_edx & 2601 HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE))) { 2602 kvm_queue_exception(vcpu, UD_VECTOR); 2603 return 1; 2604 } 2605 2606 kvm_hv_hypercall_read_xmm(&hc); 2607 } 2608 2609 switch (hc.code) { 2610 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2611 if (unlikely(hc.rep || hc.var_cnt)) { 2612 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2613 break; 2614 } 2615 kvm_vcpu_on_spin(vcpu, true); 2616 break; 2617 case HVCALL_SIGNAL_EVENT: 2618 if (unlikely(hc.rep || hc.var_cnt)) { 2619 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2620 break; 2621 } 2622 ret = kvm_hvcall_signal_event(vcpu, &hc); 2623 if (ret != HV_STATUS_INVALID_PORT_ID) 2624 break; 2625 fallthrough; /* maybe userspace knows this conn_id */ 2626 case HVCALL_POST_MESSAGE: 2627 /* don't bother userspace if it has no way to handle it */ 2628 if (unlikely(hc.rep || hc.var_cnt || !to_hv_synic(vcpu)->active)) { 2629 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2630 break; 2631 } 2632 goto hypercall_userspace_exit; 2633 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2634 if (unlikely(hc.var_cnt)) { 2635 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2636 break; 2637 } 2638 fallthrough; 2639 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2640 if (unlikely(!hc.rep_cnt || hc.rep_idx)) { 2641 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2642 break; 2643 } 2644 ret = kvm_hv_flush_tlb(vcpu, &hc); 2645 break; 2646 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2647 if (unlikely(hc.var_cnt)) { 2648 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2649 break; 2650 } 2651 fallthrough; 2652 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2653 if (unlikely(hc.rep)) { 2654 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2655 break; 2656 } 2657 ret = kvm_hv_flush_tlb(vcpu, &hc); 2658 break; 2659 case HVCALL_SEND_IPI: 2660 if (unlikely(hc.var_cnt)) { 2661 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2662 break; 2663 } 2664 fallthrough; 2665 case HVCALL_SEND_IPI_EX: 2666 if (unlikely(hc.rep)) { 2667 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2668 break; 2669 } 2670 ret = kvm_hv_send_ipi(vcpu, &hc); 2671 break; 2672 case HVCALL_POST_DEBUG_DATA: 2673 case HVCALL_RETRIEVE_DEBUG_DATA: 2674 if (unlikely(hc.fast)) { 2675 ret = HV_STATUS_INVALID_PARAMETER; 2676 break; 2677 } 2678 fallthrough; 2679 case HVCALL_RESET_DEBUG_SESSION: { 2680 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 2681 2682 if (!kvm_hv_is_syndbg_enabled(vcpu)) { 2683 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2684 break; 2685 } 2686 2687 if (!(syndbg->options & HV_X64_SYNDBG_OPTION_USE_HCALLS)) { 2688 ret = HV_STATUS_OPERATION_DENIED; 2689 break; 2690 } 2691 goto hypercall_userspace_exit; 2692 } 2693 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2694 if (unlikely(hc.fast)) { 2695 ret = HV_STATUS_INVALID_PARAMETER; 2696 break; 2697 } 2698 goto hypercall_userspace_exit; 2699 default: 2700 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2701 break; 2702 } 2703 2704 hypercall_complete: 2705 return kvm_hv_hypercall_complete(vcpu, ret); 2706 2707 hypercall_userspace_exit: 2708 vcpu->run->exit_reason = KVM_EXIT_HYPERV; 2709 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL; 2710 vcpu->run->hyperv.u.hcall.input = hc.param; 2711 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa; 2712 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa; 2713 vcpu->arch.complete_userspace_io = kvm_hv_hypercall_complete_userspace; 2714 return 0; 2715 } 2716 2717 void kvm_hv_init_vm(struct kvm *kvm) 2718 { 2719 struct kvm_hv *hv = to_kvm_hv(kvm); 2720 2721 mutex_init(&hv->hv_lock); 2722 idr_init(&hv->conn_to_evt); 2723 } 2724 2725 void kvm_hv_destroy_vm(struct kvm *kvm) 2726 { 2727 struct kvm_hv *hv = to_kvm_hv(kvm); 2728 struct eventfd_ctx *eventfd; 2729 int i; 2730 2731 idr_for_each_entry(&hv->conn_to_evt, eventfd, i) 2732 eventfd_ctx_put(eventfd); 2733 idr_destroy(&hv->conn_to_evt); 2734 } 2735 2736 static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd) 2737 { 2738 struct kvm_hv *hv = to_kvm_hv(kvm); 2739 struct eventfd_ctx *eventfd; 2740 int ret; 2741 2742 eventfd = eventfd_ctx_fdget(fd); 2743 if (IS_ERR(eventfd)) 2744 return PTR_ERR(eventfd); 2745 2746 mutex_lock(&hv->hv_lock); 2747 ret = idr_alloc(&hv->conn_to_evt, eventfd, conn_id, conn_id + 1, 2748 GFP_KERNEL_ACCOUNT); 2749 mutex_unlock(&hv->hv_lock); 2750 2751 if (ret >= 0) 2752 return 0; 2753 2754 if (ret == -ENOSPC) 2755 ret = -EEXIST; 2756 eventfd_ctx_put(eventfd); 2757 return ret; 2758 } 2759 2760 static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id) 2761 { 2762 struct kvm_hv *hv = to_kvm_hv(kvm); 2763 struct eventfd_ctx *eventfd; 2764 2765 mutex_lock(&hv->hv_lock); 2766 eventfd = idr_remove(&hv->conn_to_evt, conn_id); 2767 mutex_unlock(&hv->hv_lock); 2768 2769 if (!eventfd) 2770 return -ENOENT; 2771 2772 synchronize_srcu(&kvm->srcu); 2773 eventfd_ctx_put(eventfd); 2774 return 0; 2775 } 2776 2777 int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args) 2778 { 2779 if ((args->flags & ~KVM_HYPERV_EVENTFD_DEASSIGN) || 2780 (args->conn_id & ~KVM_HYPERV_CONN_ID_MASK)) 2781 return -EINVAL; 2782 2783 if (args->flags == KVM_HYPERV_EVENTFD_DEASSIGN) 2784 return kvm_hv_eventfd_deassign(kvm, args->conn_id); 2785 return kvm_hv_eventfd_assign(kvm, args->conn_id, args->fd); 2786 } 2787 2788 int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, 2789 struct kvm_cpuid_entry2 __user *entries) 2790 { 2791 uint16_t evmcs_ver = 0; 2792 struct kvm_cpuid_entry2 cpuid_entries[] = { 2793 { .function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS }, 2794 { .function = HYPERV_CPUID_INTERFACE }, 2795 { .function = HYPERV_CPUID_VERSION }, 2796 { .function = HYPERV_CPUID_FEATURES }, 2797 { .function = HYPERV_CPUID_ENLIGHTMENT_INFO }, 2798 { .function = HYPERV_CPUID_IMPLEMENT_LIMITS }, 2799 { .function = HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS }, 2800 { .function = HYPERV_CPUID_SYNDBG_INTERFACE }, 2801 { .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES }, 2802 { .function = HYPERV_CPUID_NESTED_FEATURES }, 2803 }; 2804 int i, nent = ARRAY_SIZE(cpuid_entries); 2805 2806 if (kvm_nested_ops.enabled) 2807 evmcs_ver = kvm_nested_call(get_evmcs_version)(vcpu); 2808 2809 if (cpuid->nent < nent) 2810 return -E2BIG; 2811 2812 if (cpuid->nent > nent) 2813 cpuid->nent = nent; 2814 2815 for (i = 0; i < nent; i++) { 2816 struct kvm_cpuid_entry2 *ent = &cpuid_entries[i]; 2817 u32 signature[3]; 2818 2819 switch (ent->function) { 2820 case HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS: 2821 memcpy(signature, "Linux KVM Hv", 12); 2822 2823 ent->eax = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES; 2824 ent->ebx = signature[0]; 2825 ent->ecx = signature[1]; 2826 ent->edx = signature[2]; 2827 break; 2828 2829 case HYPERV_CPUID_INTERFACE: 2830 ent->eax = HYPERV_CPUID_SIGNATURE_EAX; 2831 break; 2832 2833 case HYPERV_CPUID_VERSION: 2834 /* 2835 * We implement some Hyper-V 2016 functions so let's use 2836 * this version. 2837 */ 2838 ent->eax = 0x00003839; 2839 ent->ebx = 0x000A0000; 2840 break; 2841 2842 case HYPERV_CPUID_FEATURES: 2843 ent->eax |= HV_MSR_VP_RUNTIME_AVAILABLE; 2844 ent->eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE; 2845 ent->eax |= HV_MSR_SYNIC_AVAILABLE; 2846 ent->eax |= HV_MSR_SYNTIMER_AVAILABLE; 2847 ent->eax |= HV_MSR_APIC_ACCESS_AVAILABLE; 2848 ent->eax |= HV_MSR_HYPERCALL_AVAILABLE; 2849 ent->eax |= HV_MSR_VP_INDEX_AVAILABLE; 2850 ent->eax |= HV_MSR_RESET_AVAILABLE; 2851 ent->eax |= HV_MSR_REFERENCE_TSC_AVAILABLE; 2852 ent->eax |= HV_ACCESS_FREQUENCY_MSRS; 2853 ent->eax |= HV_ACCESS_REENLIGHTENMENT; 2854 ent->eax |= HV_ACCESS_TSC_INVARIANT; 2855 2856 ent->ebx |= HV_POST_MESSAGES; 2857 ent->ebx |= HV_SIGNAL_EVENTS; 2858 ent->ebx |= HV_ENABLE_EXTENDED_HYPERCALLS; 2859 2860 ent->edx |= HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE; 2861 ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE; 2862 ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 2863 2864 ent->ebx |= HV_DEBUGGING; 2865 ent->edx |= HV_X64_GUEST_DEBUGGING_AVAILABLE; 2866 ent->edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE; 2867 ent->edx |= HV_FEATURE_EXT_GVA_RANGES_FLUSH; 2868 2869 /* 2870 * Direct Synthetic timers only make sense with in-kernel 2871 * LAPIC 2872 */ 2873 if (!vcpu || lapic_in_kernel(vcpu)) 2874 ent->edx |= HV_STIMER_DIRECT_MODE_AVAILABLE; 2875 2876 break; 2877 2878 case HYPERV_CPUID_ENLIGHTMENT_INFO: 2879 ent->eax |= HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2880 ent->eax |= HV_X64_APIC_ACCESS_RECOMMENDED; 2881 ent->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED; 2882 if (!vcpu || lapic_in_kernel(vcpu)) 2883 ent->eax |= HV_X64_CLUSTER_IPI_RECOMMENDED; 2884 ent->eax |= HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED; 2885 if (evmcs_ver) 2886 ent->eax |= HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 2887 if (!cpu_smt_possible()) 2888 ent->eax |= HV_X64_NO_NONARCH_CORESHARING; 2889 2890 ent->eax |= HV_DEPRECATING_AEOI_RECOMMENDED; 2891 /* 2892 * Default number of spinlock retry attempts, matches 2893 * HyperV 2016. 2894 */ 2895 ent->ebx = 0x00000FFF; 2896 2897 break; 2898 2899 case HYPERV_CPUID_IMPLEMENT_LIMITS: 2900 /* Maximum number of virtual processors */ 2901 ent->eax = KVM_MAX_VCPUS; 2902 /* 2903 * Maximum number of logical processors, matches 2904 * HyperV 2016. 2905 */ 2906 ent->ebx = 64; 2907 2908 break; 2909 2910 case HYPERV_CPUID_NESTED_FEATURES: 2911 ent->eax = evmcs_ver; 2912 ent->eax |= HV_X64_NESTED_DIRECT_FLUSH; 2913 ent->eax |= HV_X64_NESTED_MSR_BITMAP; 2914 ent->ebx |= HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; 2915 break; 2916 2917 case HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS: 2918 memcpy(signature, "Linux KVM Hv", 12); 2919 2920 ent->eax = 0; 2921 ent->ebx = signature[0]; 2922 ent->ecx = signature[1]; 2923 ent->edx = signature[2]; 2924 break; 2925 2926 case HYPERV_CPUID_SYNDBG_INTERFACE: 2927 memcpy(signature, "VS#1\0\0\0\0\0\0\0\0", 12); 2928 ent->eax = signature[0]; 2929 break; 2930 2931 case HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES: 2932 ent->eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 2933 break; 2934 2935 default: 2936 break; 2937 } 2938 } 2939 2940 if (copy_to_user(entries, cpuid_entries, 2941 nent * sizeof(struct kvm_cpuid_entry2))) 2942 return -EFAULT; 2943 2944 return 0; 2945 } 2946