1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM Microsoft Hyper-V emulation 4 * 5 * derived from arch/x86/kvm/x86.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com> 12 * 13 * Authors: 14 * Avi Kivity <avi@qumranet.com> 15 * Yaniv Kamay <yaniv@qumranet.com> 16 * Amit Shah <amit.shah@qumranet.com> 17 * Ben-Ami Yassour <benami@il.ibm.com> 18 * Andrey Smetanin <asmetanin@virtuozzo.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include "x86.h" 23 #include "lapic.h" 24 #include "ioapic.h" 25 #include "cpuid.h" 26 #include "hyperv.h" 27 #include "mmu.h" 28 #include "xen.h" 29 30 #include <linux/cpu.h> 31 #include <linux/kvm_host.h> 32 #include <linux/highmem.h> 33 #include <linux/sched/cputime.h> 34 #include <linux/spinlock.h> 35 #include <linux/eventfd.h> 36 37 #include <asm/apicdef.h> 38 #include <asm/mshyperv.h> 39 #include <trace/events/kvm.h> 40 41 #include "trace.h" 42 #include "irq.h" 43 #include "fpu.h" 44 45 #define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, HV_VCPUS_PER_SPARSE_BANK) 46 47 /* 48 * As per Hyper-V TLFS, extended hypercalls start from 0x8001 49 * (HvExtCallQueryCapabilities). Response of this hypercalls is a 64 bit value 50 * where each bit tells which extended hypercall is available besides 51 * HvExtCallQueryCapabilities. 52 * 53 * 0x8001 - First extended hypercall, HvExtCallQueryCapabilities, no bit 54 * assigned. 55 * 56 * 0x8002 - Bit 0 57 * 0x8003 - Bit 1 58 * .. 59 * 0x8041 - Bit 63 60 * 61 * Therefore, HV_EXT_CALL_MAX = 0x8001 + 64 62 */ 63 #define HV_EXT_CALL_MAX (HV_EXT_CALL_QUERY_CAPABILITIES + 64) 64 65 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 66 bool vcpu_kick); 67 68 static inline u64 synic_read_sint(struct kvm_vcpu_hv_synic *synic, int sint) 69 { 70 return atomic64_read(&synic->sint[sint]); 71 } 72 73 static inline int synic_get_sint_vector(u64 sint_value) 74 { 75 if (sint_value & HV_SYNIC_SINT_MASKED) 76 return -1; 77 return sint_value & HV_SYNIC_SINT_VECTOR_MASK; 78 } 79 80 static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic, 81 int vector) 82 { 83 int i; 84 85 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 86 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 87 return true; 88 } 89 return false; 90 } 91 92 static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic, 93 int vector) 94 { 95 int i; 96 u64 sint_value; 97 98 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 99 sint_value = synic_read_sint(synic, i); 100 if (synic_get_sint_vector(sint_value) == vector && 101 sint_value & HV_SYNIC_SINT_AUTO_EOI) 102 return true; 103 } 104 return false; 105 } 106 107 static void synic_update_vector(struct kvm_vcpu_hv_synic *synic, 108 int vector) 109 { 110 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 111 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 112 bool auto_eoi_old, auto_eoi_new; 113 114 if (vector < HV_SYNIC_FIRST_VALID_VECTOR) 115 return; 116 117 if (synic_has_vector_connected(synic, vector)) 118 __set_bit(vector, synic->vec_bitmap); 119 else 120 __clear_bit(vector, synic->vec_bitmap); 121 122 auto_eoi_old = !bitmap_empty(synic->auto_eoi_bitmap, 256); 123 124 if (synic_has_vector_auto_eoi(synic, vector)) 125 __set_bit(vector, synic->auto_eoi_bitmap); 126 else 127 __clear_bit(vector, synic->auto_eoi_bitmap); 128 129 auto_eoi_new = !bitmap_empty(synic->auto_eoi_bitmap, 256); 130 131 if (auto_eoi_old == auto_eoi_new) 132 return; 133 134 if (!enable_apicv) 135 return; 136 137 down_write(&vcpu->kvm->arch.apicv_update_lock); 138 139 if (auto_eoi_new) 140 hv->synic_auto_eoi_used++; 141 else 142 hv->synic_auto_eoi_used--; 143 144 /* 145 * Inhibit APICv if any vCPU is using SynIC's AutoEOI, which relies on 146 * the hypervisor to manually inject IRQs. 147 */ 148 __kvm_set_or_clear_apicv_inhibit(vcpu->kvm, 149 APICV_INHIBIT_REASON_HYPERV, 150 !!hv->synic_auto_eoi_used); 151 152 up_write(&vcpu->kvm->arch.apicv_update_lock); 153 } 154 155 static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, 156 u64 data, bool host) 157 { 158 int vector, old_vector; 159 bool masked; 160 161 vector = data & HV_SYNIC_SINT_VECTOR_MASK; 162 masked = data & HV_SYNIC_SINT_MASKED; 163 164 /* 165 * Valid vectors are 16-255, however, nested Hyper-V attempts to write 166 * default '0x10000' value on boot and this should not #GP. We need to 167 * allow zero-initing the register from host as well. 168 */ 169 if (vector < HV_SYNIC_FIRST_VALID_VECTOR && !host && !masked) 170 return 1; 171 /* 172 * Guest may configure multiple SINTs to use the same vector, so 173 * we maintain a bitmap of vectors handled by synic, and a 174 * bitmap of vectors with auto-eoi behavior. The bitmaps are 175 * updated here, and atomically queried on fast paths. 176 */ 177 old_vector = synic_read_sint(synic, sint) & HV_SYNIC_SINT_VECTOR_MASK; 178 179 atomic64_set(&synic->sint[sint], data); 180 181 synic_update_vector(synic, old_vector); 182 183 synic_update_vector(synic, vector); 184 185 /* Load SynIC vectors into EOI exit bitmap */ 186 kvm_make_request(KVM_REQ_SCAN_IOAPIC, hv_synic_to_vcpu(synic)); 187 return 0; 188 } 189 190 static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx) 191 { 192 struct kvm_vcpu *vcpu = NULL; 193 unsigned long i; 194 195 if (vpidx >= KVM_MAX_VCPUS) 196 return NULL; 197 198 vcpu = kvm_get_vcpu(kvm, vpidx); 199 if (vcpu && kvm_hv_get_vpindex(vcpu) == vpidx) 200 return vcpu; 201 kvm_for_each_vcpu(i, vcpu, kvm) 202 if (kvm_hv_get_vpindex(vcpu) == vpidx) 203 return vcpu; 204 return NULL; 205 } 206 207 static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx) 208 { 209 struct kvm_vcpu_hv_synic *synic; 210 struct kvm_vcpu_hv *hv_vcpu; 211 struct kvm_vcpu *vcpu; 212 213 vcpu = get_vcpu_by_vpidx(kvm, vpidx); 214 if (!vcpu) 215 return NULL; 216 217 hv_vcpu = to_hv_vcpu_safe(vcpu); 218 if (!hv_vcpu) 219 return NULL; 220 221 synic = &hv_vcpu->synic; 222 return READ_ONCE(synic->active) ? synic : NULL; 223 } 224 225 static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint) 226 { 227 struct kvm *kvm = vcpu->kvm; 228 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 229 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 230 struct kvm_vcpu_hv_stimer *stimer; 231 int gsi, idx; 232 233 trace_kvm_hv_notify_acked_sint(vcpu->vcpu_id, sint); 234 235 /* Try to deliver pending Hyper-V SynIC timers messages */ 236 for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) { 237 stimer = &hv_vcpu->stimer[idx]; 238 if (stimer->msg_pending && stimer->config.enable && 239 !stimer->config.direct_mode && 240 stimer->config.sintx == sint) 241 stimer_mark_pending(stimer, false); 242 } 243 244 idx = srcu_read_lock(&kvm->irq_srcu); 245 gsi = atomic_read(&synic->sint_to_gsi[sint]); 246 if (gsi != -1) 247 kvm_notify_acked_gsi(kvm, gsi); 248 srcu_read_unlock(&kvm->irq_srcu, idx); 249 } 250 251 static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr) 252 { 253 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 254 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 255 256 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC; 257 hv_vcpu->exit.u.synic.msr = msr; 258 hv_vcpu->exit.u.synic.control = synic->control; 259 hv_vcpu->exit.u.synic.evt_page = synic->evt_page; 260 hv_vcpu->exit.u.synic.msg_page = synic->msg_page; 261 262 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 263 } 264 265 static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, 266 u32 msr, u64 data, bool host) 267 { 268 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 269 int ret; 270 271 if (!synic->active && (!host || data)) 272 return 1; 273 274 trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host); 275 276 ret = 0; 277 switch (msr) { 278 case HV_X64_MSR_SCONTROL: 279 synic->control = data; 280 if (!host) 281 synic_exit(synic, msr); 282 break; 283 case HV_X64_MSR_SVERSION: 284 if (!host) { 285 ret = 1; 286 break; 287 } 288 synic->version = data; 289 break; 290 case HV_X64_MSR_SIEFP: 291 if ((data & HV_SYNIC_SIEFP_ENABLE) && !host && 292 !synic->dont_zero_synic_pages) 293 if (kvm_clear_guest(vcpu->kvm, 294 data & PAGE_MASK, PAGE_SIZE)) { 295 ret = 1; 296 break; 297 } 298 synic->evt_page = data; 299 if (!host) 300 synic_exit(synic, msr); 301 break; 302 case HV_X64_MSR_SIMP: 303 if ((data & HV_SYNIC_SIMP_ENABLE) && !host && 304 !synic->dont_zero_synic_pages) 305 if (kvm_clear_guest(vcpu->kvm, 306 data & PAGE_MASK, PAGE_SIZE)) { 307 ret = 1; 308 break; 309 } 310 synic->msg_page = data; 311 if (!host) 312 synic_exit(synic, msr); 313 break; 314 case HV_X64_MSR_EOM: { 315 int i; 316 317 if (!synic->active) 318 break; 319 320 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 321 kvm_hv_notify_acked_sint(vcpu, i); 322 break; 323 } 324 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 325 ret = synic_set_sint(synic, msr - HV_X64_MSR_SINT0, data, host); 326 break; 327 default: 328 ret = 1; 329 break; 330 } 331 return ret; 332 } 333 334 static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) 335 { 336 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 337 338 return hv_vcpu->cpuid_cache.syndbg_cap_eax & 339 HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 340 } 341 342 static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) 343 { 344 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 345 346 if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL) 347 hv->hv_syndbg.control.status = 348 vcpu->run->hyperv.u.syndbg.status; 349 return 1; 350 } 351 352 static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr) 353 { 354 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 355 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 356 357 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG; 358 hv_vcpu->exit.u.syndbg.msr = msr; 359 hv_vcpu->exit.u.syndbg.control = syndbg->control.control; 360 hv_vcpu->exit.u.syndbg.send_page = syndbg->control.send_page; 361 hv_vcpu->exit.u.syndbg.recv_page = syndbg->control.recv_page; 362 hv_vcpu->exit.u.syndbg.pending_page = syndbg->control.pending_page; 363 vcpu->arch.complete_userspace_io = 364 kvm_hv_syndbg_complete_userspace; 365 366 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 367 } 368 369 static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 370 { 371 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 372 373 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 374 return 1; 375 376 trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id, 377 to_hv_vcpu(vcpu)->vp_index, msr, data); 378 switch (msr) { 379 case HV_X64_MSR_SYNDBG_CONTROL: 380 syndbg->control.control = data; 381 if (!host) 382 syndbg_exit(vcpu, msr); 383 break; 384 case HV_X64_MSR_SYNDBG_STATUS: 385 syndbg->control.status = data; 386 break; 387 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 388 syndbg->control.send_page = data; 389 break; 390 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 391 syndbg->control.recv_page = data; 392 break; 393 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 394 syndbg->control.pending_page = data; 395 if (!host) 396 syndbg_exit(vcpu, msr); 397 break; 398 case HV_X64_MSR_SYNDBG_OPTIONS: 399 syndbg->options = data; 400 break; 401 default: 402 break; 403 } 404 405 return 0; 406 } 407 408 static int syndbg_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 409 { 410 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 411 412 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 413 return 1; 414 415 switch (msr) { 416 case HV_X64_MSR_SYNDBG_CONTROL: 417 *pdata = syndbg->control.control; 418 break; 419 case HV_X64_MSR_SYNDBG_STATUS: 420 *pdata = syndbg->control.status; 421 break; 422 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 423 *pdata = syndbg->control.send_page; 424 break; 425 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 426 *pdata = syndbg->control.recv_page; 427 break; 428 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 429 *pdata = syndbg->control.pending_page; 430 break; 431 case HV_X64_MSR_SYNDBG_OPTIONS: 432 *pdata = syndbg->options; 433 break; 434 default: 435 break; 436 } 437 438 trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, kvm_hv_get_vpindex(vcpu), msr, *pdata); 439 440 return 0; 441 } 442 443 static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata, 444 bool host) 445 { 446 int ret; 447 448 if (!synic->active && !host) 449 return 1; 450 451 ret = 0; 452 switch (msr) { 453 case HV_X64_MSR_SCONTROL: 454 *pdata = synic->control; 455 break; 456 case HV_X64_MSR_SVERSION: 457 *pdata = synic->version; 458 break; 459 case HV_X64_MSR_SIEFP: 460 *pdata = synic->evt_page; 461 break; 462 case HV_X64_MSR_SIMP: 463 *pdata = synic->msg_page; 464 break; 465 case HV_X64_MSR_EOM: 466 *pdata = 0; 467 break; 468 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 469 *pdata = atomic64_read(&synic->sint[msr - HV_X64_MSR_SINT0]); 470 break; 471 default: 472 ret = 1; 473 break; 474 } 475 return ret; 476 } 477 478 static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint) 479 { 480 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 481 struct kvm_lapic_irq irq; 482 int ret, vector; 483 484 if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 485 return -EINVAL; 486 487 if (sint >= ARRAY_SIZE(synic->sint)) 488 return -EINVAL; 489 490 vector = synic_get_sint_vector(synic_read_sint(synic, sint)); 491 if (vector < 0) 492 return -ENOENT; 493 494 memset(&irq, 0, sizeof(irq)); 495 irq.shorthand = APIC_DEST_SELF; 496 irq.dest_mode = APIC_DEST_PHYSICAL; 497 irq.delivery_mode = APIC_DM_FIXED; 498 irq.vector = vector; 499 irq.level = 1; 500 501 ret = kvm_irq_delivery_to_apic(vcpu->kvm, vcpu->arch.apic, &irq); 502 trace_kvm_hv_synic_set_irq(vcpu->vcpu_id, sint, irq.vector, ret); 503 return ret; 504 } 505 506 int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 507 int irq_source_id, int level, bool line_status) 508 { 509 struct kvm_vcpu_hv_synic *synic; 510 511 if (!level) 512 return -1; 513 514 synic = synic_get(kvm, e->hv_sint.vcpu); 515 if (!synic) 516 return -EINVAL; 517 518 return synic_set_irq(synic, e->hv_sint.sint); 519 } 520 521 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector) 522 { 523 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 524 int i; 525 526 trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector); 527 528 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 529 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 530 kvm_hv_notify_acked_sint(vcpu, i); 531 } 532 533 static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi) 534 { 535 struct kvm_vcpu_hv_synic *synic; 536 537 synic = synic_get(kvm, vpidx); 538 if (!synic) 539 return -EINVAL; 540 541 if (sint >= ARRAY_SIZE(synic->sint_to_gsi)) 542 return -EINVAL; 543 544 atomic_set(&synic->sint_to_gsi[sint], gsi); 545 return 0; 546 } 547 548 void kvm_hv_irq_routing_update(struct kvm *kvm) 549 { 550 struct kvm_irq_routing_table *irq_rt; 551 struct kvm_kernel_irq_routing_entry *e; 552 u32 gsi; 553 554 irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu, 555 lockdep_is_held(&kvm->irq_lock)); 556 557 for (gsi = 0; gsi < irq_rt->nr_rt_entries; gsi++) { 558 hlist_for_each_entry(e, &irq_rt->map[gsi], link) { 559 if (e->type == KVM_IRQ_ROUTING_HV_SINT) 560 kvm_hv_set_sint_gsi(kvm, e->hv_sint.vcpu, 561 e->hv_sint.sint, gsi); 562 } 563 } 564 } 565 566 static void synic_init(struct kvm_vcpu_hv_synic *synic) 567 { 568 int i; 569 570 memset(synic, 0, sizeof(*synic)); 571 synic->version = HV_SYNIC_VERSION_1; 572 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 573 atomic64_set(&synic->sint[i], HV_SYNIC_SINT_MASKED); 574 atomic_set(&synic->sint_to_gsi[i], -1); 575 } 576 } 577 578 static u64 get_time_ref_counter(struct kvm *kvm) 579 { 580 struct kvm_hv *hv = to_kvm_hv(kvm); 581 struct kvm_vcpu *vcpu; 582 u64 tsc; 583 584 /* 585 * Fall back to get_kvmclock_ns() when TSC page hasn't been set up, 586 * is broken, disabled or being updated. 587 */ 588 if (hv->hv_tsc_page_status != HV_TSC_PAGE_SET) 589 return div_u64(get_kvmclock_ns(kvm), 100); 590 591 vcpu = kvm_get_vcpu(kvm, 0); 592 tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 593 return mul_u64_u64_shr(tsc, hv->tsc_ref.tsc_scale, 64) 594 + hv->tsc_ref.tsc_offset; 595 } 596 597 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 598 bool vcpu_kick) 599 { 600 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 601 602 set_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap); 603 kvm_make_request(KVM_REQ_HV_STIMER, vcpu); 604 if (vcpu_kick) 605 kvm_vcpu_kick(vcpu); 606 } 607 608 static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer) 609 { 610 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 611 612 trace_kvm_hv_stimer_cleanup(hv_stimer_to_vcpu(stimer)->vcpu_id, 613 stimer->index); 614 615 hrtimer_cancel(&stimer->timer); 616 clear_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap); 617 stimer->msg_pending = false; 618 stimer->exp_time = 0; 619 } 620 621 static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) 622 { 623 struct kvm_vcpu_hv_stimer *stimer; 624 625 stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer); 626 trace_kvm_hv_stimer_callback(hv_stimer_to_vcpu(stimer)->vcpu_id, 627 stimer->index); 628 stimer_mark_pending(stimer, true); 629 630 return HRTIMER_NORESTART; 631 } 632 633 /* 634 * Translate a stimer expiry given in 100ns reference ticks into an 635 * an absolute deadline. Saturates on overflow. 636 */ 637 static ktime_t stimer_add_delta(ktime_t now, u64 delta_100ns) 638 { 639 if (delta_100ns >= KTIME_MAX / 100) 640 return KTIME_MAX; 641 642 return ktime_add_safe(now, 100 * delta_100ns); 643 } 644 645 /* 646 * stimer_start() assumptions: 647 * a) stimer->count is not equal to 0 648 * b) stimer->config has HV_STIMER_ENABLE flag 649 */ 650 static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) 651 { 652 u64 time_now; 653 ktime_t ktime_now; 654 ktime_t deadline; 655 656 time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm); 657 ktime_now = ktime_get(); 658 659 if (stimer->config.periodic) { 660 if (stimer->exp_time) { 661 if (time_now >= stimer->exp_time) { 662 u64 remainder; 663 664 div64_u64_rem(time_now - stimer->exp_time, 665 stimer->count, &remainder); 666 stimer->exp_time = 667 time_now + (stimer->count - remainder); 668 } 669 } else 670 stimer->exp_time = time_now + stimer->count; 671 672 trace_kvm_hv_stimer_start_periodic( 673 hv_stimer_to_vcpu(stimer)->vcpu_id, 674 stimer->index, 675 time_now, stimer->exp_time); 676 677 deadline = stimer_add_delta(ktime_now, stimer->exp_time - time_now); 678 hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); 679 return 0; 680 } 681 stimer->exp_time = stimer->count; 682 if (time_now >= stimer->count) { 683 /* 684 * Expire timer according to Hypervisor Top-Level Functional 685 * specification v4(15.3.1): 686 * "If a one shot is enabled and the specified count is in 687 * the past, it will expire immediately." 688 */ 689 stimer_mark_pending(stimer, false); 690 return 0; 691 } 692 693 trace_kvm_hv_stimer_start_one_shot(hv_stimer_to_vcpu(stimer)->vcpu_id, 694 stimer->index, 695 time_now, stimer->count); 696 697 deadline = stimer_add_delta(ktime_now, stimer->count - time_now); 698 hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); 699 700 return 0; 701 } 702 703 static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, 704 bool host) 705 { 706 union hv_stimer_config new_config = {.as_uint64 = config}, 707 old_config = {.as_uint64 = stimer->config.as_uint64}; 708 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 709 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 710 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 711 712 if (!synic->active && (!host || config)) 713 return 1; 714 715 if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode && 716 !(hv_vcpu->cpuid_cache.features_edx & 717 HV_STIMER_DIRECT_MODE_AVAILABLE))) 718 return 1; 719 720 trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id, 721 stimer->index, config, host); 722 723 stimer_cleanup(stimer); 724 if (old_config.enable && 725 !new_config.direct_mode && new_config.sintx == 0) 726 new_config.enable = 0; 727 stimer->config.as_uint64 = new_config.as_uint64; 728 729 if (stimer->config.enable) 730 stimer_mark_pending(stimer, false); 731 732 return 0; 733 } 734 735 static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count, 736 bool host) 737 { 738 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 739 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 740 741 if (!synic->active && (!host || count)) 742 return 1; 743 744 trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id, 745 stimer->index, count, host); 746 747 stimer_cleanup(stimer); 748 stimer->count = count; 749 if (!host) { 750 if (stimer->count == 0) 751 stimer->config.enable = 0; 752 else if (stimer->config.auto_enable) 753 stimer->config.enable = 1; 754 } 755 756 if (stimer->config.enable) 757 stimer_mark_pending(stimer, false); 758 759 return 0; 760 } 761 762 static int stimer_get_config(struct kvm_vcpu_hv_stimer *stimer, u64 *pconfig) 763 { 764 *pconfig = stimer->config.as_uint64; 765 return 0; 766 } 767 768 static int stimer_get_count(struct kvm_vcpu_hv_stimer *stimer, u64 *pcount) 769 { 770 *pcount = stimer->count; 771 return 0; 772 } 773 774 static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, 775 struct hv_message *src_msg, bool no_retry) 776 { 777 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 778 int msg_off = offsetof(struct hv_message_page, sint_message[sint]); 779 gfn_t msg_page_gfn; 780 struct hv_message_header hv_hdr; 781 int r; 782 783 if (!(synic->msg_page & HV_SYNIC_SIMP_ENABLE)) 784 return -ENOENT; 785 786 msg_page_gfn = synic->msg_page >> PAGE_SHIFT; 787 788 /* 789 * Strictly following the spec-mandated ordering would assume setting 790 * .msg_pending before checking .message_type. However, this function 791 * is only called in vcpu context so the entire update is atomic from 792 * guest POV and thus the exact order here doesn't matter. 793 */ 794 r = kvm_vcpu_read_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_type, 795 msg_off + offsetof(struct hv_message, 796 header.message_type), 797 sizeof(hv_hdr.message_type)); 798 if (r < 0) 799 return r; 800 801 if (hv_hdr.message_type != HVMSG_NONE) { 802 if (no_retry) 803 return 0; 804 805 hv_hdr.message_flags.asu8 = 0; 806 hv_hdr.message_flags.msg_pending = 1; 807 808 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, 809 &hv_hdr.message_flags, 810 msg_off + 811 offsetof(struct hv_message, 812 header.message_flags), 813 sizeof(hv_hdr.message_flags)); 814 if (r < 0) 815 return r; 816 return -EAGAIN; 817 } 818 819 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, src_msg, msg_off, 820 sizeof(src_msg->header) + 821 src_msg->header.payload_size); 822 if (r < 0) 823 return r; 824 825 r = synic_set_irq(synic, sint); 826 if (r < 0) 827 return r; 828 if (r == 0) 829 return -EFAULT; 830 return 0; 831 } 832 833 static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer) 834 { 835 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 836 struct hv_message *msg = &stimer->msg; 837 struct hv_timer_message_payload *payload = 838 (struct hv_timer_message_payload *)&msg->u.payload; 839 840 /* 841 * To avoid piling up periodic ticks, don't retry message 842 * delivery for them (within "lazy" lost ticks policy). 843 */ 844 bool no_retry = stimer->config.periodic; 845 846 payload->expiration_time = stimer->exp_time; 847 payload->delivery_time = get_time_ref_counter(vcpu->kvm); 848 return synic_deliver_msg(to_hv_synic(vcpu), 849 stimer->config.sintx, msg, 850 no_retry); 851 } 852 853 static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer) 854 { 855 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 856 struct kvm_lapic_irq irq = { 857 .delivery_mode = APIC_DM_FIXED, 858 .vector = stimer->config.apic_vector 859 }; 860 861 if (lapic_in_kernel(vcpu)) 862 return !kvm_apic_set_irq(vcpu, &irq, NULL); 863 return 0; 864 } 865 866 static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) 867 { 868 int r, direct = stimer->config.direct_mode; 869 870 stimer->msg_pending = true; 871 if (!direct) 872 r = stimer_send_msg(stimer); 873 else 874 r = stimer_notify_direct(stimer); 875 trace_kvm_hv_stimer_expiration(hv_stimer_to_vcpu(stimer)->vcpu_id, 876 stimer->index, direct, r); 877 if (!r) { 878 stimer->msg_pending = false; 879 if (!(stimer->config.periodic)) 880 stimer->config.enable = 0; 881 } 882 } 883 884 void kvm_hv_process_stimers(struct kvm_vcpu *vcpu) 885 { 886 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 887 struct kvm_vcpu_hv_stimer *stimer; 888 u64 time_now, exp_time; 889 int i; 890 891 if (!hv_vcpu) 892 return; 893 894 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 895 if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) { 896 stimer = &hv_vcpu->stimer[i]; 897 if (stimer->config.enable) { 898 exp_time = stimer->exp_time; 899 900 if (exp_time) { 901 time_now = 902 get_time_ref_counter(vcpu->kvm); 903 if (time_now >= exp_time) 904 stimer_expiration(stimer); 905 } 906 907 if ((stimer->config.enable) && 908 stimer->count) { 909 if (!stimer->msg_pending) 910 stimer_start(stimer); 911 } else 912 stimer_cleanup(stimer); 913 } 914 } 915 } 916 917 void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu) 918 { 919 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 920 int i; 921 922 if (!hv_vcpu) 923 return; 924 925 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 926 stimer_cleanup(&hv_vcpu->stimer[i]); 927 928 kfree(hv_vcpu); 929 vcpu->arch.hyperv = NULL; 930 } 931 932 bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu) 933 { 934 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 935 936 if (!hv_vcpu) 937 return false; 938 939 if (!(hv_vcpu->hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) 940 return false; 941 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 942 } 943 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_hv_assist_page_enabled); 944 945 int kvm_hv_get_assist_page(struct kvm_vcpu *vcpu) 946 { 947 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 948 949 if (!hv_vcpu || !kvm_hv_assist_page_enabled(vcpu)) 950 return -EFAULT; 951 952 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, 953 &hv_vcpu->vp_assist_page, sizeof(struct hv_vp_assist_page)); 954 } 955 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_hv_get_assist_page); 956 957 static void stimer_prepare_msg(struct kvm_vcpu_hv_stimer *stimer) 958 { 959 struct hv_message *msg = &stimer->msg; 960 struct hv_timer_message_payload *payload = 961 (struct hv_timer_message_payload *)&msg->u.payload; 962 963 memset(&msg->header, 0, sizeof(msg->header)); 964 msg->header.message_type = HVMSG_TIMER_EXPIRED; 965 msg->header.payload_size = sizeof(*payload); 966 967 payload->timer_index = stimer->index; 968 payload->expiration_time = 0; 969 payload->delivery_time = 0; 970 } 971 972 static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index) 973 { 974 memset(stimer, 0, sizeof(*stimer)); 975 stimer->index = timer_index; 976 hrtimer_setup(&stimer->timer, stimer_timer_callback, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 977 stimer_prepare_msg(stimer); 978 } 979 980 int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) 981 { 982 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 983 int i; 984 985 if (hv_vcpu) 986 return 0; 987 988 hv_vcpu = kzalloc_obj(struct kvm_vcpu_hv, GFP_KERNEL_ACCOUNT); 989 if (!hv_vcpu) 990 return -ENOMEM; 991 992 hv_vcpu->vcpu = vcpu; 993 994 synic_init(&hv_vcpu->synic); 995 996 bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); 997 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 998 stimer_init(&hv_vcpu->stimer[i], i); 999 1000 hv_vcpu->vp_index = vcpu->vcpu_idx; 1001 1002 for (i = 0; i < HV_NR_TLB_FLUSH_FIFOS; i++) { 1003 INIT_KFIFO(hv_vcpu->tlb_flush_fifo[i].entries); 1004 spin_lock_init(&hv_vcpu->tlb_flush_fifo[i].write_lock); 1005 } 1006 1007 /* 1008 * Ensure the structure is fully initialized before it's visible to 1009 * other tasks, as much of the state can be legally accessed without 1010 * holding vcpu->mutex. 1011 * 1012 * Pairs with the smp_load_acquire() in to_hv_vcpu_safe(). 1013 */ 1014 smp_store_release(&vcpu->arch.hyperv, hv_vcpu); 1015 return 0; 1016 } 1017 1018 int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages) 1019 { 1020 struct kvm_vcpu_hv_synic *synic; 1021 int r; 1022 1023 r = kvm_hv_vcpu_init(vcpu); 1024 if (r) 1025 return r; 1026 1027 synic = to_hv_synic(vcpu); 1028 1029 WRITE_ONCE(synic->active, true); 1030 synic->dont_zero_synic_pages = dont_zero_synic_pages; 1031 synic->control = HV_SYNIC_CONTROL_ENABLE; 1032 return 0; 1033 } 1034 1035 static bool kvm_hv_msr_partition_wide(u32 msr) 1036 { 1037 bool r = false; 1038 1039 switch (msr) { 1040 case HV_X64_MSR_GUEST_OS_ID: 1041 case HV_X64_MSR_HYPERCALL: 1042 case HV_X64_MSR_REFERENCE_TSC: 1043 case HV_X64_MSR_TIME_REF_COUNT: 1044 case HV_X64_MSR_CRASH_CTL: 1045 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1046 case HV_X64_MSR_RESET: 1047 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1048 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1049 case HV_X64_MSR_TSC_EMULATION_STATUS: 1050 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1051 case HV_X64_MSR_SYNDBG_OPTIONS: 1052 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1053 r = true; 1054 break; 1055 } 1056 1057 return r; 1058 } 1059 1060 static int kvm_hv_msr_get_crash_data(struct kvm *kvm, u32 index, u64 *pdata) 1061 { 1062 struct kvm_hv *hv = to_kvm_hv(kvm); 1063 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1064 1065 if (WARN_ON_ONCE(index >= size)) 1066 return -EINVAL; 1067 1068 *pdata = hv->hv_crash_param[array_index_nospec(index, size)]; 1069 return 0; 1070 } 1071 1072 static int kvm_hv_msr_get_crash_ctl(struct kvm *kvm, u64 *pdata) 1073 { 1074 struct kvm_hv *hv = to_kvm_hv(kvm); 1075 1076 *pdata = hv->hv_crash_ctl; 1077 return 0; 1078 } 1079 1080 static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data) 1081 { 1082 struct kvm_hv *hv = to_kvm_hv(kvm); 1083 1084 hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY; 1085 1086 return 0; 1087 } 1088 1089 static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data) 1090 { 1091 struct kvm_hv *hv = to_kvm_hv(kvm); 1092 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1093 1094 if (WARN_ON_ONCE(index >= size)) 1095 return -EINVAL; 1096 1097 hv->hv_crash_param[array_index_nospec(index, size)] = data; 1098 return 0; 1099 } 1100 1101 /* 1102 * The kvmclock and Hyper-V TSC page use similar formulas, and converting 1103 * between them is possible: 1104 * 1105 * kvmclock formula: 1106 * nsec = (ticks - tsc_timestamp) * tsc_to_system_mul * 2^(tsc_shift-32) 1107 * + system_time 1108 * 1109 * Hyper-V formula: 1110 * nsec/100 = ticks * scale / 2^64 + offset 1111 * 1112 * When tsc_timestamp = system_time = 0, offset is zero in the Hyper-V formula. 1113 * By dividing the kvmclock formula by 100 and equating what's left we get: 1114 * ticks * scale / 2^64 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1115 * scale / 2^64 = tsc_to_system_mul * 2^(tsc_shift-32) / 100 1116 * scale = tsc_to_system_mul * 2^(32+tsc_shift) / 100 1117 * 1118 * Now expand the kvmclock formula and divide by 100: 1119 * nsec = ticks * tsc_to_system_mul * 2^(tsc_shift-32) 1120 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) 1121 * + system_time 1122 * nsec/100 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1123 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1124 * + system_time / 100 1125 * 1126 * Replace tsc_to_system_mul * 2^(tsc_shift-32) / 100 by scale / 2^64: 1127 * nsec/100 = ticks * scale / 2^64 1128 * - tsc_timestamp * scale / 2^64 1129 * + system_time / 100 1130 * 1131 * Equate with the Hyper-V formula so that ticks * scale / 2^64 cancels out: 1132 * offset = system_time / 100 - tsc_timestamp * scale / 2^64 1133 * 1134 * These two equivalencies are implemented in this function. 1135 */ 1136 static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock, 1137 struct ms_hyperv_tsc_page *tsc_ref) 1138 { 1139 u64 max_mul; 1140 1141 if (!(hv_clock->flags & PVCLOCK_TSC_STABLE_BIT)) 1142 return false; 1143 1144 /* 1145 * check if scale would overflow, if so we use the time ref counter 1146 * tsc_to_system_mul * 2^(tsc_shift+32) / 100 >= 2^64 1147 * tsc_to_system_mul / 100 >= 2^(32-tsc_shift) 1148 * tsc_to_system_mul >= 100 * 2^(32-tsc_shift) 1149 */ 1150 max_mul = 100ull << (32 - hv_clock->tsc_shift); 1151 if (hv_clock->tsc_to_system_mul >= max_mul) 1152 return false; 1153 1154 /* 1155 * Otherwise compute the scale and offset according to the formulas 1156 * derived above. 1157 */ 1158 tsc_ref->tsc_scale = 1159 mul_u64_u32_div(1ULL << (32 + hv_clock->tsc_shift), 1160 hv_clock->tsc_to_system_mul, 1161 100); 1162 1163 tsc_ref->tsc_offset = hv_clock->system_time; 1164 do_div(tsc_ref->tsc_offset, 100); 1165 tsc_ref->tsc_offset -= 1166 mul_u64_u64_shr(hv_clock->tsc_timestamp, tsc_ref->tsc_scale, 64); 1167 return true; 1168 } 1169 1170 /* 1171 * Don't touch TSC page values if the guest has opted for TSC emulation after 1172 * migration. KVM doesn't fully support reenlightenment notifications and TSC 1173 * access emulation and Hyper-V is known to expect the values in TSC page to 1174 * stay constant before TSC access emulation is disabled from guest side 1175 * (HV_X64_MSR_TSC_EMULATION_STATUS). KVM userspace is expected to preserve TSC 1176 * frequency and guest visible TSC value across migration (and prevent it when 1177 * TSC scaling is unsupported). 1178 */ 1179 static inline bool tsc_page_update_unsafe(struct kvm_hv *hv) 1180 { 1181 return (hv->hv_tsc_page_status != HV_TSC_PAGE_GUEST_CHANGED) && 1182 hv->hv_tsc_emulation_control; 1183 } 1184 1185 void kvm_hv_setup_tsc_page(struct kvm *kvm, 1186 struct pvclock_vcpu_time_info *hv_clock) 1187 { 1188 struct kvm_hv *hv = to_kvm_hv(kvm); 1189 u32 tsc_seq; 1190 u64 gfn; 1191 1192 BUILD_BUG_ON(sizeof(tsc_seq) != sizeof(hv->tsc_ref.tsc_sequence)); 1193 BUILD_BUG_ON(offsetof(struct ms_hyperv_tsc_page, tsc_sequence) != 0); 1194 1195 guard(mutex)(&hv->hv_lock); 1196 1197 if (hv->hv_tsc_page_status == HV_TSC_PAGE_BROKEN || 1198 hv->hv_tsc_page_status == HV_TSC_PAGE_SET || 1199 hv->hv_tsc_page_status == HV_TSC_PAGE_UNSET) 1200 return; 1201 1202 if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE)) 1203 return; 1204 1205 gfn = hv->hv_tsc_page >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT; 1206 /* 1207 * Because the TSC parameters only vary when there is a 1208 * change in the master clock, do not bother with caching. 1209 */ 1210 if (unlikely(kvm_read_guest(kvm, gfn_to_gpa(gfn), 1211 &tsc_seq, sizeof(tsc_seq)))) 1212 goto out_err; 1213 1214 if (tsc_seq && tsc_page_update_unsafe(hv)) { 1215 if (kvm_read_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1216 goto out_err; 1217 1218 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1219 return; 1220 } 1221 1222 /* 1223 * While we're computing and writing the parameters, force the 1224 * guest to use the time reference count MSR. 1225 */ 1226 hv->tsc_ref.tsc_sequence = 0; 1227 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1228 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1229 goto out_err; 1230 1231 if (!compute_tsc_page_parameters(hv_clock, &hv->tsc_ref)) 1232 goto out_err; 1233 1234 /* Ensure sequence is zero before writing the rest of the struct. */ 1235 smp_wmb(); 1236 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1237 goto out_err; 1238 1239 /* 1240 * Now switch to the TSC page mechanism by writing the sequence. 1241 */ 1242 tsc_seq++; 1243 if (tsc_seq == 0xFFFFFFFF || tsc_seq == 0) 1244 tsc_seq = 1; 1245 1246 /* Write the struct entirely before the non-zero sequence. */ 1247 smp_wmb(); 1248 1249 hv->tsc_ref.tsc_sequence = tsc_seq; 1250 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1251 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1252 goto out_err; 1253 1254 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1255 return; 1256 1257 out_err: 1258 hv->hv_tsc_page_status = HV_TSC_PAGE_BROKEN; 1259 } 1260 1261 void kvm_hv_request_tsc_page_update(struct kvm *kvm) 1262 { 1263 struct kvm_hv *hv = to_kvm_hv(kvm); 1264 1265 mutex_lock(&hv->hv_lock); 1266 1267 if (hv->hv_tsc_page_status == HV_TSC_PAGE_SET && 1268 !tsc_page_update_unsafe(hv)) 1269 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1270 1271 mutex_unlock(&hv->hv_lock); 1272 } 1273 1274 static bool hv_check_msr_access(struct kvm_vcpu_hv *hv_vcpu, u32 msr) 1275 { 1276 if (!hv_vcpu->enforce_cpuid) 1277 return true; 1278 1279 switch (msr) { 1280 case HV_X64_MSR_GUEST_OS_ID: 1281 case HV_X64_MSR_HYPERCALL: 1282 return hv_vcpu->cpuid_cache.features_eax & 1283 HV_MSR_HYPERCALL_AVAILABLE; 1284 case HV_X64_MSR_VP_RUNTIME: 1285 return hv_vcpu->cpuid_cache.features_eax & 1286 HV_MSR_VP_RUNTIME_AVAILABLE; 1287 case HV_X64_MSR_TIME_REF_COUNT: 1288 return hv_vcpu->cpuid_cache.features_eax & 1289 HV_MSR_TIME_REF_COUNT_AVAILABLE; 1290 case HV_X64_MSR_VP_INDEX: 1291 return hv_vcpu->cpuid_cache.features_eax & 1292 HV_MSR_VP_INDEX_AVAILABLE; 1293 case HV_X64_MSR_RESET: 1294 return hv_vcpu->cpuid_cache.features_eax & 1295 HV_MSR_RESET_AVAILABLE; 1296 case HV_X64_MSR_REFERENCE_TSC: 1297 return hv_vcpu->cpuid_cache.features_eax & 1298 HV_MSR_REFERENCE_TSC_AVAILABLE; 1299 case HV_X64_MSR_SCONTROL: 1300 case HV_X64_MSR_SVERSION: 1301 case HV_X64_MSR_SIEFP: 1302 case HV_X64_MSR_SIMP: 1303 case HV_X64_MSR_EOM: 1304 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1305 return hv_vcpu->cpuid_cache.features_eax & 1306 HV_MSR_SYNIC_AVAILABLE; 1307 case HV_X64_MSR_STIMER0_CONFIG: 1308 case HV_X64_MSR_STIMER1_CONFIG: 1309 case HV_X64_MSR_STIMER2_CONFIG: 1310 case HV_X64_MSR_STIMER3_CONFIG: 1311 case HV_X64_MSR_STIMER0_COUNT: 1312 case HV_X64_MSR_STIMER1_COUNT: 1313 case HV_X64_MSR_STIMER2_COUNT: 1314 case HV_X64_MSR_STIMER3_COUNT: 1315 return hv_vcpu->cpuid_cache.features_eax & 1316 HV_MSR_SYNTIMER_AVAILABLE; 1317 case HV_X64_MSR_EOI: 1318 case HV_X64_MSR_ICR: 1319 case HV_X64_MSR_TPR: 1320 case HV_X64_MSR_VP_ASSIST_PAGE: 1321 return hv_vcpu->cpuid_cache.features_eax & 1322 HV_MSR_APIC_ACCESS_AVAILABLE; 1323 case HV_X64_MSR_TSC_FREQUENCY: 1324 case HV_X64_MSR_APIC_FREQUENCY: 1325 return hv_vcpu->cpuid_cache.features_eax & 1326 HV_ACCESS_FREQUENCY_MSRS; 1327 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1328 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1329 case HV_X64_MSR_TSC_EMULATION_STATUS: 1330 return hv_vcpu->cpuid_cache.features_eax & 1331 HV_ACCESS_REENLIGHTENMENT; 1332 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1333 return hv_vcpu->cpuid_cache.features_eax & 1334 HV_ACCESS_TSC_INVARIANT; 1335 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1336 case HV_X64_MSR_CRASH_CTL: 1337 return hv_vcpu->cpuid_cache.features_edx & 1338 HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 1339 case HV_X64_MSR_SYNDBG_OPTIONS: 1340 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1341 return hv_vcpu->cpuid_cache.features_edx & 1342 HV_FEATURE_DEBUG_MSRS_AVAILABLE; 1343 default: 1344 break; 1345 } 1346 1347 return false; 1348 } 1349 1350 #define KVM_HV_WIN2016_GUEST_ID 0x1040a00003839 1351 #define KVM_HV_WIN2016_GUEST_ID_MASK (~GENMASK_ULL(23, 16)) /* mask out the service version */ 1352 1353 /* 1354 * Hyper-V enabled Windows Server 2016 SMP VMs fail to boot in !XSAVES && XSAVEC 1355 * configuration. 1356 * Such configuration can result from, for example, AMD Erratum 1386 workaround. 1357 * 1358 * Print a notice so users aren't left wondering what's suddenly gone wrong. 1359 */ 1360 static void __kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu) 1361 { 1362 struct kvm *kvm = vcpu->kvm; 1363 struct kvm_hv *hv = to_kvm_hv(kvm); 1364 1365 /* Check again under the hv_lock. */ 1366 if (hv->xsaves_xsavec_checked) 1367 return; 1368 1369 if ((hv->hv_guest_os_id & KVM_HV_WIN2016_GUEST_ID_MASK) != 1370 KVM_HV_WIN2016_GUEST_ID) 1371 return; 1372 1373 hv->xsaves_xsavec_checked = true; 1374 1375 /* UP configurations aren't affected */ 1376 if (atomic_read(&kvm->online_vcpus) < 2) 1377 return; 1378 1379 if (guest_cpuid_has(vcpu, X86_FEATURE_XSAVES) || 1380 !guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVEC)) 1381 return; 1382 1383 pr_notice_ratelimited("Booting SMP Windows KVM VM with !XSAVES && XSAVEC. " 1384 "If it fails to boot try disabling XSAVEC in the VM config.\n"); 1385 } 1386 1387 void kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu) 1388 { 1389 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1390 1391 if (!vcpu->arch.hyperv_enabled || 1392 hv->xsaves_xsavec_checked) 1393 return; 1394 1395 mutex_lock(&hv->hv_lock); 1396 __kvm_hv_xsaves_xsavec_maybe_warn(vcpu); 1397 mutex_unlock(&hv->hv_lock); 1398 } 1399 1400 static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, 1401 bool host) 1402 { 1403 struct kvm *kvm = vcpu->kvm; 1404 struct kvm_hv *hv = to_kvm_hv(kvm); 1405 1406 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1407 return 1; 1408 1409 switch (msr) { 1410 case HV_X64_MSR_GUEST_OS_ID: 1411 hv->hv_guest_os_id = data; 1412 /* setting guest os id to zero disables hypercall page */ 1413 if (!hv->hv_guest_os_id) 1414 hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE; 1415 break; 1416 case HV_X64_MSR_HYPERCALL: { 1417 u8 instructions[9]; 1418 int i = 0; 1419 u64 addr; 1420 1421 /* if guest os id is not set hypercall should remain disabled */ 1422 if (!hv->hv_guest_os_id) 1423 break; 1424 if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) { 1425 hv->hv_hypercall = data; 1426 break; 1427 } 1428 1429 /* 1430 * If Xen and Hyper-V hypercalls are both enabled, disambiguate 1431 * the same way Xen itself does, by setting the bit 31 of EAX 1432 * which is RsvdZ in the 32-bit Hyper-V hypercall ABI and just 1433 * going to be clobbered on 64-bit. 1434 */ 1435 if (kvm_xen_hypercall_enabled(kvm)) { 1436 /* orl $0x80000000, %eax */ 1437 instructions[i++] = 0x0d; 1438 instructions[i++] = 0x00; 1439 instructions[i++] = 0x00; 1440 instructions[i++] = 0x00; 1441 instructions[i++] = 0x80; 1442 } 1443 1444 /* vmcall/vmmcall */ 1445 kvm_x86_call(patch_hypercall)(vcpu, instructions + i); 1446 i += 3; 1447 1448 /* ret */ 1449 ((unsigned char *)instructions)[i++] = 0xc3; 1450 1451 addr = data & HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK; 1452 if (kvm_vcpu_write_guest(vcpu, addr, instructions, i)) 1453 return 1; 1454 hv->hv_hypercall = data; 1455 break; 1456 } 1457 case HV_X64_MSR_REFERENCE_TSC: 1458 hv->hv_tsc_page = data; 1459 if (hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) { 1460 if (!host) 1461 hv->hv_tsc_page_status = HV_TSC_PAGE_GUEST_CHANGED; 1462 else 1463 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1464 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1465 } else { 1466 hv->hv_tsc_page_status = HV_TSC_PAGE_UNSET; 1467 } 1468 break; 1469 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1470 return kvm_hv_msr_set_crash_data(kvm, 1471 msr - HV_X64_MSR_CRASH_P0, 1472 data); 1473 case HV_X64_MSR_CRASH_CTL: 1474 if (host) 1475 return kvm_hv_msr_set_crash_ctl(kvm, data); 1476 1477 if (data & HV_CRASH_CTL_CRASH_NOTIFY) { 1478 vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n", 1479 hv->hv_crash_param[0], 1480 hv->hv_crash_param[1], 1481 hv->hv_crash_param[2], 1482 hv->hv_crash_param[3], 1483 hv->hv_crash_param[4]); 1484 1485 /* Send notification about crash to user space */ 1486 kvm_make_request(KVM_REQ_HV_CRASH, vcpu); 1487 } 1488 break; 1489 case HV_X64_MSR_RESET: 1490 if (data == 1) { 1491 vcpu_debug(vcpu, "hyper-v reset requested\n"); 1492 kvm_make_request(KVM_REQ_HV_RESET, vcpu); 1493 } 1494 break; 1495 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1496 hv->hv_reenlightenment_control = data; 1497 break; 1498 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1499 hv->hv_tsc_emulation_control = data; 1500 break; 1501 case HV_X64_MSR_TSC_EMULATION_STATUS: 1502 if (data && !host) 1503 return 1; 1504 1505 hv->hv_tsc_emulation_status = data; 1506 break; 1507 case HV_X64_MSR_TIME_REF_COUNT: 1508 /* read-only, but still ignore it if host-initiated */ 1509 if (!host) 1510 return 1; 1511 break; 1512 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1513 /* Only bit 0 is supported */ 1514 if (data & ~HV_EXPOSE_INVARIANT_TSC) 1515 return 1; 1516 1517 /* The feature can't be disabled from the guest */ 1518 if (!host && hv->hv_invtsc_control && !data) 1519 return 1; 1520 1521 hv->hv_invtsc_control = data; 1522 break; 1523 case HV_X64_MSR_SYNDBG_OPTIONS: 1524 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1525 return syndbg_set_msr(vcpu, msr, data, host); 1526 default: 1527 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1528 return 1; 1529 } 1530 return 0; 1531 } 1532 1533 /* Calculate cpu time spent by current task in 100ns units */ 1534 static u64 current_task_runtime_100ns(void) 1535 { 1536 u64 utime, stime; 1537 1538 task_cputime_adjusted(current, &utime, &stime); 1539 1540 return div_u64(utime + stime, 100); 1541 } 1542 1543 static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1544 { 1545 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1546 1547 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1548 return 1; 1549 1550 switch (msr) { 1551 case HV_X64_MSR_VP_INDEX: { 1552 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1553 u32 new_vp_index = (u32)data; 1554 1555 if (!host || new_vp_index >= KVM_MAX_VCPUS) 1556 return 1; 1557 1558 if (new_vp_index == hv_vcpu->vp_index) 1559 return 0; 1560 1561 /* 1562 * The VP index is initialized to vcpu_index by 1563 * kvm_hv_vcpu_postcreate so they initially match. Now the 1564 * VP index is changing, adjust num_mismatched_vp_indexes if 1565 * it now matches or no longer matches vcpu_idx. 1566 */ 1567 if (hv_vcpu->vp_index == vcpu->vcpu_idx) 1568 atomic_inc(&hv->num_mismatched_vp_indexes); 1569 else if (new_vp_index == vcpu->vcpu_idx) 1570 atomic_dec(&hv->num_mismatched_vp_indexes); 1571 1572 hv_vcpu->vp_index = new_vp_index; 1573 break; 1574 } 1575 case HV_X64_MSR_VP_ASSIST_PAGE: { 1576 u64 gfn; 1577 unsigned long addr; 1578 1579 if (!(data & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) { 1580 hv_vcpu->hv_vapic = data; 1581 if (kvm_lapic_set_pv_eoi(vcpu, 0, 0)) 1582 return 1; 1583 break; 1584 } 1585 gfn = data >> HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT; 1586 addr = kvm_vcpu_gfn_to_hva(vcpu, gfn); 1587 if (kvm_is_error_hva(addr)) 1588 return 1; 1589 1590 /* 1591 * Clear apic_assist portion of struct hv_vp_assist_page 1592 * only, there can be valuable data in the rest which needs 1593 * to be preserved e.g. on migration. 1594 */ 1595 if (put_user(0, (u32 __user *)addr)) 1596 return 1; 1597 hv_vcpu->hv_vapic = data; 1598 kvm_vcpu_mark_page_dirty(vcpu, gfn); 1599 if (kvm_lapic_set_pv_eoi(vcpu, 1600 gfn_to_gpa(gfn) | KVM_MSR_ENABLED, 1601 sizeof(struct hv_vp_assist_page))) 1602 return 1; 1603 break; 1604 } 1605 case HV_X64_MSR_EOI: 1606 return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data); 1607 case HV_X64_MSR_ICR: 1608 return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data); 1609 case HV_X64_MSR_TPR: 1610 return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data); 1611 case HV_X64_MSR_VP_RUNTIME: 1612 if (!host) 1613 return 1; 1614 hv_vcpu->runtime_offset = data - current_task_runtime_100ns(); 1615 break; 1616 case HV_X64_MSR_SCONTROL: 1617 case HV_X64_MSR_SVERSION: 1618 case HV_X64_MSR_SIEFP: 1619 case HV_X64_MSR_SIMP: 1620 case HV_X64_MSR_EOM: 1621 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1622 return synic_set_msr(to_hv_synic(vcpu), msr, data, host); 1623 case HV_X64_MSR_STIMER0_CONFIG: 1624 case HV_X64_MSR_STIMER1_CONFIG: 1625 case HV_X64_MSR_STIMER2_CONFIG: 1626 case HV_X64_MSR_STIMER3_CONFIG: { 1627 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1628 1629 return stimer_set_config(to_hv_stimer(vcpu, timer_index), 1630 data, host); 1631 } 1632 case HV_X64_MSR_STIMER0_COUNT: 1633 case HV_X64_MSR_STIMER1_COUNT: 1634 case HV_X64_MSR_STIMER2_COUNT: 1635 case HV_X64_MSR_STIMER3_COUNT: { 1636 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1637 1638 return stimer_set_count(to_hv_stimer(vcpu, timer_index), 1639 data, host); 1640 } 1641 case HV_X64_MSR_TSC_FREQUENCY: 1642 case HV_X64_MSR_APIC_FREQUENCY: 1643 /* read-only, but still ignore it if host-initiated */ 1644 if (!host) 1645 return 1; 1646 break; 1647 default: 1648 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1649 return 1; 1650 } 1651 1652 return 0; 1653 } 1654 1655 static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1656 bool host) 1657 { 1658 u64 data = 0; 1659 struct kvm *kvm = vcpu->kvm; 1660 struct kvm_hv *hv = to_kvm_hv(kvm); 1661 1662 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1663 return 1; 1664 1665 switch (msr) { 1666 case HV_X64_MSR_GUEST_OS_ID: 1667 data = hv->hv_guest_os_id; 1668 break; 1669 case HV_X64_MSR_HYPERCALL: 1670 data = hv->hv_hypercall; 1671 break; 1672 case HV_X64_MSR_TIME_REF_COUNT: 1673 data = get_time_ref_counter(kvm); 1674 break; 1675 case HV_X64_MSR_REFERENCE_TSC: 1676 data = hv->hv_tsc_page; 1677 break; 1678 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1679 return kvm_hv_msr_get_crash_data(kvm, 1680 msr - HV_X64_MSR_CRASH_P0, 1681 pdata); 1682 case HV_X64_MSR_CRASH_CTL: 1683 return kvm_hv_msr_get_crash_ctl(kvm, pdata); 1684 case HV_X64_MSR_RESET: 1685 data = 0; 1686 break; 1687 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1688 data = hv->hv_reenlightenment_control; 1689 break; 1690 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1691 data = hv->hv_tsc_emulation_control; 1692 break; 1693 case HV_X64_MSR_TSC_EMULATION_STATUS: 1694 data = hv->hv_tsc_emulation_status; 1695 break; 1696 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1697 data = hv->hv_invtsc_control; 1698 break; 1699 case HV_X64_MSR_SYNDBG_OPTIONS: 1700 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1701 return syndbg_get_msr(vcpu, msr, pdata, host); 1702 default: 1703 kvm_pr_unimpl_rdmsr(vcpu, msr); 1704 return 1; 1705 } 1706 1707 *pdata = data; 1708 return 0; 1709 } 1710 1711 static int kvm_hv_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1712 bool host) 1713 { 1714 u64 data = 0; 1715 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1716 1717 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1718 return 1; 1719 1720 switch (msr) { 1721 case HV_X64_MSR_VP_INDEX: 1722 data = hv_vcpu->vp_index; 1723 break; 1724 case HV_X64_MSR_EOI: 1725 return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata); 1726 case HV_X64_MSR_ICR: 1727 return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata); 1728 case HV_X64_MSR_TPR: 1729 return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata); 1730 case HV_X64_MSR_VP_ASSIST_PAGE: 1731 data = hv_vcpu->hv_vapic; 1732 break; 1733 case HV_X64_MSR_VP_RUNTIME: 1734 data = current_task_runtime_100ns() + hv_vcpu->runtime_offset; 1735 break; 1736 case HV_X64_MSR_SCONTROL: 1737 case HV_X64_MSR_SVERSION: 1738 case HV_X64_MSR_SIEFP: 1739 case HV_X64_MSR_SIMP: 1740 case HV_X64_MSR_EOM: 1741 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1742 return synic_get_msr(to_hv_synic(vcpu), msr, pdata, host); 1743 case HV_X64_MSR_STIMER0_CONFIG: 1744 case HV_X64_MSR_STIMER1_CONFIG: 1745 case HV_X64_MSR_STIMER2_CONFIG: 1746 case HV_X64_MSR_STIMER3_CONFIG: { 1747 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1748 1749 return stimer_get_config(to_hv_stimer(vcpu, timer_index), 1750 pdata); 1751 } 1752 case HV_X64_MSR_STIMER0_COUNT: 1753 case HV_X64_MSR_STIMER1_COUNT: 1754 case HV_X64_MSR_STIMER2_COUNT: 1755 case HV_X64_MSR_STIMER3_COUNT: { 1756 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1757 1758 return stimer_get_count(to_hv_stimer(vcpu, timer_index), 1759 pdata); 1760 } 1761 case HV_X64_MSR_TSC_FREQUENCY: 1762 data = (u64)vcpu->arch.virtual_tsc_khz * 1000; 1763 break; 1764 case HV_X64_MSR_APIC_FREQUENCY: 1765 data = div64_u64(1000000000ULL, 1766 vcpu->kvm->arch.apic_bus_cycle_ns); 1767 break; 1768 default: 1769 kvm_pr_unimpl_rdmsr(vcpu, msr); 1770 return 1; 1771 } 1772 *pdata = data; 1773 return 0; 1774 } 1775 1776 int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1777 { 1778 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1779 1780 if (!host && !vcpu->arch.hyperv_enabled) 1781 return 1; 1782 1783 if (kvm_hv_vcpu_init(vcpu)) 1784 return 1; 1785 1786 if (kvm_hv_msr_partition_wide(msr)) { 1787 int r; 1788 1789 mutex_lock(&hv->hv_lock); 1790 r = kvm_hv_set_msr_pw(vcpu, msr, data, host); 1791 mutex_unlock(&hv->hv_lock); 1792 return r; 1793 } else 1794 return kvm_hv_set_msr(vcpu, msr, data, host); 1795 } 1796 1797 int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 1798 { 1799 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1800 1801 if (!host && !vcpu->arch.hyperv_enabled) 1802 return 1; 1803 1804 if (kvm_hv_vcpu_init(vcpu)) 1805 return 1; 1806 1807 if (kvm_hv_msr_partition_wide(msr)) { 1808 int r; 1809 1810 mutex_lock(&hv->hv_lock); 1811 r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host); 1812 mutex_unlock(&hv->hv_lock); 1813 return r; 1814 } else 1815 return kvm_hv_get_msr(vcpu, msr, pdata, host); 1816 } 1817 1818 static void sparse_set_to_vcpu_mask(struct kvm *kvm, u64 *sparse_banks, 1819 u64 valid_bank_mask, unsigned long *vcpu_mask) 1820 { 1821 struct kvm_hv *hv = to_kvm_hv(kvm); 1822 bool has_mismatch = atomic_read(&hv->num_mismatched_vp_indexes); 1823 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS]; 1824 struct kvm_vcpu *vcpu; 1825 int bank, sbank = 0; 1826 unsigned long i; 1827 u64 *bitmap; 1828 1829 BUILD_BUG_ON(sizeof(vp_bitmap) > 1830 sizeof(*vcpu_mask) * BITS_TO_LONGS(KVM_MAX_VCPUS)); 1831 1832 /* 1833 * If vp_index == vcpu_idx for all vCPUs, fill vcpu_mask directly, else 1834 * fill a temporary buffer and manually test each vCPU's VP index. 1835 */ 1836 if (likely(!has_mismatch)) 1837 bitmap = (u64 *)vcpu_mask; 1838 else 1839 bitmap = vp_bitmap; 1840 1841 /* 1842 * Each set of 64 VPs is packed into sparse_banks, with valid_bank_mask 1843 * having a '1' for each bank that exists in sparse_banks. Sets must 1844 * be in ascending order, i.e. bank0..bankN. 1845 */ 1846 memset(bitmap, 0, sizeof(vp_bitmap)); 1847 for_each_set_bit(bank, (unsigned long *)&valid_bank_mask, 1848 KVM_HV_MAX_SPARSE_VCPU_SET_BITS) 1849 bitmap[bank] = sparse_banks[sbank++]; 1850 1851 if (likely(!has_mismatch)) 1852 return; 1853 1854 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 1855 kvm_for_each_vcpu(i, vcpu, kvm) { 1856 if (test_bit(kvm_hv_get_vpindex(vcpu), (unsigned long *)vp_bitmap)) 1857 __set_bit(i, vcpu_mask); 1858 } 1859 } 1860 1861 static bool hv_is_vp_in_sparse_set(u32 vp_id, u64 valid_bank_mask, u64 sparse_banks[]) 1862 { 1863 int valid_bit_nr = vp_id / HV_VCPUS_PER_SPARSE_BANK; 1864 unsigned long sbank; 1865 1866 BUILD_BUG_ON(BITS_PER_TYPE(valid_bank_mask) != HV_MAX_SPARSE_VCPU_BANKS); 1867 1868 if (valid_bit_nr >= HV_MAX_SPARSE_VCPU_BANKS) 1869 return false; 1870 1871 if (!test_bit(valid_bit_nr, (unsigned long *)&valid_bank_mask)) 1872 return false; 1873 1874 /* 1875 * The index into the sparse bank is the number of preceding bits in 1876 * the valid mask. Optimize for VMs with <64 vCPUs by skipping the 1877 * fancy math if there can't possibly be preceding bits. 1878 */ 1879 if (valid_bit_nr) 1880 sbank = hweight64(valid_bank_mask & GENMASK_ULL(valid_bit_nr - 1, 0)); 1881 else 1882 sbank = 0; 1883 1884 return test_bit(vp_id % HV_VCPUS_PER_SPARSE_BANK, 1885 (unsigned long *)&sparse_banks[sbank]); 1886 } 1887 1888 struct kvm_hv_hcall { 1889 /* Hypercall input data */ 1890 u64 param; 1891 u64 ingpa; 1892 u64 outgpa; 1893 u16 code; 1894 u16 var_cnt; 1895 u16 rep_cnt; 1896 u16 rep_idx; 1897 bool fast; 1898 bool rep; 1899 sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS]; 1900 1901 /* 1902 * Current read offset when KVM reads hypercall input data gradually, 1903 * either offset in bytes from 'ingpa' for regular hypercalls or the 1904 * number of already consumed 'XMM halves' for 'fast' hypercalls. 1905 */ 1906 union { 1907 gpa_t data_offset; 1908 int consumed_xmm_halves; 1909 }; 1910 }; 1911 1912 1913 static int kvm_hv_get_hc_data(struct kvm *kvm, struct kvm_hv_hcall *hc, 1914 u16 orig_cnt, u16 cnt_cap, u64 *data) 1915 { 1916 /* 1917 * Preserve the original count when ignoring entries via a "cap", KVM 1918 * still needs to validate the guest input (though the non-XMM path 1919 * punts on the checks). 1920 */ 1921 u16 cnt = min(orig_cnt, cnt_cap); 1922 int i, j; 1923 1924 if (hc->fast) { 1925 /* 1926 * Each XMM holds two sparse banks, but do not count halves that 1927 * have already been consumed for hypercall parameters. 1928 */ 1929 if (orig_cnt > 2 * HV_HYPERCALL_MAX_XMM_REGISTERS - hc->consumed_xmm_halves) 1930 return HV_STATUS_INVALID_HYPERCALL_INPUT; 1931 1932 for (i = 0; i < cnt; i++) { 1933 j = i + hc->consumed_xmm_halves; 1934 if (j % 2) 1935 data[i] = sse128_hi(hc->xmm[j / 2]); 1936 else 1937 data[i] = sse128_lo(hc->xmm[j / 2]); 1938 } 1939 return 0; 1940 } 1941 1942 return kvm_read_guest(kvm, hc->ingpa + hc->data_offset, data, 1943 cnt * sizeof(*data)); 1944 } 1945 1946 static u64 kvm_get_sparse_vp_set(struct kvm *kvm, struct kvm_hv_hcall *hc, 1947 u64 *sparse_banks) 1948 { 1949 if (hc->var_cnt > HV_MAX_SPARSE_VCPU_BANKS) 1950 return -EINVAL; 1951 1952 /* Cap var_cnt to ignore banks that cannot contain a legal VP index. */ 1953 return kvm_hv_get_hc_data(kvm, hc, hc->var_cnt, KVM_HV_MAX_SPARSE_VCPU_SET_BITS, 1954 sparse_banks); 1955 } 1956 1957 static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc, u64 entries[]) 1958 { 1959 return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries); 1960 } 1961 1962 static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, u64 *entries, int count, 1963 bool is_guest_mode) 1964 { 1965 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 1966 u64 flush_all_entry = KVM_HV_TLB_FLUSHALL_ENTRY; 1967 1968 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode); 1969 if (!tlb_flush_fifo) 1970 return; 1971 1972 spin_lock(&tlb_flush_fifo->write_lock); 1973 1974 /* 1975 * All entries should fit on the fifo leaving one free for 'flush all' 1976 * entry in case another request comes in. In case there's not enough 1977 * space, just put 'flush all' entry there. 1978 */ 1979 if (count && entries && count < kfifo_avail(&tlb_flush_fifo->entries)) { 1980 WARN_ON(kfifo_in(&tlb_flush_fifo->entries, entries, count) != count); 1981 goto out_unlock; 1982 } 1983 1984 /* 1985 * Note: full fifo always contains 'flush all' entry, no need to check the 1986 * return value. 1987 */ 1988 kfifo_in(&tlb_flush_fifo->entries, &flush_all_entry, 1); 1989 1990 out_unlock: 1991 spin_unlock(&tlb_flush_fifo->write_lock); 1992 } 1993 1994 int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) 1995 { 1996 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 1997 u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; 1998 int i, j, count; 1999 gva_t gva; 2000 2001 if (!tdp_enabled) 2002 return -EINVAL; 2003 2004 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu)); 2005 if (!tlb_flush_fifo) 2006 return -EINVAL; 2007 2008 count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); 2009 2010 for (i = 0; i < count; i++) { 2011 if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) 2012 goto out_flush_all; 2013 2014 /* 2015 * Lower 12 bits of 'address' encode the number of additional 2016 * pages to flush. 2017 */ 2018 gva = entries[i] & PAGE_MASK; 2019 for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { 2020 if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) 2021 continue; 2022 2023 kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); 2024 } 2025 2026 ++vcpu->stat.tlb_flush; 2027 } 2028 return 0; 2029 2030 out_flush_all: 2031 kfifo_reset_out(&tlb_flush_fifo->entries); 2032 2033 /* Fall back to full flush. */ 2034 return -ENOSPC; 2035 } 2036 2037 static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2038 { 2039 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2040 unsigned long *vcpu_mask = hv_vcpu->vcpu_mask; 2041 u64 *sparse_banks = hv_vcpu->sparse_banks; 2042 struct kvm *kvm = vcpu->kvm; 2043 struct hv_tlb_flush_ex flush_ex; 2044 struct hv_tlb_flush flush; 2045 /* 2046 * Normally, there can be no more than 'KVM_HV_TLB_FLUSH_FIFO_SIZE' 2047 * entries on the TLB flush fifo. The last entry, however, needs to be 2048 * always left free for 'flush all' entry which gets placed when 2049 * there is not enough space to put all the requested entries. 2050 */ 2051 u64 __tlb_flush_entries[KVM_HV_TLB_FLUSH_FIFO_SIZE - 1]; 2052 u64 *tlb_flush_entries; 2053 u64 valid_bank_mask; 2054 struct kvm_vcpu *v; 2055 unsigned long i; 2056 bool all_cpus; 2057 2058 /* 2059 * The Hyper-V TLFS doesn't allow more than HV_MAX_SPARSE_VCPU_BANKS 2060 * sparse banks. Fail the build if KVM's max allowed number of 2061 * vCPUs (>4096) exceeds this limit. 2062 */ 2063 BUILD_BUG_ON(KVM_HV_MAX_SPARSE_VCPU_SET_BITS > HV_MAX_SPARSE_VCPU_BANKS); 2064 2065 /* 2066 * 'Slow' hypercall's first parameter is the address in guest's memory 2067 * where hypercall parameters are placed. This is either a GPA or a 2068 * nested GPA when KVM is handling the call from L2 ('direct' TLB 2069 * flush). Translate the address here so the memory can be uniformly 2070 * read with kvm_read_guest(). 2071 */ 2072 if (!hc->fast) { 2073 hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.gva_walk, hc->ingpa, 2074 PFERR_GUEST_FINAL_MASK, NULL, 0); 2075 if (unlikely(hc->ingpa == INVALID_GPA)) 2076 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2077 } 2078 2079 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST || 2080 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE) { 2081 if (hc->fast) { 2082 flush.address_space = hc->ingpa; 2083 flush.flags = hc->outgpa; 2084 flush.processor_mask = sse128_lo(hc->xmm[0]); 2085 hc->consumed_xmm_halves = 1; 2086 } else { 2087 if (unlikely(kvm_read_guest(kvm, hc->ingpa, 2088 &flush, sizeof(flush)))) 2089 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2090 hc->data_offset = sizeof(flush); 2091 } 2092 2093 trace_kvm_hv_flush_tlb(flush.processor_mask, 2094 flush.address_space, flush.flags, 2095 is_guest_mode(vcpu)); 2096 2097 valid_bank_mask = BIT_ULL(0); 2098 sparse_banks[0] = flush.processor_mask; 2099 2100 /* 2101 * Work around possible WS2012 bug: it sends hypercalls 2102 * with processor_mask = 0x0 and HV_FLUSH_ALL_PROCESSORS clear, 2103 * while also expecting us to flush something and crashing if 2104 * we don't. Let's treat processor_mask == 0 same as 2105 * HV_FLUSH_ALL_PROCESSORS. 2106 */ 2107 all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) || 2108 flush.processor_mask == 0; 2109 } else { 2110 if (hc->fast) { 2111 flush_ex.address_space = hc->ingpa; 2112 flush_ex.flags = hc->outgpa; 2113 memcpy(&flush_ex.hv_vp_set, 2114 &hc->xmm[0], sizeof(hc->xmm[0])); 2115 hc->consumed_xmm_halves = 2; 2116 } else { 2117 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex, 2118 sizeof(flush_ex)))) 2119 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2120 hc->data_offset = sizeof(flush_ex); 2121 } 2122 2123 trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask, 2124 flush_ex.hv_vp_set.format, 2125 flush_ex.address_space, 2126 flush_ex.flags, is_guest_mode(vcpu)); 2127 2128 valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask; 2129 all_cpus = flush_ex.hv_vp_set.format != 2130 HV_GENERIC_SET_SPARSE_4K; 2131 2132 if (hc->var_cnt != hweight64(valid_bank_mask)) 2133 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2134 2135 if (!all_cpus) { 2136 if (!hc->var_cnt) 2137 goto ret_success; 2138 2139 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2140 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2141 } 2142 2143 /* 2144 * Hyper-V TLFS doesn't explicitly forbid non-empty sparse vCPU 2145 * banks (and, thus, non-zero 'var_cnt') for the 'all vCPUs' 2146 * case (HV_GENERIC_SET_ALL). Always adjust data_offset and 2147 * consumed_xmm_halves to make sure TLB flush entries are read 2148 * from the correct offset. 2149 */ 2150 if (hc->fast) 2151 hc->consumed_xmm_halves += hc->var_cnt; 2152 else 2153 hc->data_offset += hc->var_cnt * sizeof(sparse_banks[0]); 2154 } 2155 2156 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE || 2157 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX || 2158 hc->rep_cnt > ARRAY_SIZE(__tlb_flush_entries)) { 2159 tlb_flush_entries = NULL; 2160 } else { 2161 if (kvm_hv_get_tlb_flush_entries(kvm, hc, __tlb_flush_entries)) 2162 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2163 tlb_flush_entries = __tlb_flush_entries; 2164 } 2165 2166 /* 2167 * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't 2168 * analyze it here, flush TLB regardless of the specified address space. 2169 */ 2170 if (all_cpus && !is_guest_mode(vcpu)) { 2171 kvm_for_each_vcpu(i, v, kvm) 2172 hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false); 2173 2174 kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH); 2175 } else if (!is_guest_mode(vcpu)) { 2176 sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, vcpu_mask); 2177 2178 for_each_set_bit(i, vcpu_mask, KVM_MAX_VCPUS) { 2179 v = kvm_get_vcpu(kvm, i); 2180 if (!v) 2181 continue; 2182 hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false); 2183 } 2184 2185 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2186 } else { 2187 struct kvm_vcpu_hv *hv_v; 2188 2189 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 2190 2191 kvm_for_each_vcpu(i, v, kvm) { 2192 hv_v = to_hv_vcpu_safe(v); 2193 2194 /* 2195 * The following check races with nested vCPUs entering/exiting 2196 * and/or migrating between L1's vCPUs, however the only case when 2197 * KVM *must* flush the TLB is when the target L2 vCPU keeps 2198 * running on the same L1 vCPU from the moment of the request until 2199 * kvm_hv_flush_tlb() returns. TLB is fully flushed in all other 2200 * cases, e.g. when the target L2 vCPU migrates to a different L1 2201 * vCPU or when the corresponding L1 vCPU temporary switches to a 2202 * different L2 vCPU while the request is being processed. 2203 */ 2204 if (!hv_v || hv_v->nested.vm_id != hv_vcpu->nested.vm_id) 2205 continue; 2206 2207 if (!all_cpus && 2208 !hv_is_vp_in_sparse_set(hv_v->nested.vp_id, valid_bank_mask, 2209 sparse_banks)) 2210 continue; 2211 2212 __set_bit(i, vcpu_mask); 2213 hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, true); 2214 } 2215 2216 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2217 } 2218 2219 ret_success: 2220 /* We always do full TLB flush, set 'Reps completed' = 'Rep Count' */ 2221 return (u64)HV_STATUS_SUCCESS | 2222 ((u64)hc->rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET); 2223 } 2224 2225 static void kvm_hv_send_ipi_to_many(struct kvm *kvm, u32 vector, 2226 u64 *sparse_banks, u64 valid_bank_mask) 2227 { 2228 struct kvm_lapic_irq irq = { 2229 .delivery_mode = APIC_DM_FIXED, 2230 .vector = vector 2231 }; 2232 struct kvm_vcpu *vcpu; 2233 unsigned long i; 2234 2235 kvm_for_each_vcpu(i, vcpu, kvm) { 2236 if (sparse_banks && 2237 !hv_is_vp_in_sparse_set(kvm_hv_get_vpindex(vcpu), 2238 valid_bank_mask, sparse_banks)) 2239 continue; 2240 2241 /* We fail only when APIC is disabled */ 2242 kvm_apic_set_irq(vcpu, &irq, NULL); 2243 } 2244 } 2245 2246 static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2247 { 2248 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2249 u64 *sparse_banks = hv_vcpu->sparse_banks; 2250 struct kvm *kvm = vcpu->kvm; 2251 struct hv_send_ipi_ex send_ipi_ex; 2252 struct hv_send_ipi send_ipi; 2253 u64 valid_bank_mask; 2254 u32 vector; 2255 bool all_cpus; 2256 2257 if (!lapic_in_kernel(vcpu)) 2258 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2259 2260 if (hc->code == HVCALL_SEND_IPI) { 2261 if (!hc->fast) { 2262 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi, 2263 sizeof(send_ipi)))) 2264 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2265 sparse_banks[0] = send_ipi.cpu_mask; 2266 vector = send_ipi.vector; 2267 } else { 2268 /* 'reserved' part of hv_send_ipi should be 0 */ 2269 if (unlikely(hc->ingpa >> 32 != 0)) 2270 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2271 sparse_banks[0] = hc->outgpa; 2272 vector = (u32)hc->ingpa; 2273 } 2274 all_cpus = false; 2275 valid_bank_mask = BIT_ULL(0); 2276 2277 trace_kvm_hv_send_ipi(vector, sparse_banks[0]); 2278 } else { 2279 if (!hc->fast) { 2280 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex, 2281 sizeof(send_ipi_ex)))) 2282 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2283 } else { 2284 send_ipi_ex.vector = (u32)hc->ingpa; 2285 send_ipi_ex.vp_set.format = hc->outgpa; 2286 send_ipi_ex.vp_set.valid_bank_mask = sse128_lo(hc->xmm[0]); 2287 } 2288 2289 trace_kvm_hv_send_ipi_ex(send_ipi_ex.vector, 2290 send_ipi_ex.vp_set.format, 2291 send_ipi_ex.vp_set.valid_bank_mask); 2292 2293 vector = send_ipi_ex.vector; 2294 valid_bank_mask = send_ipi_ex.vp_set.valid_bank_mask; 2295 all_cpus = send_ipi_ex.vp_set.format == HV_GENERIC_SET_ALL; 2296 2297 if (hc->var_cnt != hweight64(valid_bank_mask)) 2298 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2299 2300 if (all_cpus) 2301 goto check_and_send_ipi; 2302 2303 if (!hc->var_cnt) 2304 goto ret_success; 2305 2306 if (!hc->fast) 2307 hc->data_offset = offsetof(struct hv_send_ipi_ex, 2308 vp_set.bank_contents); 2309 else 2310 hc->consumed_xmm_halves = 1; 2311 2312 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2313 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2314 } 2315 2316 check_and_send_ipi: 2317 if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) 2318 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2319 2320 if (all_cpus) 2321 kvm_hv_send_ipi_to_many(kvm, vector, NULL, 0); 2322 else 2323 kvm_hv_send_ipi_to_many(kvm, vector, sparse_banks, valid_bank_mask); 2324 2325 ret_success: 2326 return HV_STATUS_SUCCESS; 2327 } 2328 2329 void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled) 2330 { 2331 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2332 struct kvm_cpuid_entry2 *entry; 2333 2334 vcpu->arch.hyperv_enabled = hyperv_enabled; 2335 2336 if (!hv_vcpu) { 2337 /* 2338 * KVM should have already allocated kvm_vcpu_hv if Hyper-V is 2339 * enabled in CPUID. 2340 */ 2341 WARN_ON_ONCE(vcpu->arch.hyperv_enabled); 2342 return; 2343 } 2344 2345 memset(&hv_vcpu->cpuid_cache, 0, sizeof(hv_vcpu->cpuid_cache)); 2346 2347 if (!vcpu->arch.hyperv_enabled) 2348 return; 2349 2350 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_FEATURES); 2351 if (entry) { 2352 hv_vcpu->cpuid_cache.features_eax = entry->eax; 2353 hv_vcpu->cpuid_cache.features_ebx = entry->ebx; 2354 hv_vcpu->cpuid_cache.features_edx = entry->edx; 2355 } 2356 2357 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_ENLIGHTMENT_INFO); 2358 if (entry) { 2359 hv_vcpu->cpuid_cache.enlightenments_eax = entry->eax; 2360 hv_vcpu->cpuid_cache.enlightenments_ebx = entry->ebx; 2361 } 2362 2363 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES); 2364 if (entry) 2365 hv_vcpu->cpuid_cache.syndbg_cap_eax = entry->eax; 2366 2367 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_NESTED_FEATURES); 2368 if (entry) { 2369 hv_vcpu->cpuid_cache.nested_eax = entry->eax; 2370 hv_vcpu->cpuid_cache.nested_ebx = entry->ebx; 2371 } 2372 } 2373 2374 int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce) 2375 { 2376 struct kvm_vcpu_hv *hv_vcpu; 2377 int ret = 0; 2378 2379 if (!to_hv_vcpu(vcpu)) { 2380 if (enforce) { 2381 ret = kvm_hv_vcpu_init(vcpu); 2382 if (ret) 2383 return ret; 2384 } else { 2385 return 0; 2386 } 2387 } 2388 2389 hv_vcpu = to_hv_vcpu(vcpu); 2390 hv_vcpu->enforce_cpuid = enforce; 2391 2392 return ret; 2393 } 2394 2395 static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) 2396 { 2397 bool longmode; 2398 2399 longmode = is_64_bit_hypercall(vcpu); 2400 if (longmode) 2401 kvm_rax_write_raw(vcpu, result); 2402 else { 2403 kvm_edx_write(vcpu, result >> 32); 2404 kvm_eax_write(vcpu, result); 2405 } 2406 } 2407 2408 static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result) 2409 { 2410 u32 tlb_lock_count = 0; 2411 int ret; 2412 2413 if (hv_result_success(result) && is_guest_mode(vcpu) && 2414 kvm_hv_is_tlb_flush_hcall(vcpu) && 2415 kvm_read_guest(vcpu->kvm, to_hv_vcpu(vcpu)->nested.pa_page_gpa, 2416 &tlb_lock_count, sizeof(tlb_lock_count))) 2417 result = HV_STATUS_INVALID_HYPERCALL_INPUT; 2418 2419 trace_kvm_hv_hypercall_done(result); 2420 kvm_hv_hypercall_set_result(vcpu, result); 2421 ++vcpu->stat.hypercalls; 2422 2423 ret = kvm_skip_emulated_instruction(vcpu); 2424 2425 if (tlb_lock_count) 2426 kvm_nested_call(hv_inject_synthetic_vmexit_post_tlb_flush)(vcpu); 2427 2428 return ret; 2429 } 2430 2431 static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu) 2432 { 2433 return kvm_hv_hypercall_complete(vcpu, vcpu->run->hyperv.u.hcall.result); 2434 } 2435 2436 static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2437 { 2438 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 2439 struct eventfd_ctx *eventfd; 2440 2441 if (unlikely(!hc->fast)) { 2442 int ret; 2443 gpa_t gpa = hc->ingpa; 2444 2445 if ((gpa & (__alignof__(hc->ingpa) - 1)) || 2446 offset_in_page(gpa) + sizeof(hc->ingpa) > PAGE_SIZE) 2447 return HV_STATUS_INVALID_ALIGNMENT; 2448 2449 ret = kvm_vcpu_read_guest(vcpu, gpa, 2450 &hc->ingpa, sizeof(hc->ingpa)); 2451 if (ret < 0) 2452 return HV_STATUS_INVALID_ALIGNMENT; 2453 } 2454 2455 /* 2456 * Per spec, bits 32-47 contain the extra "flag number". However, we 2457 * have no use for it, and in all known usecases it is zero, so just 2458 * report lookup failure if it isn't. 2459 */ 2460 if (hc->ingpa & 0xffff00000000ULL) 2461 return HV_STATUS_INVALID_PORT_ID; 2462 /* remaining bits are reserved-zero */ 2463 if (hc->ingpa & ~KVM_HYPERV_CONN_ID_MASK) 2464 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2465 2466 /* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */ 2467 rcu_read_lock(); 2468 eventfd = idr_find(&hv->conn_to_evt, hc->ingpa); 2469 rcu_read_unlock(); 2470 if (!eventfd) 2471 return HV_STATUS_INVALID_PORT_ID; 2472 2473 eventfd_signal(eventfd); 2474 return HV_STATUS_SUCCESS; 2475 } 2476 2477 static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc) 2478 { 2479 switch (hc->code) { 2480 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2481 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2482 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2483 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2484 case HVCALL_SEND_IPI_EX: 2485 return true; 2486 } 2487 2488 return false; 2489 } 2490 2491 static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc) 2492 { 2493 int reg; 2494 2495 kvm_fpu_get(); 2496 for (reg = 0; reg < HV_HYPERCALL_MAX_XMM_REGISTERS; reg++) 2497 _kvm_read_sse_reg(reg, &hc->xmm[reg]); 2498 kvm_fpu_put(); 2499 } 2500 2501 static bool hv_check_hypercall_access(struct kvm_vcpu_hv *hv_vcpu, u16 code) 2502 { 2503 if (!hv_vcpu->enforce_cpuid) 2504 return true; 2505 2506 switch (code) { 2507 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2508 return hv_vcpu->cpuid_cache.enlightenments_ebx && 2509 hv_vcpu->cpuid_cache.enlightenments_ebx != U32_MAX; 2510 case HVCALL_POST_MESSAGE: 2511 return hv_vcpu->cpuid_cache.features_ebx & HV_POST_MESSAGES; 2512 case HVCALL_SIGNAL_EVENT: 2513 return hv_vcpu->cpuid_cache.features_ebx & HV_SIGNAL_EVENTS; 2514 case HVCALL_POST_DEBUG_DATA: 2515 case HVCALL_RETRIEVE_DEBUG_DATA: 2516 case HVCALL_RESET_DEBUG_SESSION: 2517 /* 2518 * Return 'true' when SynDBG is disabled so the resulting code 2519 * will be HV_STATUS_INVALID_HYPERCALL_CODE. 2520 */ 2521 return !kvm_hv_is_syndbg_enabled(hv_vcpu->vcpu) || 2522 hv_vcpu->cpuid_cache.features_ebx & HV_DEBUGGING; 2523 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2524 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2525 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2526 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2527 return false; 2528 fallthrough; 2529 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2530 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2531 return hv_vcpu->cpuid_cache.enlightenments_eax & 2532 HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2533 case HVCALL_SEND_IPI_EX: 2534 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2535 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2536 return false; 2537 fallthrough; 2538 case HVCALL_SEND_IPI: 2539 return hv_vcpu->cpuid_cache.enlightenments_eax & 2540 HV_X64_CLUSTER_IPI_RECOMMENDED; 2541 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2542 return hv_vcpu->cpuid_cache.features_ebx & 2543 HV_ENABLE_EXTENDED_HYPERCALLS; 2544 default: 2545 break; 2546 } 2547 2548 return true; 2549 } 2550 2551 int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 2552 { 2553 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2554 struct kvm_hv_hcall hc; 2555 u64 ret = HV_STATUS_SUCCESS; 2556 2557 /* 2558 * hypercall generates UD from non zero cpl and real mode 2559 * per HYPER-V spec 2560 */ 2561 if (kvm_x86_call(get_cpl)(vcpu) != 0 || !is_protmode(vcpu)) { 2562 kvm_queue_exception(vcpu, UD_VECTOR); 2563 return 1; 2564 } 2565 2566 #ifdef CONFIG_X86_64 2567 if (is_64_bit_hypercall(vcpu)) { 2568 hc.param = kvm_rcx_read_raw(vcpu); 2569 hc.ingpa = kvm_rdx_read_raw(vcpu); 2570 hc.outgpa = kvm_r8_read_raw(vcpu); 2571 } else 2572 #endif 2573 { 2574 hc.param = ((u64)kvm_edx_read(vcpu) << 32) | kvm_eax_read(vcpu); 2575 hc.ingpa = ((u64)kvm_ebx_read(vcpu) << 32) | kvm_ecx_read(vcpu); 2576 hc.outgpa = ((u64)kvm_edi_read(vcpu) << 32) | kvm_esi_read(vcpu); 2577 } 2578 2579 hc.code = hc.param & 0xffff; 2580 hc.var_cnt = (hc.param & HV_HYPERCALL_VARHEAD_MASK) >> HV_HYPERCALL_VARHEAD_OFFSET; 2581 hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT); 2582 hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff; 2583 hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff; 2584 hc.rep = !!(hc.rep_cnt || hc.rep_idx); 2585 2586 trace_kvm_hv_hypercall(hc.code, hc.fast, hc.var_cnt, hc.rep_cnt, 2587 hc.rep_idx, hc.ingpa, hc.outgpa); 2588 2589 if (unlikely(!hv_check_hypercall_access(hv_vcpu, hc.code))) { 2590 ret = HV_STATUS_ACCESS_DENIED; 2591 goto hypercall_complete; 2592 } 2593 2594 if (unlikely(hc.param & HV_HYPERCALL_RSVD_MASK)) { 2595 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2596 goto hypercall_complete; 2597 } 2598 2599 if (hc.fast && is_xmm_fast_hypercall(&hc)) { 2600 if (unlikely(hv_vcpu->enforce_cpuid && 2601 !(hv_vcpu->cpuid_cache.features_edx & 2602 HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE))) { 2603 kvm_queue_exception(vcpu, UD_VECTOR); 2604 return 1; 2605 } 2606 2607 kvm_hv_hypercall_read_xmm(&hc); 2608 } 2609 2610 switch (hc.code) { 2611 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2612 if (unlikely(hc.rep || hc.var_cnt)) { 2613 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2614 break; 2615 } 2616 kvm_vcpu_on_spin(vcpu, true); 2617 break; 2618 case HVCALL_SIGNAL_EVENT: 2619 if (unlikely(hc.rep || hc.var_cnt)) { 2620 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2621 break; 2622 } 2623 ret = kvm_hvcall_signal_event(vcpu, &hc); 2624 if (ret != HV_STATUS_INVALID_PORT_ID) 2625 break; 2626 fallthrough; /* maybe userspace knows this conn_id */ 2627 case HVCALL_POST_MESSAGE: 2628 /* don't bother userspace if it has no way to handle it */ 2629 if (unlikely(hc.rep || hc.var_cnt || !to_hv_synic(vcpu)->active)) { 2630 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2631 break; 2632 } 2633 goto hypercall_userspace_exit; 2634 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2635 if (unlikely(hc.var_cnt)) { 2636 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2637 break; 2638 } 2639 fallthrough; 2640 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2641 if (unlikely(!hc.rep_cnt || hc.rep_idx)) { 2642 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2643 break; 2644 } 2645 ret = kvm_hv_flush_tlb(vcpu, &hc); 2646 break; 2647 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2648 if (unlikely(hc.var_cnt)) { 2649 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2650 break; 2651 } 2652 fallthrough; 2653 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2654 if (unlikely(hc.rep)) { 2655 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2656 break; 2657 } 2658 ret = kvm_hv_flush_tlb(vcpu, &hc); 2659 break; 2660 case HVCALL_SEND_IPI: 2661 if (unlikely(hc.var_cnt)) { 2662 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2663 break; 2664 } 2665 fallthrough; 2666 case HVCALL_SEND_IPI_EX: 2667 if (unlikely(hc.rep)) { 2668 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2669 break; 2670 } 2671 ret = kvm_hv_send_ipi(vcpu, &hc); 2672 break; 2673 case HVCALL_POST_DEBUG_DATA: 2674 case HVCALL_RETRIEVE_DEBUG_DATA: 2675 if (unlikely(hc.fast)) { 2676 ret = HV_STATUS_INVALID_PARAMETER; 2677 break; 2678 } 2679 fallthrough; 2680 case HVCALL_RESET_DEBUG_SESSION: { 2681 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 2682 2683 if (!kvm_hv_is_syndbg_enabled(vcpu)) { 2684 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2685 break; 2686 } 2687 2688 if (!(syndbg->options & HV_X64_SYNDBG_OPTION_USE_HCALLS)) { 2689 ret = HV_STATUS_OPERATION_DENIED; 2690 break; 2691 } 2692 goto hypercall_userspace_exit; 2693 } 2694 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2695 if (unlikely(hc.fast)) { 2696 ret = HV_STATUS_INVALID_PARAMETER; 2697 break; 2698 } 2699 goto hypercall_userspace_exit; 2700 default: 2701 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2702 break; 2703 } 2704 2705 hypercall_complete: 2706 return kvm_hv_hypercall_complete(vcpu, ret); 2707 2708 hypercall_userspace_exit: 2709 vcpu->run->exit_reason = KVM_EXIT_HYPERV; 2710 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL; 2711 vcpu->run->hyperv.u.hcall.input = hc.param; 2712 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa; 2713 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa; 2714 vcpu->arch.complete_userspace_io = kvm_hv_hypercall_complete_userspace; 2715 return 0; 2716 } 2717 2718 void kvm_hv_init_vm(struct kvm *kvm) 2719 { 2720 struct kvm_hv *hv = to_kvm_hv(kvm); 2721 2722 mutex_init(&hv->hv_lock); 2723 idr_init(&hv->conn_to_evt); 2724 } 2725 2726 void kvm_hv_destroy_vm(struct kvm *kvm) 2727 { 2728 struct kvm_hv *hv = to_kvm_hv(kvm); 2729 struct eventfd_ctx *eventfd; 2730 int i; 2731 2732 idr_for_each_entry(&hv->conn_to_evt, eventfd, i) 2733 eventfd_ctx_put(eventfd); 2734 idr_destroy(&hv->conn_to_evt); 2735 } 2736 2737 static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd) 2738 { 2739 struct kvm_hv *hv = to_kvm_hv(kvm); 2740 struct eventfd_ctx *eventfd; 2741 int ret; 2742 2743 eventfd = eventfd_ctx_fdget(fd); 2744 if (IS_ERR(eventfd)) 2745 return PTR_ERR(eventfd); 2746 2747 mutex_lock(&hv->hv_lock); 2748 ret = idr_alloc(&hv->conn_to_evt, eventfd, conn_id, conn_id + 1, 2749 GFP_KERNEL_ACCOUNT); 2750 mutex_unlock(&hv->hv_lock); 2751 2752 if (ret >= 0) 2753 return 0; 2754 2755 if (ret == -ENOSPC) 2756 ret = -EEXIST; 2757 eventfd_ctx_put(eventfd); 2758 return ret; 2759 } 2760 2761 static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id) 2762 { 2763 struct kvm_hv *hv = to_kvm_hv(kvm); 2764 struct eventfd_ctx *eventfd; 2765 2766 mutex_lock(&hv->hv_lock); 2767 eventfd = idr_remove(&hv->conn_to_evt, conn_id); 2768 mutex_unlock(&hv->hv_lock); 2769 2770 if (!eventfd) 2771 return -ENOENT; 2772 2773 synchronize_srcu(&kvm->srcu); 2774 eventfd_ctx_put(eventfd); 2775 return 0; 2776 } 2777 2778 int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args) 2779 { 2780 if ((args->flags & ~KVM_HYPERV_EVENTFD_DEASSIGN) || 2781 (args->conn_id & ~KVM_HYPERV_CONN_ID_MASK)) 2782 return -EINVAL; 2783 2784 if (args->flags == KVM_HYPERV_EVENTFD_DEASSIGN) 2785 return kvm_hv_eventfd_deassign(kvm, args->conn_id); 2786 return kvm_hv_eventfd_assign(kvm, args->conn_id, args->fd); 2787 } 2788 2789 int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, 2790 struct kvm_cpuid_entry2 __user *entries) 2791 { 2792 uint16_t evmcs_ver = 0; 2793 struct kvm_cpuid_entry2 cpuid_entries[] = { 2794 { .function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS }, 2795 { .function = HYPERV_CPUID_INTERFACE }, 2796 { .function = HYPERV_CPUID_VERSION }, 2797 { .function = HYPERV_CPUID_FEATURES }, 2798 { .function = HYPERV_CPUID_ENLIGHTMENT_INFO }, 2799 { .function = HYPERV_CPUID_IMPLEMENT_LIMITS }, 2800 { .function = HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS }, 2801 { .function = HYPERV_CPUID_SYNDBG_INTERFACE }, 2802 { .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES }, 2803 { .function = HYPERV_CPUID_NESTED_FEATURES }, 2804 }; 2805 int i, nent = ARRAY_SIZE(cpuid_entries); 2806 2807 if (kvm_nested_ops.enabled) 2808 evmcs_ver = kvm_nested_call(get_evmcs_version)(vcpu); 2809 2810 if (cpuid->nent < nent) 2811 return -E2BIG; 2812 2813 if (cpuid->nent > nent) 2814 cpuid->nent = nent; 2815 2816 for (i = 0; i < nent; i++) { 2817 struct kvm_cpuid_entry2 *ent = &cpuid_entries[i]; 2818 u32 signature[3]; 2819 2820 switch (ent->function) { 2821 case HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS: 2822 memcpy(signature, "Linux KVM Hv", 12); 2823 2824 ent->eax = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES; 2825 ent->ebx = signature[0]; 2826 ent->ecx = signature[1]; 2827 ent->edx = signature[2]; 2828 break; 2829 2830 case HYPERV_CPUID_INTERFACE: 2831 ent->eax = HYPERV_CPUID_SIGNATURE_EAX; 2832 break; 2833 2834 case HYPERV_CPUID_VERSION: 2835 /* 2836 * We implement some Hyper-V 2016 functions so let's use 2837 * this version. 2838 */ 2839 ent->eax = 0x00003839; 2840 ent->ebx = 0x000A0000; 2841 break; 2842 2843 case HYPERV_CPUID_FEATURES: 2844 ent->eax |= HV_MSR_VP_RUNTIME_AVAILABLE; 2845 ent->eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE; 2846 ent->eax |= HV_MSR_SYNIC_AVAILABLE; 2847 ent->eax |= HV_MSR_SYNTIMER_AVAILABLE; 2848 ent->eax |= HV_MSR_APIC_ACCESS_AVAILABLE; 2849 ent->eax |= HV_MSR_HYPERCALL_AVAILABLE; 2850 ent->eax |= HV_MSR_VP_INDEX_AVAILABLE; 2851 ent->eax |= HV_MSR_RESET_AVAILABLE; 2852 ent->eax |= HV_MSR_REFERENCE_TSC_AVAILABLE; 2853 ent->eax |= HV_ACCESS_FREQUENCY_MSRS; 2854 ent->eax |= HV_ACCESS_REENLIGHTENMENT; 2855 ent->eax |= HV_ACCESS_TSC_INVARIANT; 2856 2857 ent->ebx |= HV_POST_MESSAGES; 2858 ent->ebx |= HV_SIGNAL_EVENTS; 2859 ent->ebx |= HV_ENABLE_EXTENDED_HYPERCALLS; 2860 2861 ent->edx |= HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE; 2862 ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE; 2863 ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 2864 2865 ent->ebx |= HV_DEBUGGING; 2866 ent->edx |= HV_X64_GUEST_DEBUGGING_AVAILABLE; 2867 ent->edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE; 2868 ent->edx |= HV_FEATURE_EXT_GVA_RANGES_FLUSH; 2869 2870 /* 2871 * Direct Synthetic timers only make sense with in-kernel 2872 * LAPIC 2873 */ 2874 if (!vcpu || lapic_in_kernel(vcpu)) 2875 ent->edx |= HV_STIMER_DIRECT_MODE_AVAILABLE; 2876 2877 break; 2878 2879 case HYPERV_CPUID_ENLIGHTMENT_INFO: 2880 ent->eax |= HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2881 ent->eax |= HV_X64_APIC_ACCESS_RECOMMENDED; 2882 ent->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED; 2883 if (!vcpu || lapic_in_kernel(vcpu)) 2884 ent->eax |= HV_X64_CLUSTER_IPI_RECOMMENDED; 2885 ent->eax |= HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED; 2886 if (evmcs_ver) 2887 ent->eax |= HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 2888 if (!cpu_smt_possible()) 2889 ent->eax |= HV_X64_NO_NONARCH_CORESHARING; 2890 2891 ent->eax |= HV_DEPRECATING_AEOI_RECOMMENDED; 2892 /* 2893 * Default number of spinlock retry attempts, matches 2894 * HyperV 2016. 2895 */ 2896 ent->ebx = 0x00000FFF; 2897 2898 break; 2899 2900 case HYPERV_CPUID_IMPLEMENT_LIMITS: 2901 /* Maximum number of virtual processors */ 2902 ent->eax = KVM_MAX_VCPUS; 2903 /* 2904 * Maximum number of logical processors, matches 2905 * HyperV 2016. 2906 */ 2907 ent->ebx = 64; 2908 2909 break; 2910 2911 case HYPERV_CPUID_NESTED_FEATURES: 2912 ent->eax = evmcs_ver; 2913 ent->eax |= HV_X64_NESTED_DIRECT_FLUSH; 2914 ent->eax |= HV_X64_NESTED_MSR_BITMAP; 2915 ent->ebx |= HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; 2916 break; 2917 2918 case HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS: 2919 memcpy(signature, "Linux KVM Hv", 12); 2920 2921 ent->eax = 0; 2922 ent->ebx = signature[0]; 2923 ent->ecx = signature[1]; 2924 ent->edx = signature[2]; 2925 break; 2926 2927 case HYPERV_CPUID_SYNDBG_INTERFACE: 2928 memcpy(signature, "VS#1\0\0\0\0\0\0\0\0", 12); 2929 ent->eax = signature[0]; 2930 break; 2931 2932 case HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES: 2933 ent->eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 2934 break; 2935 2936 default: 2937 break; 2938 } 2939 } 2940 2941 if (copy_to_user(entries, cpuid_entries, 2942 nent * sizeof(struct kvm_cpuid_entry2))) 2943 return -EFAULT; 2944 2945 return 0; 2946 } 2947