1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * KVM Microsoft Hyper-V emulation 4 * 5 * derived from arch/x86/kvm/x86.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com> 12 * 13 * Authors: 14 * Avi Kivity <avi@qumranet.com> 15 * Yaniv Kamay <yaniv@qumranet.com> 16 * Amit Shah <amit.shah@qumranet.com> 17 * Ben-Ami Yassour <benami@il.ibm.com> 18 * Andrey Smetanin <asmetanin@virtuozzo.com> 19 */ 20 21 #ifndef __ARCH_X86_KVM_HYPERV_H__ 22 #define __ARCH_X86_KVM_HYPERV_H__ 23 24 #include <linux/kvm_host.h> 25 26 #include "regs.h" 27 28 #ifdef CONFIG_KVM_HYPERV 29 30 31 /* Hyper-V SynIC timer */ 32 struct kvm_vcpu_hv_stimer { 33 struct hrtimer timer; 34 int index; 35 union hv_stimer_config config; 36 u64 count; 37 u64 exp_time; 38 struct hv_message msg; 39 bool msg_pending; 40 }; 41 42 /* Hyper-V synthetic interrupt controller (SynIC)*/ 43 struct kvm_vcpu_hv_synic { 44 u64 version; 45 u64 control; 46 u64 msg_page; 47 u64 evt_page; 48 atomic64_t sint[HV_SYNIC_SINT_COUNT]; 49 atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT]; 50 DECLARE_BITMAP(auto_eoi_bitmap, 256); 51 DECLARE_BITMAP(vec_bitmap, 256); 52 bool active; 53 bool dont_zero_synic_pages; 54 }; 55 56 /* The maximum number of entries on the TLB flush fifo. */ 57 #define KVM_HV_TLB_FLUSH_FIFO_SIZE (16) 58 /* 59 * Note: the following 'magic' entry is made up by KVM to avoid putting 60 * anything besides GVA on the TLB flush fifo. It is theoretically possible 61 * to observe a request to flush 4095 PFNs starting from 0xfffffffffffff000 62 * which will look identical. KVM's action to 'flush everything' instead of 63 * flushing these particular addresses is, however, fully legitimate as 64 * flushing more than requested is always OK. 65 */ 66 #define KVM_HV_TLB_FLUSHALL_ENTRY ((u64)-1) 67 68 enum hv_tlb_flush_fifos { 69 HV_L1_TLB_FLUSH_FIFO, 70 HV_L2_TLB_FLUSH_FIFO, 71 HV_NR_TLB_FLUSH_FIFOS, 72 }; 73 74 struct kvm_vcpu_hv_tlb_flush_fifo { 75 spinlock_t write_lock; 76 DECLARE_KFIFO(entries, u64, KVM_HV_TLB_FLUSH_FIFO_SIZE); 77 }; 78 79 /* Hyper-V per vcpu emulation context */ 80 struct kvm_vcpu_hv { 81 struct kvm_vcpu *vcpu; 82 u32 vp_index; 83 u64 hv_vapic; 84 s64 runtime_offset; 85 struct kvm_vcpu_hv_synic synic; 86 struct kvm_hyperv_exit exit; 87 struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; 88 DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); 89 bool enforce_cpuid; 90 struct { 91 u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */ 92 u32 features_ebx; /* HYPERV_CPUID_FEATURES.EBX */ 93 u32 features_edx; /* HYPERV_CPUID_FEATURES.EDX */ 94 u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */ 95 u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */ 96 u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */ 97 u32 nested_eax; /* HYPERV_CPUID_NESTED_FEATURES.EAX */ 98 u32 nested_ebx; /* HYPERV_CPUID_NESTED_FEATURES.EBX */ 99 } cpuid_cache; 100 101 struct kvm_vcpu_hv_tlb_flush_fifo tlb_flush_fifo[HV_NR_TLB_FLUSH_FIFOS]; 102 103 /* 104 * Preallocated buffers for handling hypercalls that pass sparse vCPU 105 * sets (for high vCPU counts, they're too large to comfortably fit on 106 * the stack). 107 */ 108 u64 sparse_banks[HV_MAX_SPARSE_VCPU_BANKS]; 109 DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); 110 111 struct hv_vp_assist_page vp_assist_page; 112 113 struct { 114 u64 pa_page_gpa; 115 u64 vm_id; 116 u32 vp_id; 117 } nested; 118 }; 119 120 /* "Hv#1" signature */ 121 #define HYPERV_CPUID_SIGNATURE_EAX 0x31237648 122 123 /* 124 * The #defines related to the synthetic debugger are required by KDNet, but 125 * they are not documented in the Hyper-V TLFS because the synthetic debugger 126 * functionality has been deprecated and is subject to removal in future 127 * versions of Windows. 128 */ 129 #define HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS 0x40000080 130 #define HYPERV_CPUID_SYNDBG_INTERFACE 0x40000081 131 #define HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES 0x40000082 132 133 /* 134 * Hyper-V synthetic debugger platform capabilities 135 * These are HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX bits. 136 */ 137 #define HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING BIT(1) 138 139 /* Hyper-V Synthetic debug options MSR */ 140 #define HV_X64_MSR_SYNDBG_CONTROL 0x400000F1 141 #define HV_X64_MSR_SYNDBG_STATUS 0x400000F2 142 #define HV_X64_MSR_SYNDBG_SEND_BUFFER 0x400000F3 143 #define HV_X64_MSR_SYNDBG_RECV_BUFFER 0x400000F4 144 #define HV_X64_MSR_SYNDBG_PENDING_BUFFER 0x400000F5 145 #define HV_X64_MSR_SYNDBG_OPTIONS 0x400000FF 146 147 /* Hyper-V HV_X64_MSR_SYNDBG_OPTIONS bits */ 148 #define HV_X64_SYNDBG_OPTION_USE_HCALLS BIT(2) 149 150 static inline struct kvm_hv *to_kvm_hv(struct kvm *kvm) 151 { 152 return &kvm->arch.hyperv; 153 } 154 155 static inline struct kvm_vcpu_hv *to_hv_vcpu_safe(struct kvm_vcpu *vcpu) 156 { 157 /* 158 * Ensure the HyperV structure is fully initialized when accessing it 159 * without holding vcpu->mutex (or some other guarantee that KVM can't 160 * concurrently instantiate the structure). 161 * 162 * Pairs with the smp_store_release() in kvm_hv_vcpu_init(). 163 */ 164 return smp_load_acquire(&vcpu->arch.hyperv); 165 } 166 167 static inline struct kvm_vcpu_hv *to_hv_vcpu(struct kvm_vcpu *vcpu) 168 { 169 kvm_lockdep_assert_vcpu_is_locked_or_unreachable(vcpu); 170 171 return vcpu->arch.hyperv; 172 } 173 174 static inline struct kvm_vcpu_hv_synic *to_hv_synic(struct kvm_vcpu *vcpu) 175 { 176 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 177 178 return &hv_vcpu->synic; 179 } 180 181 static inline struct kvm_vcpu *hv_synic_to_vcpu(struct kvm_vcpu_hv_synic *synic) 182 { 183 struct kvm_vcpu_hv *hv_vcpu = container_of(synic, struct kvm_vcpu_hv, synic); 184 185 return hv_vcpu->vcpu; 186 } 187 188 static inline struct kvm_hv_syndbg *to_hv_syndbg(struct kvm_vcpu *vcpu) 189 { 190 return &vcpu->kvm->arch.hyperv.hv_syndbg; 191 } 192 193 static inline u32 kvm_hv_get_vpindex(struct kvm_vcpu *vcpu) 194 { 195 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu); 196 197 return hv_vcpu ? hv_vcpu->vp_index : vcpu->vcpu_idx; 198 } 199 200 int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host); 201 int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host); 202 203 static inline bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu) 204 { 205 return vcpu->arch.hyperv_enabled && to_kvm_hv(vcpu->kvm)->hv_guest_os_id; 206 } 207 208 int kvm_hv_hypercall(struct kvm_vcpu *vcpu); 209 210 void kvm_hv_irq_routing_update(struct kvm *kvm); 211 int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 212 int irq_source_id, int level, bool line_status); 213 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector); 214 int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages); 215 216 static inline bool kvm_hv_synic_has_vector(struct kvm_vcpu *vcpu, int vector) 217 { 218 return to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->vec_bitmap); 219 } 220 221 static inline bool kvm_hv_synic_auto_eoi_set(struct kvm_vcpu *vcpu, int vector) 222 { 223 return to_hv_vcpu(vcpu) && 224 test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap); 225 } 226 227 void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu); 228 229 bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu); 230 int kvm_hv_get_assist_page(struct kvm_vcpu *vcpu); 231 232 static inline struct kvm_vcpu_hv_stimer *to_hv_stimer(struct kvm_vcpu *vcpu, 233 int timer_index) 234 { 235 return &to_hv_vcpu(vcpu)->stimer[timer_index]; 236 } 237 238 static inline struct kvm_vcpu *hv_stimer_to_vcpu(struct kvm_vcpu_hv_stimer *stimer) 239 { 240 struct kvm_vcpu_hv *hv_vcpu; 241 242 hv_vcpu = container_of(stimer - stimer->index, struct kvm_vcpu_hv, 243 stimer[0]); 244 return hv_vcpu->vcpu; 245 } 246 247 static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu) 248 { 249 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu); 250 251 if (!hv_vcpu) 252 return false; 253 254 return !bitmap_empty(hv_vcpu->stimer_pending_bitmap, 255 HV_SYNIC_STIMER_COUNT); 256 } 257 258 /* 259 * With HV_ACCESS_TSC_INVARIANT feature, invariant TSC (CPUID.80000007H:EDX[8]) 260 * is only observed after HV_X64_MSR_TSC_INVARIANT_CONTROL was written to. 261 */ 262 static inline bool kvm_hv_invtsc_suppressed(struct kvm_vcpu *vcpu) 263 { 264 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 265 266 /* 267 * If Hyper-V's invariant TSC control is not exposed to the guest, 268 * the invariant TSC CPUID flag is not suppressed, Windows guests were 269 * observed to be able to handle it correctly. Going forward, VMMs are 270 * encouraged to enable Hyper-V's invariant TSC control when invariant 271 * TSC CPUID flag is set to make KVM's behavior match genuine Hyper-V. 272 */ 273 if (!hv_vcpu || 274 !(hv_vcpu->cpuid_cache.features_eax & HV_ACCESS_TSC_INVARIANT)) 275 return false; 276 277 /* 278 * If Hyper-V's invariant TSC control is exposed to the guest, KVM is 279 * responsible for suppressing the invariant TSC CPUID flag if the 280 * Hyper-V control is not enabled. 281 */ 282 return !(to_kvm_hv(vcpu->kvm)->hv_invtsc_control & HV_EXPOSE_INVARIANT_TSC); 283 } 284 285 void kvm_hv_process_stimers(struct kvm_vcpu *vcpu); 286 287 void kvm_hv_setup_tsc_page(struct kvm *kvm, 288 struct pvclock_vcpu_time_info *hv_clock); 289 void kvm_hv_request_tsc_page_update(struct kvm *kvm); 290 291 void kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu); 292 293 void kvm_hv_init_vm(struct kvm *kvm); 294 void kvm_hv_destroy_vm(struct kvm *kvm); 295 int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu); 296 void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled); 297 int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce); 298 int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args); 299 int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, 300 struct kvm_cpuid_entry2 __user *entries); 301 302 static inline struct kvm_vcpu_hv_tlb_flush_fifo *kvm_hv_get_tlb_flush_fifo(struct kvm_vcpu *vcpu, 303 bool is_guest_mode) 304 { 305 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu); 306 int i = is_guest_mode ? HV_L2_TLB_FLUSH_FIFO : 307 HV_L1_TLB_FLUSH_FIFO; 308 309 if (!hv_vcpu) 310 return NULL; 311 312 return &hv_vcpu->tlb_flush_fifo[i]; 313 } 314 315 static inline void kvm_hv_vcpu_purge_flush_tlb(struct kvm_vcpu *vcpu) 316 { 317 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 318 319 if (!kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) 320 return; 321 322 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu)); 323 if (!tlb_flush_fifo) 324 return; 325 326 kfifo_reset_out(&tlb_flush_fifo->entries); 327 } 328 329 static inline bool guest_hv_cpuid_has_l2_tlb_flush(struct kvm_vcpu *vcpu) 330 { 331 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 332 333 return hv_vcpu && 334 (hv_vcpu->cpuid_cache.nested_eax & HV_X64_NESTED_DIRECT_FLUSH); 335 } 336 337 static inline bool kvm_hv_is_tlb_flush_hcall(struct kvm_vcpu *vcpu) 338 { 339 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 340 u16 code; 341 342 if (!hv_vcpu) 343 return false; 344 345 code = is_64_bit_hypercall(vcpu) ? kvm_rcx_read_raw(vcpu) : 346 kvm_eax_read(vcpu); 347 348 return (code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE || 349 code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST || 350 code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX || 351 code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX); 352 } 353 354 static inline int kvm_hv_verify_vp_assist(struct kvm_vcpu *vcpu) 355 { 356 if (!to_hv_vcpu(vcpu)) 357 return 0; 358 359 if (!kvm_hv_assist_page_enabled(vcpu)) 360 return 0; 361 362 return kvm_hv_get_assist_page(vcpu); 363 } 364 365 static inline void kvm_hv_nested_transtion_tlb_flush(struct kvm_vcpu *vcpu, 366 bool tdp_enabled) 367 { 368 /* 369 * KVM_REQ_HV_TLB_FLUSH flushes entries from either L1's VP_ID or 370 * L2's VP_ID upon request from the guest. Make sure we check for 371 * pending entries in the right FIFO upon L1/L2 transition as these 372 * requests are put by other vCPUs asynchronously. 373 */ 374 if (to_hv_vcpu(vcpu) && tdp_enabled) 375 kvm_make_request(KVM_REQ_HV_TLB_FLUSH, vcpu); 376 } 377 378 int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu); 379 #else /* CONFIG_KVM_HYPERV */ 380 static inline void kvm_hv_setup_tsc_page(struct kvm *kvm, 381 struct pvclock_vcpu_time_info *hv_clock) {} 382 static inline void kvm_hv_request_tsc_page_update(struct kvm *kvm) {} 383 static inline void kvm_hv_xsaves_xsavec_maybe_warn(struct kvm_vcpu *vcpu) {} 384 static inline void kvm_hv_init_vm(struct kvm *kvm) {} 385 static inline void kvm_hv_destroy_vm(struct kvm *kvm) {} 386 static inline int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) 387 { 388 return 0; 389 } 390 static inline void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu) {} 391 static inline bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu) 392 { 393 return false; 394 } 395 static inline int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 396 { 397 return HV_STATUS_ACCESS_DENIED; 398 } 399 static inline void kvm_hv_vcpu_purge_flush_tlb(struct kvm_vcpu *vcpu) {} 400 static inline bool kvm_hv_synic_has_vector(struct kvm_vcpu *vcpu, int vector) 401 { 402 return false; 403 } 404 static inline bool kvm_hv_synic_auto_eoi_set(struct kvm_vcpu *vcpu, int vector) 405 { 406 return false; 407 } 408 static inline void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector) {} 409 static inline bool kvm_hv_invtsc_suppressed(struct kvm_vcpu *vcpu) 410 { 411 return false; 412 } 413 static inline void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled) {} 414 static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu) 415 { 416 return false; 417 } 418 static inline int kvm_hv_verify_vp_assist(struct kvm_vcpu *vcpu) 419 { 420 return 0; 421 } 422 static inline u32 kvm_hv_get_vpindex(struct kvm_vcpu *vcpu) 423 { 424 return vcpu->vcpu_idx; 425 } 426 static inline void kvm_hv_nested_transtion_tlb_flush(struct kvm_vcpu *vcpu, bool tdp_enabled) {} 427 #endif /* CONFIG_KVM_HYPERV */ 428 429 #endif /* __ARCH_X86_KVM_HYPERV_H__ */ 430