1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kvm_host.h> 3 #include <asm/intel_pt.h> 4 #include <asm/vmx.h> 5 6 #include "hyperv.h" 7 #include "lapic.h" 8 #include "msrs.h" 9 #include "pmu.h" 10 #include "trace.h" 11 #include "vmx/vmx.h" 12 #include "xen.h" 13 #include "x86.h" 14 15 bool __read_mostly ignore_msrs = 0; 16 module_param(ignore_msrs, bool, 0644); 17 18 bool __read_mostly report_ignored_msrs = true; 19 module_param(report_ignored_msrs, bool, 0644); 20 EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); 21 22 #define MAX_IO_MSRS 256 23 24 struct msr_bitmap_range { 25 u32 flags; 26 u32 nmsrs; 27 u32 base; 28 unsigned long *bitmap; 29 }; 30 31 struct kvm_x86_msr_filter { 32 u8 count; 33 bool default_allow:1; 34 struct msr_bitmap_range ranges[16]; 35 }; 36 37 /* 38 * Restoring the host value for MSRs that are only consumed when running in 39 * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU 40 * returns to userspace, i.e. the kernel can run with the guest's value. 41 */ 42 #define KVM_MAX_NR_USER_RETURN_MSRS 16 43 44 struct kvm_user_return_msrs { 45 struct user_return_notifier urn; 46 bool registered; 47 struct kvm_user_return_msr_values { 48 u64 host; 49 u64 curr; 50 } values[KVM_MAX_NR_USER_RETURN_MSRS]; 51 }; 52 53 u32 __read_mostly kvm_nr_uret_msrs; 54 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_nr_uret_msrs); 55 static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS]; 56 static DEFINE_PER_CPU(struct kvm_user_return_msrs, user_return_msrs); 57 58 void kvm_destroy_user_return_msrs(void) 59 { 60 int cpu; 61 62 for_each_possible_cpu(cpu) 63 WARN_ON_ONCE(per_cpu(user_return_msrs, cpu).registered); 64 65 kvm_nr_uret_msrs = 0; 66 } 67 68 static void kvm_on_user_return(struct user_return_notifier *urn) 69 { 70 unsigned slot; 71 struct kvm_user_return_msrs *msrs 72 = container_of(urn, struct kvm_user_return_msrs, urn); 73 struct kvm_user_return_msr_values *values; 74 75 msrs->registered = false; 76 user_return_notifier_unregister(urn); 77 78 for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) { 79 values = &msrs->values[slot]; 80 if (values->host != values->curr) { 81 wrmsrq(kvm_uret_msrs_list[slot], values->host); 82 values->curr = values->host; 83 } 84 } 85 } 86 87 static int kvm_probe_user_return_msr(u32 msr) 88 { 89 u64 val; 90 int ret; 91 92 preempt_disable(); 93 ret = rdmsrq_safe(msr, &val); 94 if (ret) 95 goto out; 96 ret = wrmsrq_safe(msr, val); 97 out: 98 preempt_enable(); 99 return ret; 100 } 101 102 int kvm_add_user_return_msr(u32 msr) 103 { 104 BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS); 105 106 if (kvm_probe_user_return_msr(msr)) 107 return -1; 108 109 kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr; 110 return kvm_nr_uret_msrs++; 111 } 112 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_add_user_return_msr); 113 114 int kvm_find_user_return_msr(u32 msr) 115 { 116 int i; 117 118 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 119 if (kvm_uret_msrs_list[i] == msr) 120 return i; 121 } 122 return -1; 123 } 124 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_find_user_return_msr); 125 126 void kvm_user_return_msr_cpu_online(void) 127 { 128 struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); 129 u64 value; 130 int i; 131 132 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 133 rdmsrq_safe(kvm_uret_msrs_list[i], &value); 134 msrs->values[i].host = value; 135 msrs->values[i].curr = value; 136 } 137 } 138 139 static void kvm_user_return_register_notifier(struct kvm_user_return_msrs *msrs) 140 { 141 if (!msrs->registered) { 142 msrs->urn.on_user_return = kvm_on_user_return; 143 user_return_notifier_register(&msrs->urn); 144 msrs->registered = true; 145 } 146 } 147 148 int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) 149 { 150 struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); 151 int err; 152 153 value = (value & mask) | (msrs->values[slot].host & ~mask); 154 if (value == msrs->values[slot].curr) 155 return 0; 156 err = wrmsrq_safe(kvm_uret_msrs_list[slot], value); 157 if (err) 158 return 1; 159 160 msrs->values[slot].curr = value; 161 kvm_user_return_register_notifier(msrs); 162 return 0; 163 } 164 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr); 165 166 u64 kvm_get_user_return_msr(unsigned int slot) 167 { 168 return this_cpu_ptr(&user_return_msrs)->values[slot].curr; 169 } 170 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_user_return_msr); 171 172 void drop_user_return_notifiers(void) 173 { 174 struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); 175 176 if (msrs->registered) 177 kvm_on_user_return(&msrs->urn); 178 } 179 180 /* 181 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) track 182 * the set of MSRs that KVM exposes to userspace through KVM_GET_MSRS, 183 * KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. msrs_to_save holds MSRs that 184 * require host support, i.e. should be probed via RDMSR. emulated_msrs holds 185 * MSRs that KVM emulates without strictly requiring host support. 186 * msr_based_features holds MSRs that enumerate features, i.e. are effectively 187 * CPUID leafs. Note, msr_based_features isn't mutually exclusive with 188 * msrs_to_save and emulated_msrs. 189 */ 190 191 static const u32 msrs_to_save_base[] = { 192 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, 193 MSR_STAR, 194 #ifdef CONFIG_X86_64 195 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, 196 #endif 197 MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA, 198 MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX, 199 MSR_IA32_SPEC_CTRL, MSR_IA32_TSX_CTRL, 200 MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH, 201 MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK, 202 MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B, 203 MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B, 204 MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B, 205 MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B, 206 MSR_IA32_UMWAIT_CONTROL, 207 208 MSR_IA32_XFD, MSR_IA32_XFD_ERR, MSR_IA32_XSS, 209 210 MSR_IA32_U_CET, MSR_IA32_S_CET, 211 MSR_IA32_PL0_SSP, MSR_IA32_PL1_SSP, MSR_IA32_PL2_SSP, 212 MSR_IA32_PL3_SSP, MSR_IA32_INT_SSP_TAB, 213 MSR_IA32_DEBUGCTLMSR, 214 MSR_IA32_LASTBRANCHFROMIP, MSR_IA32_LASTBRANCHTOIP, 215 MSR_IA32_LASTINTFROMIP, MSR_IA32_LASTINTTOIP, 216 }; 217 218 static const u32 msrs_to_save_pmu[] = { 219 MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1, 220 MSR_ARCH_PERFMON_FIXED_CTR0 + 2, 221 MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS, 222 MSR_CORE_PERF_GLOBAL_CTRL, 223 MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG, 224 225 /* This part of MSRs should match KVM_MAX_NR_INTEL_GP_COUNTERS. */ 226 MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1, 227 MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3, 228 MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5, 229 MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7, 230 MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1, 231 MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3, 232 MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5, 233 MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7, 234 235 MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3, 236 MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3, 237 238 /* This part of MSRs should match KVM_MAX_NR_AMD_GP_COUNTERS. */ 239 MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2, 240 MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5, 241 MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2, 242 MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5, 243 244 MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 245 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, 246 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, 247 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, 248 }; 249 250 static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) + 251 ARRAY_SIZE(msrs_to_save_pmu)]; 252 static unsigned num_msrs_to_save; 253 254 static const u32 emulated_msrs_all[] = { 255 MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, 256 MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, 257 258 #ifdef CONFIG_KVM_HYPERV 259 HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, 260 HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC, 261 HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY, 262 HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2, 263 HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL, 264 HV_X64_MSR_RESET, 265 HV_X64_MSR_VP_INDEX, 266 HV_X64_MSR_VP_RUNTIME, 267 HV_X64_MSR_SCONTROL, 268 HV_X64_MSR_STIMER0_CONFIG, 269 HV_X64_MSR_VP_ASSIST_PAGE, 270 HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL, 271 HV_X64_MSR_TSC_EMULATION_STATUS, HV_X64_MSR_TSC_INVARIANT_CONTROL, 272 HV_X64_MSR_SYNDBG_OPTIONS, 273 HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS, 274 HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER, 275 HV_X64_MSR_SYNDBG_PENDING_BUFFER, 276 #endif 277 278 MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, 279 MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK, 280 281 MSR_IA32_TSC_ADJUST, 282 MSR_IA32_TSC_DEADLINE, 283 MSR_IA32_ARCH_CAPABILITIES, 284 MSR_IA32_PERF_CAPABILITIES, 285 MSR_IA32_MISC_ENABLE, 286 MSR_IA32_MCG_STATUS, 287 MSR_IA32_MCG_CTL, 288 MSR_IA32_MCG_EXT_CTL, 289 MSR_IA32_SMBASE, 290 MSR_SMI_COUNT, 291 MSR_PLATFORM_INFO, 292 MSR_MISC_FEATURES_ENABLES, 293 MSR_AMD64_VIRT_SPEC_CTRL, 294 MSR_AMD64_TSC_RATIO, 295 MSR_IA32_POWER_CTL, 296 MSR_IA32_UCODE_REV, 297 298 /* 299 * KVM always supports the "true" VMX control MSRs, even if the host 300 * does not. The VMX MSRs as a whole are considered "emulated" as KVM 301 * doesn't strictly require them to exist in the host (ignoring that 302 * KVM would refuse to load in the first place if the core set of MSRs 303 * aren't supported). 304 */ 305 MSR_IA32_VMX_BASIC, 306 MSR_IA32_VMX_TRUE_PINBASED_CTLS, 307 MSR_IA32_VMX_TRUE_PROCBASED_CTLS, 308 MSR_IA32_VMX_TRUE_EXIT_CTLS, 309 MSR_IA32_VMX_TRUE_ENTRY_CTLS, 310 MSR_IA32_VMX_MISC, 311 MSR_IA32_VMX_CR0_FIXED0, 312 MSR_IA32_VMX_CR4_FIXED0, 313 MSR_IA32_VMX_VMCS_ENUM, 314 MSR_IA32_VMX_PROCBASED_CTLS2, 315 MSR_IA32_VMX_EPT_VPID_CAP, 316 MSR_IA32_VMX_VMFUNC, 317 318 MSR_K7_HWCR, 319 MSR_KVM_POLL_CONTROL, 320 }; 321 322 static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)]; 323 static unsigned num_emulated_msrs; 324 325 /* 326 * List of MSRs that control the existence of MSR-based features, i.e. MSRs 327 * that are effectively CPUID leafs. VMX MSRs are also included in the set of 328 * feature MSRs, but are handled separately to allow expedited lookups. 329 */ 330 static const u32 msr_based_features_all_except_vmx[] = { 331 MSR_AMD64_DE_CFG, 332 MSR_IA32_UCODE_REV, 333 MSR_IA32_ARCH_CAPABILITIES, 334 MSR_IA32_PERF_CAPABILITIES, 335 MSR_PLATFORM_INFO, 336 }; 337 338 static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all_except_vmx) + 339 (KVM_LAST_EMULATED_VMX_MSR - KVM_FIRST_EMULATED_VMX_MSR + 1)]; 340 static unsigned int num_msr_based_features; 341 342 int kvm_get_msr_index_list(struct kvm_msr_list __user *user_msr_list) 343 { 344 struct kvm_msr_list msr_list; 345 unsigned int n; 346 347 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) 348 return -EFAULT; 349 350 n = msr_list.nmsrs; 351 msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs; 352 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) 353 return -EFAULT; 354 355 if (n < msr_list.nmsrs) 356 return -E2BIG; 357 358 if (copy_to_user(user_msr_list->indices, &msrs_to_save, 359 num_msrs_to_save * sizeof(u32))) 360 return -EFAULT; 361 362 if (copy_to_user(user_msr_list->indices + num_msrs_to_save, 363 &emulated_msrs, num_emulated_msrs * sizeof(u32))) 364 return -EFAULT; 365 366 return 0; 367 } 368 369 int kvm_get_feature_msr_index_list(struct kvm_msr_list __user *user_msr_list) 370 { 371 struct kvm_msr_list msr_list; 372 unsigned int n; 373 374 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) 375 return -EFAULT; 376 377 n = msr_list.nmsrs; 378 msr_list.nmsrs = num_msr_based_features; 379 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) 380 return -EFAULT; 381 382 if (n < msr_list.nmsrs) 383 return -E2BIG; 384 385 if (copy_to_user(user_msr_list->indices, &msr_based_features, 386 num_msr_based_features * sizeof(u32))) 387 return -EFAULT; 388 389 return 0; 390 } 391 392 /* 393 * All feature MSRs except uCode revID, which tracks the currently loaded uCode 394 * patch, are immutable once the vCPU model is defined. 395 */ 396 static bool kvm_is_immutable_feature_msr(u32 msr) 397 { 398 int i; 399 400 if (msr >= KVM_FIRST_EMULATED_VMX_MSR && msr <= KVM_LAST_EMULATED_VMX_MSR) 401 return true; 402 403 for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) { 404 if (msr == msr_based_features_all_except_vmx[i]) 405 return msr != MSR_IA32_UCODE_REV; 406 } 407 408 return false; 409 } 410 411 static bool kvm_is_advertised_msr(u32 msr_index) 412 { 413 unsigned int i; 414 415 for (i = 0; i < num_msrs_to_save; i++) { 416 if (msrs_to_save[i] == msr_index) 417 return true; 418 } 419 420 for (i = 0; i < num_emulated_msrs; i++) { 421 if (emulated_msrs[i] == msr_index) 422 return true; 423 } 424 425 return false; 426 } 427 428 429 /* 430 * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM 431 * does not yet virtualize. These include: 432 * 10 - MISC_PACKAGE_CTRLS 433 * 11 - ENERGY_FILTERING_CTL 434 * 12 - DOITM 435 * 18 - FB_CLEAR_CTRL 436 * 21 - XAPIC_DISABLE_STATUS 437 * 23 - OVERCLOCKING_STATUS 438 */ 439 440 #define KVM_SUPPORTED_ARCH_CAP \ 441 (ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \ 442 ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \ 443 ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \ 444 ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \ 445 ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \ 446 ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO | ARCH_CAP_ITS_NO) 447 448 u64 kvm_get_arch_capabilities(void) 449 { 450 u64 data = kvm_host.arch_capabilities & KVM_SUPPORTED_ARCH_CAP; 451 452 /* 453 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that 454 * the nested hypervisor runs with NX huge pages. If it is not, 455 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other 456 * L1 guests, so it need not worry about its own (L2) guests. 457 */ 458 data |= ARCH_CAP_PSCHANGE_MC_NO; 459 460 /* 461 * If we're doing cache flushes (either "always" or "cond") 462 * we will do one whenever the guest does a vmlaunch/vmresume. 463 * If an outer hypervisor is doing the cache flush for us 464 * (ARCH_CAP_SKIP_VMENTRY_L1DFLUSH), we can safely pass that 465 * capability to the guest too, and if EPT is disabled we're not 466 * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will 467 * require a nested hypervisor to do a flush of its own. 468 */ 469 if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER) 470 data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH; 471 472 if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN)) 473 data |= ARCH_CAP_RDCL_NO; 474 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 475 data |= ARCH_CAP_SSB_NO; 476 if (!boot_cpu_has_bug(X86_BUG_MDS)) 477 data |= ARCH_CAP_MDS_NO; 478 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 479 data |= ARCH_CAP_RFDS_NO; 480 if (!boot_cpu_has_bug(X86_BUG_ITS)) 481 data |= ARCH_CAP_ITS_NO; 482 483 if (!boot_cpu_has(X86_FEATURE_RTM)) { 484 /* 485 * If RTM=0 because the kernel has disabled TSX, the host might 486 * have TAA_NO or TSX_CTRL. Clear TAA_NO (the guest sees RTM=0 487 * and therefore knows that there cannot be TAA) but keep 488 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts, 489 * and we want to allow migrating those guests to tsx=off hosts. 490 */ 491 data &= ~ARCH_CAP_TAA_NO; 492 } else if (!boot_cpu_has_bug(X86_BUG_TAA)) { 493 data |= ARCH_CAP_TAA_NO; 494 } else { 495 /* 496 * Nothing to do here; we emulate TSX_CTRL if present on the 497 * host so the guest can choose between disabling TSX or 498 * using VERW to clear CPU buffers. 499 */ 500 } 501 502 if (!boot_cpu_has_bug(X86_BUG_GDS) || gds_ucode_mitigated()) 503 data |= ARCH_CAP_GDS_NO; 504 505 return data; 506 } 507 508 static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, 509 bool host_initiated) 510 { 511 WARN_ON_ONCE(!host_initiated); 512 513 switch (index) { 514 case MSR_IA32_ARCH_CAPABILITIES: 515 *data = kvm_get_arch_capabilities(); 516 break; 517 case MSR_IA32_PERF_CAPABILITIES: 518 *data = kvm_caps.supported_perf_cap; 519 break; 520 case MSR_PLATFORM_INFO: 521 *data = MSR_PLATFORM_INFO_CPUID_FAULT; 522 break; 523 case MSR_IA32_UCODE_REV: 524 rdmsrq_safe(index, data); 525 break; 526 default: 527 return kvm_x86_call(get_feature_msr)(index, data); 528 } 529 return 0; 530 } 531 532 typedef int (*msr_access_t)(struct kvm_vcpu *vcpu, u32 index, u64 *data, 533 bool host_initiated); 534 535 static __always_inline int kvm_do_msr_access(struct kvm_vcpu *vcpu, u32 msr, 536 u64 *data, bool host_initiated, 537 enum kvm_msr_access rw, 538 msr_access_t msr_access_fn) 539 { 540 const char *op = rw == MSR_TYPE_W ? "wrmsr" : "rdmsr"; 541 int ret; 542 543 BUILD_BUG_ON(rw != MSR_TYPE_R && rw != MSR_TYPE_W); 544 545 /* 546 * Zero the data on read failures to avoid leaking stack data to the 547 * guest and/or userspace, e.g. if the failure is ignored below. 548 */ 549 ret = msr_access_fn(vcpu, msr, data, host_initiated); 550 if (ret && rw == MSR_TYPE_R) 551 *data = 0; 552 553 if (ret != KVM_MSR_RET_UNSUPPORTED) 554 return ret; 555 556 /* 557 * Userspace is allowed to read MSRs, and write '0' to MSRs, that KVM 558 * advertises to userspace, even if an MSR isn't fully supported. 559 * Simply check that @data is '0', which covers both the write '0' case 560 * and all reads (in which case @data is zeroed on failure; see above). 561 */ 562 if (host_initiated && !*data && kvm_is_advertised_msr(msr)) 563 return 0; 564 565 if (!ignore_msrs) { 566 kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n", 567 op, msr, *data); 568 return ret; 569 } 570 571 if (report_ignored_msrs) 572 kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n", op, msr, *data); 573 574 return 0; 575 } 576 577 static int do_get_feature_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 578 { 579 return kvm_do_msr_access(vcpu, index, data, true, MSR_TYPE_R, 580 kvm_get_feature_msr); 581 } 582 583 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) 584 { 585 if (efer & EFER_AUTOIBRS && !guest_cpu_cap_has(vcpu, X86_FEATURE_AUTOIBRS)) 586 return false; 587 588 if (efer & EFER_FFXSR && !guest_cpu_cap_has(vcpu, X86_FEATURE_FXSR_OPT)) 589 return false; 590 591 if (efer & EFER_SVME && !guest_cpu_cap_has(vcpu, X86_FEATURE_SVM)) 592 return false; 593 594 if (efer & (EFER_LME | EFER_LMA) && 595 !guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) 596 return false; 597 598 if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) 599 return false; 600 601 return true; 602 603 } 604 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) 605 { 606 if (efer & ~kvm_caps.supported_efer_bits) 607 return false; 608 609 return __kvm_valid_efer(vcpu, efer); 610 } 611 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_valid_efer); 612 613 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 614 { 615 u64 old_efer = vcpu->arch.efer; 616 u64 efer = msr_info->data; 617 int r; 618 619 if (efer & ~kvm_caps.supported_efer_bits) 620 return 1; 621 622 if (!msr_info->host_initiated) { 623 if (!__kvm_valid_efer(vcpu, efer)) 624 return 1; 625 626 if (is_paging(vcpu) && 627 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) 628 return 1; 629 } 630 631 efer &= ~EFER_LMA; 632 efer |= vcpu->arch.efer & EFER_LMA; 633 634 r = kvm_x86_call(set_efer)(vcpu, efer); 635 if (r) { 636 WARN_ON(r > 0); 637 return r; 638 } 639 640 if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS) 641 kvm_mmu_reset_context(vcpu); 642 643 if (!cpu_feature_enabled(X86_FEATURE_XSAVES) && 644 (efer & EFER_SVME)) 645 kvm_hv_xsaves_xsavec_maybe_warn(vcpu); 646 647 return 0; 648 } 649 650 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type) 651 { 652 struct kvm_x86_msr_filter *msr_filter; 653 struct msr_bitmap_range *ranges; 654 struct kvm *kvm = vcpu->kvm; 655 bool allowed; 656 int idx; 657 u32 i; 658 659 /* x2APIC MSRs do not support filtering. */ 660 if (index >= 0x800 && index <= 0x8ff) 661 return true; 662 663 idx = srcu_read_lock(&kvm->srcu); 664 665 msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu); 666 if (!msr_filter) { 667 allowed = true; 668 goto out; 669 } 670 671 allowed = msr_filter->default_allow; 672 ranges = msr_filter->ranges; 673 674 for (i = 0; i < msr_filter->count; i++) { 675 u32 start = ranges[i].base; 676 u32 end = start + ranges[i].nmsrs; 677 u32 flags = ranges[i].flags; 678 unsigned long *bitmap = ranges[i].bitmap; 679 680 if ((index >= start) && (index < end) && (flags & type)) { 681 allowed = test_bit(index - start, bitmap); 682 break; 683 } 684 } 685 686 out: 687 srcu_read_unlock(&kvm->srcu, idx); 688 689 return allowed; 690 } 691 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_allowed); 692 693 /* 694 * Write @data into the MSR specified by @index. Select MSR specific fault 695 * checks are bypassed if @host_initiated is %true. 696 * Returns 0 on success, non-0 otherwise. 697 * Assumes vcpu_load() was already called. 698 */ 699 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, 700 bool host_initiated) 701 { 702 struct msr_data msr; 703 704 switch (index) { 705 case MSR_FS_BASE: 706 case MSR_GS_BASE: 707 case MSR_KERNEL_GS_BASE: 708 case MSR_CSTAR: 709 case MSR_LSTAR: 710 if (is_noncanonical_msr_address(data, vcpu)) 711 return 1; 712 break; 713 case MSR_IA32_SYSENTER_EIP: 714 case MSR_IA32_SYSENTER_ESP: 715 /* 716 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if 717 * non-canonical address is written on Intel but not on 718 * AMD (which ignores the top 32-bits, because it does 719 * not implement 64-bit SYSENTER). 720 * 721 * 64-bit code should hence be able to write a non-canonical 722 * value on AMD. Making the address canonical ensures that 723 * vmentry does not fail on Intel after writing a non-canonical 724 * value, and that something deterministic happens if the guest 725 * invokes 64-bit SYSENTER. 726 */ 727 data = __canonical_address(data, max_host_virt_addr_bits()); 728 break; 729 case MSR_TSC_AUX: 730 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) 731 return 1; 732 733 if (!host_initiated && 734 !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) && 735 !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID)) 736 return 1; 737 738 /* 739 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has 740 * incomplete and conflicting architectural behavior. Current 741 * AMD CPUs completely ignore bits 63:32, i.e. they aren't 742 * reserved and always read as zeros. Enforce Intel's reserved 743 * bits check if the guest CPU is Intel compatible, otherwise 744 * clear the bits. This ensures cross-vendor migration will 745 * provide consistent behavior for the guest. 746 */ 747 if (guest_cpuid_is_intel_compatible(vcpu) && (data >> 32) != 0) 748 return 1; 749 750 data = (u32)data; 751 break; 752 case MSR_IA32_U_CET: 753 case MSR_IA32_S_CET: 754 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && 755 !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT)) 756 return KVM_MSR_RET_UNSUPPORTED; 757 if (!kvm_is_valid_u_s_cet(vcpu, data)) 758 return 1; 759 break; 760 case MSR_KVM_INTERNAL_GUEST_SSP: 761 if (!host_initiated) 762 return 1; 763 fallthrough; 764 /* 765 * Note that the MSR emulation here is flawed when a vCPU 766 * doesn't support the Intel 64 architecture. The expected 767 * architectural behavior in this case is that the upper 32 768 * bits do not exist and should always read '0'. However, 769 * because the actual hardware on which the virtual CPU is 770 * running does support Intel 64, XRSTORS/XSAVES in the 771 * guest could observe behavior that violates the 772 * architecture. Intercepting XRSTORS/XSAVES for this 773 * special case isn't deemed worthwhile. 774 */ 775 case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB: 776 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) 777 return KVM_MSR_RET_UNSUPPORTED; 778 /* 779 * MSR_IA32_INT_SSP_TAB is not present on processors that do 780 * not support Intel 64 architecture. 781 */ 782 if (index == MSR_IA32_INT_SSP_TAB && !guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) 783 return KVM_MSR_RET_UNSUPPORTED; 784 if (is_noncanonical_msr_address(data, vcpu)) 785 return 1; 786 /* All SSP MSRs except MSR_IA32_INT_SSP_TAB must be 4-byte aligned */ 787 if (index != MSR_IA32_INT_SSP_TAB && !IS_ALIGNED(data, 4)) 788 return 1; 789 break; 790 } 791 792 msr.data = data; 793 msr.index = index; 794 msr.host_initiated = host_initiated; 795 796 return kvm_x86_call(set_msr)(vcpu, &msr); 797 } 798 799 static int _kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, 800 bool host_initiated) 801 { 802 return __kvm_set_msr(vcpu, index, *data, host_initiated); 803 } 804 805 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu, 806 u32 index, u64 data, bool host_initiated) 807 { 808 return kvm_do_msr_access(vcpu, index, &data, host_initiated, MSR_TYPE_W, 809 _kvm_set_msr); 810 } 811 812 /* 813 * Read the MSR specified by @index into @data. Select MSR specific fault 814 * checks are bypassed if @host_initiated is %true. 815 * Returns 0 on success, non-0 otherwise. 816 * Assumes vcpu_load() was already called. 817 */ 818 static int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, 819 bool host_initiated) 820 { 821 struct msr_data msr; 822 int ret; 823 824 switch (index) { 825 case MSR_TSC_AUX: 826 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) 827 return 1; 828 829 if (!host_initiated && 830 !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) && 831 !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID)) 832 return 1; 833 break; 834 case MSR_IA32_U_CET: 835 case MSR_IA32_S_CET: 836 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && 837 !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT)) 838 return KVM_MSR_RET_UNSUPPORTED; 839 break; 840 case MSR_KVM_INTERNAL_GUEST_SSP: 841 if (!host_initiated) 842 return 1; 843 fallthrough; 844 case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB: 845 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) 846 return KVM_MSR_RET_UNSUPPORTED; 847 break; 848 } 849 850 msr.index = index; 851 msr.host_initiated = host_initiated; 852 853 ret = kvm_x86_call(get_msr)(vcpu, &msr); 854 if (!ret) 855 *data = msr.data; 856 return ret; 857 } 858 859 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu, 860 u32 index, u64 *data, bool host_initiated) 861 { 862 return kvm_do_msr_access(vcpu, index, data, host_initiated, MSR_TYPE_R, 863 __kvm_get_msr); 864 } 865 866 int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) 867 { 868 return __kvm_set_msr(vcpu, index, data, true); 869 } 870 871 int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) 872 { 873 return __kvm_get_msr(vcpu, index, data, true); 874 } 875 876 int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) 877 { 878 return kvm_get_msr_ignored_check(vcpu, index, data, false); 879 } 880 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_read); 881 882 int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) 883 { 884 return kvm_set_msr_ignored_check(vcpu, index, data, false); 885 } 886 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_write); 887 888 int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) 889 { 890 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ)) 891 return KVM_MSR_RET_FILTERED; 892 893 return __kvm_emulate_msr_read(vcpu, index, data); 894 } 895 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_read); 896 897 int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) 898 { 899 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE)) 900 return KVM_MSR_RET_FILTERED; 901 902 return __kvm_emulate_msr_write(vcpu, index, data); 903 } 904 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_write); 905 906 static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) 907 { 908 if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu)) 909 return EXIT_FASTPATH_NONE; 910 911 switch (msr) { 912 case APIC_BASE_MSR + (APIC_ICR >> 4): 913 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic) || 914 kvm_x2apic_icr_write_fast(vcpu->arch.apic, data)) 915 return EXIT_FASTPATH_NONE; 916 break; 917 case MSR_IA32_TSC_DEADLINE: 918 kvm_set_lapic_tscdeadline_msr(vcpu, data); 919 break; 920 default: 921 return EXIT_FASTPATH_NONE; 922 } 923 924 trace_kvm_msr_write(msr, data); 925 926 if (!kvm_skip_emulated_instruction(vcpu)) 927 return EXIT_FASTPATH_EXIT_USERSPACE; 928 929 return EXIT_FASTPATH_REENTER_GUEST; 930 } 931 932 fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu) 933 { 934 return __handle_fastpath_wrmsr(vcpu, kvm_ecx_read(vcpu), 935 kvm_read_edx_eax(vcpu)); 936 } 937 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr); 938 939 fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) 940 { 941 return __handle_fastpath_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg)); 942 } 943 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr_imm); 944 945 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu) 946 { 947 if (!vcpu->run->msr.error) { 948 kvm_eax_write(vcpu, vcpu->run->msr.data); 949 kvm_edx_write(vcpu, vcpu->run->msr.data >> 32); 950 } 951 } 952 953 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) 954 { 955 if (err) { 956 kvm_inject_gp(vcpu, 0); 957 return 1; 958 } 959 960 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP | 961 EMULTYPE_COMPLETE_USER_EXIT); 962 } 963 964 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu) 965 { 966 return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error); 967 } 968 969 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) 970 { 971 complete_userspace_rdmsr(vcpu); 972 return complete_emulated_msr_access(vcpu); 973 } 974 975 static int complete_fast_msr_access(struct kvm_vcpu *vcpu) 976 { 977 return kvm_x86_call(complete_emulated_msr)(vcpu, vcpu->run->msr.error); 978 } 979 980 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu) 981 { 982 complete_userspace_rdmsr(vcpu); 983 return complete_fast_msr_access(vcpu); 984 } 985 986 static int complete_fast_rdmsr_imm(struct kvm_vcpu *vcpu) 987 { 988 if (!vcpu->run->msr.error) 989 kvm_register_write(vcpu, vcpu->arch.cui_rdmsr_imm_reg, 990 vcpu->run->msr.data); 991 992 return complete_fast_msr_access(vcpu); 993 } 994 995 static u64 kvm_msr_reason(int r) 996 { 997 switch (r) { 998 case KVM_MSR_RET_UNSUPPORTED: 999 return KVM_MSR_EXIT_REASON_UNKNOWN; 1000 case KVM_MSR_RET_FILTERED: 1001 return KVM_MSR_EXIT_REASON_FILTER; 1002 default: 1003 return KVM_MSR_EXIT_REASON_INVAL; 1004 } 1005 } 1006 1007 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index, 1008 u32 exit_reason, u64 data, 1009 int (*completion)(struct kvm_vcpu *vcpu), 1010 int r) 1011 { 1012 u64 msr_reason = kvm_msr_reason(r); 1013 1014 /* Check if the user wanted to know about this MSR fault */ 1015 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason)) 1016 return 0; 1017 1018 vcpu->run->exit_reason = exit_reason; 1019 vcpu->run->msr.error = 0; 1020 memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad)); 1021 vcpu->run->msr.reason = msr_reason; 1022 vcpu->run->msr.index = index; 1023 vcpu->run->msr.data = data; 1024 vcpu->arch.complete_userspace_io = completion; 1025 1026 return 1; 1027 } 1028 1029 static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg, 1030 int (*complete_rdmsr)(struct kvm_vcpu *)) 1031 { 1032 u64 data; 1033 int r; 1034 1035 r = kvm_emulate_msr_read(vcpu, msr, &data); 1036 1037 if (!r) { 1038 trace_kvm_msr_read(msr, data); 1039 1040 if (reg < 0) { 1041 kvm_eax_write(vcpu, data); 1042 kvm_edx_write(vcpu, data >> 32); 1043 } else { 1044 kvm_register_write(vcpu, reg, data); 1045 } 1046 } else { 1047 /* MSR read failed? See if we should ask user space */ 1048 if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_RDMSR, 0, 1049 complete_rdmsr, r)) 1050 return 0; 1051 trace_kvm_msr_read_ex(msr); 1052 } 1053 1054 return kvm_x86_call(complete_emulated_msr)(vcpu, r); 1055 } 1056 1057 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) 1058 { 1059 return __kvm_emulate_rdmsr(vcpu, kvm_ecx_read(vcpu), -1, 1060 complete_fast_rdmsr); 1061 } 1062 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr); 1063 1064 int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) 1065 { 1066 vcpu->arch.cui_rdmsr_imm_reg = reg; 1067 1068 return __kvm_emulate_rdmsr(vcpu, msr, reg, complete_fast_rdmsr_imm); 1069 } 1070 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr_imm); 1071 1072 static int __kvm_emulate_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1073 { 1074 int r; 1075 1076 r = kvm_emulate_msr_write(vcpu, msr, data); 1077 if (!r) { 1078 trace_kvm_msr_write(msr, data); 1079 } else { 1080 /* MSR write failed? See if we should ask user space */ 1081 if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_WRMSR, data, 1082 complete_fast_msr_access, r)) 1083 return 0; 1084 /* Signal all other negative errors to userspace */ 1085 if (r < 0) 1086 return r; 1087 trace_kvm_msr_write_ex(msr, data); 1088 } 1089 1090 return kvm_x86_call(complete_emulated_msr)(vcpu, r); 1091 } 1092 1093 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) 1094 { 1095 return __kvm_emulate_wrmsr(vcpu, kvm_ecx_read(vcpu), 1096 kvm_read_edx_eax(vcpu)); 1097 } 1098 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr); 1099 1100 int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) 1101 { 1102 return __kvm_emulate_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg)); 1103 } 1104 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr_imm); 1105 1106 int kvm_emulator_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index, 1107 u64 *pdata) 1108 { 1109 int r; 1110 1111 r = kvm_emulate_msr_read(vcpu, msr_index, pdata); 1112 if (r < 0) 1113 return X86EMUL_UNHANDLEABLE; 1114 1115 if (r) { 1116 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, 1117 complete_emulated_rdmsr, r)) 1118 return X86EMUL_IO_NEEDED; 1119 1120 trace_kvm_msr_read_ex(msr_index); 1121 return X86EMUL_PROPAGATE_FAULT; 1122 } 1123 1124 trace_kvm_msr_read(msr_index, *pdata); 1125 return X86EMUL_CONTINUE; 1126 } 1127 1128 int kvm_emulator_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index, 1129 u64 data) 1130 { 1131 int r; 1132 1133 r = kvm_emulate_msr_write(vcpu, msr_index, data); 1134 if (r < 0) 1135 return X86EMUL_UNHANDLEABLE; 1136 1137 if (r) { 1138 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, 1139 complete_emulated_msr_access, r)) 1140 return X86EMUL_IO_NEEDED; 1141 1142 trace_kvm_msr_write_ex(msr_index, data); 1143 return X86EMUL_PROPAGATE_FAULT; 1144 } 1145 1146 trace_kvm_msr_write(msr_index, data); 1147 return X86EMUL_CONTINUE; 1148 } 1149 1150 int kvm_emulator_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) 1151 { 1152 /* 1153 * Treat emulator accesses to the current shadow stack pointer as host- 1154 * initiated, as they aren't true MSR accesses (SSP is a "just a reg"), 1155 * and this API is used only for implicit accesses, i.e. not RDMSR, and 1156 * so the index is fully KVM-controlled. 1157 */ 1158 if (unlikely(msr_index == MSR_KVM_INTERNAL_GUEST_SSP)) 1159 return kvm_msr_read(vcpu, msr_index, pdata); 1160 1161 return __kvm_emulate_msr_read(vcpu, msr_index, pdata); 1162 } 1163 1164 /* 1165 * Returns true if the MSR in question is managed via XSTATE, i.e. is context 1166 * switched with the rest of guest FPU state. 1167 * 1168 * Note, S_CET is _not_ saved/restored via XSAVES/XRSTORS. 1169 */ 1170 static bool is_xstate_managed_msr(struct kvm_vcpu *vcpu, u32 msr) 1171 { 1172 if (!vcpu) 1173 return false; 1174 1175 switch (msr) { 1176 case MSR_IA32_U_CET: 1177 return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) || 1178 guest_cpu_cap_has(vcpu, X86_FEATURE_IBT); 1179 case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: 1180 return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 1181 default: 1182 return false; 1183 } 1184 } 1185 1186 /* 1187 * Lock (and if necessary, re-load) the guest FPU, i.e. XSTATE, and access an 1188 * MSR that is managed via XSTATE. Note, the caller is responsible for doing 1189 * the initial FPU load, this helper only ensures that guest state is resident 1190 * in hardware (the kernel can load its FPU state in IRQ context). 1191 * 1192 * Note, loading guest values for U_CET and PL[0-3]_SSP while executing in the 1193 * kernel is safe, as U_CET is specific to userspace, and PL[0-3]_SSP are only 1194 * consumed when transitioning to lower privilege levels, i.e. are effectively 1195 * only consumed by userspace as well. 1196 */ 1197 static __always_inline void kvm_access_xstate_msr(struct kvm_vcpu *vcpu, 1198 struct msr_data *msr_info, 1199 int access) 1200 { 1201 BUILD_BUG_ON(access != MSR_TYPE_R && access != MSR_TYPE_W); 1202 1203 KVM_BUG_ON(!is_xstate_managed_msr(vcpu, msr_info->index), vcpu->kvm); 1204 KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm); 1205 1206 kvm_fpu_get(); 1207 if (access == MSR_TYPE_R) 1208 rdmsrq(msr_info->index, msr_info->data); 1209 else 1210 wrmsrq(msr_info->index, msr_info->data); 1211 kvm_fpu_put(); 1212 } 1213 1214 static void kvm_set_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1215 { 1216 kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_W); 1217 } 1218 1219 static void kvm_get_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1220 { 1221 kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_R); 1222 } 1223 1224 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs) 1225 { 1226 int version; 1227 int r; 1228 struct pvclock_wall_clock wc; 1229 u32 wc_sec_hi; 1230 u64 wall_nsec; 1231 1232 if (!wall_clock) 1233 return; 1234 1235 r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version)); 1236 if (r) 1237 return; 1238 1239 if (version & 1) 1240 ++version; /* first time write, random junk */ 1241 1242 ++version; 1243 1244 if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version))) 1245 return; 1246 1247 wall_nsec = kvm_get_wall_clock_epoch(kvm); 1248 1249 wc.nsec = do_div(wall_nsec, NSEC_PER_SEC); 1250 wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */ 1251 wc.version = version; 1252 1253 kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc)); 1254 1255 if (sec_hi_ofs) { 1256 wc_sec_hi = wall_nsec >> 32; 1257 kvm_write_guest(kvm, wall_clock + sec_hi_ofs, 1258 &wc_sec_hi, sizeof(wc_sec_hi)); 1259 } 1260 1261 version++; 1262 kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); 1263 } 1264 1265 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time, 1266 bool old_msr, bool host_initiated) 1267 { 1268 struct kvm_arch *ka = &vcpu->kvm->arch; 1269 1270 if (vcpu->vcpu_id == 0 && !host_initiated) { 1271 if (ka->boot_vcpu_runs_old_kvmclock != old_msr) 1272 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1273 1274 ka->boot_vcpu_runs_old_kvmclock = old_msr; 1275 } 1276 1277 vcpu->arch.time = system_time; 1278 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); 1279 1280 /* we verify if the enable bit is set... */ 1281 if (system_time & 1) 1282 kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL, 1283 sizeof(struct pvclock_vcpu_time_info)); 1284 else 1285 kvm_gpc_deactivate(&vcpu->arch.pv_time); 1286 1287 return; 1288 } 1289 1290 /* These helpers are safe iff @msr is known to be an MCx bank MSR. */ 1291 static bool is_mci_control_msr(u32 msr) 1292 { 1293 return (msr & 3) == 0; 1294 } 1295 static bool is_mci_status_msr(u32 msr) 1296 { 1297 return (msr & 3) == 1; 1298 } 1299 1300 /* 1301 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP. 1302 */ 1303 static bool can_set_mci_status(struct kvm_vcpu *vcpu) 1304 { 1305 /* McStatusWrEn enabled? */ 1306 if (guest_cpuid_is_amd_compatible(vcpu)) 1307 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18)); 1308 1309 return false; 1310 } 1311 1312 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1313 { 1314 u64 mcg_cap = vcpu->arch.mcg_cap; 1315 unsigned bank_num = mcg_cap & 0xff; 1316 u32 msr = msr_info->index; 1317 u64 data = msr_info->data; 1318 u32 offset, last_msr; 1319 1320 switch (msr) { 1321 case MSR_IA32_MCG_STATUS: 1322 vcpu->arch.mcg_status = data; 1323 break; 1324 case MSR_IA32_MCG_CTL: 1325 if (!(mcg_cap & MCG_CTL_P) && 1326 (data || !msr_info->host_initiated)) 1327 return 1; 1328 if (data != 0 && data != ~(u64)0) 1329 return 1; 1330 vcpu->arch.mcg_ctl = data; 1331 break; 1332 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 1333 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; 1334 if (msr > last_msr) 1335 return 1; 1336 1337 if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated)) 1338 return 1; 1339 /* An attempt to write a 1 to a reserved bit raises #GP */ 1340 if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK)) 1341 return 1; 1342 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, 1343 last_msr + 1 - MSR_IA32_MC0_CTL2); 1344 vcpu->arch.mci_ctl2_banks[offset] = data; 1345 break; 1346 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 1347 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; 1348 if (msr > last_msr) 1349 return 1; 1350 1351 /* 1352 * Only 0 or all 1s can be written to IA32_MCi_CTL, all other 1353 * values are architecturally undefined. But, some Linux 1354 * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB 1355 * issue on AMD K8s, allow bit 10 to be clear when setting all 1356 * other bits in order to avoid an uncaught #GP in the guest. 1357 * 1358 * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable, 1359 * single-bit ECC data errors. 1360 */ 1361 if (is_mci_control_msr(msr) && 1362 data != 0 && (data | (1 << 10) | 1) != ~(u64)0) 1363 return 1; 1364 1365 /* 1366 * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR. 1367 * AMD-based CPUs allow non-zero values, but if and only if 1368 * HWCR[McStatusWrEn] is set. 1369 */ 1370 if (!msr_info->host_initiated && is_mci_status_msr(msr) && 1371 data != 0 && !can_set_mci_status(vcpu)) 1372 return 1; 1373 1374 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, 1375 last_msr + 1 - MSR_IA32_MC0_CTL); 1376 vcpu->arch.mce_banks[offset] = data; 1377 break; 1378 default: 1379 return 1; 1380 } 1381 return 0; 1382 } 1383 1384 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) 1385 { 1386 gpa_t gpa = data & ~0x3f; 1387 1388 /* Bits 4:5 are reserved, Should be zero */ 1389 if (data & 0x30) 1390 return 1; 1391 1392 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) && 1393 (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT)) 1394 return 1; 1395 1396 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) && 1397 (data & KVM_ASYNC_PF_DELIVERY_AS_INT)) 1398 return 1; 1399 1400 if (!lapic_in_kernel(vcpu)) 1401 return data ? 1 : 0; 1402 1403 if (__kvm_pv_async_pf_enabled(data) && 1404 kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, 1405 sizeof(u64))) 1406 return 1; 1407 1408 vcpu->arch.apf.msr_en_val = data; 1409 1410 if (__kvm_pv_async_pf_enabled(data)) { 1411 kvm_async_pf_wakeup_all(vcpu); 1412 } else { 1413 kvm_clear_async_pf_completion_queue(vcpu); 1414 kvm_async_pf_hash_reset(vcpu); 1415 } 1416 return 0; 1417 } 1418 1419 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) 1420 { 1421 /* Bits 8-63 are reserved */ 1422 if (data >> 8) 1423 return 1; 1424 1425 if (!lapic_in_kernel(vcpu)) 1426 return 1; 1427 1428 vcpu->arch.apf.msr_int_val = data; 1429 1430 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK; 1431 1432 return 0; 1433 } 1434 1435 #ifdef CONFIG_X86_64 1436 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) 1437 { 1438 return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC; 1439 } 1440 #endif 1441 1442 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1443 { 1444 u32 msr = msr_info->index; 1445 u64 data = msr_info->data; 1446 1447 /* 1448 * Do not allow host-initiated writes to trigger the Xen hypercall 1449 * page setup; it could incur locking paths which are not expected 1450 * if userspace sets the MSR in an unusual location. 1451 */ 1452 if (kvm_xen_is_hypercall_page_msr(vcpu->kvm, msr) && 1453 !msr_info->host_initiated) 1454 return kvm_xen_write_hypercall_page(vcpu, data); 1455 1456 switch (msr) { 1457 case MSR_AMD64_NB_CFG: 1458 case MSR_IA32_UCODE_WRITE: 1459 case MSR_VM_HSAVE_PA: 1460 case MSR_AMD64_PATCH_LOADER: 1461 case MSR_AMD64_BU_CFG2: 1462 case MSR_AMD64_DC_CFG: 1463 case MSR_AMD64_TW_CFG: 1464 case MSR_F15H_EX_CFG: 1465 break; 1466 1467 case MSR_IA32_UCODE_REV: 1468 if (msr_info->host_initiated) 1469 vcpu->arch.microcode_version = data; 1470 break; 1471 case MSR_IA32_ARCH_CAPABILITIES: 1472 if (!msr_info->host_initiated || 1473 !guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) 1474 return KVM_MSR_RET_UNSUPPORTED; 1475 vcpu->arch.arch_capabilities = data; 1476 break; 1477 case MSR_IA32_PERF_CAPABILITIES: 1478 if (!msr_info->host_initiated || 1479 !guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) 1480 return KVM_MSR_RET_UNSUPPORTED; 1481 1482 if (data & ~kvm_caps.supported_perf_cap) 1483 return 1; 1484 1485 /* 1486 * Note, this is not just a performance optimization! KVM 1487 * disallows changing feature MSRs after the vCPU has run; PMU 1488 * refresh will bug the VM if called after the vCPU has run. 1489 */ 1490 if (vcpu->arch.perf_capabilities == data) 1491 break; 1492 1493 vcpu->arch.perf_capabilities = data; 1494 kvm_pmu_refresh(vcpu); 1495 kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu); 1496 break; 1497 case MSR_IA32_PRED_CMD: { 1498 u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB); 1499 1500 if (!msr_info->host_initiated) { 1501 if ((!guest_has_pred_cmd_msr(vcpu))) 1502 return 1; 1503 1504 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SPEC_CTRL) && 1505 !guest_cpu_cap_has(vcpu, X86_FEATURE_AMD_IBPB)) 1506 reserved_bits |= PRED_CMD_IBPB; 1507 1508 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SBPB)) 1509 reserved_bits |= PRED_CMD_SBPB; 1510 } 1511 1512 if (!boot_cpu_has(X86_FEATURE_IBPB)) 1513 reserved_bits |= PRED_CMD_IBPB; 1514 1515 if (!boot_cpu_has(X86_FEATURE_SBPB)) 1516 reserved_bits |= PRED_CMD_SBPB; 1517 1518 if (data & reserved_bits) 1519 return 1; 1520 1521 if (!data) 1522 break; 1523 1524 wrmsrq(MSR_IA32_PRED_CMD, data); 1525 break; 1526 } 1527 case MSR_IA32_FLUSH_CMD: 1528 if (!msr_info->host_initiated && 1529 !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D)) 1530 return 1; 1531 1532 if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D) || (data & ~L1D_FLUSH)) 1533 return 1; 1534 if (!data) 1535 break; 1536 1537 wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 1538 break; 1539 case MSR_EFER: 1540 return set_efer(vcpu, msr_info); 1541 case MSR_K7_HWCR: { 1542 /* 1543 * Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2 1544 * through at least v6.6 whine if TscFreqSel is clear, 1545 * depending on F/M/S. 1546 */ 1547 u64 valid = BIT_ULL(18) | BIT_ULL(24); 1548 1549 data &= ~(u64)0x40; /* ignore flush filter disable */ 1550 data &= ~(u64)0x100; /* ignore ignne emulation enable */ 1551 data &= ~(u64)0x8; /* ignore TLB cache disable */ 1552 1553 if (guest_cpu_cap_has(vcpu, X86_FEATURE_GP_ON_USER_CPUID)) 1554 valid |= MSR_K7_HWCR_CPUID_USER_DIS; 1555 1556 if (data & ~valid) { 1557 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1558 return 1; 1559 } 1560 vcpu->arch.msr_hwcr = data; 1561 break; 1562 } 1563 case MSR_FAM10H_MMIO_CONF_BASE: 1564 if (data != 0) { 1565 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1566 return 1; 1567 } 1568 break; 1569 case MSR_IA32_CR_PAT: 1570 if (!kvm_pat_valid(data)) 1571 return 1; 1572 1573 vcpu->arch.pat = data; 1574 break; 1575 case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: 1576 case MSR_MTRRdefType: 1577 return kvm_mtrr_set_msr(vcpu, msr, data); 1578 case MSR_IA32_APICBASE: 1579 return kvm_apic_set_base(vcpu, data, msr_info->host_initiated); 1580 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: 1581 return kvm_x2apic_msr_write(vcpu, msr, data); 1582 case MSR_IA32_TSC_DEADLINE: 1583 kvm_set_lapic_tscdeadline_msr(vcpu, data); 1584 break; 1585 case MSR_IA32_TSC_ADJUST: 1586 if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSC_ADJUST)) { 1587 if (!msr_info->host_initiated) { 1588 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr; 1589 adjust_tsc_offset_guest(vcpu, adj); 1590 /* Before back to guest, tsc_timestamp must be adjusted 1591 * as well, otherwise guest's percpu pvclock time could jump. 1592 */ 1593 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1594 } 1595 vcpu->arch.ia32_tsc_adjust_msr = data; 1596 } 1597 break; 1598 case MSR_IA32_MISC_ENABLE: { 1599 u64 old_val = vcpu->arch.ia32_misc_enable_msr; 1600 1601 if (!msr_info->host_initiated) { 1602 /* RO bits */ 1603 if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK) 1604 return 1; 1605 1606 /* R bits, i.e. writes are ignored, but don't fault. */ 1607 data = data & ~MSR_IA32_MISC_ENABLE_EMON; 1608 data |= old_val & MSR_IA32_MISC_ENABLE_EMON; 1609 } 1610 1611 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) && 1612 ((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) { 1613 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XMM3)) 1614 return 1; 1615 vcpu->arch.ia32_misc_enable_msr = data; 1616 vcpu->arch.cpuid_dynamic_bits_dirty = true; 1617 } else { 1618 vcpu->arch.ia32_misc_enable_msr = data; 1619 } 1620 break; 1621 } 1622 case MSR_IA32_SMBASE: 1623 if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) 1624 return 1; 1625 vcpu->arch.smbase = data; 1626 break; 1627 case MSR_IA32_POWER_CTL: 1628 vcpu->arch.msr_ia32_power_ctl = data; 1629 break; 1630 case MSR_IA32_TSC: 1631 if (msr_info->host_initiated) { 1632 kvm_synchronize_tsc(vcpu, &data); 1633 } else if (!vcpu->arch.guest_tsc_protected) { 1634 u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; 1635 adjust_tsc_offset_guest(vcpu, adj); 1636 vcpu->arch.ia32_tsc_adjust_msr += adj; 1637 } 1638 break; 1639 case MSR_IA32_XSS: 1640 if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) 1641 return KVM_MSR_RET_UNSUPPORTED; 1642 1643 if (data & ~vcpu->arch.guest_supported_xss) 1644 return 1; 1645 if (vcpu->arch.ia32_xss == data) 1646 break; 1647 vcpu->arch.ia32_xss = data; 1648 vcpu->arch.cpuid_dynamic_bits_dirty = true; 1649 break; 1650 case MSR_SMI_COUNT: 1651 if (!msr_info->host_initiated) 1652 return 1; 1653 vcpu->arch.smi_count = data; 1654 break; 1655 case MSR_KVM_WALL_CLOCK_NEW: 1656 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 1657 return KVM_MSR_RET_UNSUPPORTED; 1658 1659 vcpu->kvm->arch.wall_clock = data; 1660 kvm_write_wall_clock(vcpu->kvm, data, 0); 1661 break; 1662 case MSR_KVM_WALL_CLOCK: 1663 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 1664 return KVM_MSR_RET_UNSUPPORTED; 1665 1666 vcpu->kvm->arch.wall_clock = data; 1667 kvm_write_wall_clock(vcpu->kvm, data, 0); 1668 break; 1669 case MSR_KVM_SYSTEM_TIME_NEW: 1670 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 1671 return KVM_MSR_RET_UNSUPPORTED; 1672 1673 kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); 1674 break; 1675 case MSR_KVM_SYSTEM_TIME: 1676 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 1677 return KVM_MSR_RET_UNSUPPORTED; 1678 1679 kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); 1680 break; 1681 case MSR_KVM_ASYNC_PF_EN: 1682 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) 1683 return KVM_MSR_RET_UNSUPPORTED; 1684 1685 if (kvm_pv_enable_async_pf(vcpu, data)) 1686 return 1; 1687 break; 1688 case MSR_KVM_ASYNC_PF_INT: 1689 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 1690 return KVM_MSR_RET_UNSUPPORTED; 1691 1692 if (kvm_pv_enable_async_pf_int(vcpu, data)) 1693 return 1; 1694 break; 1695 case MSR_KVM_ASYNC_PF_ACK: 1696 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 1697 return KVM_MSR_RET_UNSUPPORTED; 1698 if (data & 0x1) { 1699 /* 1700 * Pairs with the smp_mb__after_atomic() in 1701 * kvm_arch_async_page_present_queued(). 1702 */ 1703 smp_store_mb(vcpu->arch.apf.pageready_pending, false); 1704 1705 kvm_check_async_pf_completion(vcpu); 1706 } 1707 break; 1708 case MSR_KVM_STEAL_TIME: 1709 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) 1710 return KVM_MSR_RET_UNSUPPORTED; 1711 1712 if (unlikely(!sched_info_on())) 1713 return 1; 1714 1715 if (data & KVM_STEAL_RESERVED_MASK) 1716 return 1; 1717 1718 vcpu->arch.st.msr_val = data; 1719 1720 if (!(data & KVM_MSR_ENABLED)) 1721 break; 1722 1723 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 1724 1725 break; 1726 case MSR_KVM_PV_EOI_EN: 1727 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) 1728 return KVM_MSR_RET_UNSUPPORTED; 1729 1730 if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) 1731 return 1; 1732 break; 1733 1734 case MSR_KVM_POLL_CONTROL: 1735 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) 1736 return KVM_MSR_RET_UNSUPPORTED; 1737 1738 /* only enable bit supported */ 1739 if (data & (-1ULL << 1)) 1740 return 1; 1741 1742 vcpu->arch.msr_kvm_poll_control = data; 1743 break; 1744 1745 case MSR_IA32_MCG_CTL: 1746 case MSR_IA32_MCG_STATUS: 1747 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 1748 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 1749 return set_msr_mce(vcpu, msr_info); 1750 1751 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: 1752 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: 1753 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: 1754 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: 1755 if (kvm_pmu_is_valid_msr(vcpu, msr)) 1756 return kvm_pmu_set_msr(vcpu, msr_info); 1757 1758 if (data) 1759 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1760 break; 1761 case MSR_K7_CLK_CTL: 1762 /* 1763 * Ignore all writes to this no longer documented MSR. 1764 * Writes are only relevant for old K7 processors, 1765 * all pre-dating SVM, but a recommended workaround from 1766 * AMD for these chips. It is possible to specify the 1767 * affected processor models on the command line, hence 1768 * the need to ignore the workaround. 1769 */ 1770 break; 1771 #ifdef CONFIG_KVM_HYPERV 1772 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 1773 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1774 case HV_X64_MSR_SYNDBG_OPTIONS: 1775 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1776 case HV_X64_MSR_CRASH_CTL: 1777 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: 1778 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1779 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1780 case HV_X64_MSR_TSC_EMULATION_STATUS: 1781 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1782 return kvm_hv_set_msr_common(vcpu, msr, data, 1783 msr_info->host_initiated); 1784 #endif 1785 case MSR_IA32_BBL_CR_CTL3: 1786 /* Drop writes to this legacy MSR -- see rdmsr 1787 * counterpart for further detail. 1788 */ 1789 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1790 break; 1791 case MSR_AMD64_OSVW_ID_LENGTH: 1792 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) 1793 return 1; 1794 vcpu->arch.osvw.length = data; 1795 break; 1796 case MSR_AMD64_OSVW_STATUS: 1797 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) 1798 return 1; 1799 vcpu->arch.osvw.status = data; 1800 break; 1801 case MSR_PLATFORM_INFO: 1802 if (!msr_info->host_initiated) 1803 return 1; 1804 vcpu->arch.msr_platform_info = data; 1805 break; 1806 case MSR_MISC_FEATURES_ENABLES: 1807 if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT || 1808 (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT && 1809 !(vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT))) 1810 return 1; 1811 vcpu->arch.msr_misc_features_enables = data; 1812 break; 1813 #ifdef CONFIG_X86_64 1814 case MSR_IA32_XFD: 1815 if (!msr_info->host_initiated && 1816 !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) 1817 return 1; 1818 1819 if (data & ~kvm_guest_supported_xfd(vcpu)) 1820 return 1; 1821 1822 fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data); 1823 break; 1824 case MSR_IA32_XFD_ERR: 1825 if (!msr_info->host_initiated && 1826 !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) 1827 return 1; 1828 1829 if (data & ~kvm_guest_supported_xfd(vcpu)) 1830 return 1; 1831 1832 vcpu->arch.guest_fpu.xfd_err = data; 1833 break; 1834 #endif 1835 case MSR_IA32_U_CET: 1836 case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: 1837 kvm_set_xstate_msr(vcpu, msr_info); 1838 break; 1839 default: 1840 if (kvm_pmu_is_valid_msr(vcpu, msr)) 1841 return kvm_pmu_set_msr(vcpu, msr_info); 1842 1843 return KVM_MSR_RET_UNSUPPORTED; 1844 } 1845 return 0; 1846 } 1847 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_msr_common); 1848 1849 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 1850 { 1851 u64 data; 1852 u64 mcg_cap = vcpu->arch.mcg_cap; 1853 unsigned bank_num = mcg_cap & 0xff; 1854 u32 offset, last_msr; 1855 1856 switch (msr) { 1857 case MSR_IA32_P5_MC_ADDR: 1858 case MSR_IA32_P5_MC_TYPE: 1859 data = 0; 1860 break; 1861 case MSR_IA32_MCG_CAP: 1862 data = vcpu->arch.mcg_cap; 1863 break; 1864 case MSR_IA32_MCG_CTL: 1865 if (!(mcg_cap & MCG_CTL_P) && !host) 1866 return 1; 1867 data = vcpu->arch.mcg_ctl; 1868 break; 1869 case MSR_IA32_MCG_STATUS: 1870 data = vcpu->arch.mcg_status; 1871 break; 1872 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 1873 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; 1874 if (msr > last_msr) 1875 return 1; 1876 1877 if (!(mcg_cap & MCG_CMCI_P) && !host) 1878 return 1; 1879 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, 1880 last_msr + 1 - MSR_IA32_MC0_CTL2); 1881 data = vcpu->arch.mci_ctl2_banks[offset]; 1882 break; 1883 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 1884 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; 1885 if (msr > last_msr) 1886 return 1; 1887 1888 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, 1889 last_msr + 1 - MSR_IA32_MC0_CTL); 1890 data = vcpu->arch.mce_banks[offset]; 1891 break; 1892 default: 1893 return 1; 1894 } 1895 *pdata = data; 1896 return 0; 1897 } 1898 1899 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1900 { 1901 switch (msr_info->index) { 1902 case MSR_IA32_PLATFORM_ID: 1903 case MSR_IA32_EBL_CR_POWERON: 1904 case MSR_IA32_LASTBRANCHFROMIP: 1905 case MSR_IA32_LASTBRANCHTOIP: 1906 case MSR_IA32_LASTINTFROMIP: 1907 case MSR_IA32_LASTINTTOIP: 1908 case MSR_AMD64_SYSCFG: 1909 case MSR_K8_TSEG_ADDR: 1910 case MSR_K8_TSEG_MASK: 1911 case MSR_VM_HSAVE_PA: 1912 case MSR_K8_INT_PENDING_MSG: 1913 case MSR_AMD64_NB_CFG: 1914 case MSR_FAM10H_MMIO_CONF_BASE: 1915 case MSR_AMD64_BU_CFG2: 1916 case MSR_IA32_PERF_CTL: 1917 case MSR_AMD64_DC_CFG: 1918 case MSR_AMD64_TW_CFG: 1919 case MSR_F15H_EX_CFG: 1920 /* 1921 * Intel Sandy Bridge CPUs must support the RAPL (running average power 1922 * limit) MSRs. Just return 0, as we do not want to expose the host 1923 * data here. Do not conditionalize this on CPUID, as KVM does not do 1924 * so for existing CPU-specific MSRs. 1925 */ 1926 case MSR_RAPL_POWER_UNIT: 1927 case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */ 1928 case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */ 1929 case MSR_PKG_ENERGY_STATUS: /* Total package */ 1930 case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */ 1931 msr_info->data = 0; 1932 break; 1933 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: 1934 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: 1935 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: 1936 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: 1937 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) 1938 return kvm_pmu_get_msr(vcpu, msr_info); 1939 msr_info->data = 0; 1940 break; 1941 case MSR_IA32_UCODE_REV: 1942 msr_info->data = vcpu->arch.microcode_version; 1943 break; 1944 case MSR_IA32_ARCH_CAPABILITIES: 1945 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) 1946 return KVM_MSR_RET_UNSUPPORTED; 1947 msr_info->data = vcpu->arch.arch_capabilities; 1948 break; 1949 case MSR_IA32_PERF_CAPABILITIES: 1950 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) 1951 return KVM_MSR_RET_UNSUPPORTED; 1952 msr_info->data = vcpu->arch.perf_capabilities; 1953 break; 1954 case MSR_IA32_POWER_CTL: 1955 msr_info->data = vcpu->arch.msr_ia32_power_ctl; 1956 break; 1957 case MSR_IA32_TSC: { 1958 /* 1959 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset 1960 * even when not intercepted. AMD manual doesn't explicitly 1961 * state this but appears to behave the same. 1962 * 1963 * On userspace reads and writes, however, we unconditionally 1964 * return L1's TSC value to ensure backwards-compatible 1965 * behavior for migration. 1966 */ 1967 u64 offset, ratio; 1968 1969 if (msr_info->host_initiated) { 1970 offset = vcpu->arch.l1_tsc_offset; 1971 ratio = vcpu->arch.l1_tsc_scaling_ratio; 1972 } else { 1973 offset = vcpu->arch.tsc_offset; 1974 ratio = vcpu->arch.tsc_scaling_ratio; 1975 } 1976 1977 msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset; 1978 break; 1979 } 1980 case MSR_IA32_CR_PAT: 1981 msr_info->data = vcpu->arch.pat; 1982 break; 1983 case MSR_MTRRcap: 1984 case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: 1985 case MSR_MTRRdefType: 1986 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data); 1987 case 0xcd: /* fsb frequency */ 1988 msr_info->data = 3; 1989 break; 1990 /* 1991 * MSR_EBC_FREQUENCY_ID 1992 * Conservative value valid for even the basic CPU models. 1993 * Models 0,1: 000 in bits 23:21 indicating a bus speed of 1994 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz, 1995 * and 266MHz for model 3, or 4. Set Core Clock 1996 * Frequency to System Bus Frequency Ratio to 1 (bits 1997 * 31:24) even though these are only valid for CPU 1998 * models > 2, however guests may end up dividing or 1999 * multiplying by zero otherwise. 2000 */ 2001 case MSR_EBC_FREQUENCY_ID: 2002 msr_info->data = 1 << 24; 2003 break; 2004 case MSR_IA32_APICBASE: 2005 msr_info->data = vcpu->arch.apic_base; 2006 break; 2007 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: 2008 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data); 2009 case MSR_IA32_TSC_DEADLINE: 2010 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu); 2011 break; 2012 case MSR_IA32_TSC_ADJUST: 2013 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr; 2014 break; 2015 case MSR_IA32_MISC_ENABLE: 2016 msr_info->data = vcpu->arch.ia32_misc_enable_msr; 2017 break; 2018 case MSR_IA32_SMBASE: 2019 if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) 2020 return 1; 2021 msr_info->data = vcpu->arch.smbase; 2022 break; 2023 case MSR_SMI_COUNT: 2024 msr_info->data = vcpu->arch.smi_count; 2025 break; 2026 case MSR_IA32_PERF_STATUS: 2027 /* TSC increment by tick */ 2028 msr_info->data = 1000ULL; 2029 /* CPU multiplier */ 2030 msr_info->data |= (((uint64_t)4ULL) << 40); 2031 break; 2032 case MSR_EFER: 2033 msr_info->data = vcpu->arch.efer; 2034 break; 2035 case MSR_KVM_WALL_CLOCK: 2036 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 2037 return KVM_MSR_RET_UNSUPPORTED; 2038 2039 msr_info->data = vcpu->kvm->arch.wall_clock; 2040 break; 2041 case MSR_KVM_WALL_CLOCK_NEW: 2042 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 2043 return KVM_MSR_RET_UNSUPPORTED; 2044 2045 msr_info->data = vcpu->kvm->arch.wall_clock; 2046 break; 2047 case MSR_KVM_SYSTEM_TIME: 2048 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 2049 return KVM_MSR_RET_UNSUPPORTED; 2050 2051 msr_info->data = vcpu->arch.time; 2052 break; 2053 case MSR_KVM_SYSTEM_TIME_NEW: 2054 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 2055 return KVM_MSR_RET_UNSUPPORTED; 2056 2057 msr_info->data = vcpu->arch.time; 2058 break; 2059 case MSR_KVM_ASYNC_PF_EN: 2060 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) 2061 return KVM_MSR_RET_UNSUPPORTED; 2062 2063 msr_info->data = vcpu->arch.apf.msr_en_val; 2064 break; 2065 case MSR_KVM_ASYNC_PF_INT: 2066 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 2067 return KVM_MSR_RET_UNSUPPORTED; 2068 2069 msr_info->data = vcpu->arch.apf.msr_int_val; 2070 break; 2071 case MSR_KVM_ASYNC_PF_ACK: 2072 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 2073 return KVM_MSR_RET_UNSUPPORTED; 2074 2075 msr_info->data = 0; 2076 break; 2077 case MSR_KVM_STEAL_TIME: 2078 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) 2079 return KVM_MSR_RET_UNSUPPORTED; 2080 2081 msr_info->data = vcpu->arch.st.msr_val; 2082 break; 2083 case MSR_KVM_PV_EOI_EN: 2084 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) 2085 return KVM_MSR_RET_UNSUPPORTED; 2086 2087 msr_info->data = vcpu->arch.pv_eoi.msr_val; 2088 break; 2089 case MSR_KVM_POLL_CONTROL: 2090 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) 2091 return KVM_MSR_RET_UNSUPPORTED; 2092 2093 msr_info->data = vcpu->arch.msr_kvm_poll_control; 2094 break; 2095 case MSR_IA32_P5_MC_ADDR: 2096 case MSR_IA32_P5_MC_TYPE: 2097 case MSR_IA32_MCG_CAP: 2098 case MSR_IA32_MCG_CTL: 2099 case MSR_IA32_MCG_STATUS: 2100 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 2101 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 2102 return get_msr_mce(vcpu, msr_info->index, &msr_info->data, 2103 msr_info->host_initiated); 2104 case MSR_IA32_XSS: 2105 if (!msr_info->host_initiated && 2106 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) 2107 return 1; 2108 msr_info->data = vcpu->arch.ia32_xss; 2109 break; 2110 case MSR_K7_CLK_CTL: 2111 /* 2112 * Provide expected ramp-up count for K7. All other 2113 * are set to zero, indicating minimum divisors for 2114 * every field. 2115 * 2116 * This prevents guest kernels on AMD host with CPU 2117 * type 6, model 8 and higher from exploding due to 2118 * the rdmsr failing. 2119 */ 2120 msr_info->data = 0x20000000; 2121 break; 2122 #ifdef CONFIG_KVM_HYPERV 2123 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 2124 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 2125 case HV_X64_MSR_SYNDBG_OPTIONS: 2126 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 2127 case HV_X64_MSR_CRASH_CTL: 2128 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: 2129 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 2130 case HV_X64_MSR_TSC_EMULATION_CONTROL: 2131 case HV_X64_MSR_TSC_EMULATION_STATUS: 2132 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 2133 return kvm_hv_get_msr_common(vcpu, 2134 msr_info->index, &msr_info->data, 2135 msr_info->host_initiated); 2136 #endif 2137 case MSR_IA32_BBL_CR_CTL3: 2138 /* This legacy MSR exists but isn't fully documented in current 2139 * silicon. It is however accessed by winxp in very narrow 2140 * scenarios where it sets bit #19, itself documented as 2141 * a "reserved" bit. Best effort attempt to source coherent 2142 * read data here should the balance of the register be 2143 * interpreted by the guest: 2144 * 2145 * L2 cache control register 3: 64GB range, 256KB size, 2146 * enabled, latency 0x1, configured 2147 */ 2148 msr_info->data = 0xbe702111; 2149 break; 2150 case MSR_AMD64_OSVW_ID_LENGTH: 2151 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) 2152 return 1; 2153 msr_info->data = vcpu->arch.osvw.length; 2154 break; 2155 case MSR_AMD64_OSVW_STATUS: 2156 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) 2157 return 1; 2158 msr_info->data = vcpu->arch.osvw.status; 2159 break; 2160 case MSR_PLATFORM_INFO: 2161 if (!msr_info->host_initiated && 2162 !vcpu->kvm->arch.guest_can_read_msr_platform_info) 2163 return 1; 2164 msr_info->data = vcpu->arch.msr_platform_info; 2165 break; 2166 case MSR_MISC_FEATURES_ENABLES: 2167 msr_info->data = vcpu->arch.msr_misc_features_enables; 2168 break; 2169 case MSR_K7_HWCR: 2170 msr_info->data = vcpu->arch.msr_hwcr; 2171 break; 2172 #ifdef CONFIG_X86_64 2173 case MSR_IA32_XFD: 2174 if (!msr_info->host_initiated && 2175 !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) 2176 return 1; 2177 2178 msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd; 2179 break; 2180 case MSR_IA32_XFD_ERR: 2181 if (!msr_info->host_initiated && 2182 !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) 2183 return 1; 2184 2185 msr_info->data = vcpu->arch.guest_fpu.xfd_err; 2186 break; 2187 #endif 2188 case MSR_IA32_U_CET: 2189 case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: 2190 kvm_get_xstate_msr(vcpu, msr_info); 2191 break; 2192 default: 2193 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) 2194 return kvm_pmu_get_msr(vcpu, msr_info); 2195 2196 return KVM_MSR_RET_UNSUPPORTED; 2197 } 2198 return 0; 2199 } 2200 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_msr_common); 2201 2202 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 2203 { 2204 return kvm_get_msr_ignored_check(vcpu, index, data, true); 2205 } 2206 2207 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 2208 { 2209 u64 val; 2210 2211 /* 2212 * Reject writes to immutable feature MSRs if the vCPU model is frozen, 2213 * as KVM doesn't support modifying the guest vCPU model on the fly, 2214 * e.g. changing the VMX capabilities MSRs while L2 is active is 2215 * nonsensical. Allow writes of the same value, e.g. so that userspace 2216 * can blindly stuff all MSRs when emulating RESET. 2217 */ 2218 if (!kvm_can_set_cpuid_and_feature_msrs(vcpu) && 2219 kvm_is_immutable_feature_msr(index) && 2220 (do_get_msr(vcpu, index, &val) || *data != val)) 2221 return -EINVAL; 2222 2223 return kvm_set_msr_ignored_check(vcpu, index, *data, true); 2224 } 2225 2226 /* 2227 * Read or write a bunch of msrs. All parameters are kernel addresses. 2228 * 2229 * @return number of msrs set successfully. 2230 */ 2231 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, 2232 struct kvm_msr_entry *entries, 2233 int (*do_msr)(struct kvm_vcpu *vcpu, 2234 unsigned index, u64 *data)) 2235 { 2236 bool fpu_loaded = false; 2237 int i; 2238 2239 for (i = 0; i < msrs->nmsrs; ++i) { 2240 /* 2241 * If userspace is accessing one or more XSTATE-managed MSRs, 2242 * temporarily load the guest's FPU state so that the guest's 2243 * MSR value(s) is resident in hardware and thus can be accessed 2244 * via RDMSR/WRMSR. 2245 */ 2246 if (!fpu_loaded && is_xstate_managed_msr(vcpu, entries[i].index)) { 2247 kvm_load_guest_fpu(vcpu); 2248 fpu_loaded = true; 2249 } 2250 if (do_msr(vcpu, entries[i].index, &entries[i].data)) 2251 break; 2252 } 2253 if (fpu_loaded) 2254 kvm_put_guest_fpu(vcpu); 2255 2256 return i; 2257 } 2258 2259 /* 2260 * Read or write a bunch of msrs. Parameters are user addresses. 2261 * 2262 * @return number of msrs set successfully. 2263 */ 2264 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, 2265 int (*do_msr)(struct kvm_vcpu *vcpu, 2266 unsigned index, u64 *data), 2267 int writeback) 2268 { 2269 struct kvm_msrs msrs; 2270 struct kvm_msr_entry *entries; 2271 unsigned size; 2272 int r; 2273 2274 r = -EFAULT; 2275 if (copy_from_user(&msrs, user_msrs, sizeof(msrs))) 2276 goto out; 2277 2278 r = -E2BIG; 2279 if (msrs.nmsrs >= MAX_IO_MSRS) 2280 goto out; 2281 2282 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; 2283 entries = memdup_user(user_msrs->entries, size); 2284 if (IS_ERR(entries)) { 2285 r = PTR_ERR(entries); 2286 goto out; 2287 } 2288 2289 r = __msr_io(vcpu, &msrs, entries, do_msr); 2290 2291 if (writeback && copy_to_user(user_msrs->entries, entries, size)) 2292 r = -EFAULT; 2293 2294 kfree(entries); 2295 out: 2296 return r; 2297 } 2298 2299 int kvm_get_feature_msrs(struct kvm_msrs __user *user_msrs) 2300 { 2301 return msr_io(NULL, user_msrs, do_get_feature_msr, 1); 2302 } 2303 2304 int kvm_get_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs) 2305 { 2306 guard(srcu)(&vcpu->kvm->srcu); 2307 2308 return msr_io(vcpu, user_msrs, do_get_msr, 1); 2309 } 2310 2311 int kvm_set_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs) 2312 { 2313 guard(srcu)(&vcpu->kvm->srcu); 2314 2315 return msr_io(vcpu, user_msrs, do_set_msr, 0); 2316 } 2317 2318 static int kvm_get_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val) 2319 { 2320 u64 val; 2321 2322 if (do_get_msr(vcpu, msr, &val)) 2323 return -EINVAL; 2324 2325 if (put_user(val, user_val)) 2326 return -EFAULT; 2327 2328 return 0; 2329 } 2330 2331 static int kvm_set_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val) 2332 { 2333 u64 val; 2334 2335 if (get_user(val, user_val)) 2336 return -EFAULT; 2337 2338 if (do_set_msr(vcpu, msr, &val)) 2339 return -EINVAL; 2340 2341 return 0; 2342 } 2343 2344 struct kvm_x86_reg_id { 2345 __u32 index; 2346 __u8 type; 2347 __u8 rsvd1; 2348 __u8 rsvd2:4; 2349 __u8 size:4; 2350 __u8 x86; 2351 }; 2352 2353 static int kvm_translate_kvm_reg(struct kvm_vcpu *vcpu, 2354 struct kvm_x86_reg_id *reg) 2355 { 2356 switch (reg->index) { 2357 case KVM_REG_GUEST_SSP: 2358 /* 2359 * FIXME: If host-initiated accesses are ever exempted from 2360 * ignore_msrs (in kvm_do_msr_access()), drop this manual check 2361 * and rely on KVM's standard checks to reject accesses to regs 2362 * that don't exist. 2363 */ 2364 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) 2365 return -EINVAL; 2366 2367 reg->type = KVM_X86_REG_TYPE_MSR; 2368 reg->index = MSR_KVM_INTERNAL_GUEST_SSP; 2369 break; 2370 default: 2371 return -EINVAL; 2372 } 2373 return 0; 2374 } 2375 2376 int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, 2377 void __user *argp) 2378 { 2379 struct kvm_one_reg one_reg; 2380 struct kvm_x86_reg_id *reg; 2381 u64 __user *user_val; 2382 bool load_fpu; 2383 int r; 2384 2385 if (copy_from_user(&one_reg, argp, sizeof(one_reg))) 2386 return -EFAULT; 2387 2388 if ((one_reg.id & KVM_REG_ARCH_MASK) != KVM_REG_X86) 2389 return -EINVAL; 2390 2391 reg = (struct kvm_x86_reg_id *)&one_reg.id; 2392 if (reg->rsvd1 || reg->rsvd2) 2393 return -EINVAL; 2394 2395 if (reg->type == KVM_X86_REG_TYPE_KVM) { 2396 r = kvm_translate_kvm_reg(vcpu, reg); 2397 if (r) 2398 return r; 2399 } 2400 2401 if (reg->type != KVM_X86_REG_TYPE_MSR) 2402 return -EINVAL; 2403 2404 if ((one_reg.id & KVM_REG_SIZE_MASK) != KVM_REG_SIZE_U64) 2405 return -EINVAL; 2406 2407 guard(srcu)(&vcpu->kvm->srcu); 2408 2409 load_fpu = is_xstate_managed_msr(vcpu, reg->index); 2410 if (load_fpu) 2411 kvm_load_guest_fpu(vcpu); 2412 2413 user_val = u64_to_user_ptr(one_reg.addr); 2414 if (ioctl == KVM_GET_ONE_REG) 2415 r = kvm_get_one_msr(vcpu, reg->index, user_val); 2416 else 2417 r = kvm_set_one_msr(vcpu, reg->index, user_val); 2418 2419 if (load_fpu) 2420 kvm_put_guest_fpu(vcpu); 2421 return r; 2422 } 2423 2424 int kvm_get_reg_list(struct kvm_vcpu *vcpu, 2425 struct kvm_reg_list __user *user_list) 2426 { 2427 u64 nr_regs = guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ? 1 : 0; 2428 u64 user_nr_regs; 2429 2430 if (get_user(user_nr_regs, &user_list->n)) 2431 return -EFAULT; 2432 2433 if (put_user(nr_regs, &user_list->n)) 2434 return -EFAULT; 2435 2436 if (user_nr_regs < nr_regs) 2437 return -E2BIG; 2438 2439 if (nr_regs && 2440 put_user(KVM_X86_REG_KVM(KVM_REG_GUEST_SSP), &user_list->reg[0])) 2441 return -EFAULT; 2442 2443 return 0; 2444 } 2445 2446 static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow) 2447 { 2448 struct kvm_x86_msr_filter *msr_filter; 2449 2450 msr_filter = kzalloc_obj(*msr_filter, GFP_KERNEL_ACCOUNT); 2451 if (!msr_filter) 2452 return NULL; 2453 2454 msr_filter->default_allow = default_allow; 2455 return msr_filter; 2456 } 2457 2458 void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter) 2459 { 2460 u32 i; 2461 2462 if (!msr_filter) 2463 return; 2464 2465 for (i = 0; i < msr_filter->count; i++) 2466 kfree(msr_filter->ranges[i].bitmap); 2467 2468 kfree(msr_filter); 2469 } 2470 2471 static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter, 2472 struct kvm_msr_filter_range *user_range) 2473 { 2474 unsigned long *bitmap; 2475 size_t bitmap_size; 2476 2477 if (!user_range->nmsrs) 2478 return 0; 2479 2480 if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK) 2481 return -EINVAL; 2482 2483 if (!user_range->flags) 2484 return -EINVAL; 2485 2486 bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long); 2487 if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE) 2488 return -EINVAL; 2489 2490 bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size); 2491 if (IS_ERR(bitmap)) 2492 return PTR_ERR(bitmap); 2493 2494 msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) { 2495 .flags = user_range->flags, 2496 .base = user_range->base, 2497 .nmsrs = user_range->nmsrs, 2498 .bitmap = bitmap, 2499 }; 2500 2501 msr_filter->count++; 2502 return 0; 2503 } 2504 2505 int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, struct kvm_msr_filter *filter) 2506 { 2507 struct kvm_x86_msr_filter *new_filter, *old_filter; 2508 bool default_allow; 2509 bool empty = true; 2510 int r; 2511 u32 i; 2512 2513 if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK) 2514 return -EINVAL; 2515 2516 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) 2517 empty &= !filter->ranges[i].nmsrs; 2518 2519 default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY); 2520 if (empty && !default_allow) 2521 return -EINVAL; 2522 2523 new_filter = kvm_alloc_msr_filter(default_allow); 2524 if (!new_filter) 2525 return -ENOMEM; 2526 2527 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) { 2528 r = kvm_add_msr_filter(new_filter, &filter->ranges[i]); 2529 if (r) { 2530 kvm_free_msr_filter(new_filter); 2531 return r; 2532 } 2533 } 2534 2535 mutex_lock(&kvm->lock); 2536 old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter, 2537 mutex_is_locked(&kvm->lock)); 2538 mutex_unlock(&kvm->lock); 2539 synchronize_srcu(&kvm->srcu); 2540 2541 kvm_free_msr_filter(old_filter); 2542 2543 /* 2544 * Recalc MSR intercepts as userspace may want to intercept accesses to 2545 * MSRs that KVM would otherwise pass through to the guest. 2546 */ 2547 kvm_make_all_cpus_request(kvm, KVM_REQ_RECALC_INTERCEPTS); 2548 2549 return 0; 2550 } 2551 2552 2553 static void kvm_probe_feature_msr(u32 msr_index) 2554 { 2555 u64 data; 2556 2557 if (kvm_get_feature_msr(NULL, msr_index, &data, true)) 2558 return; 2559 2560 msr_based_features[num_msr_based_features++] = msr_index; 2561 } 2562 2563 static void kvm_probe_msr_to_save(u32 msr_index) 2564 { 2565 u64 dummy; 2566 2567 if (rdmsrq_safe(msr_index, &dummy)) 2568 return; 2569 2570 /* 2571 * Even MSRs that are valid in the host may not be exposed to guests in 2572 * some cases. 2573 */ 2574 switch (msr_index) { 2575 case MSR_IA32_BNDCFGS: 2576 if (!kvm_mpx_supported()) 2577 return; 2578 break; 2579 case MSR_TSC_AUX: 2580 if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) && 2581 !kvm_cpu_cap_has(X86_FEATURE_RDPID)) 2582 return; 2583 break; 2584 case MSR_IA32_UMWAIT_CONTROL: 2585 if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG)) 2586 return; 2587 break; 2588 case MSR_IA32_RTIT_CTL: 2589 case MSR_IA32_RTIT_STATUS: 2590 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) 2591 return; 2592 break; 2593 case MSR_IA32_RTIT_CR3_MATCH: 2594 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || 2595 !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering)) 2596 return; 2597 break; 2598 case MSR_IA32_RTIT_OUTPUT_BASE: 2599 case MSR_IA32_RTIT_OUTPUT_MASK: 2600 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || 2601 (!intel_pt_validate_hw_cap(PT_CAP_topa_output) && 2602 !intel_pt_validate_hw_cap(PT_CAP_single_range_output))) 2603 return; 2604 break; 2605 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2606 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || 2607 (msr_index - MSR_IA32_RTIT_ADDR0_A >= 2608 intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)) 2609 return; 2610 break; 2611 case MSR_ARCH_PERFMON_PERFCTR0 ... 2612 MSR_ARCH_PERFMON_PERFCTR0 + KVM_MAX_NR_GP_COUNTERS - 1: 2613 if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >= 2614 kvm_pmu_cap.num_counters_gp) 2615 return; 2616 break; 2617 case MSR_ARCH_PERFMON_EVENTSEL0 ... 2618 MSR_ARCH_PERFMON_EVENTSEL0 + KVM_MAX_NR_GP_COUNTERS - 1: 2619 if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >= 2620 kvm_pmu_cap.num_counters_gp) 2621 return; 2622 break; 2623 case MSR_ARCH_PERFMON_FIXED_CTR0 ... 2624 MSR_ARCH_PERFMON_FIXED_CTR0 + KVM_MAX_NR_FIXED_COUNTERS - 1: 2625 if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >= 2626 kvm_pmu_cap.num_counters_fixed) 2627 return; 2628 break; 2629 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 2630 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 2631 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 2632 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 2633 if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) 2634 return; 2635 break; 2636 case MSR_IA32_XFD: 2637 case MSR_IA32_XFD_ERR: 2638 if (!kvm_cpu_cap_has(X86_FEATURE_XFD)) 2639 return; 2640 break; 2641 case MSR_IA32_TSX_CTRL: 2642 if (!(kvm_get_arch_capabilities() & ARCH_CAP_TSX_CTRL_MSR)) 2643 return; 2644 break; 2645 case MSR_IA32_XSS: 2646 if (!kvm_caps.supported_xss) 2647 return; 2648 break; 2649 case MSR_IA32_U_CET: 2650 case MSR_IA32_S_CET: 2651 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && 2652 !kvm_cpu_cap_has(X86_FEATURE_IBT)) 2653 return; 2654 break; 2655 case MSR_IA32_INT_SSP_TAB: 2656 if (!kvm_cpu_cap_has(X86_FEATURE_LM)) 2657 return; 2658 fallthrough; 2659 case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: 2660 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK)) 2661 return; 2662 break; 2663 default: 2664 break; 2665 } 2666 2667 msrs_to_save[num_msrs_to_save++] = msr_index; 2668 } 2669 2670 void kvm_init_msr_lists(void) 2671 { 2672 unsigned i; 2673 2674 BUILD_BUG_ON_MSG(KVM_MAX_NR_FIXED_COUNTERS != 3, 2675 "Please update the fixed PMCs in msrs_to_save_pmu[]"); 2676 2677 num_msrs_to_save = 0; 2678 num_emulated_msrs = 0; 2679 num_msr_based_features = 0; 2680 2681 for (i = 0; i < ARRAY_SIZE(msrs_to_save_base); i++) 2682 kvm_probe_msr_to_save(msrs_to_save_base[i]); 2683 2684 if (enable_pmu) { 2685 for (i = 0; i < ARRAY_SIZE(msrs_to_save_pmu); i++) 2686 kvm_probe_msr_to_save(msrs_to_save_pmu[i]); 2687 } 2688 2689 for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) { 2690 if (!kvm_x86_call(has_emulated_msr)(NULL, 2691 emulated_msrs_all[i])) 2692 continue; 2693 2694 emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i]; 2695 } 2696 2697 for (i = KVM_FIRST_EMULATED_VMX_MSR; i <= KVM_LAST_EMULATED_VMX_MSR; i++) 2698 kvm_probe_feature_msr(i); 2699 2700 for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) 2701 kvm_probe_feature_msr(msr_based_features_all_except_vmx[i]); 2702 } 2703 2704 int kvm_spec_ctrl_test_value(u64 value) 2705 { 2706 /* 2707 * test that setting IA32_SPEC_CTRL to given value 2708 * is allowed by the host processor 2709 */ 2710 2711 u64 saved_value; 2712 unsigned long flags; 2713 int ret = 0; 2714 2715 local_irq_save(flags); 2716 2717 if (rdmsrq_safe(MSR_IA32_SPEC_CTRL, &saved_value)) 2718 ret = 1; 2719 else if (wrmsrq_safe(MSR_IA32_SPEC_CTRL, value)) 2720 ret = 1; 2721 else 2722 wrmsrq(MSR_IA32_SPEC_CTRL, saved_value); 2723 2724 local_irq_restore(flags); 2725 2726 return ret; 2727 } 2728 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spec_ctrl_test_value); 2729