1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2018, Google LLC. 4 */ 5 6 #ifndef SELFTEST_KVM_PROCESSOR_H 7 #define SELFTEST_KVM_PROCESSOR_H 8 9 #include <assert.h> 10 #include <stdint.h> 11 #include <syscall.h> 12 13 #include <asm/msr-index.h> 14 #include <asm/prctl.h> 15 16 #include <linux/kvm_para.h> 17 #include <linux/stringify.h> 18 19 #include "kvm_util.h" 20 #include "ucall_common.h" 21 22 extern bool host_cpu_is_intel; 23 extern bool host_cpu_is_amd; 24 extern bool host_cpu_is_hygon; 25 extern bool host_cpu_is_amd_compatible; 26 extern bool host_cpu_is_zhaoxin; 27 extern bool host_cpu_is_intel_compatible; 28 extern u64 guest_tsc_khz; 29 extern struct kvm_mmu guest_mmu; 30 31 #ifndef MAX_NR_CPUID_ENTRIES 32 #define MAX_NR_CPUID_ENTRIES 100 33 #endif 34 35 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 36 37 /* Forced emulation prefix, used to invoke the emulator unconditionally. */ 38 #define KVM_FEP "ud2; .byte 'k', 'v', 'm';" 39 40 #define NMI_VECTOR 0x02 41 42 const char *ex_str(int vector); 43 44 #define X86_EFLAGS_CF BIT(0) /* Carry Flag */ 45 #define X86_EFLAGS_FIXED BIT(1) /* Bit 1 - always on */ 46 #define X86_EFLAGS_PF BIT(2) /* Parity Flag */ 47 #define X86_EFLAGS_AF BIT(4) /* Auxiliary carry Flag */ 48 #define X86_EFLAGS_ZF BIT(6) /* Zero Flag */ 49 #define X86_EFLAGS_SF BIT(7) /* Sign Flag */ 50 #define X86_EFLAGS_TF BIT(8) /* Trap Flag */ 51 #define X86_EFLAGS_IF BIT(9) /* Interrupt Flag */ 52 #define X86_EFLAGS_DF BIT(10) /* Direction Flag */ 53 #define X86_EFLAGS_OF BIT(11) /* Overflow Flag */ 54 #define X86_EFLAGS_IOPL BIT(12) /* I/O Privilege Level (2 bits) */ 55 #define X86_EFLAGS_NT BIT(14) /* Nested Task */ 56 #define X86_EFLAGS_RF BIT(16) /* Resume Flag */ 57 #define X86_EFLAGS_VM BIT(17) /* Virtual Mode */ 58 #define X86_EFLAGS_AC BIT(18) /* Alignment Check/Access Control */ 59 #define X86_EFLAGS_VIF BIT(19) /* Virtual Interrupt Flag */ 60 #define X86_EFLAGS_VIP BIT(20) /* Virtual Interrupt Pending */ 61 #define X86_EFLAGS_ID BIT(21) /* CPUID detection */ 62 63 #define X86_CR4_VME (1ul << 0) 64 #define X86_CR4_PVI (1ul << 1) 65 #define X86_CR4_TSD (1ul << 2) 66 #define X86_CR4_DE (1ul << 3) 67 #define X86_CR4_PSE (1ul << 4) 68 #define X86_CR4_PAE (1ul << 5) 69 #define X86_CR4_MCE (1ul << 6) 70 #define X86_CR4_PGE (1ul << 7) 71 #define X86_CR4_PCE (1ul << 8) 72 #define X86_CR4_OSFXSR (1ul << 9) 73 #define X86_CR4_OSXMMEXCPT (1ul << 10) 74 #define X86_CR4_UMIP (1ul << 11) 75 #define X86_CR4_LA57 (1ul << 12) 76 #define X86_CR4_VMXE (1ul << 13) 77 #define X86_CR4_SMXE (1ul << 14) 78 #define X86_CR4_FSGSBASE (1ul << 16) 79 #define X86_CR4_PCIDE (1ul << 17) 80 #define X86_CR4_OSXSAVE (1ul << 18) 81 #define X86_CR4_SMEP (1ul << 20) 82 #define X86_CR4_SMAP (1ul << 21) 83 #define X86_CR4_PKE (1ul << 22) 84 85 struct xstate_header { 86 u64 xstate_bv; 87 u64 xcomp_bv; 88 u64 reserved[6]; 89 } __attribute__((packed)); 90 91 struct xstate { 92 u8 i387[512]; 93 struct xstate_header header; 94 u8 extended_state_area[0]; 95 } __attribute__ ((packed, aligned (64))); 96 97 #define XFEATURE_MASK_FP BIT_ULL(0) 98 #define XFEATURE_MASK_SSE BIT_ULL(1) 99 #define XFEATURE_MASK_YMM BIT_ULL(2) 100 #define XFEATURE_MASK_BNDREGS BIT_ULL(3) 101 #define XFEATURE_MASK_BNDCSR BIT_ULL(4) 102 #define XFEATURE_MASK_OPMASK BIT_ULL(5) 103 #define XFEATURE_MASK_ZMM_Hi256 BIT_ULL(6) 104 #define XFEATURE_MASK_Hi16_ZMM BIT_ULL(7) 105 #define XFEATURE_MASK_PT BIT_ULL(8) 106 #define XFEATURE_MASK_PKRU BIT_ULL(9) 107 #define XFEATURE_MASK_PASID BIT_ULL(10) 108 #define XFEATURE_MASK_CET_USER BIT_ULL(11) 109 #define XFEATURE_MASK_CET_KERNEL BIT_ULL(12) 110 #define XFEATURE_MASK_LBR BIT_ULL(15) 111 #define XFEATURE_MASK_XTILE_CFG BIT_ULL(17) 112 #define XFEATURE_MASK_XTILE_DATA BIT_ULL(18) 113 114 #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK | \ 115 XFEATURE_MASK_ZMM_Hi256 | \ 116 XFEATURE_MASK_Hi16_ZMM) 117 #define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA | \ 118 XFEATURE_MASK_XTILE_CFG) 119 120 /* Note, these are ordered alphabetically to match kvm_cpuid_entry2. Eww. */ 121 enum cpuid_output_regs { 122 KVM_CPUID_EAX, 123 KVM_CPUID_EBX, 124 KVM_CPUID_ECX, 125 KVM_CPUID_EDX 126 }; 127 128 /* 129 * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be 130 * passed by value with no overhead. 131 */ 132 struct kvm_x86_cpu_feature { 133 u32 function; 134 u16 index; 135 u8 reg; 136 u8 bit; 137 }; 138 #define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit) \ 139 ({ \ 140 struct kvm_x86_cpu_feature feature = { \ 141 .function = fn, \ 142 .index = idx, \ 143 .reg = KVM_CPUID_##gpr, \ 144 .bit = __bit, \ 145 }; \ 146 \ 147 kvm_static_assert((fn & 0xc0000000) == 0 || \ 148 (fn & 0xc0000000) == 0x40000000 || \ 149 (fn & 0xc0000000) == 0x80000000 || \ 150 (fn & 0xc0000000) == 0xc0000000); \ 151 kvm_static_assert(idx < BIT(sizeof(feature.index) * BITS_PER_BYTE)); \ 152 feature; \ 153 }) 154 155 /* 156 * Basic Leafs, a.k.a. Intel defined 157 */ 158 #define X86_FEATURE_MWAIT KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3) 159 #define X86_FEATURE_VMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5) 160 #define X86_FEATURE_SMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6) 161 #define X86_FEATURE_PDCM KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15) 162 #define X86_FEATURE_PCID KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17) 163 #define X86_FEATURE_X2APIC KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21) 164 #define X86_FEATURE_MOVBE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22) 165 #define X86_FEATURE_TSC_DEADLINE_TIMER KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24) 166 #define X86_FEATURE_XSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26) 167 #define X86_FEATURE_OSXSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27) 168 #define X86_FEATURE_RDRAND KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30) 169 #define X86_FEATURE_HYPERVISOR KVM_X86_CPU_FEATURE(0x1, 0, ECX, 31) 170 #define X86_FEATURE_PAE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 6) 171 #define X86_FEATURE_MCE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7) 172 #define X86_FEATURE_APIC KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9) 173 #define X86_FEATURE_CLFLUSH KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19) 174 #define X86_FEATURE_XMM KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25) 175 #define X86_FEATURE_XMM2 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26) 176 #define X86_FEATURE_FSGSBASE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0) 177 #define X86_FEATURE_TSC_ADJUST KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1) 178 #define X86_FEATURE_SGX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 2) 179 #define X86_FEATURE_HLE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4) 180 #define X86_FEATURE_SMEP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7) 181 #define X86_FEATURE_INVPCID KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10) 182 #define X86_FEATURE_RTM KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11) 183 #define X86_FEATURE_MPX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14) 184 #define X86_FEATURE_SMAP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20) 185 #define X86_FEATURE_PCOMMIT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22) 186 #define X86_FEATURE_CLFLUSHOPT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23) 187 #define X86_FEATURE_CLWB KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24) 188 #define X86_FEATURE_UMIP KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2) 189 #define X86_FEATURE_PKU KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3) 190 #define X86_FEATURE_OSPKE KVM_X86_CPU_FEATURE(0x7, 0, ECX, 4) 191 #define X86_FEATURE_LA57 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16) 192 #define X86_FEATURE_RDPID KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22) 193 #define X86_FEATURE_SGX_LC KVM_X86_CPU_FEATURE(0x7, 0, ECX, 30) 194 #define X86_FEATURE_SHSTK KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7) 195 #define X86_FEATURE_IBT KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20) 196 #define X86_FEATURE_AMX_TILE KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24) 197 #define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26) 198 #define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29) 199 #define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31) 200 #define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17) 201 #define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18) 202 #define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3) 203 #define X86_FEATURE_XFD KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4) 204 #define X86_FEATURE_XTILEDATA_XFD KVM_X86_CPU_FEATURE(0xD, 18, ECX, 2) 205 206 /* 207 * Extended Leafs, a.k.a. AMD defined 208 */ 209 #define X86_FEATURE_SVM KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2) 210 #define X86_FEATURE_PERFCTR_CORE KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 23) 211 #define X86_FEATURE_PERFCTR_NB KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 24) 212 #define X86_FEATURE_PERFCTR_LLC KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 28) 213 #define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20) 214 #define X86_FEATURE_FXSR_OPT KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 25) 215 #define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26) 216 #define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27) 217 #define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29) 218 #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) 219 #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) 220 #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) 221 #define X86_FEATURE_EFER_LMSLE_MBZ KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 20) 222 #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) 223 #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) 224 #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) 225 #define X86_FEATURE_TSCRATEMSR KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4) 226 #define X86_FEATURE_PAUSEFILTER KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10) 227 #define X86_FEATURE_PFTHRESHOLD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12) 228 #define X86_FEATURE_V_VMSAVE_VMLOAD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 15) 229 #define X86_FEATURE_VGIF KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16) 230 #define X86_FEATURE_IDLE_HLT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 30) 231 #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1) 232 #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3) 233 #define X86_FEATURE_SEV_SNP KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 4) 234 #define X86_FEATURE_AUTOIBRS KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 8) 235 #define X86_FEATURE_GP_ON_USER_CPUID KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 17) 236 #define X86_FEATURE_PERFMON_V2 KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 0) 237 #define X86_FEATURE_LBR_PMC_FREEZE KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 2) 238 239 /* 240 * KVM defined paravirt features. 241 */ 242 #define X86_FEATURE_KVM_CLOCKSOURCE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0) 243 #define X86_FEATURE_KVM_NOP_IO_DELAY KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1) 244 #define X86_FEATURE_KVM_MMU_OP KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2) 245 #define X86_FEATURE_KVM_CLOCKSOURCE2 KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3) 246 #define X86_FEATURE_KVM_ASYNC_PF KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4) 247 #define X86_FEATURE_KVM_STEAL_TIME KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5) 248 #define X86_FEATURE_KVM_PV_EOI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6) 249 #define X86_FEATURE_KVM_PV_UNHALT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7) 250 /* Bit 8 apparently isn't used?!?! */ 251 #define X86_FEATURE_KVM_PV_TLB_FLUSH KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9) 252 #define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10) 253 #define X86_FEATURE_KVM_PV_SEND_IPI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11) 254 #define X86_FEATURE_KVM_POLL_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12) 255 #define X86_FEATURE_KVM_PV_SCHED_YIELD KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13) 256 #define X86_FEATURE_KVM_ASYNC_PF_INT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14) 257 #define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15) 258 #define X86_FEATURE_KVM_HC_MAP_GPA_RANGE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16) 259 #define X86_FEATURE_KVM_MIGRATION_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17) 260 261 /* 262 * Same idea as X86_FEATURE_XXX, but X86_PROPERTY_XXX retrieves a multi-bit 263 * value/property as opposed to a single-bit feature. Again, pack the info 264 * into a 64-bit value to pass by value with no overhead. 265 */ 266 struct kvm_x86_cpu_property { 267 u32 function; 268 u8 index; 269 u8 reg; 270 u8 lo_bit; 271 u8 hi_bit; 272 }; 273 #define KVM_X86_CPU_PROPERTY(fn, idx, gpr, low_bit, high_bit) \ 274 ({ \ 275 struct kvm_x86_cpu_property property = { \ 276 .function = fn, \ 277 .index = idx, \ 278 .reg = KVM_CPUID_##gpr, \ 279 .lo_bit = low_bit, \ 280 .hi_bit = high_bit, \ 281 }; \ 282 \ 283 kvm_static_assert(low_bit < high_bit); \ 284 kvm_static_assert((fn & 0xc0000000) == 0 || \ 285 (fn & 0xc0000000) == 0x40000000 || \ 286 (fn & 0xc0000000) == 0x80000000 || \ 287 (fn & 0xc0000000) == 0xc0000000); \ 288 kvm_static_assert(idx < BIT(sizeof(property.index) * BITS_PER_BYTE)); \ 289 property; \ 290 }) 291 292 #define X86_PROPERTY_MAX_BASIC_LEAF KVM_X86_CPU_PROPERTY(0, 0, EAX, 0, 31) 293 #define X86_PROPERTY_PMU_VERSION KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 0, 7) 294 #define X86_PROPERTY_PMU_NR_GP_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 8, 15) 295 #define X86_PROPERTY_PMU_GP_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 16, 23) 296 #define X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 24, 31) 297 #define X86_PROPERTY_PMU_EVENTS_MASK KVM_X86_CPU_PROPERTY(0xa, 0, EBX, 0, 12) 298 #define X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK KVM_X86_CPU_PROPERTY(0xa, 0, ECX, 0, 31) 299 #define X86_PROPERTY_PMU_NR_FIXED_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 0, 4) 300 #define X86_PROPERTY_PMU_FIXED_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 5, 12) 301 302 #define X86_PROPERTY_SUPPORTED_XCR0_LO KVM_X86_CPU_PROPERTY(0xd, 0, EAX, 0, 31) 303 #define X86_PROPERTY_XSTATE_MAX_SIZE_XCR0 KVM_X86_CPU_PROPERTY(0xd, 0, EBX, 0, 31) 304 #define X86_PROPERTY_XSTATE_MAX_SIZE KVM_X86_CPU_PROPERTY(0xd, 0, ECX, 0, 31) 305 #define X86_PROPERTY_SUPPORTED_XCR0_HI KVM_X86_CPU_PROPERTY(0xd, 0, EDX, 0, 31) 306 307 #define X86_PROPERTY_XSTATE_TILE_SIZE KVM_X86_CPU_PROPERTY(0xd, 18, EAX, 0, 31) 308 #define X86_PROPERTY_XSTATE_TILE_OFFSET KVM_X86_CPU_PROPERTY(0xd, 18, EBX, 0, 31) 309 #define X86_PROPERTY_AMX_MAX_PALETTE_TABLES KVM_X86_CPU_PROPERTY(0x1d, 0, EAX, 0, 31) 310 #define X86_PROPERTY_AMX_TOTAL_TILE_BYTES KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 0, 15) 311 #define X86_PROPERTY_AMX_BYTES_PER_TILE KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 16, 31) 312 #define X86_PROPERTY_AMX_BYTES_PER_ROW KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 0, 15) 313 #define X86_PROPERTY_AMX_NR_TILE_REGS KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 16, 31) 314 #define X86_PROPERTY_AMX_MAX_ROWS KVM_X86_CPU_PROPERTY(0x1d, 1, ECX, 0, 15) 315 316 #define X86_PROPERTY_MAX_KVM_LEAF KVM_X86_CPU_PROPERTY(0x40000000, 0, EAX, 0, 31) 317 318 #define X86_PROPERTY_MAX_EXT_LEAF KVM_X86_CPU_PROPERTY(0x80000000, 0, EAX, 0, 31) 319 #define X86_PROPERTY_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 0, 7) 320 #define X86_PROPERTY_MAX_VIRT_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 8, 15) 321 #define X86_PROPERTY_GUEST_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 16, 23) 322 #define X86_PROPERTY_SEV_C_BIT KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 0, 5) 323 #define X86_PROPERTY_PHYS_ADDR_REDUCTION KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 6, 11) 324 #define X86_PROPERTY_NR_PERFCTR_CORE KVM_X86_CPU_PROPERTY(0x80000022, 0, EBX, 0, 3) 325 #define X86_PROPERTY_NR_PERFCTR_NB KVM_X86_CPU_PROPERTY(0x80000022, 0, EBX, 10, 15) 326 327 #define X86_PROPERTY_MAX_CENTAUR_LEAF KVM_X86_CPU_PROPERTY(0xC0000000, 0, EAX, 0, 31) 328 329 /* 330 * Intel's architectural PMU events are bizarre. They have a "feature" bit 331 * that indicates the feature is _not_ supported, and a property that states 332 * the length of the bit mask of unsupported features. A feature is supported 333 * if the size of the bit mask is larger than the "unavailable" bit, and said 334 * bit is not set. Fixed counters also bizarre enumeration, but inverted from 335 * arch events for general purpose counters. Fixed counters are supported if a 336 * feature flag is set **OR** the total number of fixed counters is greater 337 * than index of the counter. 338 * 339 * Wrap the events for general purpose and fixed counters to simplify checking 340 * whether or not a given architectural event is supported. 341 */ 342 struct kvm_x86_pmu_feature { 343 struct kvm_x86_cpu_feature f; 344 }; 345 #define KVM_X86_PMU_FEATURE(__reg, __bit) \ 346 ({ \ 347 struct kvm_x86_pmu_feature feature = { \ 348 .f = KVM_X86_CPU_FEATURE(0xa, 0, __reg, __bit), \ 349 }; \ 350 \ 351 kvm_static_assert(KVM_CPUID_##__reg == KVM_CPUID_EBX || \ 352 KVM_CPUID_##__reg == KVM_CPUID_ECX); \ 353 feature; \ 354 }) 355 356 #define X86_PMU_FEATURE_CPU_CYCLES KVM_X86_PMU_FEATURE(EBX, 0) 357 #define X86_PMU_FEATURE_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 1) 358 #define X86_PMU_FEATURE_REFERENCE_CYCLES KVM_X86_PMU_FEATURE(EBX, 2) 359 #define X86_PMU_FEATURE_LLC_REFERENCES KVM_X86_PMU_FEATURE(EBX, 3) 360 #define X86_PMU_FEATURE_LLC_MISSES KVM_X86_PMU_FEATURE(EBX, 4) 361 #define X86_PMU_FEATURE_BRANCH_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 5) 362 #define X86_PMU_FEATURE_BRANCHES_MISPREDICTED KVM_X86_PMU_FEATURE(EBX, 6) 363 #define X86_PMU_FEATURE_TOPDOWN_SLOTS KVM_X86_PMU_FEATURE(EBX, 7) 364 #define X86_PMU_FEATURE_TOPDOWN_BE_BOUND KVM_X86_PMU_FEATURE(EBX, 8) 365 #define X86_PMU_FEATURE_TOPDOWN_BAD_SPEC KVM_X86_PMU_FEATURE(EBX, 9) 366 #define X86_PMU_FEATURE_TOPDOWN_FE_BOUND KVM_X86_PMU_FEATURE(EBX, 10) 367 #define X86_PMU_FEATURE_TOPDOWN_RETIRING KVM_X86_PMU_FEATURE(EBX, 11) 368 #define X86_PMU_FEATURE_LBR_INSERTS KVM_X86_PMU_FEATURE(EBX, 12) 369 370 #define X86_PMU_FEATURE_INSNS_RETIRED_FIXED KVM_X86_PMU_FEATURE(ECX, 0) 371 #define X86_PMU_FEATURE_CPU_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 1) 372 #define X86_PMU_FEATURE_REFERENCE_TSC_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 2) 373 #define X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED KVM_X86_PMU_FEATURE(ECX, 3) 374 375 static inline unsigned int x86_family(unsigned int eax) 376 { 377 unsigned int x86; 378 379 x86 = (eax >> 8) & 0xf; 380 381 if (x86 == 0xf) 382 x86 += (eax >> 20) & 0xff; 383 384 return x86; 385 } 386 387 static inline unsigned int x86_model(unsigned int eax) 388 { 389 return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f); 390 } 391 392 #define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12) 393 394 #define PAGE_SHIFT 12 395 #define PAGE_SIZE (1ULL << PAGE_SHIFT) 396 #define PAGE_MASK (~(PAGE_SIZE-1) & PHYSICAL_PAGE_MASK) 397 398 #define HUGEPAGE_SHIFT(x) (PAGE_SHIFT + (((x) - 1) * 9)) 399 #define HUGEPAGE_SIZE(x) (1UL << HUGEPAGE_SHIFT(x)) 400 #define HUGEPAGE_MASK(x) (~(HUGEPAGE_SIZE(x) - 1) & PHYSICAL_PAGE_MASK) 401 402 #define PTE_GET_PA(pte) ((pte) & PHYSICAL_PAGE_MASK) 403 #define PTE_GET_PFN(pte) (PTE_GET_PA(pte) >> PAGE_SHIFT) 404 405 struct guest_regs { 406 u64 rax; 407 u64 rcx; 408 u64 rdx; 409 u64 rbx; 410 u64 rsp; 411 u64 rbp; 412 u64 rsi; 413 u64 rdi; 414 u64 r8; 415 u64 r9; 416 u64 r10; 417 u64 r11; 418 u64 r12; 419 u64 r13; 420 u64 r14; 421 u64 r15; 422 u64 rflags; 423 }; 424 425 extern struct guest_regs guest_regs; 426 427 #define GUEST_REG_OFFSET(name) \ 428 [off_##name] "i" (offsetof(struct guest_regs, name)) 429 430 #define GUEST_REGS_OFFSETS \ 431 GUEST_REG_OFFSET(rax), \ 432 GUEST_REG_OFFSET(rcx), \ 433 GUEST_REG_OFFSET(rdx), \ 434 GUEST_REG_OFFSET(rbx), \ 435 GUEST_REG_OFFSET(rsp), \ 436 GUEST_REG_OFFSET(rbp), \ 437 GUEST_REG_OFFSET(rsi), \ 438 GUEST_REG_OFFSET(rdi), \ 439 GUEST_REG_OFFSET(r8), \ 440 GUEST_REG_OFFSET(r9), \ 441 GUEST_REG_OFFSET(r10), \ 442 GUEST_REG_OFFSET(r11), \ 443 GUEST_REG_OFFSET(r12), \ 444 GUEST_REG_OFFSET(r13), \ 445 GUEST_REG_OFFSET(r14), \ 446 GUEST_REG_OFFSET(r15), \ 447 GUEST_REG_OFFSET(rflags) 448 449 #define GUEST_REG(name) "guest_regs + %c[off_" #name "]" 450 451 #define GUEST_SWITCH_GPR_ASM(name) \ 452 "xchg %%" #name ", " GUEST_REG(name) "\n\t" 453 454 struct desc64 { 455 u16 limit0; 456 u16 base0; 457 unsigned base1:8, type:4, s:1, dpl:2, p:1; 458 unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8; 459 u32 base3; 460 u32 zero1; 461 } __attribute__((packed)); 462 463 struct desc_ptr { 464 u16 size; 465 u64 address; 466 } __attribute__((packed)); 467 468 struct kvm_x86_state { 469 struct kvm_xsave *xsave; 470 struct kvm_vcpu_events events; 471 struct kvm_mp_state mp_state; 472 struct kvm_regs regs; 473 struct kvm_xcrs xcrs; 474 struct kvm_sregs sregs; 475 struct kvm_debugregs debugregs; 476 union { 477 struct kvm_nested_state nested; 478 char nested_[16384]; 479 }; 480 struct kvm_msrs msrs; 481 }; 482 483 static inline u64 get_desc64_base(const struct desc64 *desc) 484 { 485 return (u64)desc->base3 << 32 | 486 (u64)desc->base2 << 24 | 487 (u64)desc->base1 << 16 | 488 (u64)desc->base0; 489 } 490 491 static inline u64 rdtsc(void) 492 { 493 u32 eax, edx; 494 u64 tsc_val; 495 /* 496 * The lfence is to wait (on Intel CPUs) until all previous 497 * instructions have been executed. If software requires RDTSC to be 498 * executed prior to execution of any subsequent instruction, it can 499 * execute LFENCE immediately after RDTSC 500 */ 501 __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 502 tsc_val = ((u64)edx) << 32 | eax; 503 return tsc_val; 504 } 505 506 static inline u64 rdtscp(u32 *aux) 507 { 508 u32 eax, edx; 509 510 __asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux)); 511 return ((u64)edx) << 32 | eax; 512 } 513 514 static inline u64 rdmsr(u32 msr) 515 { 516 u32 a, d; 517 518 __asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory"); 519 520 return a | ((u64)d << 32); 521 } 522 523 static inline void wrmsr(u32 msr, u64 value) 524 { 525 u32 a = value; 526 u32 d = value >> 32; 527 528 __asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory"); 529 } 530 531 532 static inline u16 inw(u16 port) 533 { 534 u16 tmp; 535 536 __asm__ __volatile__("in %%dx, %%ax" 537 : /* output */ "=a" (tmp) 538 : /* input */ "d" (port)); 539 540 return tmp; 541 } 542 543 static inline u16 get_es(void) 544 { 545 u16 es; 546 547 __asm__ __volatile__("mov %%es, %[es]" 548 : /* output */ [es]"=rm"(es)); 549 return es; 550 } 551 552 static inline u16 get_cs(void) 553 { 554 u16 cs; 555 556 __asm__ __volatile__("mov %%cs, %[cs]" 557 : /* output */ [cs]"=rm"(cs)); 558 return cs; 559 } 560 561 static inline u16 get_ss(void) 562 { 563 u16 ss; 564 565 __asm__ __volatile__("mov %%ss, %[ss]" 566 : /* output */ [ss]"=rm"(ss)); 567 return ss; 568 } 569 570 static inline u16 get_ds(void) 571 { 572 u16 ds; 573 574 __asm__ __volatile__("mov %%ds, %[ds]" 575 : /* output */ [ds]"=rm"(ds)); 576 return ds; 577 } 578 579 static inline u16 get_fs(void) 580 { 581 u16 fs; 582 583 __asm__ __volatile__("mov %%fs, %[fs]" 584 : /* output */ [fs]"=rm"(fs)); 585 return fs; 586 } 587 588 static inline u16 get_gs(void) 589 { 590 u16 gs; 591 592 __asm__ __volatile__("mov %%gs, %[gs]" 593 : /* output */ [gs]"=rm"(gs)); 594 return gs; 595 } 596 597 static inline u16 get_tr(void) 598 { 599 u16 tr; 600 601 __asm__ __volatile__("str %[tr]" 602 : /* output */ [tr]"=rm"(tr)); 603 return tr; 604 } 605 606 static inline u64 get_cr0(void) 607 { 608 u64 cr0; 609 610 __asm__ __volatile__("mov %%cr0, %[cr0]" 611 : /* output */ [cr0]"=r"(cr0)); 612 return cr0; 613 } 614 615 static inline void set_cr0(u64 val) 616 { 617 __asm__ __volatile__("mov %0, %%cr0" : : "r" (val) : "memory"); 618 } 619 620 static inline u64 get_cr2(void) 621 { 622 u64 cr2; 623 624 __asm__ __volatile__("mov %%cr2, %[cr2]" : [cr2]"=r"(cr2)); 625 return cr2; 626 } 627 628 static inline u64 get_cr3(void) 629 { 630 u64 cr3; 631 632 __asm__ __volatile__("mov %%cr3, %[cr3]" 633 : /* output */ [cr3]"=r"(cr3)); 634 return cr3; 635 } 636 637 static inline void set_cr3(u64 val) 638 { 639 __asm__ __volatile__("mov %0, %%cr3" : : "r" (val) : "memory"); 640 } 641 642 static inline u64 get_cr4(void) 643 { 644 u64 cr4; 645 646 __asm__ __volatile__("mov %%cr4, %[cr4]" 647 : /* output */ [cr4]"=r"(cr4)); 648 return cr4; 649 } 650 651 static inline void set_cr4(u64 val) 652 { 653 __asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory"); 654 } 655 656 static inline u64 get_cr8(void) 657 { 658 u64 cr8; 659 660 __asm__ __volatile__("mov %%cr8, %[cr8]" : [cr8]"=r"(cr8)); 661 return cr8; 662 } 663 664 static inline void set_cr8(u64 val) 665 { 666 __asm__ __volatile__("mov %0, %%cr8" : : "r" (val) : "memory"); 667 } 668 669 static inline void set_idt(const struct desc_ptr *idt_desc) 670 { 671 __asm__ __volatile__("lidt %0"::"m"(*idt_desc)); 672 } 673 674 static inline u64 xgetbv(u32 index) 675 { 676 u32 eax, edx; 677 678 __asm__ __volatile__("xgetbv;" 679 : "=a" (eax), "=d" (edx) 680 : "c" (index)); 681 return eax | ((u64)edx << 32); 682 } 683 684 static inline void xsetbv(u32 index, u64 value) 685 { 686 u32 eax = value; 687 u32 edx = value >> 32; 688 689 __asm__ __volatile__("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); 690 } 691 692 static inline void wrpkru(u32 pkru) 693 { 694 /* Note, ECX and EDX are architecturally required to be '0'. */ 695 asm volatile(".byte 0x0f,0x01,0xef\n\t" 696 : : "a" (pkru), "c"(0), "d"(0)); 697 } 698 699 static inline struct desc_ptr get_gdt(void) 700 { 701 struct desc_ptr gdt; 702 __asm__ __volatile__("sgdt %[gdt]" 703 : /* output */ [gdt]"=m"(gdt)); 704 return gdt; 705 } 706 707 static inline struct desc_ptr get_idt(void) 708 { 709 struct desc_ptr idt; 710 __asm__ __volatile__("sidt %[idt]" 711 : /* output */ [idt]"=m"(idt)); 712 return idt; 713 } 714 715 static inline void outl(u16 port, u32 value) 716 { 717 __asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value)); 718 } 719 720 static inline void __cpuid(u32 function, u32 index, 721 u32 *eax, u32 *ebx, 722 u32 *ecx, u32 *edx) 723 { 724 *eax = function; 725 *ecx = index; 726 727 asm volatile("cpuid" 728 : "=a" (*eax), 729 "=b" (*ebx), 730 "=c" (*ecx), 731 "=d" (*edx) 732 : "0" (*eax), "2" (*ecx) 733 : "memory"); 734 } 735 736 static inline void cpuid(u32 function, 737 u32 *eax, u32 *ebx, 738 u32 *ecx, u32 *edx) 739 { 740 return __cpuid(function, 0, eax, ebx, ecx, edx); 741 } 742 743 static inline u32 this_cpu_fms(void) 744 { 745 u32 eax, ebx, ecx, edx; 746 747 cpuid(1, &eax, &ebx, &ecx, &edx); 748 return eax; 749 } 750 751 static inline u32 this_cpu_family(void) 752 { 753 return x86_family(this_cpu_fms()); 754 } 755 756 static inline u32 this_cpu_model(void) 757 { 758 return x86_model(this_cpu_fms()); 759 } 760 761 static inline bool this_cpu_vendor_string_is(const char *vendor) 762 { 763 const u32 *chunk = (const u32 *)vendor; 764 u32 eax, ebx, ecx, edx; 765 766 cpuid(0, &eax, &ebx, &ecx, &edx); 767 return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]); 768 } 769 770 static inline bool this_cpu_is_intel(void) 771 { 772 return this_cpu_vendor_string_is("GenuineIntel"); 773 } 774 775 /* 776 * Exclude early K5 samples with a vendor string of "AMDisbetter!" 777 */ 778 static inline bool this_cpu_is_amd(void) 779 { 780 return this_cpu_vendor_string_is("AuthenticAMD"); 781 } 782 783 static inline bool this_cpu_is_hygon(void) 784 { 785 return this_cpu_vendor_string_is("HygonGenuine"); 786 } 787 788 static inline bool this_cpu_is_zhaoxin(void) 789 { 790 return this_cpu_vendor_string_is("CentaurHauls") || 791 this_cpu_vendor_string_is(" Shanghai "); 792 } 793 794 static inline u32 __this_cpu_has(u32 function, u32 index, u8 reg, u8 lo, u8 hi) 795 { 796 u32 gprs[4]; 797 798 __cpuid(function, index, 799 &gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX], 800 &gprs[KVM_CPUID_ECX], &gprs[KVM_CPUID_EDX]); 801 802 return (gprs[reg] & GENMASK(hi, lo)) >> lo; 803 } 804 805 static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature) 806 { 807 return __this_cpu_has(feature.function, feature.index, 808 feature.reg, feature.bit, feature.bit); 809 } 810 811 static inline u32 this_cpu_property(struct kvm_x86_cpu_property property) 812 { 813 return __this_cpu_has(property.function, property.index, 814 property.reg, property.lo_bit, property.hi_bit); 815 } 816 817 static __always_inline bool this_cpu_has_p(struct kvm_x86_cpu_property property) 818 { 819 u32 max_leaf; 820 821 switch (property.function & 0xc0000000) { 822 case 0: 823 max_leaf = this_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF); 824 break; 825 case 0x40000000: 826 max_leaf = this_cpu_property(X86_PROPERTY_MAX_KVM_LEAF); 827 break; 828 case 0x80000000: 829 max_leaf = this_cpu_property(X86_PROPERTY_MAX_EXT_LEAF); 830 break; 831 case 0xc0000000: 832 max_leaf = this_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF); 833 } 834 return max_leaf >= property.function; 835 } 836 837 static inline bool this_pmu_has(struct kvm_x86_pmu_feature feature) 838 { 839 u32 nr_bits; 840 841 if (feature.f.reg == KVM_CPUID_EBX) { 842 nr_bits = this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH); 843 return nr_bits > feature.f.bit && !this_cpu_has(feature.f); 844 } 845 846 GUEST_ASSERT(feature.f.reg == KVM_CPUID_ECX); 847 nr_bits = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 848 return nr_bits > feature.f.bit || this_cpu_has(feature.f); 849 } 850 851 static __always_inline u64 this_cpu_supported_xcr0(void) 852 { 853 if (!this_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO)) 854 return 0; 855 856 return this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) | 857 ((u64)this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32); 858 } 859 860 typedef u32 __attribute__((vector_size(16))) sse128_t; 861 #define __sse128_u union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; } 862 #define sse128_lo(x) ({ __sse128_u t; t.vec = x; t.as_u64[0]; }) 863 #define sse128_hi(x) ({ __sse128_u t; t.vec = x; t.as_u64[1]; }) 864 865 static inline void read_sse_reg(int reg, sse128_t *data) 866 { 867 switch (reg) { 868 case 0: 869 asm("movdqa %%xmm0, %0" : "=m"(*data)); 870 break; 871 case 1: 872 asm("movdqa %%xmm1, %0" : "=m"(*data)); 873 break; 874 case 2: 875 asm("movdqa %%xmm2, %0" : "=m"(*data)); 876 break; 877 case 3: 878 asm("movdqa %%xmm3, %0" : "=m"(*data)); 879 break; 880 case 4: 881 asm("movdqa %%xmm4, %0" : "=m"(*data)); 882 break; 883 case 5: 884 asm("movdqa %%xmm5, %0" : "=m"(*data)); 885 break; 886 case 6: 887 asm("movdqa %%xmm6, %0" : "=m"(*data)); 888 break; 889 case 7: 890 asm("movdqa %%xmm7, %0" : "=m"(*data)); 891 break; 892 default: 893 BUG(); 894 } 895 } 896 897 static inline void write_sse_reg(int reg, const sse128_t *data) 898 { 899 switch (reg) { 900 case 0: 901 asm("movdqa %0, %%xmm0" : : "m"(*data)); 902 break; 903 case 1: 904 asm("movdqa %0, %%xmm1" : : "m"(*data)); 905 break; 906 case 2: 907 asm("movdqa %0, %%xmm2" : : "m"(*data)); 908 break; 909 case 3: 910 asm("movdqa %0, %%xmm3" : : "m"(*data)); 911 break; 912 case 4: 913 asm("movdqa %0, %%xmm4" : : "m"(*data)); 914 break; 915 case 5: 916 asm("movdqa %0, %%xmm5" : : "m"(*data)); 917 break; 918 case 6: 919 asm("movdqa %0, %%xmm6" : : "m"(*data)); 920 break; 921 case 7: 922 asm("movdqa %0, %%xmm7" : : "m"(*data)); 923 break; 924 default: 925 BUG(); 926 } 927 } 928 929 static inline void invlpg(u64 addr) 930 { 931 __asm__ __volatile__("invlpg (%0)" : : "r"(addr) : "memory"); 932 } 933 934 static inline void cpu_relax(void) 935 { 936 asm volatile("rep; nop" ::: "memory"); 937 } 938 939 static inline void udelay(unsigned long usec) 940 { 941 u64 start, now, cycles; 942 943 GUEST_ASSERT(guest_tsc_khz); 944 cycles = guest_tsc_khz / 1000 * usec; 945 946 /* 947 * Deliberately don't PAUSE, a.k.a. cpu_relax(), so that the delay is 948 * as accurate as possible, e.g. doesn't trigger PAUSE-Loop VM-Exits. 949 */ 950 start = rdtsc(); 951 do { 952 now = rdtsc(); 953 } while (now - start < cycles); 954 } 955 956 #define ud2() \ 957 __asm__ __volatile__( \ 958 "ud2\n" \ 959 ) 960 961 #define hlt() \ 962 __asm__ __volatile__( \ 963 "hlt\n" \ 964 ) 965 966 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu); 967 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state); 968 void kvm_x86_state_cleanup(struct kvm_x86_state *state); 969 970 static inline bool kvm_x86_state_is_guest_mode(struct kvm_x86_state *state) 971 { 972 return state->nested.size && (state->nested.flags & KVM_STATE_NESTED_GUEST_MODE); 973 } 974 975 const struct kvm_msr_list *kvm_get_msr_index_list(void); 976 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void); 977 bool kvm_msr_is_in_save_restore_list(u32 msr_index); 978 u64 kvm_get_feature_msr(u64 msr_index); 979 980 static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu, 981 struct kvm_msrs *msrs) 982 { 983 int r = __vcpu_ioctl(vcpu, KVM_GET_MSRS, msrs); 984 985 TEST_ASSERT(r == msrs->nmsrs, 986 "KVM_GET_MSRS failed, r: %i (failed on MSR %x)", 987 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index); 988 } 989 static inline void vcpu_msrs_set(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs) 990 { 991 int r = __vcpu_ioctl(vcpu, KVM_SET_MSRS, msrs); 992 993 TEST_ASSERT(r == msrs->nmsrs, 994 "KVM_SET_MSRS failed, r: %i (failed on MSR %x)", 995 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index); 996 } 997 static inline void vcpu_debugregs_get(struct kvm_vcpu *vcpu, 998 struct kvm_debugregs *debugregs) 999 { 1000 vcpu_ioctl(vcpu, KVM_GET_DEBUGREGS, debugregs); 1001 } 1002 static inline void vcpu_debugregs_set(struct kvm_vcpu *vcpu, 1003 struct kvm_debugregs *debugregs) 1004 { 1005 vcpu_ioctl(vcpu, KVM_SET_DEBUGREGS, debugregs); 1006 } 1007 static inline void vcpu_xsave_get(struct kvm_vcpu *vcpu, 1008 struct kvm_xsave *xsave) 1009 { 1010 vcpu_ioctl(vcpu, KVM_GET_XSAVE, xsave); 1011 } 1012 static inline void vcpu_xsave2_get(struct kvm_vcpu *vcpu, 1013 struct kvm_xsave *xsave) 1014 { 1015 vcpu_ioctl(vcpu, KVM_GET_XSAVE2, xsave); 1016 } 1017 static inline void vcpu_xsave_set(struct kvm_vcpu *vcpu, 1018 struct kvm_xsave *xsave) 1019 { 1020 vcpu_ioctl(vcpu, KVM_SET_XSAVE, xsave); 1021 } 1022 static inline void vcpu_xcrs_get(struct kvm_vcpu *vcpu, 1023 struct kvm_xcrs *xcrs) 1024 { 1025 vcpu_ioctl(vcpu, KVM_GET_XCRS, xcrs); 1026 } 1027 static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs) 1028 { 1029 vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs); 1030 } 1031 1032 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid, 1033 u32 function, u32 index); 1034 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void); 1035 1036 static inline u32 kvm_cpu_fms(void) 1037 { 1038 return get_cpuid_entry(kvm_get_supported_cpuid(), 0x1, 0)->eax; 1039 } 1040 1041 static inline u32 kvm_cpu_family(void) 1042 { 1043 return x86_family(kvm_cpu_fms()); 1044 } 1045 1046 static inline u32 kvm_cpu_model(void) 1047 { 1048 return x86_model(kvm_cpu_fms()); 1049 } 1050 1051 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid, 1052 struct kvm_x86_cpu_feature feature); 1053 1054 static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature) 1055 { 1056 return kvm_cpuid_has(kvm_get_supported_cpuid(), feature); 1057 } 1058 1059 u32 kvm_cpuid_property(const struct kvm_cpuid2 *cpuid, 1060 struct kvm_x86_cpu_property property); 1061 1062 static inline u32 kvm_cpu_property(struct kvm_x86_cpu_property property) 1063 { 1064 return kvm_cpuid_property(kvm_get_supported_cpuid(), property); 1065 } 1066 1067 static __always_inline bool kvm_cpu_has_p(struct kvm_x86_cpu_property property) 1068 { 1069 u32 max_leaf; 1070 1071 switch (property.function & 0xc0000000) { 1072 case 0: 1073 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF); 1074 break; 1075 case 0x40000000: 1076 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_KVM_LEAF); 1077 break; 1078 case 0x80000000: 1079 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_EXT_LEAF); 1080 break; 1081 case 0xc0000000: 1082 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF); 1083 } 1084 return max_leaf >= property.function; 1085 } 1086 1087 static inline bool kvm_pmu_has(struct kvm_x86_pmu_feature feature) 1088 { 1089 u32 nr_bits; 1090 1091 if (feature.f.reg == KVM_CPUID_EBX) { 1092 nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH); 1093 return nr_bits > feature.f.bit && !kvm_cpu_has(feature.f); 1094 } 1095 1096 TEST_ASSERT_EQ(feature.f.reg, KVM_CPUID_ECX); 1097 nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 1098 return nr_bits > feature.f.bit || kvm_cpu_has(feature.f); 1099 } 1100 1101 static __always_inline u64 kvm_cpu_supported_xcr0(void) 1102 { 1103 if (!kvm_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO)) 1104 return 0; 1105 1106 return kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) | 1107 ((u64)kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32); 1108 } 1109 1110 static inline size_t kvm_cpuid2_size(int nr_entries) 1111 { 1112 return sizeof(struct kvm_cpuid2) + 1113 sizeof(struct kvm_cpuid_entry2) * nr_entries; 1114 } 1115 1116 /* 1117 * Allocate a "struct kvm_cpuid2* instance, with the 0-length arrary of 1118 * entries sized to hold @nr_entries. The caller is responsible for freeing 1119 * the struct. 1120 */ 1121 static inline struct kvm_cpuid2 *allocate_kvm_cpuid2(int nr_entries) 1122 { 1123 struct kvm_cpuid2 *cpuid; 1124 1125 cpuid = malloc(kvm_cpuid2_size(nr_entries)); 1126 TEST_ASSERT(cpuid, "-ENOMEM when allocating kvm_cpuid2"); 1127 1128 cpuid->nent = nr_entries; 1129 1130 return cpuid; 1131 } 1132 1133 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid); 1134 1135 static inline void vcpu_get_cpuid(struct kvm_vcpu *vcpu) 1136 { 1137 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid); 1138 } 1139 1140 static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu, 1141 u32 function, 1142 u32 index) 1143 { 1144 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first (or equivalent)"); 1145 1146 vcpu_get_cpuid(vcpu); 1147 1148 return (struct kvm_cpuid_entry2 *)get_cpuid_entry(vcpu->cpuid, 1149 function, index); 1150 } 1151 1152 static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu, 1153 u32 function) 1154 { 1155 return __vcpu_get_cpuid_entry(vcpu, function, 0); 1156 } 1157 1158 static inline int __vcpu_set_cpuid(struct kvm_vcpu *vcpu) 1159 { 1160 int r; 1161 1162 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first"); 1163 r = __vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid); 1164 if (r) 1165 return r; 1166 1167 /* On success, refresh the cache to pick up adjustments made by KVM. */ 1168 vcpu_get_cpuid(vcpu); 1169 return 0; 1170 } 1171 1172 static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu) 1173 { 1174 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first"); 1175 vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid); 1176 1177 /* Refresh the cache to pick up adjustments made by KVM. */ 1178 vcpu_get_cpuid(vcpu); 1179 } 1180 1181 void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu, 1182 struct kvm_x86_cpu_property property, 1183 u32 value); 1184 1185 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function); 1186 1187 static inline bool vcpu_cpuid_has(struct kvm_vcpu *vcpu, 1188 struct kvm_x86_cpu_feature feature) 1189 { 1190 struct kvm_cpuid_entry2 *entry; 1191 1192 entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index); 1193 return *((&entry->eax) + feature.reg) & BIT(feature.bit); 1194 } 1195 1196 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu, 1197 struct kvm_x86_cpu_feature feature, 1198 bool set); 1199 1200 static inline void vcpu_set_cpuid_feature(struct kvm_vcpu *vcpu, 1201 struct kvm_x86_cpu_feature feature) 1202 { 1203 vcpu_set_or_clear_cpuid_feature(vcpu, feature, true); 1204 1205 } 1206 1207 static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu, 1208 struct kvm_x86_cpu_feature feature) 1209 { 1210 vcpu_set_or_clear_cpuid_feature(vcpu, feature, false); 1211 } 1212 1213 u64 vcpu_get_msr(struct kvm_vcpu *vcpu, u64 msr_index); 1214 int _vcpu_set_msr(struct kvm_vcpu *vcpu, u64 msr_index, u64 msr_value); 1215 1216 /* 1217 * Assert on an MSR access(es) and pretty print the MSR name when possible. 1218 * Note, the caller provides the stringified name so that the name of macro is 1219 * printed, not the value the macro resolves to (due to macro expansion). 1220 */ 1221 #define TEST_ASSERT_MSR(cond, fmt, msr, str, args...) \ 1222 do { \ 1223 if (__builtin_constant_p(msr)) { \ 1224 TEST_ASSERT(cond, fmt, str, args); \ 1225 } else if (!(cond)) { \ 1226 char buf[16]; \ 1227 \ 1228 snprintf(buf, sizeof(buf), "MSR 0x%x", msr); \ 1229 TEST_ASSERT(cond, fmt, buf, args); \ 1230 } \ 1231 } while (0) 1232 1233 /* 1234 * Returns true if KVM should return the last written value when reading an MSR 1235 * from userspace, e.g. the MSR isn't a command MSR, doesn't emulate state that 1236 * is changing, etc. This is NOT an exhaustive list! The intent is to filter 1237 * out MSRs that are not durable _and_ that a selftest wants to write. 1238 */ 1239 static inline bool is_durable_msr(u32 msr) 1240 { 1241 return msr != MSR_IA32_TSC; 1242 } 1243 1244 #define vcpu_set_msr(vcpu, msr, val) \ 1245 do { \ 1246 u64 r, v = val; \ 1247 \ 1248 TEST_ASSERT_MSR(_vcpu_set_msr(vcpu, msr, v) == 1, \ 1249 "KVM_SET_MSRS failed on %s, value = 0x%lx", msr, #msr, v); \ 1250 if (!is_durable_msr(msr)) \ 1251 break; \ 1252 r = vcpu_get_msr(vcpu, msr); \ 1253 TEST_ASSERT_MSR(r == v, "Set %s to '0x%lx', got back '0x%lx'", msr, #msr, v, r);\ 1254 } while (0) 1255 1256 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits); 1257 void kvm_init_vm_address_properties(struct kvm_vm *vm); 1258 1259 struct ex_regs { 1260 u64 rax, rcx, rdx, rbx; 1261 u64 rbp, rsi, rdi; 1262 u64 r8, r9, r10, r11; 1263 u64 r12, r13, r14, r15; 1264 u64 vector; 1265 u64 error_code; 1266 u64 rip; 1267 u64 cs; 1268 u64 rflags; 1269 }; 1270 1271 struct idt_entry { 1272 u16 offset0; 1273 u16 selector; 1274 u16 ist : 3; 1275 u16 : 5; 1276 u16 type : 4; 1277 u16 : 1; 1278 u16 dpl : 2; 1279 u16 p : 1; 1280 u16 offset1; 1281 u32 offset2; u32 reserved; 1282 }; 1283 1284 void vm_install_exception_handler(struct kvm_vm *vm, int vector, 1285 void (*handler)(struct ex_regs *)); 1286 1287 gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages); 1288 1289 /* 1290 * Exception fixup morphs #DE to an arbitrary magic vector so that '0' can be 1291 * used to signal "no expcetion". 1292 */ 1293 #define KVM_MAGIC_DE_VECTOR 0xff 1294 1295 /* If a toddler were to say "abracadabra". */ 1296 #define KVM_EXCEPTION_MAGIC 0xabacadabaULL 1297 1298 /* 1299 * KVM selftest exception fixup uses registers to coordinate with the exception 1300 * handler, versus the kernel's in-memory tables and KVM-Unit-Tests's in-memory 1301 * per-CPU data. Using only registers avoids having to map memory into the 1302 * guest, doesn't require a valid, stable GS.base, and reduces the risk of 1303 * for recursive faults when accessing memory in the handler. The downside to 1304 * using registers is that it restricts what registers can be used by the actual 1305 * instruction. But, selftests are 64-bit only, making register* pressure a 1306 * minor concern. Use r9-r11 as they are volatile, i.e. don't need to be saved 1307 * by the callee, and except for r11 are not implicit parameters to any 1308 * instructions. Ideally, fixup would use r8-r10 and thus avoid implicit 1309 * parameters entirely, but Hyper-V's hypercall ABI uses r8 and testing Hyper-V 1310 * is higher priority than testing non-faulting SYSCALL/SYSRET. 1311 * 1312 * Note, the fixup handler deliberately does not handle #DE, i.e. the vector 1313 * is guaranteed to be non-zero on fault. 1314 * 1315 * REGISTER INPUTS: 1316 * r9 = MAGIC 1317 * r10 = RIP 1318 * r11 = new RIP on fault 1319 * 1320 * REGISTER OUTPUTS: 1321 * r9 = exception vector (non-zero) 1322 * r10 = error code 1323 */ 1324 #define __KVM_ASM_SAFE(insn, fep) \ 1325 "mov $" __stringify(KVM_EXCEPTION_MAGIC) ", %%r9\n\t" \ 1326 "lea 1f(%%rip), %%r10\n\t" \ 1327 "lea 2f(%%rip), %%r11\n\t" \ 1328 fep "1: " insn "\n\t" \ 1329 "mov $0, %%r9d\n\t" \ 1330 "2:\n\t" \ 1331 "mov %%r9b, %[vector]\n\t" \ 1332 "mov %%r10, %[error_code]\n\t" 1333 1334 #define KVM_ASM_SAFE(insn) __KVM_ASM_SAFE(insn, "") 1335 #define KVM_ASM_SAFE_FEP(insn) __KVM_ASM_SAFE(insn, KVM_FEP) 1336 1337 #define KVM_ASM_SAFE_OUTPUTS(v, ec) [vector] "=qm"(v), [error_code] "=rm"(ec) 1338 #define KVM_ASM_SAFE_CLOBBERS "r9", "r10", "r11" 1339 1340 #define kvm_asm_safe(insn, inputs...) \ 1341 ({ \ 1342 u64 ign_error_code; \ 1343 u8 vector; \ 1344 \ 1345 asm volatile(KVM_ASM_SAFE(insn) \ 1346 : KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \ 1347 : inputs \ 1348 : KVM_ASM_SAFE_CLOBBERS); \ 1349 vector; \ 1350 }) 1351 1352 #define kvm_asm_safe_ec(insn, error_code, inputs...) \ 1353 ({ \ 1354 u8 vector; \ 1355 \ 1356 asm volatile(KVM_ASM_SAFE(insn) \ 1357 : KVM_ASM_SAFE_OUTPUTS(vector, error_code) \ 1358 : inputs \ 1359 : KVM_ASM_SAFE_CLOBBERS); \ 1360 vector; \ 1361 }) 1362 1363 #define kvm_asm_safe_fep(insn, inputs...) \ 1364 ({ \ 1365 u64 ign_error_code; \ 1366 u8 vector; \ 1367 \ 1368 asm volatile(KVM_ASM_SAFE_FEP(insn) \ 1369 : KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \ 1370 : inputs \ 1371 : KVM_ASM_SAFE_CLOBBERS); \ 1372 vector; \ 1373 }) 1374 1375 #define kvm_asm_safe_ec_fep(insn, error_code, inputs...) \ 1376 ({ \ 1377 u8 vector; \ 1378 \ 1379 asm volatile(KVM_ASM_SAFE_FEP(insn) \ 1380 : KVM_ASM_SAFE_OUTPUTS(vector, error_code) \ 1381 : inputs \ 1382 : KVM_ASM_SAFE_CLOBBERS); \ 1383 vector; \ 1384 }) 1385 1386 #define BUILD_READ_U64_SAFE_HELPER(insn, _fep, _FEP) \ 1387 static inline u8 insn##_safe ##_fep(u32 idx, u64 *val) \ 1388 { \ 1389 u64 error_code; \ 1390 u8 vector; \ 1391 u32 a, d; \ 1392 \ 1393 asm volatile(KVM_ASM_SAFE##_FEP(#insn) \ 1394 : "=a"(a), "=d"(d), \ 1395 KVM_ASM_SAFE_OUTPUTS(vector, error_code) \ 1396 : "c"(idx) \ 1397 : KVM_ASM_SAFE_CLOBBERS); \ 1398 \ 1399 *val = (u64)a | ((u64)d << 32); \ 1400 return vector; \ 1401 } 1402 1403 /* 1404 * Generate {insn}_safe() and {insn}_safe_fep() helpers for instructions that 1405 * use ECX as in input index, and EDX:EAX as a 64-bit output. 1406 */ 1407 #define BUILD_READ_U64_SAFE_HELPERS(insn) \ 1408 BUILD_READ_U64_SAFE_HELPER(insn, , ) \ 1409 BUILD_READ_U64_SAFE_HELPER(insn, _fep, _FEP) \ 1410 1411 BUILD_READ_U64_SAFE_HELPERS(rdmsr) 1412 BUILD_READ_U64_SAFE_HELPERS(rdpmc) 1413 BUILD_READ_U64_SAFE_HELPERS(xgetbv) 1414 1415 static inline u8 wrmsr_safe(u32 msr, u64 val) 1416 { 1417 return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr)); 1418 } 1419 1420 static inline u8 xsetbv_safe(u32 index, u64 value) 1421 { 1422 u32 eax = value; 1423 u32 edx = value >> 32; 1424 1425 return kvm_asm_safe("xsetbv", "a" (eax), "d" (edx), "c" (index)); 1426 } 1427 1428 bool kvm_is_tdp_enabled(void); 1429 1430 static inline bool get_kvm_intel_param_bool(const char *param) 1431 { 1432 return kvm_get_module_param_bool("kvm_intel", param); 1433 } 1434 1435 static inline bool get_kvm_amd_param_bool(const char *param) 1436 { 1437 return kvm_get_module_param_bool("kvm_amd", param); 1438 } 1439 1440 static inline int get_kvm_intel_param_integer(const char *param) 1441 { 1442 return kvm_get_module_param_integer("kvm_intel", param); 1443 } 1444 1445 static inline int get_kvm_amd_param_integer(const char *param) 1446 { 1447 return kvm_get_module_param_integer("kvm_amd", param); 1448 } 1449 1450 static inline bool kvm_is_pmu_enabled(void) 1451 { 1452 return get_kvm_param_bool("enable_pmu"); 1453 } 1454 1455 static inline bool kvm_is_mediated_pmu_enabled(void) 1456 { 1457 if (host_cpu_is_intel_compatible) 1458 return get_kvm_intel_param_bool("enable_mediated_pmu"); 1459 1460 return get_kvm_amd_param_bool("enable_mediated_pmu"); 1461 } 1462 1463 static inline bool kvm_is_forced_emulation_enabled(void) 1464 { 1465 return !!get_kvm_param_integer("force_emulation_prefix"); 1466 } 1467 1468 static inline bool kvm_is_unrestricted_guest_enabled(void) 1469 { 1470 return get_kvm_intel_param_bool("unrestricted_guest"); 1471 } 1472 1473 static inline bool kvm_is_ignore_msrs(void) 1474 { 1475 return get_kvm_param_bool("ignore_msrs"); 1476 } 1477 1478 static inline bool kvm_is_lbrv_enabled(void) 1479 { 1480 return !!get_kvm_amd_param_integer("lbrv"); 1481 } 1482 1483 /* 1484 * Do NOT use this to check for nVMX or nSVM support. Querying kvm_cpu_has() 1485 * for either of X86_FEATURE_{VMX,SVM} is the idiomatic way to check for nested 1486 * virtualization support. Use this *only* to validate KVM's own enumeration. 1487 */ 1488 static inline bool kvm_is_nested_virtualization_enabled(void) 1489 { 1490 if (host_cpu_is_intel) 1491 return get_kvm_intel_param_bool("nested"); 1492 1493 return get_kvm_amd_param_integer("nested"); 1494 } 1495 1496 u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva); 1497 1498 u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3); 1499 u64 __xen_hypercall(u64 nr, u64 a0, void *a1); 1500 void xen_hypercall(u64 nr, u64 a0, void *a1); 1501 1502 static inline u64 __kvm_hypercall_map_gpa_range(gpa_t gpa, u64 size, u64 flags) 1503 { 1504 return kvm_hypercall(KVM_HC_MAP_GPA_RANGE, gpa, size >> PAGE_SHIFT, flags, 0); 1505 } 1506 1507 static inline void kvm_hypercall_map_gpa_range(gpa_t gpa, u64 size, u64 flags) 1508 { 1509 u64 ret = __kvm_hypercall_map_gpa_range(gpa, size, flags); 1510 1511 GUEST_ASSERT(!ret); 1512 } 1513 1514 /* 1515 * Execute HLT in an STI interrupt shadow to ensure that a pending IRQ that's 1516 * intended to be a wake event arrives *after* HLT is executed. Modern CPUs, 1517 * except for a few oddballs that KVM is unlikely to run on, block IRQs for one 1518 * instruction after STI, *if* RFLAGS.IF=0 before STI. Note, Intel CPUs may 1519 * block other events beyond regular IRQs, e.g. may block NMIs and SMIs too. 1520 */ 1521 static inline void safe_halt(void) 1522 { 1523 asm volatile("sti; hlt"); 1524 } 1525 1526 /* 1527 * Enable interrupts and ensure that interrupts are evaluated upon return from 1528 * this function, i.e. execute a nop to consume the STi interrupt shadow. 1529 */ 1530 static inline void sti_nop(void) 1531 { 1532 asm volatile ("sti; nop"); 1533 } 1534 1535 /* 1536 * Enable interrupts for one instruction (nop), to allow the CPU to process all 1537 * interrupts that are already pending. 1538 */ 1539 static inline void sti_nop_cli(void) 1540 { 1541 asm volatile ("sti; nop; cli"); 1542 } 1543 1544 static inline void sti(void) 1545 { 1546 asm volatile("sti"); 1547 } 1548 1549 static inline void cli(void) 1550 { 1551 asm volatile ("cli"); 1552 } 1553 1554 void __vm_xsave_require_permission(u64 xfeature, const char *name); 1555 1556 #define vm_xsave_require_permission(xfeature) \ 1557 __vm_xsave_require_permission(xfeature, #xfeature) 1558 1559 enum pg_level { 1560 PG_LEVEL_NONE, 1561 PG_LEVEL_4K, 1562 PG_LEVEL_2M, 1563 PG_LEVEL_1G, 1564 PG_LEVEL_512G, 1565 PG_LEVEL_256T 1566 }; 1567 1568 #define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12) 1569 #define PG_LEVEL_SIZE(_level) (1ull << PG_LEVEL_SHIFT(_level)) 1570 1571 #define PG_SIZE_4K PG_LEVEL_SIZE(PG_LEVEL_4K) 1572 #define PG_SIZE_2M PG_LEVEL_SIZE(PG_LEVEL_2M) 1573 #define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G) 1574 1575 #define PTE_PRESENT_MASK(mmu) ((mmu)->arch.pte_masks.present) 1576 #define PTE_WRITABLE_MASK(mmu) ((mmu)->arch.pte_masks.writable) 1577 #define PTE_USER_MASK(mmu) ((mmu)->arch.pte_masks.user) 1578 #define PTE_READABLE_MASK(mmu) ((mmu)->arch.pte_masks.readable) 1579 #define PTE_EXECUTABLE_MASK(mmu) ((mmu)->arch.pte_masks.executable) 1580 #define PTE_ACCESSED_MASK(mmu) ((mmu)->arch.pte_masks.accessed) 1581 #define PTE_DIRTY_MASK(mmu) ((mmu)->arch.pte_masks.dirty) 1582 #define PTE_HUGE_MASK(mmu) ((mmu)->arch.pte_masks.huge) 1583 #define PTE_NX_MASK(mmu) ((mmu)->arch.pte_masks.nx) 1584 #define PTE_C_BIT_MASK(mmu) ((mmu)->arch.pte_masks.c) 1585 #define PTE_S_BIT_MASK(mmu) ((mmu)->arch.pte_masks.s) 1586 #define PTE_ALWAYS_SET_MASK(mmu) ((mmu)->arch.pte_masks.always_set) 1587 1588 /* 1589 * For PTEs without a PRESENT bit (i.e. EPT entries), treat the PTE as present 1590 * if it's executable or readable, as EPT supports execute-only PTEs, but not 1591 * write-only PTEs. 1592 */ 1593 #define is_present_pte(mmu, pte) \ 1594 (PTE_PRESENT_MASK(mmu) ? \ 1595 !!(*(pte) & PTE_PRESENT_MASK(mmu)) : \ 1596 !!(*(pte) & (PTE_READABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu)))) 1597 #define is_executable_pte(mmu, pte) \ 1598 ((*(pte) & (PTE_EXECUTABLE_MASK(mmu) | PTE_NX_MASK(mmu))) == PTE_EXECUTABLE_MASK(mmu)) 1599 #define is_writable_pte(mmu, pte) (!!(*(pte) & PTE_WRITABLE_MASK(mmu))) 1600 #define is_user_pte(mmu, pte) (!!(*(pte) & PTE_USER_MASK(mmu))) 1601 #define is_accessed_pte(mmu, pte) (!!(*(pte) & PTE_ACCESSED_MASK(mmu))) 1602 #define is_dirty_pte(mmu, pte) (!!(*(pte) & PTE_DIRTY_MASK(mmu))) 1603 #define is_huge_pte(mmu, pte) (!!(*(pte) & PTE_HUGE_MASK(mmu))) 1604 #define is_nx_pte(mmu, pte) (!is_executable_pte(mmu, pte)) 1605 1606 void tdp_mmu_init(struct kvm_vm *vm, int pgtable_levels, 1607 struct pte_masks *pte_masks); 1608 1609 void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, gva_t gva, 1610 gpa_t gpa, int level); 1611 void virt_map_level(struct kvm_vm *vm, gva_t gva, gpa_t gpa, 1612 u64 nr_bytes, int level); 1613 1614 void vm_enable_tdp(struct kvm_vm *vm); 1615 bool kvm_cpu_has_tdp(void); 1616 void tdp_map(struct kvm_vm *vm, gpa_t l2_gpa, gpa_t gpa, u64 size); 1617 void tdp_identity_map_default_memslots(struct kvm_vm *vm); 1618 void tdp_identity_map_1g(struct kvm_vm *vm, u64 addr, u64 size); 1619 u64 *tdp_get_pte(struct kvm_vm *vm, u64 l2_gpa); 1620 1621 /* 1622 * Basic CPU control in CR0 1623 */ 1624 #define X86_CR0_PE (1UL<<0) /* Protection Enable */ 1625 #define X86_CR0_MP (1UL<<1) /* Monitor Coprocessor */ 1626 #define X86_CR0_EM (1UL<<2) /* Emulation */ 1627 #define X86_CR0_TS (1UL<<3) /* Task Switched */ 1628 #define X86_CR0_ET (1UL<<4) /* Extension Type */ 1629 #define X86_CR0_NE (1UL<<5) /* Numeric Error */ 1630 #define X86_CR0_WP (1UL<<16) /* Write Protect */ 1631 #define X86_CR0_AM (1UL<<18) /* Alignment Mask */ 1632 #define X86_CR0_NW (1UL<<29) /* Not Write-through */ 1633 #define X86_CR0_CD (1UL<<30) /* Cache Disable */ 1634 #define X86_CR0_PG (1UL<<31) /* Paging */ 1635 1636 #define PFERR_PRESENT_BIT 0 1637 #define PFERR_WRITE_BIT 1 1638 #define PFERR_USER_BIT 2 1639 #define PFERR_RSVD_BIT 3 1640 #define PFERR_FETCH_BIT 4 1641 #define PFERR_PK_BIT 5 1642 #define PFERR_SGX_BIT 15 1643 #define PFERR_GUEST_FINAL_BIT 32 1644 #define PFERR_GUEST_PAGE_BIT 33 1645 #define PFERR_IMPLICIT_ACCESS_BIT 48 1646 1647 #define PFERR_PRESENT_MASK BIT(PFERR_PRESENT_BIT) 1648 #define PFERR_WRITE_MASK BIT(PFERR_WRITE_BIT) 1649 #define PFERR_USER_MASK BIT(PFERR_USER_BIT) 1650 #define PFERR_RSVD_MASK BIT(PFERR_RSVD_BIT) 1651 #define PFERR_FETCH_MASK BIT(PFERR_FETCH_BIT) 1652 #define PFERR_PK_MASK BIT(PFERR_PK_BIT) 1653 #define PFERR_SGX_MASK BIT(PFERR_SGX_BIT) 1654 #define PFERR_GUEST_FINAL_MASK BIT_ULL(PFERR_GUEST_FINAL_BIT) 1655 #define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT) 1656 #define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT) 1657 1658 #define EPT_VIOLATION_ACC_READ BIT(0) 1659 #define EPT_VIOLATION_ACC_WRITE BIT(1) 1660 #define EPT_VIOLATION_ACC_INSTR BIT(2) 1661 #define EPT_VIOLATION_PROT_READ BIT(3) 1662 #define EPT_VIOLATION_PROT_WRITE BIT(4) 1663 #define EPT_VIOLATION_PROT_EXEC BIT(5) 1664 #define EPT_VIOLATION_GVA_IS_VALID BIT(7) 1665 #define EPT_VIOLATION_GVA_TRANSLATED BIT(8) 1666 1667 bool sys_clocksource_is_based_on_tsc(void); 1668 1669 #endif /* SELFTEST_KVM_PROCESSOR_H */ 1670