1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef ARCH_X86_KVM_X86_H 3 #define ARCH_X86_KVM_X86_H 4 5 #include <asm/processor.h> 6 #include <asm/mwait.h> 7 #include <linux/kvm_host.h> 8 #include <asm/pvclock.h> 9 #include "kvm_cache_regs.h" 10 11 #define MSR_IA32_CR_PAT_DEFAULT 0x0007040600070406ULL 12 13 static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu) 14 { 15 vcpu->arch.exception.injected = false; 16 } 17 18 static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector, 19 bool soft) 20 { 21 vcpu->arch.interrupt.pending = true; 22 vcpu->arch.interrupt.soft = soft; 23 vcpu->arch.interrupt.nr = vector; 24 } 25 26 static inline void kvm_clear_interrupt_queue(struct kvm_vcpu *vcpu) 27 { 28 vcpu->arch.interrupt.pending = false; 29 } 30 31 static inline bool kvm_event_needs_reinjection(struct kvm_vcpu *vcpu) 32 { 33 return vcpu->arch.exception.injected || vcpu->arch.interrupt.pending || 34 vcpu->arch.nmi_injected; 35 } 36 37 static inline bool kvm_exception_is_soft(unsigned int nr) 38 { 39 return (nr == BP_VECTOR) || (nr == OF_VECTOR); 40 } 41 42 static inline bool is_protmode(struct kvm_vcpu *vcpu) 43 { 44 return kvm_read_cr0_bits(vcpu, X86_CR0_PE); 45 } 46 47 static inline int is_long_mode(struct kvm_vcpu *vcpu) 48 { 49 #ifdef CONFIG_X86_64 50 return vcpu->arch.efer & EFER_LMA; 51 #else 52 return 0; 53 #endif 54 } 55 56 static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) 57 { 58 int cs_db, cs_l; 59 60 if (!is_long_mode(vcpu)) 61 return false; 62 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 63 return cs_l; 64 } 65 66 static inline bool is_la57_mode(struct kvm_vcpu *vcpu) 67 { 68 #ifdef CONFIG_X86_64 69 return (vcpu->arch.efer & EFER_LMA) && 70 kvm_read_cr4_bits(vcpu, X86_CR4_LA57); 71 #else 72 return 0; 73 #endif 74 } 75 76 static inline bool mmu_is_nested(struct kvm_vcpu *vcpu) 77 { 78 return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu; 79 } 80 81 static inline int is_pae(struct kvm_vcpu *vcpu) 82 { 83 return kvm_read_cr4_bits(vcpu, X86_CR4_PAE); 84 } 85 86 static inline int is_pse(struct kvm_vcpu *vcpu) 87 { 88 return kvm_read_cr4_bits(vcpu, X86_CR4_PSE); 89 } 90 91 static inline int is_paging(struct kvm_vcpu *vcpu) 92 { 93 return likely(kvm_read_cr0_bits(vcpu, X86_CR0_PG)); 94 } 95 96 static inline u32 bit(int bitno) 97 { 98 return 1 << (bitno & 31); 99 } 100 101 static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu) 102 { 103 return kvm_read_cr4_bits(vcpu, X86_CR4_LA57) ? 57 : 48; 104 } 105 106 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt) 107 { 108 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; 109 } 110 111 static inline u64 get_canonical(u64 la, u8 vaddr_bits) 112 { 113 return ((int64_t)la << (64 - vaddr_bits)) >> (64 - vaddr_bits); 114 } 115 116 static inline bool is_noncanonical_address(u64 la, struct kvm_vcpu *vcpu) 117 { 118 #ifdef CONFIG_X86_64 119 return get_canonical(la, vcpu_virt_addr_bits(vcpu)) != la; 120 #else 121 return false; 122 #endif 123 } 124 125 static inline bool emul_is_noncanonical_address(u64 la, 126 struct x86_emulate_ctxt *ctxt) 127 { 128 #ifdef CONFIG_X86_64 129 return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la; 130 #else 131 return false; 132 #endif 133 } 134 135 static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, 136 gva_t gva, gfn_t gfn, unsigned access) 137 { 138 /* 139 * If this is a shadow nested page table, the "GVA" is 140 * actually a nGPA. 141 */ 142 vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK; 143 vcpu->arch.access = access; 144 vcpu->arch.mmio_gfn = gfn; 145 vcpu->arch.mmio_gen = kvm_memslots(vcpu->kvm)->generation; 146 } 147 148 static inline bool vcpu_match_mmio_gen(struct kvm_vcpu *vcpu) 149 { 150 return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation; 151 } 152 153 /* 154 * Clear the mmio cache info for the given gva. If gva is MMIO_GVA_ANY, we 155 * clear all mmio cache info. 156 */ 157 #define MMIO_GVA_ANY (~(gva_t)0) 158 159 static inline void vcpu_clear_mmio_info(struct kvm_vcpu *vcpu, gva_t gva) 160 { 161 if (gva != MMIO_GVA_ANY && vcpu->arch.mmio_gva != (gva & PAGE_MASK)) 162 return; 163 164 vcpu->arch.mmio_gva = 0; 165 } 166 167 static inline bool vcpu_match_mmio_gva(struct kvm_vcpu *vcpu, unsigned long gva) 168 { 169 if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gva && 170 vcpu->arch.mmio_gva == (gva & PAGE_MASK)) 171 return true; 172 173 return false; 174 } 175 176 static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) 177 { 178 if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gfn && 179 vcpu->arch.mmio_gfn == gpa >> PAGE_SHIFT) 180 return true; 181 182 return false; 183 } 184 185 static inline unsigned long kvm_register_readl(struct kvm_vcpu *vcpu, 186 enum kvm_reg reg) 187 { 188 unsigned long val = kvm_register_read(vcpu, reg); 189 190 return is_64_bit_mode(vcpu) ? val : (u32)val; 191 } 192 193 static inline void kvm_register_writel(struct kvm_vcpu *vcpu, 194 enum kvm_reg reg, 195 unsigned long val) 196 { 197 if (!is_64_bit_mode(vcpu)) 198 val = (u32)val; 199 return kvm_register_write(vcpu, reg, val); 200 } 201 202 static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) 203 { 204 return !(kvm->arch.disabled_quirks & quirk); 205 } 206 207 void kvm_before_handle_nmi(struct kvm_vcpu *vcpu); 208 void kvm_after_handle_nmi(struct kvm_vcpu *vcpu); 209 void kvm_set_pending_timer(struct kvm_vcpu *vcpu); 210 int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip); 211 212 void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr); 213 u64 get_kvmclock_ns(struct kvm *kvm); 214 215 int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt, 216 gva_t addr, void *val, unsigned int bytes, 217 struct x86_exception *exception); 218 219 int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt, 220 gva_t addr, void *val, unsigned int bytes, 221 struct x86_exception *exception); 222 223 void kvm_vcpu_mtrr_init(struct kvm_vcpu *vcpu); 224 u8 kvm_mtrr_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn); 225 bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data); 226 int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); 227 int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); 228 bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, 229 int page_num); 230 bool kvm_vector_hashing_enabled(void); 231 232 #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ 233 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ 234 | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ 235 | XFEATURE_MASK_PKRU) 236 extern u64 host_xcr0; 237 238 extern u64 kvm_supported_xcr0(void); 239 240 extern unsigned int min_timer_period_us; 241 242 extern unsigned int lapic_timer_advance_ns; 243 244 extern struct static_key kvm_no_apic_vcpu; 245 246 static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) 247 { 248 return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult, 249 vcpu->arch.virtual_tsc_shift); 250 } 251 252 /* Same "calling convention" as do_div: 253 * - divide (n << 32) by base 254 * - put result in n 255 * - return remainder 256 */ 257 #define do_shl32_div32(n, base) \ 258 ({ \ 259 u32 __quot, __rem; \ 260 asm("divl %2" : "=a" (__quot), "=d" (__rem) \ 261 : "rm" (base), "0" (0), "1" ((u32) n)); \ 262 n = __quot; \ 263 __rem; \ 264 }) 265 266 static inline bool kvm_mwait_in_guest(void) 267 { 268 unsigned int eax, ebx, ecx, edx; 269 270 if (!cpu_has(&boot_cpu_data, X86_FEATURE_MWAIT)) 271 return false; 272 273 switch (boot_cpu_data.x86_vendor) { 274 case X86_VENDOR_AMD: 275 /* All AMD CPUs have a working MWAIT implementation */ 276 return true; 277 case X86_VENDOR_INTEL: 278 /* Handle Intel below */ 279 break; 280 default: 281 return false; 282 } 283 284 /* 285 * Intel CPUs without CPUID5_ECX_INTERRUPT_BREAK are problematic as 286 * they would allow guest to stop the CPU completely by disabling 287 * interrupts then invoking MWAIT. 288 */ 289 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 290 return false; 291 292 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx); 293 294 if (!(ecx & CPUID5_ECX_INTERRUPT_BREAK)) 295 return false; 296 297 return true; 298 } 299 300 #endif 301