1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * tools/testing/selftests/kvm/include/x86_64/processor.h 4 * 5 * Copyright (C) 2018, Google LLC. 6 */ 7 8 #ifndef SELFTEST_KVM_PROCESSOR_H 9 #define SELFTEST_KVM_PROCESSOR_H 10 11 #include <assert.h> 12 #include <stdint.h> 13 #include <syscall.h> 14 15 #include <asm/msr-index.h> 16 #include <asm/prctl.h> 17 18 #include "../kvm_util.h" 19 20 #define X86_EFLAGS_FIXED (1u << 1) 21 22 #define X86_CR4_VME (1ul << 0) 23 #define X86_CR4_PVI (1ul << 1) 24 #define X86_CR4_TSD (1ul << 2) 25 #define X86_CR4_DE (1ul << 3) 26 #define X86_CR4_PSE (1ul << 4) 27 #define X86_CR4_PAE (1ul << 5) 28 #define X86_CR4_MCE (1ul << 6) 29 #define X86_CR4_PGE (1ul << 7) 30 #define X86_CR4_PCE (1ul << 8) 31 #define X86_CR4_OSFXSR (1ul << 9) 32 #define X86_CR4_OSXMMEXCPT (1ul << 10) 33 #define X86_CR4_UMIP (1ul << 11) 34 #define X86_CR4_LA57 (1ul << 12) 35 #define X86_CR4_VMXE (1ul << 13) 36 #define X86_CR4_SMXE (1ul << 14) 37 #define X86_CR4_FSGSBASE (1ul << 16) 38 #define X86_CR4_PCIDE (1ul << 17) 39 #define X86_CR4_OSXSAVE (1ul << 18) 40 #define X86_CR4_SMEP (1ul << 20) 41 #define X86_CR4_SMAP (1ul << 21) 42 #define X86_CR4_PKE (1ul << 22) 43 44 /* CPUID.1.ECX */ 45 #define CPUID_VMX (1ul << 5) 46 #define CPUID_SMX (1ul << 6) 47 #define CPUID_PCID (1ul << 17) 48 #define CPUID_XSAVE (1ul << 26) 49 50 /* CPUID.7.EBX */ 51 #define CPUID_FSGSBASE (1ul << 0) 52 #define CPUID_SMEP (1ul << 7) 53 #define CPUID_SMAP (1ul << 20) 54 55 /* CPUID.7.ECX */ 56 #define CPUID_UMIP (1ul << 2) 57 #define CPUID_PKU (1ul << 3) 58 #define CPUID_LA57 (1ul << 16) 59 60 /* CPUID.0x8000_0001.EDX */ 61 #define CPUID_GBPAGES (1ul << 26) 62 63 /* Page table bitfield declarations */ 64 #define PTE_PRESENT_MASK BIT_ULL(0) 65 #define PTE_WRITABLE_MASK BIT_ULL(1) 66 #define PTE_USER_MASK BIT_ULL(2) 67 #define PTE_ACCESSED_MASK BIT_ULL(5) 68 #define PTE_DIRTY_MASK BIT_ULL(6) 69 #define PTE_LARGE_MASK BIT_ULL(7) 70 #define PTE_GLOBAL_MASK BIT_ULL(8) 71 #define PTE_NX_MASK BIT_ULL(63) 72 73 #define PAGE_SHIFT 12 74 #define PAGE_SIZE (1ULL << PAGE_SHIFT) 75 #define PAGE_MASK (~(PAGE_SIZE-1)) 76 77 #define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12) 78 #define PTE_GET_PFN(pte) (((pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) 79 80 /* General Registers in 64-Bit Mode */ 81 struct gpr64_regs { 82 u64 rax; 83 u64 rcx; 84 u64 rdx; 85 u64 rbx; 86 u64 rsp; 87 u64 rbp; 88 u64 rsi; 89 u64 rdi; 90 u64 r8; 91 u64 r9; 92 u64 r10; 93 u64 r11; 94 u64 r12; 95 u64 r13; 96 u64 r14; 97 u64 r15; 98 }; 99 100 struct desc64 { 101 uint16_t limit0; 102 uint16_t base0; 103 unsigned base1:8, type:4, s:1, dpl:2, p:1; 104 unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8; 105 uint32_t base3; 106 uint32_t zero1; 107 } __attribute__((packed)); 108 109 struct desc_ptr { 110 uint16_t size; 111 uint64_t address; 112 } __attribute__((packed)); 113 114 struct kvm_x86_state { 115 struct kvm_xsave *xsave; 116 struct kvm_vcpu_events events; 117 struct kvm_mp_state mp_state; 118 struct kvm_regs regs; 119 struct kvm_xcrs xcrs; 120 struct kvm_sregs sregs; 121 struct kvm_debugregs debugregs; 122 union { 123 struct kvm_nested_state nested; 124 char nested_[16384]; 125 }; 126 struct kvm_msrs msrs; 127 }; 128 129 static inline uint64_t get_desc64_base(const struct desc64 *desc) 130 { 131 return ((uint64_t)desc->base3 << 32) | 132 (desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24)); 133 } 134 135 static inline uint64_t rdtsc(void) 136 { 137 uint32_t eax, edx; 138 uint64_t tsc_val; 139 /* 140 * The lfence is to wait (on Intel CPUs) until all previous 141 * instructions have been executed. If software requires RDTSC to be 142 * executed prior to execution of any subsequent instruction, it can 143 * execute LFENCE immediately after RDTSC 144 */ 145 __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 146 tsc_val = ((uint64_t)edx) << 32 | eax; 147 return tsc_val; 148 } 149 150 static inline uint64_t rdtscp(uint32_t *aux) 151 { 152 uint32_t eax, edx; 153 154 __asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux)); 155 return ((uint64_t)edx) << 32 | eax; 156 } 157 158 static inline uint64_t rdmsr(uint32_t msr) 159 { 160 uint32_t a, d; 161 162 __asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory"); 163 164 return a | ((uint64_t) d << 32); 165 } 166 167 static inline void wrmsr(uint32_t msr, uint64_t value) 168 { 169 uint32_t a = value; 170 uint32_t d = value >> 32; 171 172 __asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory"); 173 } 174 175 176 static inline uint16_t inw(uint16_t port) 177 { 178 uint16_t tmp; 179 180 __asm__ __volatile__("in %%dx, %%ax" 181 : /* output */ "=a" (tmp) 182 : /* input */ "d" (port)); 183 184 return tmp; 185 } 186 187 static inline uint16_t get_es(void) 188 { 189 uint16_t es; 190 191 __asm__ __volatile__("mov %%es, %[es]" 192 : /* output */ [es]"=rm"(es)); 193 return es; 194 } 195 196 static inline uint16_t get_cs(void) 197 { 198 uint16_t cs; 199 200 __asm__ __volatile__("mov %%cs, %[cs]" 201 : /* output */ [cs]"=rm"(cs)); 202 return cs; 203 } 204 205 static inline uint16_t get_ss(void) 206 { 207 uint16_t ss; 208 209 __asm__ __volatile__("mov %%ss, %[ss]" 210 : /* output */ [ss]"=rm"(ss)); 211 return ss; 212 } 213 214 static inline uint16_t get_ds(void) 215 { 216 uint16_t ds; 217 218 __asm__ __volatile__("mov %%ds, %[ds]" 219 : /* output */ [ds]"=rm"(ds)); 220 return ds; 221 } 222 223 static inline uint16_t get_fs(void) 224 { 225 uint16_t fs; 226 227 __asm__ __volatile__("mov %%fs, %[fs]" 228 : /* output */ [fs]"=rm"(fs)); 229 return fs; 230 } 231 232 static inline uint16_t get_gs(void) 233 { 234 uint16_t gs; 235 236 __asm__ __volatile__("mov %%gs, %[gs]" 237 : /* output */ [gs]"=rm"(gs)); 238 return gs; 239 } 240 241 static inline uint16_t get_tr(void) 242 { 243 uint16_t tr; 244 245 __asm__ __volatile__("str %[tr]" 246 : /* output */ [tr]"=rm"(tr)); 247 return tr; 248 } 249 250 static inline uint64_t get_cr0(void) 251 { 252 uint64_t cr0; 253 254 __asm__ __volatile__("mov %%cr0, %[cr0]" 255 : /* output */ [cr0]"=r"(cr0)); 256 return cr0; 257 } 258 259 static inline uint64_t get_cr3(void) 260 { 261 uint64_t cr3; 262 263 __asm__ __volatile__("mov %%cr3, %[cr3]" 264 : /* output */ [cr3]"=r"(cr3)); 265 return cr3; 266 } 267 268 static inline uint64_t get_cr4(void) 269 { 270 uint64_t cr4; 271 272 __asm__ __volatile__("mov %%cr4, %[cr4]" 273 : /* output */ [cr4]"=r"(cr4)); 274 return cr4; 275 } 276 277 static inline void set_cr4(uint64_t val) 278 { 279 __asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory"); 280 } 281 282 static inline struct desc_ptr get_gdt(void) 283 { 284 struct desc_ptr gdt; 285 __asm__ __volatile__("sgdt %[gdt]" 286 : /* output */ [gdt]"=m"(gdt)); 287 return gdt; 288 } 289 290 static inline struct desc_ptr get_idt(void) 291 { 292 struct desc_ptr idt; 293 __asm__ __volatile__("sidt %[idt]" 294 : /* output */ [idt]"=m"(idt)); 295 return idt; 296 } 297 298 static inline void outl(uint16_t port, uint32_t value) 299 { 300 __asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value)); 301 } 302 303 static inline void cpuid(uint32_t *eax, uint32_t *ebx, 304 uint32_t *ecx, uint32_t *edx) 305 { 306 /* ecx is often an input as well as an output. */ 307 asm volatile("cpuid" 308 : "=a" (*eax), 309 "=b" (*ebx), 310 "=c" (*ecx), 311 "=d" (*edx) 312 : "0" (*eax), "2" (*ecx) 313 : "memory"); 314 } 315 316 #define SET_XMM(__var, __xmm) \ 317 asm volatile("movq %0, %%"#__xmm : : "r"(__var) : #__xmm) 318 319 static inline void set_xmm(int n, unsigned long val) 320 { 321 switch (n) { 322 case 0: 323 SET_XMM(val, xmm0); 324 break; 325 case 1: 326 SET_XMM(val, xmm1); 327 break; 328 case 2: 329 SET_XMM(val, xmm2); 330 break; 331 case 3: 332 SET_XMM(val, xmm3); 333 break; 334 case 4: 335 SET_XMM(val, xmm4); 336 break; 337 case 5: 338 SET_XMM(val, xmm5); 339 break; 340 case 6: 341 SET_XMM(val, xmm6); 342 break; 343 case 7: 344 SET_XMM(val, xmm7); 345 break; 346 } 347 } 348 349 #define GET_XMM(__xmm) \ 350 ({ \ 351 unsigned long __val; \ 352 asm volatile("movq %%"#__xmm", %0" : "=r"(__val)); \ 353 __val; \ 354 }) 355 356 static inline unsigned long get_xmm(int n) 357 { 358 assert(n >= 0 && n <= 7); 359 360 switch (n) { 361 case 0: 362 return GET_XMM(xmm0); 363 case 1: 364 return GET_XMM(xmm1); 365 case 2: 366 return GET_XMM(xmm2); 367 case 3: 368 return GET_XMM(xmm3); 369 case 4: 370 return GET_XMM(xmm4); 371 case 5: 372 return GET_XMM(xmm5); 373 case 6: 374 return GET_XMM(xmm6); 375 case 7: 376 return GET_XMM(xmm7); 377 } 378 379 /* never reached */ 380 return 0; 381 } 382 383 static inline void cpu_relax(void) 384 { 385 asm volatile("rep; nop" ::: "memory"); 386 } 387 388 bool is_intel_cpu(void); 389 bool is_amd_cpu(void); 390 391 static inline unsigned int x86_family(unsigned int eax) 392 { 393 unsigned int x86; 394 395 x86 = (eax >> 8) & 0xf; 396 397 if (x86 == 0xf) 398 x86 += (eax >> 20) & 0xff; 399 400 return x86; 401 } 402 403 static inline unsigned int x86_model(unsigned int eax) 404 { 405 return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f); 406 } 407 408 struct kvm_x86_state *vcpu_save_state(struct kvm_vm *vm, uint32_t vcpuid); 409 void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, 410 struct kvm_x86_state *state); 411 void kvm_x86_state_cleanup(struct kvm_x86_state *state); 412 413 struct kvm_msr_list *kvm_get_msr_index_list(void); 414 uint64_t kvm_get_feature_msr(uint64_t msr_index); 415 struct kvm_cpuid2 *kvm_get_supported_cpuid(void); 416 417 struct kvm_cpuid2 *vcpu_get_cpuid(struct kvm_vm *vm, uint32_t vcpuid); 418 int __vcpu_set_cpuid(struct kvm_vm *vm, uint32_t vcpuid, 419 struct kvm_cpuid2 *cpuid); 420 void vcpu_set_cpuid(struct kvm_vm *vm, uint32_t vcpuid, 421 struct kvm_cpuid2 *cpuid); 422 423 struct kvm_cpuid_entry2 * 424 kvm_get_supported_cpuid_index(uint32_t function, uint32_t index); 425 426 static inline struct kvm_cpuid_entry2 * 427 kvm_get_supported_cpuid_entry(uint32_t function) 428 { 429 return kvm_get_supported_cpuid_index(function, 0); 430 } 431 432 uint64_t vcpu_get_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index); 433 int _vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index, 434 uint64_t msr_value); 435 void vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index, 436 uint64_t msr_value); 437 438 uint32_t kvm_get_cpuid_max_basic(void); 439 uint32_t kvm_get_cpuid_max_extended(void); 440 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits); 441 442 struct ex_regs { 443 uint64_t rax, rcx, rdx, rbx; 444 uint64_t rbp, rsi, rdi; 445 uint64_t r8, r9, r10, r11; 446 uint64_t r12, r13, r14, r15; 447 uint64_t vector; 448 uint64_t error_code; 449 uint64_t rip; 450 uint64_t cs; 451 uint64_t rflags; 452 }; 453 454 void vm_init_descriptor_tables(struct kvm_vm *vm); 455 void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid); 456 void vm_install_exception_handler(struct kvm_vm *vm, int vector, 457 void (*handler)(struct ex_regs *)); 458 459 uint64_t vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr); 460 void vm_set_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr, 461 uint64_t pte); 462 463 /* 464 * get_cpuid() - find matching CPUID entry and return pointer to it. 465 */ 466 struct kvm_cpuid_entry2 *get_cpuid(struct kvm_cpuid2 *cpuid, uint32_t function, 467 uint32_t index); 468 /* 469 * set_cpuid() - overwrites a matching cpuid entry with the provided value. 470 * matches based on ent->function && ent->index. returns true 471 * if a match was found and successfully overwritten. 472 * @cpuid: the kvm cpuid list to modify. 473 * @ent: cpuid entry to insert 474 */ 475 bool set_cpuid(struct kvm_cpuid2 *cpuid, struct kvm_cpuid_entry2 *ent); 476 477 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, 478 uint64_t a3); 479 480 struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void); 481 void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid); 482 struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid); 483 void vm_xsave_req_perm(int bit); 484 485 enum x86_page_size { 486 X86_PAGE_SIZE_4K = 0, 487 X86_PAGE_SIZE_2M, 488 X86_PAGE_SIZE_1G, 489 }; 490 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 491 enum x86_page_size page_size); 492 493 /* 494 * Basic CPU control in CR0 495 */ 496 #define X86_CR0_PE (1UL<<0) /* Protection Enable */ 497 #define X86_CR0_MP (1UL<<1) /* Monitor Coprocessor */ 498 #define X86_CR0_EM (1UL<<2) /* Emulation */ 499 #define X86_CR0_TS (1UL<<3) /* Task Switched */ 500 #define X86_CR0_ET (1UL<<4) /* Extension Type */ 501 #define X86_CR0_NE (1UL<<5) /* Numeric Error */ 502 #define X86_CR0_WP (1UL<<16) /* Write Protect */ 503 #define X86_CR0_AM (1UL<<18) /* Alignment Mask */ 504 #define X86_CR0_NW (1UL<<29) /* Not Write-through */ 505 #define X86_CR0_CD (1UL<<30) /* Cache Disable */ 506 #define X86_CR0_PG (1UL<<31) /* Paging */ 507 508 /* VMX_EPT_VPID_CAP bits */ 509 #define VMX_EPT_VPID_CAP_AD_BITS (1ULL << 21) 510 511 #define XSTATE_XTILE_CFG_BIT 17 512 #define XSTATE_XTILE_DATA_BIT 18 513 514 #define XSTATE_XTILE_CFG_MASK (1ULL << XSTATE_XTILE_CFG_BIT) 515 #define XSTATE_XTILE_DATA_MASK (1ULL << XSTATE_XTILE_DATA_BIT) 516 #define XFEATURE_XTILE_MASK (XSTATE_XTILE_CFG_MASK | \ 517 XSTATE_XTILE_DATA_MASK) 518 #endif /* SELFTEST_KVM_PROCESSOR_H */ 519