1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018, Google LLC. 4 */ 5 6 #include "linux/bitmap.h" 7 #include "test_util.h" 8 #include "kvm_util.h" 9 #include "pmu.h" 10 #include "processor.h" 11 #include "sev.h" 12 13 #ifndef NUM_INTERRUPTS 14 #define NUM_INTERRUPTS 256 15 #endif 16 17 #define KERNEL_CS 0x8 18 #define KERNEL_DS 0x10 19 #define KERNEL_TSS 0x18 20 21 vm_vaddr_t exception_handlers; 22 bool host_cpu_is_amd; 23 bool host_cpu_is_intel; 24 bool is_forced_emulation_enabled; 25 uint64_t guest_tsc_khz; 26 27 const char *ex_str(int vector) 28 { 29 switch (vector) { 30 #define VEC_STR(v) case v##_VECTOR: return "#" #v 31 case DE_VECTOR: return "no exception"; 32 case KVM_MAGIC_DE_VECTOR: return "#DE"; 33 VEC_STR(DB); 34 VEC_STR(NMI); 35 VEC_STR(BP); 36 VEC_STR(OF); 37 VEC_STR(BR); 38 VEC_STR(UD); 39 VEC_STR(NM); 40 VEC_STR(DF); 41 VEC_STR(TS); 42 VEC_STR(NP); 43 VEC_STR(SS); 44 VEC_STR(GP); 45 VEC_STR(PF); 46 VEC_STR(MF); 47 VEC_STR(AC); 48 VEC_STR(MC); 49 VEC_STR(XM); 50 VEC_STR(VE); 51 VEC_STR(CP); 52 VEC_STR(HV); 53 VEC_STR(VC); 54 VEC_STR(SX); 55 default: return "#??"; 56 #undef VEC_STR 57 } 58 } 59 60 static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent) 61 { 62 fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx " 63 "rcx: 0x%.16llx rdx: 0x%.16llx\n", 64 indent, "", 65 regs->rax, regs->rbx, regs->rcx, regs->rdx); 66 fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx " 67 "rsp: 0x%.16llx rbp: 0x%.16llx\n", 68 indent, "", 69 regs->rsi, regs->rdi, regs->rsp, regs->rbp); 70 fprintf(stream, "%*sr8: 0x%.16llx r9: 0x%.16llx " 71 "r10: 0x%.16llx r11: 0x%.16llx\n", 72 indent, "", 73 regs->r8, regs->r9, regs->r10, regs->r11); 74 fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx " 75 "r14: 0x%.16llx r15: 0x%.16llx\n", 76 indent, "", 77 regs->r12, regs->r13, regs->r14, regs->r15); 78 fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n", 79 indent, "", 80 regs->rip, regs->rflags); 81 } 82 83 static void segment_dump(FILE *stream, struct kvm_segment *segment, 84 uint8_t indent) 85 { 86 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x " 87 "selector: 0x%.4x type: 0x%.2x\n", 88 indent, "", segment->base, segment->limit, 89 segment->selector, segment->type); 90 fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x " 91 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n", 92 indent, "", segment->present, segment->dpl, 93 segment->db, segment->s, segment->l); 94 fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x " 95 "unusable: 0x%.2x padding: 0x%.2x\n", 96 indent, "", segment->g, segment->avl, 97 segment->unusable, segment->padding); 98 } 99 100 static void dtable_dump(FILE *stream, struct kvm_dtable *dtable, 101 uint8_t indent) 102 { 103 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x " 104 "padding: 0x%.4x 0x%.4x 0x%.4x\n", 105 indent, "", dtable->base, dtable->limit, 106 dtable->padding[0], dtable->padding[1], dtable->padding[2]); 107 } 108 109 static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent) 110 { 111 unsigned int i; 112 113 fprintf(stream, "%*scs:\n", indent, ""); 114 segment_dump(stream, &sregs->cs, indent + 2); 115 fprintf(stream, "%*sds:\n", indent, ""); 116 segment_dump(stream, &sregs->ds, indent + 2); 117 fprintf(stream, "%*ses:\n", indent, ""); 118 segment_dump(stream, &sregs->es, indent + 2); 119 fprintf(stream, "%*sfs:\n", indent, ""); 120 segment_dump(stream, &sregs->fs, indent + 2); 121 fprintf(stream, "%*sgs:\n", indent, ""); 122 segment_dump(stream, &sregs->gs, indent + 2); 123 fprintf(stream, "%*sss:\n", indent, ""); 124 segment_dump(stream, &sregs->ss, indent + 2); 125 fprintf(stream, "%*str:\n", indent, ""); 126 segment_dump(stream, &sregs->tr, indent + 2); 127 fprintf(stream, "%*sldt:\n", indent, ""); 128 segment_dump(stream, &sregs->ldt, indent + 2); 129 130 fprintf(stream, "%*sgdt:\n", indent, ""); 131 dtable_dump(stream, &sregs->gdt, indent + 2); 132 fprintf(stream, "%*sidt:\n", indent, ""); 133 dtable_dump(stream, &sregs->idt, indent + 2); 134 135 fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx " 136 "cr3: 0x%.16llx cr4: 0x%.16llx\n", 137 indent, "", 138 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4); 139 fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx " 140 "apic_base: 0x%.16llx\n", 141 indent, "", 142 sregs->cr8, sregs->efer, sregs->apic_base); 143 144 fprintf(stream, "%*sinterrupt_bitmap:\n", indent, ""); 145 for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) { 146 fprintf(stream, "%*s%.16llx\n", indent + 2, "", 147 sregs->interrupt_bitmap[i]); 148 } 149 } 150 151 bool kvm_is_tdp_enabled(void) 152 { 153 if (host_cpu_is_intel) 154 return get_kvm_intel_param_bool("ept"); 155 else 156 return get_kvm_amd_param_bool("npt"); 157 } 158 159 void virt_arch_pgd_alloc(struct kvm_vm *vm) 160 { 161 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K, 162 "Unknown or unsupported guest mode: 0x%x", vm->mode); 163 164 /* If needed, create the top-level page table. */ 165 if (!vm->pgd_created) { 166 vm->pgd = vm_alloc_page_table(vm); 167 vm->pgd_created = true; 168 } 169 } 170 171 static void *virt_get_pte(struct kvm_vm *vm, uint64_t *parent_pte, 172 uint64_t vaddr, int level) 173 { 174 uint64_t pt_gpa = PTE_GET_PA(*parent_pte); 175 uint64_t *page_table = addr_gpa2hva(vm, pt_gpa); 176 int index = (vaddr >> PG_LEVEL_SHIFT(level)) & 0x1ffu; 177 178 TEST_ASSERT((*parent_pte & PTE_PRESENT_MASK) || parent_pte == &vm->pgd, 179 "Parent PTE (level %d) not PRESENT for gva: 0x%08lx", 180 level + 1, vaddr); 181 182 return &page_table[index]; 183 } 184 185 static uint64_t *virt_create_upper_pte(struct kvm_vm *vm, 186 uint64_t *parent_pte, 187 uint64_t vaddr, 188 uint64_t paddr, 189 int current_level, 190 int target_level) 191 { 192 uint64_t *pte = virt_get_pte(vm, parent_pte, vaddr, current_level); 193 194 paddr = vm_untag_gpa(vm, paddr); 195 196 if (!(*pte & PTE_PRESENT_MASK)) { 197 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK; 198 if (current_level == target_level) 199 *pte |= PTE_LARGE_MASK | (paddr & PHYSICAL_PAGE_MASK); 200 else 201 *pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK; 202 } else { 203 /* 204 * Entry already present. Assert that the caller doesn't want 205 * a hugepage at this level, and that there isn't a hugepage at 206 * this level. 207 */ 208 TEST_ASSERT(current_level != target_level, 209 "Cannot create hugepage at level: %u, vaddr: 0x%lx", 210 current_level, vaddr); 211 TEST_ASSERT(!(*pte & PTE_LARGE_MASK), 212 "Cannot create page table at level: %u, vaddr: 0x%lx", 213 current_level, vaddr); 214 } 215 return pte; 216 } 217 218 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level) 219 { 220 const uint64_t pg_size = PG_LEVEL_SIZE(level); 221 uint64_t *pte = &vm->pgd; 222 int current_level; 223 224 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K, 225 "Unknown or unsupported guest mode: 0x%x", vm->mode); 226 227 TEST_ASSERT((vaddr % pg_size) == 0, 228 "Virtual address not aligned,\n" 229 "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size); 230 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)), 231 "Invalid virtual address, vaddr: 0x%lx", vaddr); 232 TEST_ASSERT((paddr % pg_size) == 0, 233 "Physical address not aligned,\n" 234 " paddr: 0x%lx page size: 0x%lx", paddr, pg_size); 235 TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn, 236 "Physical address beyond maximum supported,\n" 237 " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", 238 paddr, vm->max_gfn, vm->page_size); 239 TEST_ASSERT(vm_untag_gpa(vm, paddr) == paddr, 240 "Unexpected bits in paddr: %lx", paddr); 241 242 /* 243 * Allocate upper level page tables, if not already present. Return 244 * early if a hugepage was created. 245 */ 246 for (current_level = vm->pgtable_levels; 247 current_level > PG_LEVEL_4K; 248 current_level--) { 249 pte = virt_create_upper_pte(vm, pte, vaddr, paddr, 250 current_level, level); 251 if (*pte & PTE_LARGE_MASK) 252 return; 253 } 254 255 /* Fill in page table entry. */ 256 pte = virt_get_pte(vm, pte, vaddr, PG_LEVEL_4K); 257 TEST_ASSERT(!(*pte & PTE_PRESENT_MASK), 258 "PTE already present for 4k page at vaddr: 0x%lx", vaddr); 259 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK | (paddr & PHYSICAL_PAGE_MASK); 260 261 /* 262 * Neither SEV nor TDX supports shared page tables, so only the final 263 * leaf PTE needs manually set the C/S-bit. 264 */ 265 if (vm_is_gpa_protected(vm, paddr)) 266 *pte |= vm->arch.c_bit; 267 else 268 *pte |= vm->arch.s_bit; 269 } 270 271 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 272 { 273 __virt_pg_map(vm, vaddr, paddr, PG_LEVEL_4K); 274 } 275 276 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 277 uint64_t nr_bytes, int level) 278 { 279 uint64_t pg_size = PG_LEVEL_SIZE(level); 280 uint64_t nr_pages = nr_bytes / pg_size; 281 int i; 282 283 TEST_ASSERT(nr_bytes % pg_size == 0, 284 "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx", 285 nr_bytes, pg_size); 286 287 for (i = 0; i < nr_pages; i++) { 288 __virt_pg_map(vm, vaddr, paddr, level); 289 290 vaddr += pg_size; 291 paddr += pg_size; 292 } 293 } 294 295 static bool vm_is_target_pte(uint64_t *pte, int *level, int current_level) 296 { 297 if (*pte & PTE_LARGE_MASK) { 298 TEST_ASSERT(*level == PG_LEVEL_NONE || 299 *level == current_level, 300 "Unexpected hugepage at level %d", current_level); 301 *level = current_level; 302 } 303 304 return *level == current_level; 305 } 306 307 uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr, 308 int *level) 309 { 310 int va_width = 12 + (vm->pgtable_levels) * 9; 311 uint64_t *pte = &vm->pgd; 312 int current_level; 313 314 TEST_ASSERT(!vm->arch.is_pt_protected, 315 "Walking page tables of protected guests is impossible"); 316 317 TEST_ASSERT(*level >= PG_LEVEL_NONE && *level <= vm->pgtable_levels, 318 "Invalid PG_LEVEL_* '%d'", *level); 319 320 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K, 321 "Unknown or unsupported guest mode: 0x%x", vm->mode); 322 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, 323 (vaddr >> vm->page_shift)), 324 "Invalid virtual address, vaddr: 0x%lx", 325 vaddr); 326 /* 327 * Check that the vaddr is a sign-extended va_width value. 328 */ 329 TEST_ASSERT(vaddr == 330 (((int64_t)vaddr << (64 - va_width) >> (64 - va_width))), 331 "Canonical check failed. The virtual address is invalid."); 332 333 for (current_level = vm->pgtable_levels; 334 current_level > PG_LEVEL_4K; 335 current_level--) { 336 pte = virt_get_pte(vm, pte, vaddr, current_level); 337 if (vm_is_target_pte(pte, level, current_level)) 338 return pte; 339 } 340 341 return virt_get_pte(vm, pte, vaddr, PG_LEVEL_4K); 342 } 343 344 uint64_t *vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr) 345 { 346 int level = PG_LEVEL_4K; 347 348 return __vm_get_page_table_entry(vm, vaddr, &level); 349 } 350 351 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 352 { 353 uint64_t *pml4e, *pml4e_start; 354 uint64_t *pdpe, *pdpe_start; 355 uint64_t *pde, *pde_start; 356 uint64_t *pte, *pte_start; 357 358 if (!vm->pgd_created) 359 return; 360 361 fprintf(stream, "%*s " 362 " no\n", indent, ""); 363 fprintf(stream, "%*s index hvaddr gpaddr " 364 "addr w exec dirty\n", 365 indent, ""); 366 pml4e_start = (uint64_t *) addr_gpa2hva(vm, vm->pgd); 367 for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) { 368 pml4e = &pml4e_start[n1]; 369 if (!(*pml4e & PTE_PRESENT_MASK)) 370 continue; 371 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u " 372 " %u\n", 373 indent, "", 374 pml4e - pml4e_start, pml4e, 375 addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e), 376 !!(*pml4e & PTE_WRITABLE_MASK), !!(*pml4e & PTE_NX_MASK)); 377 378 pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK); 379 for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) { 380 pdpe = &pdpe_start[n2]; 381 if (!(*pdpe & PTE_PRESENT_MASK)) 382 continue; 383 fprintf(stream, "%*spdpe 0x%-3zx %p 0x%-12lx 0x%-10llx " 384 "%u %u\n", 385 indent, "", 386 pdpe - pdpe_start, pdpe, 387 addr_hva2gpa(vm, pdpe), 388 PTE_GET_PFN(*pdpe), !!(*pdpe & PTE_WRITABLE_MASK), 389 !!(*pdpe & PTE_NX_MASK)); 390 391 pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK); 392 for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) { 393 pde = &pde_start[n3]; 394 if (!(*pde & PTE_PRESENT_MASK)) 395 continue; 396 fprintf(stream, "%*spde 0x%-3zx %p " 397 "0x%-12lx 0x%-10llx %u %u\n", 398 indent, "", pde - pde_start, pde, 399 addr_hva2gpa(vm, pde), 400 PTE_GET_PFN(*pde), !!(*pde & PTE_WRITABLE_MASK), 401 !!(*pde & PTE_NX_MASK)); 402 403 pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK); 404 for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) { 405 pte = &pte_start[n4]; 406 if (!(*pte & PTE_PRESENT_MASK)) 407 continue; 408 fprintf(stream, "%*spte 0x%-3zx %p " 409 "0x%-12lx 0x%-10llx %u %u " 410 " %u 0x%-10lx\n", 411 indent, "", 412 pte - pte_start, pte, 413 addr_hva2gpa(vm, pte), 414 PTE_GET_PFN(*pte), 415 !!(*pte & PTE_WRITABLE_MASK), 416 !!(*pte & PTE_NX_MASK), 417 !!(*pte & PTE_DIRTY_MASK), 418 ((uint64_t) n1 << 27) 419 | ((uint64_t) n2 << 18) 420 | ((uint64_t) n3 << 9) 421 | ((uint64_t) n4)); 422 } 423 } 424 } 425 } 426 } 427 428 /* 429 * Set Unusable Segment 430 * 431 * Input Args: None 432 * 433 * Output Args: 434 * segp - Pointer to segment register 435 * 436 * Return: None 437 * 438 * Sets the segment register pointed to by @segp to an unusable state. 439 */ 440 static void kvm_seg_set_unusable(struct kvm_segment *segp) 441 { 442 memset(segp, 0, sizeof(*segp)); 443 segp->unusable = true; 444 } 445 446 static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp) 447 { 448 void *gdt = addr_gva2hva(vm, vm->arch.gdt); 449 struct desc64 *desc = gdt + (segp->selector >> 3) * 8; 450 451 desc->limit0 = segp->limit & 0xFFFF; 452 desc->base0 = segp->base & 0xFFFF; 453 desc->base1 = segp->base >> 16; 454 desc->type = segp->type; 455 desc->s = segp->s; 456 desc->dpl = segp->dpl; 457 desc->p = segp->present; 458 desc->limit1 = segp->limit >> 16; 459 desc->avl = segp->avl; 460 desc->l = segp->l; 461 desc->db = segp->db; 462 desc->g = segp->g; 463 desc->base2 = segp->base >> 24; 464 if (!segp->s) 465 desc->base3 = segp->base >> 32; 466 } 467 468 static void kvm_seg_set_kernel_code_64bit(struct kvm_segment *segp) 469 { 470 memset(segp, 0, sizeof(*segp)); 471 segp->selector = KERNEL_CS; 472 segp->limit = 0xFFFFFFFFu; 473 segp->s = 0x1; /* kTypeCodeData */ 474 segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed 475 * | kFlagCodeReadable 476 */ 477 segp->g = true; 478 segp->l = true; 479 segp->present = 1; 480 } 481 482 static void kvm_seg_set_kernel_data_64bit(struct kvm_segment *segp) 483 { 484 memset(segp, 0, sizeof(*segp)); 485 segp->selector = KERNEL_DS; 486 segp->limit = 0xFFFFFFFFu; 487 segp->s = 0x1; /* kTypeCodeData */ 488 segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed 489 * | kFlagDataWritable 490 */ 491 segp->g = true; 492 segp->present = true; 493 } 494 495 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 496 { 497 int level = PG_LEVEL_NONE; 498 uint64_t *pte = __vm_get_page_table_entry(vm, gva, &level); 499 500 TEST_ASSERT(*pte & PTE_PRESENT_MASK, 501 "Leaf PTE not PRESENT for gva: 0x%08lx", gva); 502 503 /* 504 * No need for a hugepage mask on the PTE, x86-64 requires the "unused" 505 * address bits to be zero. 506 */ 507 return vm_untag_gpa(vm, PTE_GET_PA(*pte)) | (gva & ~HUGEPAGE_MASK(level)); 508 } 509 510 static void kvm_seg_set_tss_64bit(vm_vaddr_t base, struct kvm_segment *segp) 511 { 512 memset(segp, 0, sizeof(*segp)); 513 segp->base = base; 514 segp->limit = 0x67; 515 segp->selector = KERNEL_TSS; 516 segp->type = 0xb; 517 segp->present = 1; 518 } 519 520 static void vcpu_init_sregs(struct kvm_vm *vm, struct kvm_vcpu *vcpu) 521 { 522 struct kvm_sregs sregs; 523 524 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K, 525 "Unknown or unsupported guest mode: 0x%x", vm->mode); 526 527 /* Set mode specific system register values. */ 528 vcpu_sregs_get(vcpu, &sregs); 529 530 sregs.idt.base = vm->arch.idt; 531 sregs.idt.limit = NUM_INTERRUPTS * sizeof(struct idt_entry) - 1; 532 sregs.gdt.base = vm->arch.gdt; 533 sregs.gdt.limit = getpagesize() - 1; 534 535 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG; 536 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR; 537 if (kvm_cpu_has(X86_FEATURE_XSAVE)) 538 sregs.cr4 |= X86_CR4_OSXSAVE; 539 if (vm->pgtable_levels == 5) 540 sregs.cr4 |= X86_CR4_LA57; 541 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX); 542 543 kvm_seg_set_unusable(&sregs.ldt); 544 kvm_seg_set_kernel_code_64bit(&sregs.cs); 545 kvm_seg_set_kernel_data_64bit(&sregs.ds); 546 kvm_seg_set_kernel_data_64bit(&sregs.es); 547 kvm_seg_set_kernel_data_64bit(&sregs.gs); 548 kvm_seg_set_tss_64bit(vm->arch.tss, &sregs.tr); 549 550 sregs.cr3 = vm->pgd; 551 vcpu_sregs_set(vcpu, &sregs); 552 } 553 554 static void vcpu_init_xcrs(struct kvm_vm *vm, struct kvm_vcpu *vcpu) 555 { 556 struct kvm_xcrs xcrs = { 557 .nr_xcrs = 1, 558 .xcrs[0].xcr = 0, 559 .xcrs[0].value = kvm_cpu_supported_xcr0(), 560 }; 561 562 if (!kvm_cpu_has(X86_FEATURE_XSAVE)) 563 return; 564 565 vcpu_xcrs_set(vcpu, &xcrs); 566 } 567 568 static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr, 569 int dpl, unsigned short selector) 570 { 571 struct idt_entry *base = 572 (struct idt_entry *)addr_gva2hva(vm, vm->arch.idt); 573 struct idt_entry *e = &base[vector]; 574 575 memset(e, 0, sizeof(*e)); 576 e->offset0 = addr; 577 e->selector = selector; 578 e->ist = 0; 579 e->type = 14; 580 e->dpl = dpl; 581 e->p = 1; 582 e->offset1 = addr >> 16; 583 e->offset2 = addr >> 32; 584 } 585 586 static bool kvm_fixup_exception(struct ex_regs *regs) 587 { 588 if (regs->r9 != KVM_EXCEPTION_MAGIC || regs->rip != regs->r10) 589 return false; 590 591 if (regs->vector == DE_VECTOR) 592 regs->vector = KVM_MAGIC_DE_VECTOR; 593 594 regs->rip = regs->r11; 595 regs->r9 = regs->vector; 596 regs->r10 = regs->error_code; 597 return true; 598 } 599 600 void route_exception(struct ex_regs *regs) 601 { 602 typedef void(*handler)(struct ex_regs *); 603 handler *handlers = (handler *)exception_handlers; 604 605 if (handlers && handlers[regs->vector]) { 606 handlers[regs->vector](regs); 607 return; 608 } 609 610 if (kvm_fixup_exception(regs)) 611 return; 612 613 GUEST_FAIL("Unhandled exception '0x%lx' at guest RIP '0x%lx'", 614 regs->vector, regs->rip); 615 } 616 617 static void vm_init_descriptor_tables(struct kvm_vm *vm) 618 { 619 extern void *idt_handlers; 620 struct kvm_segment seg; 621 int i; 622 623 vm->arch.gdt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 624 vm->arch.idt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 625 vm->handlers = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 626 vm->arch.tss = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 627 628 /* Handlers have the same address in both address spaces.*/ 629 for (i = 0; i < NUM_INTERRUPTS; i++) 630 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0, KERNEL_CS); 631 632 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers; 633 634 kvm_seg_set_kernel_code_64bit(&seg); 635 kvm_seg_fill_gdt_64bit(vm, &seg); 636 637 kvm_seg_set_kernel_data_64bit(&seg); 638 kvm_seg_fill_gdt_64bit(vm, &seg); 639 640 kvm_seg_set_tss_64bit(vm->arch.tss, &seg); 641 kvm_seg_fill_gdt_64bit(vm, &seg); 642 } 643 644 void vm_install_exception_handler(struct kvm_vm *vm, int vector, 645 void (*handler)(struct ex_regs *)) 646 { 647 vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers); 648 649 handlers[vector] = (vm_vaddr_t)handler; 650 } 651 652 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) 653 { 654 struct ucall uc; 655 656 if (get_ucall(vcpu, &uc) == UCALL_ABORT) 657 REPORT_GUEST_ASSERT(uc); 658 } 659 660 void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) 661 { 662 int r; 663 664 TEST_ASSERT(kvm_has_cap(KVM_CAP_GET_TSC_KHZ), 665 "Require KVM_GET_TSC_KHZ to provide udelay() to guest."); 666 667 vm_create_irqchip(vm); 668 vm_init_descriptor_tables(vm); 669 670 sync_global_to_guest(vm, host_cpu_is_intel); 671 sync_global_to_guest(vm, host_cpu_is_amd); 672 sync_global_to_guest(vm, is_forced_emulation_enabled); 673 sync_global_to_guest(vm, pmu_errata_mask); 674 675 if (is_sev_vm(vm)) { 676 struct kvm_sev_init init = { 0 }; 677 678 vm_sev_ioctl(vm, KVM_SEV_INIT2, &init); 679 } 680 681 r = __vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL); 682 TEST_ASSERT(r > 0, "KVM_GET_TSC_KHZ did not provide a valid TSC frequency."); 683 guest_tsc_khz = r; 684 sync_global_to_guest(vm, guest_tsc_khz); 685 } 686 687 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) 688 { 689 struct kvm_regs regs; 690 691 vcpu_regs_get(vcpu, ®s); 692 regs.rip = (unsigned long) guest_code; 693 vcpu_regs_set(vcpu, ®s); 694 } 695 696 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id) 697 { 698 struct kvm_mp_state mp_state; 699 struct kvm_regs regs; 700 vm_vaddr_t stack_vaddr; 701 struct kvm_vcpu *vcpu; 702 703 stack_vaddr = __vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(), 704 DEFAULT_GUEST_STACK_VADDR_MIN, 705 MEM_REGION_DATA); 706 707 stack_vaddr += DEFAULT_STACK_PGS * getpagesize(); 708 709 /* 710 * Align stack to match calling sequence requirements in section "The 711 * Stack Frame" of the System V ABI AMD64 Architecture Processor 712 * Supplement, which requires the value (%rsp + 8) to be a multiple of 713 * 16 when control is transferred to the function entry point. 714 * 715 * If this code is ever used to launch a vCPU with 32-bit entry point it 716 * may need to subtract 4 bytes instead of 8 bytes. 717 */ 718 TEST_ASSERT(IS_ALIGNED(stack_vaddr, PAGE_SIZE), 719 "__vm_vaddr_alloc() did not provide a page-aligned address"); 720 stack_vaddr -= 8; 721 722 vcpu = __vm_vcpu_add(vm, vcpu_id); 723 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid()); 724 vcpu_init_sregs(vm, vcpu); 725 vcpu_init_xcrs(vm, vcpu); 726 727 /* Setup guest general purpose registers */ 728 vcpu_regs_get(vcpu, ®s); 729 regs.rflags = regs.rflags | 0x2; 730 regs.rsp = stack_vaddr; 731 vcpu_regs_set(vcpu, ®s); 732 733 /* Setup the MP state */ 734 mp_state.mp_state = 0; 735 vcpu_mp_state_set(vcpu, &mp_state); 736 737 /* 738 * Refresh CPUID after setting SREGS and XCR0, so that KVM's "runtime" 739 * updates to guest CPUID, e.g. for OSXSAVE and XSAVE state size, are 740 * reflected into selftests' vCPU CPUID cache, i.e. so that the cache 741 * is consistent with vCPU state. 742 */ 743 vcpu_get_cpuid(vcpu); 744 return vcpu; 745 } 746 747 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id) 748 { 749 struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id); 750 751 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid()); 752 753 return vcpu; 754 } 755 756 void vcpu_arch_free(struct kvm_vcpu *vcpu) 757 { 758 if (vcpu->cpuid) 759 free(vcpu->cpuid); 760 } 761 762 /* Do not use kvm_supported_cpuid directly except for validity checks. */ 763 static void *kvm_supported_cpuid; 764 765 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void) 766 { 767 int kvm_fd; 768 769 if (kvm_supported_cpuid) 770 return kvm_supported_cpuid; 771 772 kvm_supported_cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES); 773 kvm_fd = open_kvm_dev_path_or_exit(); 774 775 kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID, 776 (struct kvm_cpuid2 *)kvm_supported_cpuid); 777 778 close(kvm_fd); 779 return kvm_supported_cpuid; 780 } 781 782 static uint32_t __kvm_cpu_has(const struct kvm_cpuid2 *cpuid, 783 uint32_t function, uint32_t index, 784 uint8_t reg, uint8_t lo, uint8_t hi) 785 { 786 const struct kvm_cpuid_entry2 *entry; 787 int i; 788 789 for (i = 0; i < cpuid->nent; i++) { 790 entry = &cpuid->entries[i]; 791 792 /* 793 * The output registers in kvm_cpuid_entry2 are in alphabetical 794 * order, but kvm_x86_cpu_feature matches that mess, so yay 795 * pointer shenanigans! 796 */ 797 if (entry->function == function && entry->index == index) 798 return ((&entry->eax)[reg] & GENMASK(hi, lo)) >> lo; 799 } 800 801 return 0; 802 } 803 804 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid, 805 struct kvm_x86_cpu_feature feature) 806 { 807 return __kvm_cpu_has(cpuid, feature.function, feature.index, 808 feature.reg, feature.bit, feature.bit); 809 } 810 811 uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid, 812 struct kvm_x86_cpu_property property) 813 { 814 return __kvm_cpu_has(cpuid, property.function, property.index, 815 property.reg, property.lo_bit, property.hi_bit); 816 } 817 818 uint64_t kvm_get_feature_msr(uint64_t msr_index) 819 { 820 struct { 821 struct kvm_msrs header; 822 struct kvm_msr_entry entry; 823 } buffer = {}; 824 int r, kvm_fd; 825 826 buffer.header.nmsrs = 1; 827 buffer.entry.index = msr_index; 828 kvm_fd = open_kvm_dev_path_or_exit(); 829 830 r = __kvm_ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header); 831 TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_GET_MSRS, r)); 832 833 close(kvm_fd); 834 return buffer.entry.data; 835 } 836 837 void __vm_xsave_require_permission(uint64_t xfeature, const char *name) 838 { 839 int kvm_fd; 840 u64 bitmask; 841 long rc; 842 struct kvm_device_attr attr = { 843 .group = 0, 844 .attr = KVM_X86_XCOMP_GUEST_SUPP, 845 .addr = (unsigned long) &bitmask, 846 }; 847 848 TEST_ASSERT(!kvm_supported_cpuid, 849 "kvm_get_supported_cpuid() cannot be used before ARCH_REQ_XCOMP_GUEST_PERM"); 850 851 TEST_ASSERT(is_power_of_2(xfeature), 852 "Dynamic XFeatures must be enabled one at a time"); 853 854 kvm_fd = open_kvm_dev_path_or_exit(); 855 rc = __kvm_ioctl(kvm_fd, KVM_GET_DEVICE_ATTR, &attr); 856 close(kvm_fd); 857 858 if (rc == -1 && (errno == ENXIO || errno == EINVAL)) 859 __TEST_REQUIRE(0, "KVM_X86_XCOMP_GUEST_SUPP not supported"); 860 861 TEST_ASSERT(rc == 0, "KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) error: %ld", rc); 862 863 __TEST_REQUIRE(bitmask & xfeature, 864 "Required XSAVE feature '%s' not supported", name); 865 866 TEST_REQUIRE(!syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, ilog2(xfeature))); 867 868 rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_GUEST_PERM, &bitmask); 869 TEST_ASSERT(rc == 0, "prctl(ARCH_GET_XCOMP_GUEST_PERM) error: %ld", rc); 870 TEST_ASSERT(bitmask & xfeature, 871 "'%s' (0x%lx) not permitted after prctl(ARCH_REQ_XCOMP_GUEST_PERM) permitted=0x%lx", 872 name, xfeature, bitmask); 873 } 874 875 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid) 876 { 877 TEST_ASSERT(cpuid != vcpu->cpuid, "@cpuid can't be the vCPU's CPUID"); 878 879 /* Allow overriding the default CPUID. */ 880 if (vcpu->cpuid && vcpu->cpuid->nent < cpuid->nent) { 881 free(vcpu->cpuid); 882 vcpu->cpuid = NULL; 883 } 884 885 if (!vcpu->cpuid) 886 vcpu->cpuid = allocate_kvm_cpuid2(cpuid->nent); 887 888 memcpy(vcpu->cpuid, cpuid, kvm_cpuid2_size(cpuid->nent)); 889 vcpu_set_cpuid(vcpu); 890 } 891 892 void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu, 893 struct kvm_x86_cpu_property property, 894 uint32_t value) 895 { 896 struct kvm_cpuid_entry2 *entry; 897 898 entry = __vcpu_get_cpuid_entry(vcpu, property.function, property.index); 899 900 (&entry->eax)[property.reg] &= ~GENMASK(property.hi_bit, property.lo_bit); 901 (&entry->eax)[property.reg] |= value << property.lo_bit; 902 903 vcpu_set_cpuid(vcpu); 904 905 /* Sanity check that @value doesn't exceed the bounds in any way. */ 906 TEST_ASSERT_EQ(kvm_cpuid_property(vcpu->cpuid, property), value); 907 } 908 909 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function) 910 { 911 struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, function); 912 913 entry->eax = 0; 914 entry->ebx = 0; 915 entry->ecx = 0; 916 entry->edx = 0; 917 vcpu_set_cpuid(vcpu); 918 } 919 920 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu, 921 struct kvm_x86_cpu_feature feature, 922 bool set) 923 { 924 struct kvm_cpuid_entry2 *entry; 925 u32 *reg; 926 927 entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index); 928 reg = (&entry->eax) + feature.reg; 929 930 if (set) 931 *reg |= BIT(feature.bit); 932 else 933 *reg &= ~BIT(feature.bit); 934 935 vcpu_set_cpuid(vcpu); 936 } 937 938 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index) 939 { 940 struct { 941 struct kvm_msrs header; 942 struct kvm_msr_entry entry; 943 } buffer = {}; 944 945 buffer.header.nmsrs = 1; 946 buffer.entry.index = msr_index; 947 948 vcpu_msrs_get(vcpu, &buffer.header); 949 950 return buffer.entry.data; 951 } 952 953 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value) 954 { 955 struct { 956 struct kvm_msrs header; 957 struct kvm_msr_entry entry; 958 } buffer = {}; 959 960 memset(&buffer, 0, sizeof(buffer)); 961 buffer.header.nmsrs = 1; 962 buffer.entry.index = msr_index; 963 buffer.entry.data = msr_value; 964 965 return __vcpu_ioctl(vcpu, KVM_SET_MSRS, &buffer.header); 966 } 967 968 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) 969 { 970 va_list ap; 971 struct kvm_regs regs; 972 973 TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n" 974 " num: %u", 975 num); 976 977 va_start(ap, num); 978 vcpu_regs_get(vcpu, ®s); 979 980 if (num >= 1) 981 regs.rdi = va_arg(ap, uint64_t); 982 983 if (num >= 2) 984 regs.rsi = va_arg(ap, uint64_t); 985 986 if (num >= 3) 987 regs.rdx = va_arg(ap, uint64_t); 988 989 if (num >= 4) 990 regs.rcx = va_arg(ap, uint64_t); 991 992 if (num >= 5) 993 regs.r8 = va_arg(ap, uint64_t); 994 995 if (num >= 6) 996 regs.r9 = va_arg(ap, uint64_t); 997 998 vcpu_regs_set(vcpu, ®s); 999 va_end(ap); 1000 } 1001 1002 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) 1003 { 1004 struct kvm_regs regs; 1005 struct kvm_sregs sregs; 1006 1007 fprintf(stream, "%*svCPU ID: %u\n", indent, "", vcpu->id); 1008 1009 fprintf(stream, "%*sregs:\n", indent + 2, ""); 1010 vcpu_regs_get(vcpu, ®s); 1011 regs_dump(stream, ®s, indent + 4); 1012 1013 fprintf(stream, "%*ssregs:\n", indent + 2, ""); 1014 vcpu_sregs_get(vcpu, &sregs); 1015 sregs_dump(stream, &sregs, indent + 4); 1016 } 1017 1018 static struct kvm_msr_list *__kvm_get_msr_index_list(bool feature_msrs) 1019 { 1020 struct kvm_msr_list *list; 1021 struct kvm_msr_list nmsrs; 1022 int kvm_fd, r; 1023 1024 kvm_fd = open_kvm_dev_path_or_exit(); 1025 1026 nmsrs.nmsrs = 0; 1027 if (!feature_msrs) 1028 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs); 1029 else 1030 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, &nmsrs); 1031 1032 TEST_ASSERT(r == -1 && errno == E2BIG, 1033 "Expected -E2BIG, got rc: %i errno: %i (%s)", 1034 r, errno, strerror(errno)); 1035 1036 list = malloc(sizeof(*list) + nmsrs.nmsrs * sizeof(list->indices[0])); 1037 TEST_ASSERT(list, "-ENOMEM when allocating MSR index list"); 1038 list->nmsrs = nmsrs.nmsrs; 1039 1040 if (!feature_msrs) 1041 kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, list); 1042 else 1043 kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, list); 1044 close(kvm_fd); 1045 1046 TEST_ASSERT(list->nmsrs == nmsrs.nmsrs, 1047 "Number of MSRs in list changed, was %d, now %d", 1048 nmsrs.nmsrs, list->nmsrs); 1049 return list; 1050 } 1051 1052 const struct kvm_msr_list *kvm_get_msr_index_list(void) 1053 { 1054 static const struct kvm_msr_list *list; 1055 1056 if (!list) 1057 list = __kvm_get_msr_index_list(false); 1058 return list; 1059 } 1060 1061 1062 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void) 1063 { 1064 static const struct kvm_msr_list *list; 1065 1066 if (!list) 1067 list = __kvm_get_msr_index_list(true); 1068 return list; 1069 } 1070 1071 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index) 1072 { 1073 const struct kvm_msr_list *list = kvm_get_msr_index_list(); 1074 int i; 1075 1076 for (i = 0; i < list->nmsrs; ++i) { 1077 if (list->indices[i] == msr_index) 1078 return true; 1079 } 1080 1081 return false; 1082 } 1083 1084 static void vcpu_save_xsave_state(struct kvm_vcpu *vcpu, 1085 struct kvm_x86_state *state) 1086 { 1087 int size = vm_check_cap(vcpu->vm, KVM_CAP_XSAVE2); 1088 1089 if (size) { 1090 state->xsave = malloc(size); 1091 vcpu_xsave2_get(vcpu, state->xsave); 1092 } else { 1093 state->xsave = malloc(sizeof(struct kvm_xsave)); 1094 vcpu_xsave_get(vcpu, state->xsave); 1095 } 1096 } 1097 1098 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu) 1099 { 1100 const struct kvm_msr_list *msr_list = kvm_get_msr_index_list(); 1101 struct kvm_x86_state *state; 1102 int i; 1103 1104 static int nested_size = -1; 1105 1106 if (nested_size == -1) { 1107 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE); 1108 TEST_ASSERT(nested_size <= sizeof(state->nested_), 1109 "Nested state size too big, %i > %zi", 1110 nested_size, sizeof(state->nested_)); 1111 } 1112 1113 /* 1114 * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees 1115 * guest state is consistent only after userspace re-enters the 1116 * kernel with KVM_RUN. Complete IO prior to migrating state 1117 * to a new VM. 1118 */ 1119 vcpu_run_complete_io(vcpu); 1120 1121 state = malloc(sizeof(*state) + msr_list->nmsrs * sizeof(state->msrs.entries[0])); 1122 TEST_ASSERT(state, "-ENOMEM when allocating kvm state"); 1123 1124 vcpu_events_get(vcpu, &state->events); 1125 vcpu_mp_state_get(vcpu, &state->mp_state); 1126 vcpu_regs_get(vcpu, &state->regs); 1127 vcpu_save_xsave_state(vcpu, state); 1128 1129 if (kvm_has_cap(KVM_CAP_XCRS)) 1130 vcpu_xcrs_get(vcpu, &state->xcrs); 1131 1132 vcpu_sregs_get(vcpu, &state->sregs); 1133 1134 if (nested_size) { 1135 state->nested.size = sizeof(state->nested_); 1136 1137 vcpu_nested_state_get(vcpu, &state->nested); 1138 TEST_ASSERT(state->nested.size <= nested_size, 1139 "Nested state size too big, %i (KVM_CHECK_CAP gave %i)", 1140 state->nested.size, nested_size); 1141 } else { 1142 state->nested.size = 0; 1143 } 1144 1145 state->msrs.nmsrs = msr_list->nmsrs; 1146 for (i = 0; i < msr_list->nmsrs; i++) 1147 state->msrs.entries[i].index = msr_list->indices[i]; 1148 vcpu_msrs_get(vcpu, &state->msrs); 1149 1150 vcpu_debugregs_get(vcpu, &state->debugregs); 1151 1152 return state; 1153 } 1154 1155 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state) 1156 { 1157 vcpu_sregs_set(vcpu, &state->sregs); 1158 vcpu_msrs_set(vcpu, &state->msrs); 1159 1160 if (kvm_has_cap(KVM_CAP_XCRS)) 1161 vcpu_xcrs_set(vcpu, &state->xcrs); 1162 1163 vcpu_xsave_set(vcpu, state->xsave); 1164 vcpu_events_set(vcpu, &state->events); 1165 vcpu_mp_state_set(vcpu, &state->mp_state); 1166 vcpu_debugregs_set(vcpu, &state->debugregs); 1167 vcpu_regs_set(vcpu, &state->regs); 1168 1169 if (state->nested.size) 1170 vcpu_nested_state_set(vcpu, &state->nested); 1171 } 1172 1173 void kvm_x86_state_cleanup(struct kvm_x86_state *state) 1174 { 1175 free(state->xsave); 1176 free(state); 1177 } 1178 1179 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits) 1180 { 1181 if (!kvm_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) { 1182 *pa_bits = kvm_cpu_has(X86_FEATURE_PAE) ? 36 : 32; 1183 *va_bits = 32; 1184 } else { 1185 *pa_bits = kvm_cpu_property(X86_PROPERTY_MAX_PHY_ADDR); 1186 *va_bits = kvm_cpu_property(X86_PROPERTY_MAX_VIRT_ADDR); 1187 } 1188 } 1189 1190 void kvm_init_vm_address_properties(struct kvm_vm *vm) 1191 { 1192 if (is_sev_vm(vm)) { 1193 vm->arch.sev_fd = open_sev_dev_path_or_exit(); 1194 vm->arch.c_bit = BIT_ULL(this_cpu_property(X86_PROPERTY_SEV_C_BIT)); 1195 vm->gpa_tag_mask = vm->arch.c_bit; 1196 } else { 1197 vm->arch.sev_fd = -1; 1198 } 1199 } 1200 1201 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid, 1202 uint32_t function, uint32_t index) 1203 { 1204 int i; 1205 1206 for (i = 0; i < cpuid->nent; i++) { 1207 if (cpuid->entries[i].function == function && 1208 cpuid->entries[i].index == index) 1209 return &cpuid->entries[i]; 1210 } 1211 1212 TEST_FAIL("CPUID function 0x%x index 0x%x not found ", function, index); 1213 1214 return NULL; 1215 } 1216 1217 #define X86_HYPERCALL(inputs...) \ 1218 ({ \ 1219 uint64_t r; \ 1220 \ 1221 asm volatile("test %[use_vmmcall], %[use_vmmcall]\n\t" \ 1222 "jnz 1f\n\t" \ 1223 "vmcall\n\t" \ 1224 "jmp 2f\n\t" \ 1225 "1: vmmcall\n\t" \ 1226 "2:" \ 1227 : "=a"(r) \ 1228 : [use_vmmcall] "r" (host_cpu_is_amd), inputs); \ 1229 \ 1230 r; \ 1231 }) 1232 1233 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, 1234 uint64_t a3) 1235 { 1236 return X86_HYPERCALL("a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3)); 1237 } 1238 1239 uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1) 1240 { 1241 return X86_HYPERCALL("a"(nr), "D"(a0), "S"(a1)); 1242 } 1243 1244 void xen_hypercall(uint64_t nr, uint64_t a0, void *a1) 1245 { 1246 GUEST_ASSERT(!__xen_hypercall(nr, a0, a1)); 1247 } 1248 1249 unsigned long vm_compute_max_gfn(struct kvm_vm *vm) 1250 { 1251 const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */ 1252 unsigned long ht_gfn, max_gfn, max_pfn; 1253 uint8_t maxphyaddr, guest_maxphyaddr; 1254 1255 /* 1256 * Use "guest MAXPHYADDR" from KVM if it's available. Guest MAXPHYADDR 1257 * enumerates the max _mappable_ GPA, which can be less than the raw 1258 * MAXPHYADDR, e.g. if MAXPHYADDR=52, KVM is using TDP, and the CPU 1259 * doesn't support 5-level TDP. 1260 */ 1261 guest_maxphyaddr = kvm_cpu_property(X86_PROPERTY_GUEST_MAX_PHY_ADDR); 1262 guest_maxphyaddr = guest_maxphyaddr ?: vm->pa_bits; 1263 TEST_ASSERT(guest_maxphyaddr <= vm->pa_bits, 1264 "Guest MAXPHYADDR should never be greater than raw MAXPHYADDR"); 1265 1266 max_gfn = (1ULL << (guest_maxphyaddr - vm->page_shift)) - 1; 1267 1268 /* Avoid reserved HyperTransport region on AMD processors. */ 1269 if (!host_cpu_is_amd) 1270 return max_gfn; 1271 1272 /* On parts with <40 physical address bits, the area is fully hidden */ 1273 if (vm->pa_bits < 40) 1274 return max_gfn; 1275 1276 /* Before family 17h, the HyperTransport area is just below 1T. */ 1277 ht_gfn = (1 << 28) - num_ht_pages; 1278 if (this_cpu_family() < 0x17) 1279 goto done; 1280 1281 /* 1282 * Otherwise it's at the top of the physical address space, possibly 1283 * reduced due to SME by bits 11:6 of CPUID[0x8000001f].EBX. Use 1284 * the old conservative value if MAXPHYADDR is not enumerated. 1285 */ 1286 if (!this_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) 1287 goto done; 1288 1289 maxphyaddr = this_cpu_property(X86_PROPERTY_MAX_PHY_ADDR); 1290 max_pfn = (1ULL << (maxphyaddr - vm->page_shift)) - 1; 1291 1292 if (this_cpu_has_p(X86_PROPERTY_PHYS_ADDR_REDUCTION)) 1293 max_pfn >>= this_cpu_property(X86_PROPERTY_PHYS_ADDR_REDUCTION); 1294 1295 ht_gfn = max_pfn - num_ht_pages; 1296 done: 1297 return min(max_gfn, ht_gfn - 1); 1298 } 1299 1300 void kvm_selftest_arch_init(void) 1301 { 1302 host_cpu_is_intel = this_cpu_is_intel(); 1303 host_cpu_is_amd = this_cpu_is_amd(); 1304 is_forced_emulation_enabled = kvm_is_forced_emulation_enabled(); 1305 1306 kvm_init_pmu_errata(); 1307 } 1308 1309 bool sys_clocksource_is_based_on_tsc(void) 1310 { 1311 char *clk_name = sys_get_cur_clocksource(); 1312 bool ret = !strcmp(clk_name, "tsc\n") || 1313 !strcmp(clk_name, "hyperv_clocksource_tsc_page\n"); 1314 1315 free(clk_name); 1316 1317 return ret; 1318 } 1319 1320 bool kvm_arch_has_default_irqchip(void) 1321 { 1322 return true; 1323 } 1324