1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2018, Google LLC. 4 */ 5 #ifndef SELFTEST_KVM_UTIL_H 6 #define SELFTEST_KVM_UTIL_H 7 8 #include "test_util.h" 9 10 #include <linux/compiler.h> 11 #include "linux/hashtable.h" 12 #include "linux/list.h" 13 #include <linux/kernel.h> 14 #include <linux/kvm.h> 15 #include "linux/rbtree.h" 16 #include <linux/types.h> 17 18 #include <asm/atomic.h> 19 #include <asm/kvm.h> 20 21 #include <sys/ioctl.h> 22 23 #include "kvm_util_arch.h" 24 #include "kvm_util_types.h" 25 #include "sparsebit.h" 26 27 #define KVM_DEV_PATH "/dev/kvm" 28 #define KVM_MAX_VCPUS 512 29 30 #define NSEC_PER_SEC 1000000000L 31 32 struct userspace_mem_region { 33 struct kvm_userspace_memory_region2 region; 34 struct sparsebit *unused_phy_pages; 35 struct sparsebit *protected_phy_pages; 36 int fd; 37 off_t offset; 38 enum vm_mem_backing_src_type backing_src_type; 39 void *host_mem; 40 void *host_alias; 41 void *mmap_start; 42 void *mmap_alias; 43 size_t mmap_size; 44 struct rb_node gpa_node; 45 struct rb_node hva_node; 46 struct hlist_node slot_node; 47 }; 48 49 struct kvm_vcpu { 50 struct list_head list; 51 uint32_t id; 52 int fd; 53 struct kvm_vm *vm; 54 struct kvm_run *run; 55 #ifdef __x86_64__ 56 struct kvm_cpuid2 *cpuid; 57 #endif 58 struct kvm_dirty_gfn *dirty_gfns; 59 uint32_t fetch_index; 60 uint32_t dirty_gfns_count; 61 }; 62 63 struct userspace_mem_regions { 64 struct rb_root gpa_tree; 65 struct rb_root hva_tree; 66 DECLARE_HASHTABLE(slot_hash, 9); 67 }; 68 69 enum kvm_mem_region_type { 70 MEM_REGION_CODE, 71 MEM_REGION_DATA, 72 MEM_REGION_PT, 73 MEM_REGION_TEST_DATA, 74 NR_MEM_REGIONS, 75 }; 76 77 struct kvm_vm { 78 int mode; 79 unsigned long type; 80 int kvm_fd; 81 int fd; 82 unsigned int pgtable_levels; 83 unsigned int page_size; 84 unsigned int page_shift; 85 unsigned int pa_bits; 86 unsigned int va_bits; 87 uint64_t max_gfn; 88 struct list_head vcpus; 89 struct userspace_mem_regions regions; 90 struct sparsebit *vpages_valid; 91 struct sparsebit *vpages_mapped; 92 bool has_irqchip; 93 bool pgd_created; 94 vm_paddr_t ucall_mmio_addr; 95 vm_paddr_t pgd; 96 vm_vaddr_t handlers; 97 uint32_t dirty_ring_size; 98 uint64_t gpa_tag_mask; 99 100 struct kvm_vm_arch arch; 101 102 /* Cache of information for binary stats interface */ 103 int stats_fd; 104 struct kvm_stats_header stats_header; 105 struct kvm_stats_desc *stats_desc; 106 107 /* 108 * KVM region slots. These are the default memslots used by page 109 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] 110 * memslot. 111 */ 112 uint32_t memslots[NR_MEM_REGIONS]; 113 }; 114 115 struct vcpu_reg_sublist { 116 const char *name; 117 long capability; 118 int feature; 119 int feature_type; 120 bool finalize; 121 __u64 *regs; 122 __u64 regs_n; 123 __u64 *rejects_set; 124 __u64 rejects_set_n; 125 __u64 *skips_set; 126 __u64 skips_set_n; 127 }; 128 129 struct vcpu_reg_list { 130 char *name; 131 struct vcpu_reg_sublist sublists[]; 132 }; 133 134 #define for_each_sublist(c, s) \ 135 for ((s) = &(c)->sublists[0]; (s)->regs; ++(s)) 136 137 #define kvm_for_each_vcpu(vm, i, vcpu) \ 138 for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \ 139 if (!((vcpu) = vm->vcpus[i])) \ 140 continue; \ 141 else 142 143 struct userspace_mem_region * 144 memslot2region(struct kvm_vm *vm, uint32_t memslot); 145 146 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, 147 enum kvm_mem_region_type type) 148 { 149 assert(type < NR_MEM_REGIONS); 150 return memslot2region(vm, vm->memslots[type]); 151 } 152 153 /* Minimum allocated guest virtual and physical addresses */ 154 #define KVM_UTIL_MIN_VADDR 0x2000 155 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 156 157 #define DEFAULT_GUEST_STACK_VADDR_MIN 0xab6000 158 #define DEFAULT_STACK_PGS 5 159 160 enum vm_guest_mode { 161 VM_MODE_P52V48_4K, 162 VM_MODE_P52V48_16K, 163 VM_MODE_P52V48_64K, 164 VM_MODE_P48V48_4K, 165 VM_MODE_P48V48_16K, 166 VM_MODE_P48V48_64K, 167 VM_MODE_P40V48_4K, 168 VM_MODE_P40V48_16K, 169 VM_MODE_P40V48_64K, 170 VM_MODE_PXXV48_4K, /* For 48bits VA but ANY bits PA */ 171 VM_MODE_P47V64_4K, 172 VM_MODE_P44V64_4K, 173 VM_MODE_P36V48_4K, 174 VM_MODE_P36V48_16K, 175 VM_MODE_P36V48_64K, 176 VM_MODE_P36V47_16K, 177 NUM_VM_MODES, 178 }; 179 180 struct vm_shape { 181 uint32_t type; 182 uint8_t mode; 183 uint8_t pad0; 184 uint16_t pad1; 185 }; 186 187 kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t)); 188 189 #define VM_TYPE_DEFAULT 0 190 191 #define VM_SHAPE(__mode) \ 192 ({ \ 193 struct vm_shape shape = { \ 194 .mode = (__mode), \ 195 .type = VM_TYPE_DEFAULT \ 196 }; \ 197 \ 198 shape; \ 199 }) 200 201 #if defined(__aarch64__) 202 203 extern enum vm_guest_mode vm_mode_default; 204 205 #define VM_MODE_DEFAULT vm_mode_default 206 #define MIN_PAGE_SHIFT 12U 207 #define ptes_per_page(page_size) ((page_size) / 8) 208 209 #elif defined(__x86_64__) 210 211 #define VM_MODE_DEFAULT VM_MODE_PXXV48_4K 212 #define MIN_PAGE_SHIFT 12U 213 #define ptes_per_page(page_size) ((page_size) / 8) 214 215 #elif defined(__s390x__) 216 217 #define VM_MODE_DEFAULT VM_MODE_P44V64_4K 218 #define MIN_PAGE_SHIFT 12U 219 #define ptes_per_page(page_size) ((page_size) / 16) 220 221 #elif defined(__riscv) 222 223 #if __riscv_xlen == 32 224 #error "RISC-V 32-bit kvm selftests not supported" 225 #endif 226 227 #define VM_MODE_DEFAULT VM_MODE_P40V48_4K 228 #define MIN_PAGE_SHIFT 12U 229 #define ptes_per_page(page_size) ((page_size) / 8) 230 231 #endif 232 233 #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) 234 235 #define MIN_PAGE_SIZE (1U << MIN_PAGE_SHIFT) 236 #define PTES_PER_MIN_PAGE ptes_per_page(MIN_PAGE_SIZE) 237 238 struct vm_guest_mode_params { 239 unsigned int pa_bits; 240 unsigned int va_bits; 241 unsigned int page_size; 242 unsigned int page_shift; 243 }; 244 extern const struct vm_guest_mode_params vm_guest_mode_params[]; 245 246 int open_path_or_exit(const char *path, int flags); 247 int open_kvm_dev_path_or_exit(void); 248 249 bool get_kvm_param_bool(const char *param); 250 bool get_kvm_intel_param_bool(const char *param); 251 bool get_kvm_amd_param_bool(const char *param); 252 253 int get_kvm_param_integer(const char *param); 254 int get_kvm_intel_param_integer(const char *param); 255 int get_kvm_amd_param_integer(const char *param); 256 257 unsigned int kvm_check_cap(long cap); 258 259 static inline bool kvm_has_cap(long cap) 260 { 261 return kvm_check_cap(cap); 262 } 263 264 #define __KVM_SYSCALL_ERROR(_name, _ret) \ 265 "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno) 266 267 /* 268 * Use the "inner", double-underscore macro when reporting errors from within 269 * other macros so that the name of ioctl() and not its literal numeric value 270 * is printed on error. The "outer" macro is strongly preferred when reporting 271 * errors "directly", i.e. without an additional layer of macros, as it reduces 272 * the probability of passing in the wrong string. 273 */ 274 #define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret) 275 #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret) 276 277 #define kvm_do_ioctl(fd, cmd, arg) \ 278 ({ \ 279 kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd)); \ 280 ioctl(fd, cmd, arg); \ 281 }) 282 283 #define __kvm_ioctl(kvm_fd, cmd, arg) \ 284 kvm_do_ioctl(kvm_fd, cmd, arg) 285 286 #define kvm_ioctl(kvm_fd, cmd, arg) \ 287 ({ \ 288 int ret = __kvm_ioctl(kvm_fd, cmd, arg); \ 289 \ 290 TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ 291 }) 292 293 static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } 294 295 #define __vm_ioctl(vm, cmd, arg) \ 296 ({ \ 297 static_assert_is_vm(vm); \ 298 kvm_do_ioctl((vm)->fd, cmd, arg); \ 299 }) 300 301 /* 302 * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if 303 * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM, 304 * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before 305 * selftests existed and (b) should never outright fail, i.e. is supposed to 306 * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the 307 * VM and its vCPUs, including KVM_CHECK_EXTENSION. 308 */ 309 #define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \ 310 do { \ 311 int __errno = errno; \ 312 \ 313 static_assert_is_vm(vm); \ 314 \ 315 if (cond) \ 316 break; \ 317 \ 318 if (errno == EIO && \ 319 __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \ 320 TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \ 321 TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \ 322 } \ 323 errno = __errno; \ 324 TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \ 325 } while (0) 326 327 #define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm) \ 328 __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm) 329 330 #define vm_ioctl(vm, cmd, arg) \ 331 ({ \ 332 int ret = __vm_ioctl(vm, cmd, arg); \ 333 \ 334 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \ 335 }) 336 337 static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } 338 339 #define __vcpu_ioctl(vcpu, cmd, arg) \ 340 ({ \ 341 static_assert_is_vcpu(vcpu); \ 342 kvm_do_ioctl((vcpu)->fd, cmd, arg); \ 343 }) 344 345 #define vcpu_ioctl(vcpu, cmd, arg) \ 346 ({ \ 347 int ret = __vcpu_ioctl(vcpu, cmd, arg); \ 348 \ 349 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ 350 }) 351 352 /* 353 * Looks up and returns the value corresponding to the capability 354 * (KVM_CAP_*) given by cap. 355 */ 356 static inline int vm_check_cap(struct kvm_vm *vm, long cap) 357 { 358 int ret = __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap); 359 360 TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm); 361 return ret; 362 } 363 364 static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 365 { 366 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 367 368 return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 369 } 370 static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 371 { 372 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 373 374 vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 375 } 376 377 static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, 378 uint64_t size, uint64_t attributes) 379 { 380 struct kvm_memory_attributes attr = { 381 .attributes = attributes, 382 .address = gpa, 383 .size = size, 384 .flags = 0, 385 }; 386 387 /* 388 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows 389 * need significant enhancements to support multiple attributes. 390 */ 391 TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, 392 "Update me to support multiple attributes!"); 393 394 vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); 395 } 396 397 398 static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa, 399 uint64_t size) 400 { 401 vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); 402 } 403 404 static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, 405 uint64_t size) 406 { 407 vm_set_memory_attributes(vm, gpa, size, 0); 408 } 409 410 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, 411 bool punch_hole); 412 413 static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, 414 uint64_t size) 415 { 416 vm_guest_mem_fallocate(vm, gpa, size, true); 417 } 418 419 static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa, 420 uint64_t size) 421 { 422 vm_guest_mem_fallocate(vm, gpa, size, false); 423 } 424 425 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size); 426 const char *vm_guest_mode_string(uint32_t i); 427 428 void kvm_vm_free(struct kvm_vm *vmp); 429 void kvm_vm_restart(struct kvm_vm *vmp); 430 void kvm_vm_release(struct kvm_vm *vmp); 431 void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename); 432 int kvm_memfd_alloc(size_t size, bool hugepages); 433 434 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 435 436 static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log) 437 { 438 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot }; 439 440 vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args); 441 } 442 443 static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log, 444 uint64_t first_page, uint32_t num_pages) 445 { 446 struct kvm_clear_dirty_log args = { 447 .dirty_bitmap = log, 448 .slot = slot, 449 .first_page = first_page, 450 .num_pages = num_pages 451 }; 452 453 vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args); 454 } 455 456 static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm) 457 { 458 return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL); 459 } 460 461 static inline void kvm_vm_register_coalesced_io(struct kvm_vm *vm, 462 uint64_t address, 463 uint64_t size, bool pio) 464 { 465 struct kvm_coalesced_mmio_zone zone = { 466 .addr = address, 467 .size = size, 468 .pio = pio, 469 }; 470 471 vm_ioctl(vm, KVM_REGISTER_COALESCED_MMIO, &zone); 472 } 473 474 static inline void kvm_vm_unregister_coalesced_io(struct kvm_vm *vm, 475 uint64_t address, 476 uint64_t size, bool pio) 477 { 478 struct kvm_coalesced_mmio_zone zone = { 479 .addr = address, 480 .size = size, 481 .pio = pio, 482 }; 483 484 vm_ioctl(vm, KVM_UNREGISTER_COALESCED_MMIO, &zone); 485 } 486 487 static inline int vm_get_stats_fd(struct kvm_vm *vm) 488 { 489 int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL); 490 491 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm); 492 return fd; 493 } 494 495 static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header) 496 { 497 ssize_t ret; 498 499 ret = pread(stats_fd, header, sizeof(*header), 0); 500 TEST_ASSERT(ret == sizeof(*header), 501 "Failed to read '%lu' header bytes, ret = '%ld'", 502 sizeof(*header), ret); 503 } 504 505 struct kvm_stats_desc *read_stats_descriptors(int stats_fd, 506 struct kvm_stats_header *header); 507 508 static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header) 509 { 510 /* 511 * The base size of the descriptor is defined by KVM's ABI, but the 512 * size of the name field is variable, as far as KVM's ABI is 513 * concerned. For a given instance of KVM, the name field is the same 514 * size for all stats and is provided in the overall stats header. 515 */ 516 return sizeof(struct kvm_stats_desc) + header->name_size; 517 } 518 519 static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats, 520 int index, 521 struct kvm_stats_header *header) 522 { 523 /* 524 * Note, size_desc includes the size of the name field, which is 525 * variable. i.e. this is NOT equivalent to &stats_desc[i]. 526 */ 527 return (void *)stats + index * get_stats_descriptor_size(header); 528 } 529 530 void read_stat_data(int stats_fd, struct kvm_stats_header *header, 531 struct kvm_stats_desc *desc, uint64_t *data, 532 size_t max_elements); 533 534 void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data, 535 size_t max_elements); 536 537 static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name) 538 { 539 uint64_t data; 540 541 __vm_get_stat(vm, stat_name, &data, 1); 542 return data; 543 } 544 545 void vm_create_irqchip(struct kvm_vm *vm); 546 547 static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 548 uint64_t flags) 549 { 550 struct kvm_create_guest_memfd guest_memfd = { 551 .size = size, 552 .flags = flags, 553 }; 554 555 return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd); 556 } 557 558 static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 559 uint64_t flags) 560 { 561 int fd = __vm_create_guest_memfd(vm, size, flags); 562 563 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd)); 564 return fd; 565 } 566 567 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 568 uint64_t gpa, uint64_t size, void *hva); 569 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 570 uint64_t gpa, uint64_t size, void *hva); 571 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 572 uint64_t gpa, uint64_t size, void *hva, 573 uint32_t guest_memfd, uint64_t guest_memfd_offset); 574 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 575 uint64_t gpa, uint64_t size, void *hva, 576 uint32_t guest_memfd, uint64_t guest_memfd_offset); 577 578 void vm_userspace_mem_region_add(struct kvm_vm *vm, 579 enum vm_mem_backing_src_type src_type, 580 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 581 uint32_t flags); 582 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, 583 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 584 uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset); 585 586 #ifndef vm_arch_has_protected_memory 587 static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) 588 { 589 return false; 590 } 591 #endif 592 593 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); 594 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); 595 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); 596 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 597 void vm_populate_vaddr_bitmap(struct kvm_vm *vm); 598 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 599 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 600 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, 601 enum kvm_mem_region_type type); 602 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz, 603 vm_vaddr_t vaddr_min, 604 enum kvm_mem_region_type type); 605 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages); 606 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, 607 enum kvm_mem_region_type type); 608 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm); 609 610 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 611 unsigned int npages); 612 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa); 613 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva); 614 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva); 615 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa); 616 617 #ifndef vcpu_arch_put_guest 618 #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) 619 #endif 620 621 static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa) 622 { 623 return gpa & ~vm->gpa_tag_mask; 624 } 625 626 void vcpu_run(struct kvm_vcpu *vcpu); 627 int _vcpu_run(struct kvm_vcpu *vcpu); 628 629 static inline int __vcpu_run(struct kvm_vcpu *vcpu) 630 { 631 return __vcpu_ioctl(vcpu, KVM_RUN, NULL); 632 } 633 634 void vcpu_run_complete_io(struct kvm_vcpu *vcpu); 635 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu); 636 637 static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap, 638 uint64_t arg0) 639 { 640 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 641 642 vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap); 643 } 644 645 static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu, 646 struct kvm_guest_debug *debug) 647 { 648 vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug); 649 } 650 651 static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu, 652 struct kvm_mp_state *mp_state) 653 { 654 vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state); 655 } 656 static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu, 657 struct kvm_mp_state *mp_state) 658 { 659 vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state); 660 } 661 662 static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 663 { 664 vcpu_ioctl(vcpu, KVM_GET_REGS, regs); 665 } 666 667 static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 668 { 669 vcpu_ioctl(vcpu, KVM_SET_REGS, regs); 670 } 671 static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 672 { 673 vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs); 674 675 } 676 static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 677 { 678 vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 679 } 680 static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 681 { 682 return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 683 } 684 static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 685 { 686 vcpu_ioctl(vcpu, KVM_GET_FPU, fpu); 687 } 688 static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 689 { 690 vcpu_ioctl(vcpu, KVM_SET_FPU, fpu); 691 } 692 693 static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 694 { 695 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 696 697 return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 698 } 699 static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 700 { 701 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 702 703 return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 704 } 705 static inline uint64_t vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id) 706 { 707 uint64_t val; 708 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 709 710 TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id); 711 712 vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 713 return val; 714 } 715 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 716 { 717 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 718 719 TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id); 720 721 vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 722 } 723 724 #ifdef __KVM_HAVE_VCPU_EVENTS 725 static inline void vcpu_events_get(struct kvm_vcpu *vcpu, 726 struct kvm_vcpu_events *events) 727 { 728 vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events); 729 } 730 static inline void vcpu_events_set(struct kvm_vcpu *vcpu, 731 struct kvm_vcpu_events *events) 732 { 733 vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events); 734 } 735 #endif 736 #ifdef __x86_64__ 737 static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu, 738 struct kvm_nested_state *state) 739 { 740 vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state); 741 } 742 static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu, 743 struct kvm_nested_state *state) 744 { 745 return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 746 } 747 748 static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu, 749 struct kvm_nested_state *state) 750 { 751 vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 752 } 753 #endif 754 static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu) 755 { 756 int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL); 757 758 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm); 759 return fd; 760 } 761 762 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr); 763 764 static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr) 765 { 766 int ret = __kvm_has_device_attr(dev_fd, group, attr); 767 768 TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno); 769 } 770 771 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val); 772 773 static inline void kvm_device_attr_get(int dev_fd, uint32_t group, 774 uint64_t attr, void *val) 775 { 776 int ret = __kvm_device_attr_get(dev_fd, group, attr, val); 777 778 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret)); 779 } 780 781 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val); 782 783 static inline void kvm_device_attr_set(int dev_fd, uint32_t group, 784 uint64_t attr, void *val) 785 { 786 int ret = __kvm_device_attr_set(dev_fd, group, attr, val); 787 788 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret)); 789 } 790 791 static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 792 uint64_t attr) 793 { 794 return __kvm_has_device_attr(vcpu->fd, group, attr); 795 } 796 797 static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 798 uint64_t attr) 799 { 800 kvm_has_device_attr(vcpu->fd, group, attr); 801 } 802 803 static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 804 uint64_t attr, void *val) 805 { 806 return __kvm_device_attr_get(vcpu->fd, group, attr, val); 807 } 808 809 static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 810 uint64_t attr, void *val) 811 { 812 kvm_device_attr_get(vcpu->fd, group, attr, val); 813 } 814 815 static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 816 uint64_t attr, void *val) 817 { 818 return __kvm_device_attr_set(vcpu->fd, group, attr, val); 819 } 820 821 static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 822 uint64_t attr, void *val) 823 { 824 kvm_device_attr_set(vcpu->fd, group, attr, val); 825 } 826 827 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type); 828 int __kvm_create_device(struct kvm_vm *vm, uint64_t type); 829 830 static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type) 831 { 832 int fd = __kvm_create_device(vm, type); 833 834 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd)); 835 return fd; 836 } 837 838 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu); 839 840 /* 841 * VM VCPU Args Set 842 * 843 * Input Args: 844 * vm - Virtual Machine 845 * num - number of arguments 846 * ... - arguments, each of type uint64_t 847 * 848 * Output Args: None 849 * 850 * Return: None 851 * 852 * Sets the first @num input parameters for the function at @vcpu's entry point, 853 * per the C calling convention of the architecture, to the values given as 854 * variable args. Each of the variable args is expected to be of type uint64_t. 855 * The maximum @num can be is specific to the architecture. 856 */ 857 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...); 858 859 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 860 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 861 862 #define KVM_MAX_IRQ_ROUTES 4096 863 864 struct kvm_irq_routing *kvm_gsi_routing_create(void); 865 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing, 866 uint32_t gsi, uint32_t pin); 867 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 868 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 869 870 const char *exit_reason_str(unsigned int exit_reason); 871 872 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, 873 uint32_t memslot); 874 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 875 vm_paddr_t paddr_min, uint32_t memslot, 876 bool protected); 877 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); 878 879 static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 880 vm_paddr_t paddr_min, uint32_t memslot) 881 { 882 /* 883 * By default, allocate memory as protected for VMs that support 884 * protected memory, as the majority of memory for such VMs is 885 * protected, i.e. using shared memory is effectively opt-in. 886 */ 887 return __vm_phy_pages_alloc(vm, num, paddr_min, memslot, 888 vm_arch_has_protected_memory(vm)); 889 } 890 891 /* 892 * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also 893 * loads the test binary into guest memory and creates an IRQ chip (x86 only). 894 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to 895 * calculate the amount of memory needed for per-vCPU data, e.g. stacks. 896 */ 897 struct kvm_vm *____vm_create(struct vm_shape shape); 898 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, 899 uint64_t nr_extra_pages); 900 901 static inline struct kvm_vm *vm_create_barebones(void) 902 { 903 return ____vm_create(VM_SHAPE_DEFAULT); 904 } 905 906 static inline struct kvm_vm *vm_create_barebones_type(unsigned long type) 907 { 908 const struct vm_shape shape = { 909 .mode = VM_MODE_DEFAULT, 910 .type = type, 911 }; 912 913 return ____vm_create(shape); 914 } 915 916 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) 917 { 918 return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0); 919 } 920 921 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, 922 uint64_t extra_mem_pages, 923 void *guest_code, struct kvm_vcpu *vcpus[]); 924 925 static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus, 926 void *guest_code, 927 struct kvm_vcpu *vcpus[]) 928 { 929 return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0, 930 guest_code, vcpus); 931 } 932 933 934 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, 935 struct kvm_vcpu **vcpu, 936 uint64_t extra_mem_pages, 937 void *guest_code); 938 939 /* 940 * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages 941 * additional pages of guest memory. Returns the VM and vCPU (via out param). 942 */ 943 static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 944 uint64_t extra_mem_pages, 945 void *guest_code) 946 { 947 return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu, 948 extra_mem_pages, guest_code); 949 } 950 951 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 952 void *guest_code) 953 { 954 return __vm_create_with_one_vcpu(vcpu, 0, guest_code); 955 } 956 957 static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape, 958 struct kvm_vcpu **vcpu, 959 void *guest_code) 960 { 961 return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code); 962 } 963 964 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm); 965 966 void kvm_pin_this_task_to_pcpu(uint32_t pcpu); 967 void kvm_print_vcpu_pinning_help(void); 968 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[], 969 int nr_vcpus); 970 971 unsigned long vm_compute_max_gfn(struct kvm_vm *vm); 972 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size); 973 unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages); 974 unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages); 975 static inline unsigned int 976 vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) 977 { 978 unsigned int n; 979 n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages)); 980 #ifdef __s390x__ 981 /* s390 requires 1M aligned guest sizes */ 982 n = (n + 255) & ~255; 983 #endif 984 return n; 985 } 986 987 #define sync_global_to_guest(vm, g) ({ \ 988 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 989 memcpy(_p, &(g), sizeof(g)); \ 990 }) 991 992 #define sync_global_from_guest(vm, g) ({ \ 993 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 994 memcpy(&(g), _p, sizeof(g)); \ 995 }) 996 997 /* 998 * Write a global value, but only in the VM's (guest's) domain. Primarily used 999 * for "globals" that hold per-VM values (VMs always duplicate code and global 1000 * data into their own region of physical memory), but can be used anytime it's 1001 * undesirable to change the host's copy of the global. 1002 */ 1003 #define write_guest_global(vm, g, val) ({ \ 1004 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 1005 typeof(g) _val = val; \ 1006 \ 1007 memcpy(_p, &(_val), sizeof(g)); \ 1008 }) 1009 1010 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu); 1011 1012 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, 1013 uint8_t indent); 1014 1015 static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu, 1016 uint8_t indent) 1017 { 1018 vcpu_arch_dump(stream, vcpu, indent); 1019 } 1020 1021 /* 1022 * Adds a vCPU with reasonable defaults (e.g. a stack) 1023 * 1024 * Input Args: 1025 * vm - Virtual Machine 1026 * vcpu_id - The id of the VCPU to add to the VM. 1027 */ 1028 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 1029 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code); 1030 1031 static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, 1032 void *guest_code) 1033 { 1034 struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id); 1035 1036 vcpu_arch_set_entry_point(vcpu, guest_code); 1037 1038 return vcpu; 1039 } 1040 1041 /* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */ 1042 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id); 1043 1044 static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm, 1045 uint32_t vcpu_id) 1046 { 1047 return vm_arch_vcpu_recreate(vm, vcpu_id); 1048 } 1049 1050 void vcpu_arch_free(struct kvm_vcpu *vcpu); 1051 1052 void virt_arch_pgd_alloc(struct kvm_vm *vm); 1053 1054 static inline void virt_pgd_alloc(struct kvm_vm *vm) 1055 { 1056 virt_arch_pgd_alloc(vm); 1057 } 1058 1059 /* 1060 * VM Virtual Page Map 1061 * 1062 * Input Args: 1063 * vm - Virtual Machine 1064 * vaddr - VM Virtual Address 1065 * paddr - VM Physical Address 1066 * memslot - Memory region slot for new virtual translation tables 1067 * 1068 * Output Args: None 1069 * 1070 * Return: None 1071 * 1072 * Within @vm, creates a virtual translation for the page starting 1073 * at @vaddr to the page starting at @paddr. 1074 */ 1075 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr); 1076 1077 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 1078 { 1079 virt_arch_pg_map(vm, vaddr, paddr); 1080 } 1081 1082 1083 /* 1084 * Address Guest Virtual to Guest Physical 1085 * 1086 * Input Args: 1087 * vm - Virtual Machine 1088 * gva - VM virtual address 1089 * 1090 * Output Args: None 1091 * 1092 * Return: 1093 * Equivalent VM physical address 1094 * 1095 * Returns the VM physical address of the translated VM virtual 1096 * address given by @gva. 1097 */ 1098 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva); 1099 1100 static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 1101 { 1102 return addr_arch_gva2gpa(vm, gva); 1103 } 1104 1105 /* 1106 * Virtual Translation Tables Dump 1107 * 1108 * Input Args: 1109 * stream - Output FILE stream 1110 * vm - Virtual Machine 1111 * indent - Left margin indent amount 1112 * 1113 * Output Args: None 1114 * 1115 * Return: None 1116 * 1117 * Dumps to the FILE stream given by @stream, the contents of all the 1118 * virtual translation tables for the VM given by @vm. 1119 */ 1120 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 1121 1122 static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 1123 { 1124 virt_arch_dump(stream, vm, indent); 1125 } 1126 1127 1128 static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm) 1129 { 1130 return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0); 1131 } 1132 1133 /* 1134 * Arch hook that is invoked via a constructor, i.e. before exeucting main(), 1135 * to allow for arch-specific setup that is common to all tests, e.g. computing 1136 * the default guest "mode". 1137 */ 1138 void kvm_selftest_arch_init(void); 1139 1140 void kvm_arch_vm_post_create(struct kvm_vm *vm); 1141 1142 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); 1143 1144 uint32_t guest_get_vcpuid(void); 1145 1146 #endif /* SELFTEST_KVM_UTIL_H */ 1147