1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2018, Google LLC. 4 */ 5 #ifndef SELFTEST_KVM_UTIL_H 6 #define SELFTEST_KVM_UTIL_H 7 8 #include "test_util.h" 9 10 #include <linux/compiler.h> 11 #include "linux/hashtable.h" 12 #include "linux/list.h" 13 #include <linux/kernel.h> 14 #include <linux/kvm.h> 15 #include "linux/rbtree.h" 16 #include <linux/types.h> 17 18 #include <asm/atomic.h> 19 #include <asm/kvm.h> 20 21 #include <sys/ioctl.h> 22 23 #include "kvm_util_arch.h" 24 #include "sparsebit.h" 25 26 /* 27 * Provide a version of static_assert() that is guaranteed to have an optional 28 * message param. _GNU_SOURCE is defined for all KVM selftests, _GNU_SOURCE 29 * implies _ISOC11_SOURCE, and if _ISOC11_SOURCE is defined, glibc #undefs and 30 * #defines static_assert() as a direct alias to _Static_assert() (see 31 * usr/include/assert.h). Define a custom macro instead of redefining 32 * static_assert() to avoid creating non-deterministic behavior that is 33 * dependent on include order. 34 */ 35 #define __kvm_static_assert(expr, msg, ...) _Static_assert(expr, msg) 36 #define kvm_static_assert(expr, ...) __kvm_static_assert(expr, ##__VA_ARGS__, #expr) 37 38 #define KVM_DEV_PATH "/dev/kvm" 39 #define KVM_MAX_VCPUS 512 40 41 #define NSEC_PER_SEC 1000000000L 42 43 typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */ 44 typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */ 45 46 struct userspace_mem_region { 47 struct kvm_userspace_memory_region2 region; 48 struct sparsebit *unused_phy_pages; 49 struct sparsebit *protected_phy_pages; 50 int fd; 51 off_t offset; 52 enum vm_mem_backing_src_type backing_src_type; 53 void *host_mem; 54 void *host_alias; 55 void *mmap_start; 56 void *mmap_alias; 57 size_t mmap_size; 58 struct rb_node gpa_node; 59 struct rb_node hva_node; 60 struct hlist_node slot_node; 61 }; 62 63 struct kvm_vcpu { 64 struct list_head list; 65 uint32_t id; 66 int fd; 67 struct kvm_vm *vm; 68 struct kvm_run *run; 69 #ifdef __x86_64__ 70 struct kvm_cpuid2 *cpuid; 71 #endif 72 struct kvm_dirty_gfn *dirty_gfns; 73 uint32_t fetch_index; 74 uint32_t dirty_gfns_count; 75 }; 76 77 struct userspace_mem_regions { 78 struct rb_root gpa_tree; 79 struct rb_root hva_tree; 80 DECLARE_HASHTABLE(slot_hash, 9); 81 }; 82 83 enum kvm_mem_region_type { 84 MEM_REGION_CODE, 85 MEM_REGION_DATA, 86 MEM_REGION_PT, 87 MEM_REGION_TEST_DATA, 88 NR_MEM_REGIONS, 89 }; 90 91 struct kvm_vm { 92 int mode; 93 unsigned long type; 94 int kvm_fd; 95 int fd; 96 unsigned int pgtable_levels; 97 unsigned int page_size; 98 unsigned int page_shift; 99 unsigned int pa_bits; 100 unsigned int va_bits; 101 uint64_t max_gfn; 102 struct list_head vcpus; 103 struct userspace_mem_regions regions; 104 struct sparsebit *vpages_valid; 105 struct sparsebit *vpages_mapped; 106 bool has_irqchip; 107 bool pgd_created; 108 vm_paddr_t ucall_mmio_addr; 109 vm_paddr_t pgd; 110 vm_vaddr_t gdt; 111 vm_vaddr_t tss; 112 vm_vaddr_t idt; 113 vm_vaddr_t handlers; 114 uint32_t dirty_ring_size; 115 uint64_t gpa_tag_mask; 116 117 struct kvm_vm_arch arch; 118 119 /* Cache of information for binary stats interface */ 120 int stats_fd; 121 struct kvm_stats_header stats_header; 122 struct kvm_stats_desc *stats_desc; 123 124 /* 125 * KVM region slots. These are the default memslots used by page 126 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] 127 * memslot. 128 */ 129 uint32_t memslots[NR_MEM_REGIONS]; 130 }; 131 132 struct vcpu_reg_sublist { 133 const char *name; 134 long capability; 135 int feature; 136 int feature_type; 137 bool finalize; 138 __u64 *regs; 139 __u64 regs_n; 140 __u64 *rejects_set; 141 __u64 rejects_set_n; 142 __u64 *skips_set; 143 __u64 skips_set_n; 144 }; 145 146 struct vcpu_reg_list { 147 char *name; 148 struct vcpu_reg_sublist sublists[]; 149 }; 150 151 #define for_each_sublist(c, s) \ 152 for ((s) = &(c)->sublists[0]; (s)->regs; ++(s)) 153 154 #define kvm_for_each_vcpu(vm, i, vcpu) \ 155 for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \ 156 if (!((vcpu) = vm->vcpus[i])) \ 157 continue; \ 158 else 159 160 struct userspace_mem_region * 161 memslot2region(struct kvm_vm *vm, uint32_t memslot); 162 163 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, 164 enum kvm_mem_region_type type) 165 { 166 assert(type < NR_MEM_REGIONS); 167 return memslot2region(vm, vm->memslots[type]); 168 } 169 170 /* Minimum allocated guest virtual and physical addresses */ 171 #define KVM_UTIL_MIN_VADDR 0x2000 172 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 173 174 #define DEFAULT_GUEST_STACK_VADDR_MIN 0xab6000 175 #define DEFAULT_STACK_PGS 5 176 177 enum vm_guest_mode { 178 VM_MODE_P52V48_4K, 179 VM_MODE_P52V48_16K, 180 VM_MODE_P52V48_64K, 181 VM_MODE_P48V48_4K, 182 VM_MODE_P48V48_16K, 183 VM_MODE_P48V48_64K, 184 VM_MODE_P40V48_4K, 185 VM_MODE_P40V48_16K, 186 VM_MODE_P40V48_64K, 187 VM_MODE_PXXV48_4K, /* For 48bits VA but ANY bits PA */ 188 VM_MODE_P47V64_4K, 189 VM_MODE_P44V64_4K, 190 VM_MODE_P36V48_4K, 191 VM_MODE_P36V48_16K, 192 VM_MODE_P36V48_64K, 193 VM_MODE_P36V47_16K, 194 NUM_VM_MODES, 195 }; 196 197 struct vm_shape { 198 uint32_t type; 199 uint8_t mode; 200 uint8_t pad0; 201 uint16_t pad1; 202 }; 203 204 kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t)); 205 206 #define VM_TYPE_DEFAULT 0 207 208 #define VM_SHAPE(__mode) \ 209 ({ \ 210 struct vm_shape shape = { \ 211 .mode = (__mode), \ 212 .type = VM_TYPE_DEFAULT \ 213 }; \ 214 \ 215 shape; \ 216 }) 217 218 #if defined(__aarch64__) 219 220 extern enum vm_guest_mode vm_mode_default; 221 222 #define VM_MODE_DEFAULT vm_mode_default 223 #define MIN_PAGE_SHIFT 12U 224 #define ptes_per_page(page_size) ((page_size) / 8) 225 226 #elif defined(__x86_64__) 227 228 #define VM_MODE_DEFAULT VM_MODE_PXXV48_4K 229 #define MIN_PAGE_SHIFT 12U 230 #define ptes_per_page(page_size) ((page_size) / 8) 231 232 #elif defined(__s390x__) 233 234 #define VM_MODE_DEFAULT VM_MODE_P44V64_4K 235 #define MIN_PAGE_SHIFT 12U 236 #define ptes_per_page(page_size) ((page_size) / 16) 237 238 #elif defined(__riscv) 239 240 #if __riscv_xlen == 32 241 #error "RISC-V 32-bit kvm selftests not supported" 242 #endif 243 244 #define VM_MODE_DEFAULT VM_MODE_P40V48_4K 245 #define MIN_PAGE_SHIFT 12U 246 #define ptes_per_page(page_size) ((page_size) / 8) 247 248 #endif 249 250 #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) 251 252 #define MIN_PAGE_SIZE (1U << MIN_PAGE_SHIFT) 253 #define PTES_PER_MIN_PAGE ptes_per_page(MIN_PAGE_SIZE) 254 255 struct vm_guest_mode_params { 256 unsigned int pa_bits; 257 unsigned int va_bits; 258 unsigned int page_size; 259 unsigned int page_shift; 260 }; 261 extern const struct vm_guest_mode_params vm_guest_mode_params[]; 262 263 int open_path_or_exit(const char *path, int flags); 264 int open_kvm_dev_path_or_exit(void); 265 266 bool get_kvm_param_bool(const char *param); 267 bool get_kvm_intel_param_bool(const char *param); 268 bool get_kvm_amd_param_bool(const char *param); 269 270 int get_kvm_param_integer(const char *param); 271 int get_kvm_intel_param_integer(const char *param); 272 int get_kvm_amd_param_integer(const char *param); 273 274 unsigned int kvm_check_cap(long cap); 275 276 static inline bool kvm_has_cap(long cap) 277 { 278 return kvm_check_cap(cap); 279 } 280 281 #define __KVM_SYSCALL_ERROR(_name, _ret) \ 282 "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno) 283 284 /* 285 * Use the "inner", double-underscore macro when reporting errors from within 286 * other macros so that the name of ioctl() and not its literal numeric value 287 * is printed on error. The "outer" macro is strongly preferred when reporting 288 * errors "directly", i.e. without an additional layer of macros, as it reduces 289 * the probability of passing in the wrong string. 290 */ 291 #define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret) 292 #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret) 293 294 #define kvm_do_ioctl(fd, cmd, arg) \ 295 ({ \ 296 kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd)); \ 297 ioctl(fd, cmd, arg); \ 298 }) 299 300 #define __kvm_ioctl(kvm_fd, cmd, arg) \ 301 kvm_do_ioctl(kvm_fd, cmd, arg) 302 303 #define kvm_ioctl(kvm_fd, cmd, arg) \ 304 ({ \ 305 int ret = __kvm_ioctl(kvm_fd, cmd, arg); \ 306 \ 307 TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ 308 }) 309 310 static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } 311 312 #define __vm_ioctl(vm, cmd, arg) \ 313 ({ \ 314 static_assert_is_vm(vm); \ 315 kvm_do_ioctl((vm)->fd, cmd, arg); \ 316 }) 317 318 /* 319 * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if 320 * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM, 321 * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before 322 * selftests existed and (b) should never outright fail, i.e. is supposed to 323 * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the 324 * VM and its vCPUs, including KVM_CHECK_EXTENSION. 325 */ 326 #define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \ 327 do { \ 328 int __errno = errno; \ 329 \ 330 static_assert_is_vm(vm); \ 331 \ 332 if (cond) \ 333 break; \ 334 \ 335 if (errno == EIO && \ 336 __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \ 337 TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \ 338 TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \ 339 } \ 340 errno = __errno; \ 341 TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \ 342 } while (0) 343 344 #define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm) \ 345 __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm) 346 347 #define vm_ioctl(vm, cmd, arg) \ 348 ({ \ 349 int ret = __vm_ioctl(vm, cmd, arg); \ 350 \ 351 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \ 352 }) 353 354 static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } 355 356 #define __vcpu_ioctl(vcpu, cmd, arg) \ 357 ({ \ 358 static_assert_is_vcpu(vcpu); \ 359 kvm_do_ioctl((vcpu)->fd, cmd, arg); \ 360 }) 361 362 #define vcpu_ioctl(vcpu, cmd, arg) \ 363 ({ \ 364 int ret = __vcpu_ioctl(vcpu, cmd, arg); \ 365 \ 366 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ 367 }) 368 369 /* 370 * Looks up and returns the value corresponding to the capability 371 * (KVM_CAP_*) given by cap. 372 */ 373 static inline int vm_check_cap(struct kvm_vm *vm, long cap) 374 { 375 int ret = __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap); 376 377 TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm); 378 return ret; 379 } 380 381 static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 382 { 383 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 384 385 return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 386 } 387 static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 388 { 389 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 390 391 vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 392 } 393 394 static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, 395 uint64_t size, uint64_t attributes) 396 { 397 struct kvm_memory_attributes attr = { 398 .attributes = attributes, 399 .address = gpa, 400 .size = size, 401 .flags = 0, 402 }; 403 404 /* 405 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows 406 * need significant enhancements to support multiple attributes. 407 */ 408 TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, 409 "Update me to support multiple attributes!"); 410 411 vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); 412 } 413 414 415 static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa, 416 uint64_t size) 417 { 418 vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); 419 } 420 421 static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, 422 uint64_t size) 423 { 424 vm_set_memory_attributes(vm, gpa, size, 0); 425 } 426 427 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, 428 bool punch_hole); 429 430 static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, 431 uint64_t size) 432 { 433 vm_guest_mem_fallocate(vm, gpa, size, true); 434 } 435 436 static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa, 437 uint64_t size) 438 { 439 vm_guest_mem_fallocate(vm, gpa, size, false); 440 } 441 442 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size); 443 const char *vm_guest_mode_string(uint32_t i); 444 445 void kvm_vm_free(struct kvm_vm *vmp); 446 void kvm_vm_restart(struct kvm_vm *vmp); 447 void kvm_vm_release(struct kvm_vm *vmp); 448 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, const vm_vaddr_t gva, 449 size_t len); 450 void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename); 451 int kvm_memfd_alloc(size_t size, bool hugepages); 452 453 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 454 455 static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log) 456 { 457 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot }; 458 459 vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args); 460 } 461 462 static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log, 463 uint64_t first_page, uint32_t num_pages) 464 { 465 struct kvm_clear_dirty_log args = { 466 .dirty_bitmap = log, 467 .slot = slot, 468 .first_page = first_page, 469 .num_pages = num_pages 470 }; 471 472 vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args); 473 } 474 475 static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm) 476 { 477 return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL); 478 } 479 480 static inline int vm_get_stats_fd(struct kvm_vm *vm) 481 { 482 int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL); 483 484 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm); 485 return fd; 486 } 487 488 static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header) 489 { 490 ssize_t ret; 491 492 ret = pread(stats_fd, header, sizeof(*header), 0); 493 TEST_ASSERT(ret == sizeof(*header), 494 "Failed to read '%lu' header bytes, ret = '%ld'", 495 sizeof(*header), ret); 496 } 497 498 struct kvm_stats_desc *read_stats_descriptors(int stats_fd, 499 struct kvm_stats_header *header); 500 501 static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header) 502 { 503 /* 504 * The base size of the descriptor is defined by KVM's ABI, but the 505 * size of the name field is variable, as far as KVM's ABI is 506 * concerned. For a given instance of KVM, the name field is the same 507 * size for all stats and is provided in the overall stats header. 508 */ 509 return sizeof(struct kvm_stats_desc) + header->name_size; 510 } 511 512 static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats, 513 int index, 514 struct kvm_stats_header *header) 515 { 516 /* 517 * Note, size_desc includes the size of the name field, which is 518 * variable. i.e. this is NOT equivalent to &stats_desc[i]. 519 */ 520 return (void *)stats + index * get_stats_descriptor_size(header); 521 } 522 523 void read_stat_data(int stats_fd, struct kvm_stats_header *header, 524 struct kvm_stats_desc *desc, uint64_t *data, 525 size_t max_elements); 526 527 void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data, 528 size_t max_elements); 529 530 static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name) 531 { 532 uint64_t data; 533 534 __vm_get_stat(vm, stat_name, &data, 1); 535 return data; 536 } 537 538 void vm_create_irqchip(struct kvm_vm *vm); 539 540 static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 541 uint64_t flags) 542 { 543 struct kvm_create_guest_memfd guest_memfd = { 544 .size = size, 545 .flags = flags, 546 }; 547 548 return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd); 549 } 550 551 static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 552 uint64_t flags) 553 { 554 int fd = __vm_create_guest_memfd(vm, size, flags); 555 556 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd)); 557 return fd; 558 } 559 560 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 561 uint64_t gpa, uint64_t size, void *hva); 562 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 563 uint64_t gpa, uint64_t size, void *hva); 564 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 565 uint64_t gpa, uint64_t size, void *hva, 566 uint32_t guest_memfd, uint64_t guest_memfd_offset); 567 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 568 uint64_t gpa, uint64_t size, void *hva, 569 uint32_t guest_memfd, uint64_t guest_memfd_offset); 570 571 void vm_userspace_mem_region_add(struct kvm_vm *vm, 572 enum vm_mem_backing_src_type src_type, 573 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 574 uint32_t flags); 575 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, 576 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 577 uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset); 578 579 #ifndef vm_arch_has_protected_memory 580 static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) 581 { 582 return false; 583 } 584 #endif 585 586 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); 587 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); 588 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); 589 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 590 void vm_populate_vaddr_bitmap(struct kvm_vm *vm); 591 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 592 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 593 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, 594 enum kvm_mem_region_type type); 595 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz, 596 vm_vaddr_t vaddr_min, 597 enum kvm_mem_region_type type); 598 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages); 599 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, 600 enum kvm_mem_region_type type); 601 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm); 602 603 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 604 unsigned int npages); 605 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa); 606 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva); 607 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva); 608 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa); 609 610 #ifndef vcpu_arch_put_guest 611 #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) 612 #endif 613 614 static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa) 615 { 616 return gpa & ~vm->gpa_tag_mask; 617 } 618 619 void vcpu_run(struct kvm_vcpu *vcpu); 620 int _vcpu_run(struct kvm_vcpu *vcpu); 621 622 static inline int __vcpu_run(struct kvm_vcpu *vcpu) 623 { 624 return __vcpu_ioctl(vcpu, KVM_RUN, NULL); 625 } 626 627 void vcpu_run_complete_io(struct kvm_vcpu *vcpu); 628 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu); 629 630 static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap, 631 uint64_t arg0) 632 { 633 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 634 635 vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap); 636 } 637 638 static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu, 639 struct kvm_guest_debug *debug) 640 { 641 vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug); 642 } 643 644 static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu, 645 struct kvm_mp_state *mp_state) 646 { 647 vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state); 648 } 649 static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu, 650 struct kvm_mp_state *mp_state) 651 { 652 vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state); 653 } 654 655 static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 656 { 657 vcpu_ioctl(vcpu, KVM_GET_REGS, regs); 658 } 659 660 static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 661 { 662 vcpu_ioctl(vcpu, KVM_SET_REGS, regs); 663 } 664 static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 665 { 666 vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs); 667 668 } 669 static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 670 { 671 vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 672 } 673 static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 674 { 675 return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 676 } 677 static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 678 { 679 vcpu_ioctl(vcpu, KVM_GET_FPU, fpu); 680 } 681 static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 682 { 683 vcpu_ioctl(vcpu, KVM_SET_FPU, fpu); 684 } 685 686 static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 687 { 688 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 689 690 return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 691 } 692 static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 693 { 694 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 695 696 return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 697 } 698 static inline void vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 699 { 700 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 701 702 vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 703 } 704 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 705 { 706 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 707 708 vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 709 } 710 711 #ifdef __KVM_HAVE_VCPU_EVENTS 712 static inline void vcpu_events_get(struct kvm_vcpu *vcpu, 713 struct kvm_vcpu_events *events) 714 { 715 vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events); 716 } 717 static inline void vcpu_events_set(struct kvm_vcpu *vcpu, 718 struct kvm_vcpu_events *events) 719 { 720 vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events); 721 } 722 #endif 723 #ifdef __x86_64__ 724 static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu, 725 struct kvm_nested_state *state) 726 { 727 vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state); 728 } 729 static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu, 730 struct kvm_nested_state *state) 731 { 732 return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 733 } 734 735 static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu, 736 struct kvm_nested_state *state) 737 { 738 vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 739 } 740 #endif 741 static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu) 742 { 743 int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL); 744 745 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm); 746 return fd; 747 } 748 749 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr); 750 751 static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr) 752 { 753 int ret = __kvm_has_device_attr(dev_fd, group, attr); 754 755 TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno); 756 } 757 758 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val); 759 760 static inline void kvm_device_attr_get(int dev_fd, uint32_t group, 761 uint64_t attr, void *val) 762 { 763 int ret = __kvm_device_attr_get(dev_fd, group, attr, val); 764 765 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret)); 766 } 767 768 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val); 769 770 static inline void kvm_device_attr_set(int dev_fd, uint32_t group, 771 uint64_t attr, void *val) 772 { 773 int ret = __kvm_device_attr_set(dev_fd, group, attr, val); 774 775 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret)); 776 } 777 778 static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 779 uint64_t attr) 780 { 781 return __kvm_has_device_attr(vcpu->fd, group, attr); 782 } 783 784 static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 785 uint64_t attr) 786 { 787 kvm_has_device_attr(vcpu->fd, group, attr); 788 } 789 790 static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 791 uint64_t attr, void *val) 792 { 793 return __kvm_device_attr_get(vcpu->fd, group, attr, val); 794 } 795 796 static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 797 uint64_t attr, void *val) 798 { 799 kvm_device_attr_get(vcpu->fd, group, attr, val); 800 } 801 802 static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 803 uint64_t attr, void *val) 804 { 805 return __kvm_device_attr_set(vcpu->fd, group, attr, val); 806 } 807 808 static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 809 uint64_t attr, void *val) 810 { 811 kvm_device_attr_set(vcpu->fd, group, attr, val); 812 } 813 814 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type); 815 int __kvm_create_device(struct kvm_vm *vm, uint64_t type); 816 817 static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type) 818 { 819 int fd = __kvm_create_device(vm, type); 820 821 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd)); 822 return fd; 823 } 824 825 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu); 826 827 /* 828 * VM VCPU Args Set 829 * 830 * Input Args: 831 * vm - Virtual Machine 832 * num - number of arguments 833 * ... - arguments, each of type uint64_t 834 * 835 * Output Args: None 836 * 837 * Return: None 838 * 839 * Sets the first @num input parameters for the function at @vcpu's entry point, 840 * per the C calling convention of the architecture, to the values given as 841 * variable args. Each of the variable args is expected to be of type uint64_t. 842 * The maximum @num can be is specific to the architecture. 843 */ 844 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...); 845 846 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 847 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 848 849 #define KVM_MAX_IRQ_ROUTES 4096 850 851 struct kvm_irq_routing *kvm_gsi_routing_create(void); 852 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing, 853 uint32_t gsi, uint32_t pin); 854 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 855 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 856 857 const char *exit_reason_str(unsigned int exit_reason); 858 859 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, 860 uint32_t memslot); 861 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 862 vm_paddr_t paddr_min, uint32_t memslot, 863 bool protected); 864 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); 865 866 static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 867 vm_paddr_t paddr_min, uint32_t memslot) 868 { 869 /* 870 * By default, allocate memory as protected for VMs that support 871 * protected memory, as the majority of memory for such VMs is 872 * protected, i.e. using shared memory is effectively opt-in. 873 */ 874 return __vm_phy_pages_alloc(vm, num, paddr_min, memslot, 875 vm_arch_has_protected_memory(vm)); 876 } 877 878 /* 879 * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also 880 * loads the test binary into guest memory and creates an IRQ chip (x86 only). 881 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to 882 * calculate the amount of memory needed for per-vCPU data, e.g. stacks. 883 */ 884 struct kvm_vm *____vm_create(struct vm_shape shape); 885 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, 886 uint64_t nr_extra_pages); 887 888 static inline struct kvm_vm *vm_create_barebones(void) 889 { 890 return ____vm_create(VM_SHAPE_DEFAULT); 891 } 892 893 static inline struct kvm_vm *vm_create_barebones_type(unsigned long type) 894 { 895 const struct vm_shape shape = { 896 .mode = VM_MODE_DEFAULT, 897 .type = type, 898 }; 899 900 return ____vm_create(shape); 901 } 902 903 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) 904 { 905 return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0); 906 } 907 908 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, 909 uint64_t extra_mem_pages, 910 void *guest_code, struct kvm_vcpu *vcpus[]); 911 912 static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus, 913 void *guest_code, 914 struct kvm_vcpu *vcpus[]) 915 { 916 return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0, 917 guest_code, vcpus); 918 } 919 920 921 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, 922 struct kvm_vcpu **vcpu, 923 uint64_t extra_mem_pages, 924 void *guest_code); 925 926 /* 927 * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages 928 * additional pages of guest memory. Returns the VM and vCPU (via out param). 929 */ 930 static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 931 uint64_t extra_mem_pages, 932 void *guest_code) 933 { 934 return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu, 935 extra_mem_pages, guest_code); 936 } 937 938 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 939 void *guest_code) 940 { 941 return __vm_create_with_one_vcpu(vcpu, 0, guest_code); 942 } 943 944 static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape, 945 struct kvm_vcpu **vcpu, 946 void *guest_code) 947 { 948 return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code); 949 } 950 951 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm); 952 953 void kvm_pin_this_task_to_pcpu(uint32_t pcpu); 954 void kvm_print_vcpu_pinning_help(void); 955 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[], 956 int nr_vcpus); 957 958 unsigned long vm_compute_max_gfn(struct kvm_vm *vm); 959 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size); 960 unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages); 961 unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages); 962 static inline unsigned int 963 vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) 964 { 965 unsigned int n; 966 n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages)); 967 #ifdef __s390x__ 968 /* s390 requires 1M aligned guest sizes */ 969 n = (n + 255) & ~255; 970 #endif 971 return n; 972 } 973 974 #define sync_global_to_guest(vm, g) ({ \ 975 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 976 memcpy(_p, &(g), sizeof(g)); \ 977 }) 978 979 #define sync_global_from_guest(vm, g) ({ \ 980 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 981 memcpy(&(g), _p, sizeof(g)); \ 982 }) 983 984 /* 985 * Write a global value, but only in the VM's (guest's) domain. Primarily used 986 * for "globals" that hold per-VM values (VMs always duplicate code and global 987 * data into their own region of physical memory), but can be used anytime it's 988 * undesirable to change the host's copy of the global. 989 */ 990 #define write_guest_global(vm, g, val) ({ \ 991 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 992 typeof(g) _val = val; \ 993 \ 994 memcpy(_p, &(_val), sizeof(g)); \ 995 }) 996 997 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu); 998 999 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, 1000 uint8_t indent); 1001 1002 static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu, 1003 uint8_t indent) 1004 { 1005 vcpu_arch_dump(stream, vcpu, indent); 1006 } 1007 1008 /* 1009 * Adds a vCPU with reasonable defaults (e.g. a stack) 1010 * 1011 * Input Args: 1012 * vm - Virtual Machine 1013 * vcpu_id - The id of the VCPU to add to the VM. 1014 */ 1015 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 1016 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code); 1017 1018 static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, 1019 void *guest_code) 1020 { 1021 struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id); 1022 1023 vcpu_arch_set_entry_point(vcpu, guest_code); 1024 1025 return vcpu; 1026 } 1027 1028 /* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */ 1029 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id); 1030 1031 static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm, 1032 uint32_t vcpu_id) 1033 { 1034 return vm_arch_vcpu_recreate(vm, vcpu_id); 1035 } 1036 1037 void vcpu_arch_free(struct kvm_vcpu *vcpu); 1038 1039 void virt_arch_pgd_alloc(struct kvm_vm *vm); 1040 1041 static inline void virt_pgd_alloc(struct kvm_vm *vm) 1042 { 1043 virt_arch_pgd_alloc(vm); 1044 } 1045 1046 /* 1047 * VM Virtual Page Map 1048 * 1049 * Input Args: 1050 * vm - Virtual Machine 1051 * vaddr - VM Virtual Address 1052 * paddr - VM Physical Address 1053 * memslot - Memory region slot for new virtual translation tables 1054 * 1055 * Output Args: None 1056 * 1057 * Return: None 1058 * 1059 * Within @vm, creates a virtual translation for the page starting 1060 * at @vaddr to the page starting at @paddr. 1061 */ 1062 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr); 1063 1064 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 1065 { 1066 virt_arch_pg_map(vm, vaddr, paddr); 1067 } 1068 1069 1070 /* 1071 * Address Guest Virtual to Guest Physical 1072 * 1073 * Input Args: 1074 * vm - Virtual Machine 1075 * gva - VM virtual address 1076 * 1077 * Output Args: None 1078 * 1079 * Return: 1080 * Equivalent VM physical address 1081 * 1082 * Returns the VM physical address of the translated VM virtual 1083 * address given by @gva. 1084 */ 1085 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva); 1086 1087 static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 1088 { 1089 return addr_arch_gva2gpa(vm, gva); 1090 } 1091 1092 /* 1093 * Virtual Translation Tables Dump 1094 * 1095 * Input Args: 1096 * stream - Output FILE stream 1097 * vm - Virtual Machine 1098 * indent - Left margin indent amount 1099 * 1100 * Output Args: None 1101 * 1102 * Return: None 1103 * 1104 * Dumps to the FILE stream given by @stream, the contents of all the 1105 * virtual translation tables for the VM given by @vm. 1106 */ 1107 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 1108 1109 static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 1110 { 1111 virt_arch_dump(stream, vm, indent); 1112 } 1113 1114 1115 static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm) 1116 { 1117 return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0); 1118 } 1119 1120 /* 1121 * Arch hook that is invoked via a constructor, i.e. before exeucting main(), 1122 * to allow for arch-specific setup that is common to all tests, e.g. computing 1123 * the default guest "mode". 1124 */ 1125 void kvm_selftest_arch_init(void); 1126 1127 void kvm_arch_vm_post_create(struct kvm_vm *vm); 1128 1129 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); 1130 1131 uint32_t guest_get_vcpuid(void); 1132 1133 #endif /* SELFTEST_KVM_UTIL_H */ 1134