xref: /linux/tools/testing/selftests/kvm/include/kvm_util.h (revision 02e5f74ef08d3e6afec438d571487d0d0cec3c48)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2018, Google LLC.
4  */
5 #ifndef SELFTEST_KVM_UTIL_H
6 #define SELFTEST_KVM_UTIL_H
7 
8 #include "test_util.h"
9 
10 #include <linux/compiler.h>
11 #include "linux/hashtable.h"
12 #include "linux/list.h"
13 #include <linux/kernel.h>
14 #include <linux/kvm.h>
15 #include "linux/rbtree.h"
16 #include <linux/types.h>
17 
18 #include <asm/atomic.h>
19 #include <asm/kvm.h>
20 
21 #include <sys/eventfd.h>
22 #include <sys/ioctl.h>
23 
24 #include <pthread.h>
25 
26 #include "kvm_util_arch.h"
27 #include "kvm_util_types.h"
28 #include "sparsebit.h"
29 
30 #define KVM_DEV_PATH "/dev/kvm"
31 #define KVM_MAX_VCPUS 512
32 
33 #define NSEC_PER_SEC 1000000000L
34 
35 struct userspace_mem_region {
36 	struct kvm_userspace_memory_region2 region;
37 	struct sparsebit *unused_phy_pages;
38 	struct sparsebit *protected_phy_pages;
39 	int fd;
40 	off_t offset;
41 	enum vm_mem_backing_src_type backing_src_type;
42 	void *host_mem;
43 	void *host_alias;
44 	void *mmap_start;
45 	void *mmap_alias;
46 	size_t mmap_size;
47 	struct rb_node gpa_node;
48 	struct rb_node hva_node;
49 	struct hlist_node slot_node;
50 };
51 
52 struct kvm_binary_stats {
53 	int fd;
54 	struct kvm_stats_header header;
55 	struct kvm_stats_desc *desc;
56 };
57 
58 struct kvm_vcpu {
59 	struct list_head list;
60 	uint32_t id;
61 	int fd;
62 	struct kvm_vm *vm;
63 	struct kvm_run *run;
64 #ifdef __x86_64__
65 	struct kvm_cpuid2 *cpuid;
66 #endif
67 #ifdef __aarch64__
68 	struct kvm_vcpu_init init;
69 #endif
70 	struct kvm_binary_stats stats;
71 	struct kvm_dirty_gfn *dirty_gfns;
72 	uint32_t fetch_index;
73 	uint32_t dirty_gfns_count;
74 };
75 
76 struct userspace_mem_regions {
77 	struct rb_root gpa_tree;
78 	struct rb_root hva_tree;
79 	DECLARE_HASHTABLE(slot_hash, 9);
80 };
81 
82 enum kvm_mem_region_type {
83 	MEM_REGION_CODE,
84 	MEM_REGION_DATA,
85 	MEM_REGION_PT,
86 	MEM_REGION_TEST_DATA,
87 	NR_MEM_REGIONS,
88 };
89 
90 struct kvm_vm {
91 	int mode;
92 	unsigned long type;
93 	int kvm_fd;
94 	int fd;
95 	unsigned int pgtable_levels;
96 	unsigned int page_size;
97 	unsigned int page_shift;
98 	unsigned int pa_bits;
99 	unsigned int va_bits;
100 	uint64_t max_gfn;
101 	struct list_head vcpus;
102 	struct userspace_mem_regions regions;
103 	struct sparsebit *vpages_valid;
104 	struct sparsebit *vpages_mapped;
105 	bool has_irqchip;
106 	bool pgd_created;
107 	vm_paddr_t ucall_mmio_addr;
108 	vm_paddr_t pgd;
109 	vm_vaddr_t handlers;
110 	uint32_t dirty_ring_size;
111 	uint64_t gpa_tag_mask;
112 
113 	struct kvm_vm_arch arch;
114 
115 	struct kvm_binary_stats stats;
116 
117 	/*
118 	 * KVM region slots. These are the default memslots used by page
119 	 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
120 	 * memslot.
121 	 */
122 	uint32_t memslots[NR_MEM_REGIONS];
123 };
124 
125 struct vcpu_reg_sublist {
126 	const char *name;
127 	long capability;
128 	int feature;
129 	int feature_type;
130 	bool finalize;
131 	__u64 *regs;
132 	__u64 regs_n;
133 	__u64 *rejects_set;
134 	__u64 rejects_set_n;
135 	__u64 *skips_set;
136 	__u64 skips_set_n;
137 };
138 
139 struct vcpu_reg_list {
140 	char *name;
141 	struct vcpu_reg_sublist sublists[];
142 };
143 
144 #define for_each_sublist(c, s)		\
145 	for ((s) = &(c)->sublists[0]; (s)->regs; ++(s))
146 
147 #define kvm_for_each_vcpu(vm, i, vcpu)			\
148 	for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++)	\
149 		if (!((vcpu) = vm->vcpus[i]))		\
150 			continue;			\
151 		else
152 
153 struct userspace_mem_region *
154 memslot2region(struct kvm_vm *vm, uint32_t memslot);
155 
vm_get_mem_region(struct kvm_vm * vm,enum kvm_mem_region_type type)156 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
157 							     enum kvm_mem_region_type type)
158 {
159 	assert(type < NR_MEM_REGIONS);
160 	return memslot2region(vm, vm->memslots[type]);
161 }
162 
163 /* Minimum allocated guest virtual and physical addresses */
164 #define KVM_UTIL_MIN_VADDR		0x2000
165 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR	0x180000
166 
167 #define DEFAULT_GUEST_STACK_VADDR_MIN	0xab6000
168 #define DEFAULT_STACK_PGS		5
169 
170 enum vm_guest_mode {
171 	VM_MODE_P52V48_4K,
172 	VM_MODE_P52V48_16K,
173 	VM_MODE_P52V48_64K,
174 	VM_MODE_P48V48_4K,
175 	VM_MODE_P48V48_16K,
176 	VM_MODE_P48V48_64K,
177 	VM_MODE_P40V48_4K,
178 	VM_MODE_P40V48_16K,
179 	VM_MODE_P40V48_64K,
180 	VM_MODE_PXXV48_4K,	/* For 48bits VA but ANY bits PA */
181 	VM_MODE_P47V64_4K,
182 	VM_MODE_P44V64_4K,
183 	VM_MODE_P36V48_4K,
184 	VM_MODE_P36V48_16K,
185 	VM_MODE_P36V48_64K,
186 	VM_MODE_P47V47_16K,
187 	VM_MODE_P36V47_16K,
188 	NUM_VM_MODES,
189 };
190 
191 struct vm_shape {
192 	uint32_t type;
193 	uint8_t  mode;
194 	uint8_t  pad0;
195 	uint16_t pad1;
196 };
197 
198 kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t));
199 
200 #define VM_TYPE_DEFAULT			0
201 
202 #define VM_SHAPE(__mode)			\
203 ({						\
204 	struct vm_shape shape = {		\
205 		.mode = (__mode),		\
206 		.type = VM_TYPE_DEFAULT		\
207 	};					\
208 						\
209 	shape;					\
210 })
211 
212 #if defined(__aarch64__)
213 
214 extern enum vm_guest_mode vm_mode_default;
215 
216 #define VM_MODE_DEFAULT			vm_mode_default
217 #define MIN_PAGE_SHIFT			12U
218 #define ptes_per_page(page_size)	((page_size) / 8)
219 
220 #elif defined(__x86_64__)
221 
222 #define VM_MODE_DEFAULT			VM_MODE_PXXV48_4K
223 #define MIN_PAGE_SHIFT			12U
224 #define ptes_per_page(page_size)	((page_size) / 8)
225 
226 #elif defined(__s390x__)
227 
228 #define VM_MODE_DEFAULT			VM_MODE_P44V64_4K
229 #define MIN_PAGE_SHIFT			12U
230 #define ptes_per_page(page_size)	((page_size) / 16)
231 
232 #elif defined(__riscv)
233 
234 #if __riscv_xlen == 32
235 #error "RISC-V 32-bit kvm selftests not supported"
236 #endif
237 
238 #define VM_MODE_DEFAULT			VM_MODE_P40V48_4K
239 #define MIN_PAGE_SHIFT			12U
240 #define ptes_per_page(page_size)	((page_size) / 8)
241 
242 #elif defined(__loongarch__)
243 #define VM_MODE_DEFAULT			VM_MODE_P47V47_16K
244 #define MIN_PAGE_SHIFT			12U
245 #define ptes_per_page(page_size)	((page_size) / 8)
246 
247 #endif
248 
249 #define VM_SHAPE_DEFAULT	VM_SHAPE(VM_MODE_DEFAULT)
250 
251 #define MIN_PAGE_SIZE		(1U << MIN_PAGE_SHIFT)
252 #define PTES_PER_MIN_PAGE	ptes_per_page(MIN_PAGE_SIZE)
253 
254 struct vm_guest_mode_params {
255 	unsigned int pa_bits;
256 	unsigned int va_bits;
257 	unsigned int page_size;
258 	unsigned int page_shift;
259 };
260 extern const struct vm_guest_mode_params vm_guest_mode_params[];
261 
262 int __open_path_or_exit(const char *path, int flags, const char *enoent_help);
263 int open_path_or_exit(const char *path, int flags);
264 int open_kvm_dev_path_or_exit(void);
265 
266 int kvm_get_module_param_integer(const char *module_name, const char *param);
267 bool kvm_get_module_param_bool(const char *module_name, const char *param);
268 
get_kvm_param_bool(const char * param)269 static inline bool get_kvm_param_bool(const char *param)
270 {
271 	return kvm_get_module_param_bool("kvm", param);
272 }
273 
get_kvm_param_integer(const char * param)274 static inline int get_kvm_param_integer(const char *param)
275 {
276 	return kvm_get_module_param_integer("kvm", param);
277 }
278 
279 unsigned int kvm_check_cap(long cap);
280 
kvm_has_cap(long cap)281 static inline bool kvm_has_cap(long cap)
282 {
283 	return kvm_check_cap(cap);
284 }
285 
286 #define __KVM_SYSCALL_ERROR(_name, _ret) \
287 	"%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno)
288 
__kvm_mmap(size_t size,int prot,int flags,int fd,off_t offset)289 static inline void *__kvm_mmap(size_t size, int prot, int flags, int fd,
290 			       off_t offset)
291 {
292 	void *mem;
293 
294 	mem = mmap(NULL, size, prot, flags, fd, offset);
295 	TEST_ASSERT(mem != MAP_FAILED, __KVM_SYSCALL_ERROR("mmap()",
296 		    (int)(unsigned long)MAP_FAILED));
297 
298 	return mem;
299 }
300 
kvm_mmap(size_t size,int prot,int flags,int fd)301 static inline void *kvm_mmap(size_t size, int prot, int flags, int fd)
302 {
303 	return __kvm_mmap(size, prot, flags, fd, 0);
304 }
305 
kvm_munmap(void * mem,size_t size)306 static inline void kvm_munmap(void *mem, size_t size)
307 {
308 	int ret;
309 
310 	ret = munmap(mem, size);
311 	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
312 }
313 
314 /*
315  * Use the "inner", double-underscore macro when reporting errors from within
316  * other macros so that the name of ioctl() and not its literal numeric value
317  * is printed on error.  The "outer" macro is strongly preferred when reporting
318  * errors "directly", i.e. without an additional layer of macros, as it reduces
319  * the probability of passing in the wrong string.
320  */
321 #define __KVM_IOCTL_ERROR(_name, _ret)	__KVM_SYSCALL_ERROR(_name, _ret)
322 #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret)
323 
324 #define kvm_do_ioctl(fd, cmd, arg)						\
325 ({										\
326 	kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd));	\
327 	ioctl(fd, cmd, arg);							\
328 })
329 
330 #define __kvm_ioctl(kvm_fd, cmd, arg)				\
331 	kvm_do_ioctl(kvm_fd, cmd, arg)
332 
333 #define kvm_ioctl(kvm_fd, cmd, arg)				\
334 ({								\
335 	int ret = __kvm_ioctl(kvm_fd, cmd, arg);		\
336 								\
337 	TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret));	\
338 })
339 
static_assert_is_vm(struct kvm_vm * vm)340 static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }
341 
342 #define __vm_ioctl(vm, cmd, arg)				\
343 ({								\
344 	static_assert_is_vm(vm);				\
345 	kvm_do_ioctl((vm)->fd, cmd, arg);			\
346 })
347 
348 /*
349  * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if
350  * the ioctl() failed because KVM killed/bugged the VM.  To detect a dead VM,
351  * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before
352  * selftests existed and (b) should never outright fail, i.e. is supposed to
353  * return 0 or 1.  If KVM kills a VM, KVM returns -EIO for all ioctl()s for the
354  * VM and its vCPUs, including KVM_CHECK_EXTENSION.
355  */
356 #define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm)				\
357 do {											\
358 	int __errno = errno;								\
359 											\
360 	static_assert_is_vm(vm);							\
361 											\
362 	if (cond)									\
363 		break;									\
364 											\
365 	if (errno == EIO &&								\
366 	    __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) {	\
367 		TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO");	\
368 		TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues");	\
369 	}										\
370 	errno = __errno;								\
371 	TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret));				\
372 } while (0)
373 
374 #define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm)		\
375 	__TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm)
376 
377 #define vm_ioctl(vm, cmd, arg)					\
378 ({								\
379 	int ret = __vm_ioctl(vm, cmd, arg);			\
380 								\
381 	__TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm);		\
382 })
383 
static_assert_is_vcpu(struct kvm_vcpu * vcpu)384 static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
385 
386 #define __vcpu_ioctl(vcpu, cmd, arg)				\
387 ({								\
388 	static_assert_is_vcpu(vcpu);				\
389 	kvm_do_ioctl((vcpu)->fd, cmd, arg);			\
390 })
391 
392 #define vcpu_ioctl(vcpu, cmd, arg)				\
393 ({								\
394 	int ret = __vcpu_ioctl(vcpu, cmd, arg);			\
395 								\
396 	__TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm);	\
397 })
398 
399 /*
400  * Looks up and returns the value corresponding to the capability
401  * (KVM_CAP_*) given by cap.
402  */
vm_check_cap(struct kvm_vm * vm,long cap)403 static inline int vm_check_cap(struct kvm_vm *vm, long cap)
404 {
405 	int ret =  __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap);
406 
407 	TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm);
408 	return ret;
409 }
410 
__vm_enable_cap(struct kvm_vm * vm,uint32_t cap,uint64_t arg0)411 static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
412 {
413 	struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
414 
415 	return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
416 }
vm_enable_cap(struct kvm_vm * vm,uint32_t cap,uint64_t arg0)417 static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
418 {
419 	struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
420 
421 	vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
422 }
423 
vm_set_memory_attributes(struct kvm_vm * vm,uint64_t gpa,uint64_t size,uint64_t attributes)424 static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa,
425 					    uint64_t size, uint64_t attributes)
426 {
427 	struct kvm_memory_attributes attr = {
428 		.attributes = attributes,
429 		.address = gpa,
430 		.size = size,
431 		.flags = 0,
432 	};
433 
434 	/*
435 	 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes.  These flows
436 	 * need significant enhancements to support multiple attributes.
437 	 */
438 	TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE,
439 		    "Update me to support multiple attributes!");
440 
441 	vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr);
442 }
443 
444 
vm_mem_set_private(struct kvm_vm * vm,uint64_t gpa,uint64_t size)445 static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa,
446 				      uint64_t size)
447 {
448 	vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
449 }
450 
vm_mem_set_shared(struct kvm_vm * vm,uint64_t gpa,uint64_t size)451 static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa,
452 				     uint64_t size)
453 {
454 	vm_set_memory_attributes(vm, gpa, size, 0);
455 }
456 
457 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size,
458 			    bool punch_hole);
459 
vm_guest_mem_punch_hole(struct kvm_vm * vm,uint64_t gpa,uint64_t size)460 static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa,
461 					   uint64_t size)
462 {
463 	vm_guest_mem_fallocate(vm, gpa, size, true);
464 }
465 
vm_guest_mem_allocate(struct kvm_vm * vm,uint64_t gpa,uint64_t size)466 static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa,
467 					 uint64_t size)
468 {
469 	vm_guest_mem_fallocate(vm, gpa, size, false);
470 }
471 
472 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);
473 const char *vm_guest_mode_string(uint32_t i);
474 
475 void kvm_vm_free(struct kvm_vm *vmp);
476 void kvm_vm_restart(struct kvm_vm *vmp);
477 void kvm_vm_release(struct kvm_vm *vmp);
478 void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename);
479 int kvm_memfd_alloc(size_t size, bool hugepages);
480 
481 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
482 
kvm_vm_get_dirty_log(struct kvm_vm * vm,int slot,void * log)483 static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
484 {
485 	struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
486 
487 	vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args);
488 }
489 
kvm_vm_clear_dirty_log(struct kvm_vm * vm,int slot,void * log,uint64_t first_page,uint32_t num_pages)490 static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
491 					  uint64_t first_page, uint32_t num_pages)
492 {
493 	struct kvm_clear_dirty_log args = {
494 		.dirty_bitmap = log,
495 		.slot = slot,
496 		.first_page = first_page,
497 		.num_pages = num_pages
498 	};
499 
500 	vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args);
501 }
502 
kvm_vm_reset_dirty_ring(struct kvm_vm * vm)503 static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
504 {
505 	return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL);
506 }
507 
kvm_vm_register_coalesced_io(struct kvm_vm * vm,uint64_t address,uint64_t size,bool pio)508 static inline void kvm_vm_register_coalesced_io(struct kvm_vm *vm,
509 						uint64_t address,
510 						uint64_t size, bool pio)
511 {
512 	struct kvm_coalesced_mmio_zone zone = {
513 		.addr = address,
514 		.size = size,
515 		.pio  = pio,
516 	};
517 
518 	vm_ioctl(vm, KVM_REGISTER_COALESCED_MMIO, &zone);
519 }
520 
kvm_vm_unregister_coalesced_io(struct kvm_vm * vm,uint64_t address,uint64_t size,bool pio)521 static inline void kvm_vm_unregister_coalesced_io(struct kvm_vm *vm,
522 						  uint64_t address,
523 						  uint64_t size, bool pio)
524 {
525 	struct kvm_coalesced_mmio_zone zone = {
526 		.addr = address,
527 		.size = size,
528 		.pio  = pio,
529 	};
530 
531 	vm_ioctl(vm, KVM_UNREGISTER_COALESCED_MMIO, &zone);
532 }
533 
vm_get_stats_fd(struct kvm_vm * vm)534 static inline int vm_get_stats_fd(struct kvm_vm *vm)
535 {
536 	int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL);
537 
538 	TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm);
539 	return fd;
540 }
541 
__kvm_irqfd(struct kvm_vm * vm,uint32_t gsi,int eventfd,uint32_t flags)542 static inline int __kvm_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd,
543 			      uint32_t flags)
544 {
545 	struct kvm_irqfd irqfd = {
546 		.fd = eventfd,
547 		.gsi = gsi,
548 		.flags = flags,
549 		.resamplefd = -1,
550 	};
551 
552 	return __vm_ioctl(vm, KVM_IRQFD, &irqfd);
553 }
554 
kvm_irqfd(struct kvm_vm * vm,uint32_t gsi,int eventfd,uint32_t flags)555 static inline void kvm_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd,
556 			      uint32_t flags)
557 {
558 	int ret = __kvm_irqfd(vm, gsi, eventfd, flags);
559 
560 	TEST_ASSERT_VM_VCPU_IOCTL(!ret, KVM_IRQFD, ret, vm);
561 }
562 
kvm_assign_irqfd(struct kvm_vm * vm,uint32_t gsi,int eventfd)563 static inline void kvm_assign_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd)
564 {
565 	kvm_irqfd(vm, gsi, eventfd, 0);
566 }
567 
kvm_deassign_irqfd(struct kvm_vm * vm,uint32_t gsi,int eventfd)568 static inline void kvm_deassign_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd)
569 {
570 	kvm_irqfd(vm, gsi, eventfd, KVM_IRQFD_FLAG_DEASSIGN);
571 }
572 
kvm_new_eventfd(void)573 static inline int kvm_new_eventfd(void)
574 {
575 	int fd = eventfd(0, 0);
576 
577 	TEST_ASSERT(fd >= 0, __KVM_SYSCALL_ERROR("eventfd()", fd));
578 	return fd;
579 }
580 
read_stats_header(int stats_fd,struct kvm_stats_header * header)581 static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header)
582 {
583 	ssize_t ret;
584 
585 	ret = pread(stats_fd, header, sizeof(*header), 0);
586 	TEST_ASSERT(ret == sizeof(*header),
587 		    "Failed to read '%lu' header bytes, ret = '%ld'",
588 		    sizeof(*header), ret);
589 }
590 
591 struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
592 					      struct kvm_stats_header *header);
593 
get_stats_descriptor_size(struct kvm_stats_header * header)594 static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header)
595 {
596 	 /*
597 	  * The base size of the descriptor is defined by KVM's ABI, but the
598 	  * size of the name field is variable, as far as KVM's ABI is
599 	  * concerned. For a given instance of KVM, the name field is the same
600 	  * size for all stats and is provided in the overall stats header.
601 	  */
602 	return sizeof(struct kvm_stats_desc) + header->name_size;
603 }
604 
get_stats_descriptor(struct kvm_stats_desc * stats,int index,struct kvm_stats_header * header)605 static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats,
606 							  int index,
607 							  struct kvm_stats_header *header)
608 {
609 	/*
610 	 * Note, size_desc includes the size of the name field, which is
611 	 * variable. i.e. this is NOT equivalent to &stats_desc[i].
612 	 */
613 	return (void *)stats + index * get_stats_descriptor_size(header);
614 }
615 
616 void read_stat_data(int stats_fd, struct kvm_stats_header *header,
617 		    struct kvm_stats_desc *desc, uint64_t *data,
618 		    size_t max_elements);
619 
620 void kvm_get_stat(struct kvm_binary_stats *stats, const char *name,
621 		  uint64_t *data, size_t max_elements);
622 
623 #define __get_stat(stats, stat)							\
624 ({										\
625 	uint64_t data;								\
626 										\
627 	kvm_get_stat(stats, #stat, &data, 1);					\
628 	data;									\
629 })
630 
631 #define vm_get_stat(vm, stat) __get_stat(&(vm)->stats, stat)
632 #define vcpu_get_stat(vcpu, stat) __get_stat(&(vcpu)->stats, stat)
633 
read_smt_control(char * buf,size_t buf_size)634 static inline bool read_smt_control(char *buf, size_t buf_size)
635 {
636 	FILE *f = fopen("/sys/devices/system/cpu/smt/control", "r");
637 	bool ret;
638 
639 	if (!f)
640 		return false;
641 
642 	ret = fread(buf, sizeof(*buf), buf_size, f) > 0;
643 	fclose(f);
644 
645 	return ret;
646 }
647 
is_smt_possible(void)648 static inline bool is_smt_possible(void)
649 {
650 	char buf[16];
651 
652 	if (read_smt_control(buf, sizeof(buf)) &&
653 	    (!strncmp(buf, "forceoff", 8) || !strncmp(buf, "notsupported", 12)))
654 		return false;
655 
656 	return true;
657 }
658 
is_smt_on(void)659 static inline bool is_smt_on(void)
660 {
661 	char buf[16];
662 
663 	if (read_smt_control(buf, sizeof(buf)) && !strncmp(buf, "on", 2))
664 		return true;
665 
666 	return false;
667 }
668 
669 void vm_create_irqchip(struct kvm_vm *vm);
670 
__vm_create_guest_memfd(struct kvm_vm * vm,uint64_t size,uint64_t flags)671 static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
672 					uint64_t flags)
673 {
674 	struct kvm_create_guest_memfd guest_memfd = {
675 		.size = size,
676 		.flags = flags,
677 	};
678 
679 	return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd);
680 }
681 
vm_create_guest_memfd(struct kvm_vm * vm,uint64_t size,uint64_t flags)682 static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
683 					uint64_t flags)
684 {
685 	int fd = __vm_create_guest_memfd(vm, size, flags);
686 
687 	TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd));
688 	return fd;
689 }
690 
691 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
692 			       uint64_t gpa, uint64_t size, void *hva);
693 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
694 				uint64_t gpa, uint64_t size, void *hva);
695 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
696 				uint64_t gpa, uint64_t size, void *hva,
697 				uint32_t guest_memfd, uint64_t guest_memfd_offset);
698 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
699 				 uint64_t gpa, uint64_t size, void *hva,
700 				 uint32_t guest_memfd, uint64_t guest_memfd_offset);
701 
702 void vm_userspace_mem_region_add(struct kvm_vm *vm,
703 	enum vm_mem_backing_src_type src_type,
704 	uint64_t guest_paddr, uint32_t slot, uint64_t npages,
705 	uint32_t flags);
706 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
707 		uint64_t guest_paddr, uint32_t slot, uint64_t npages,
708 		uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset);
709 
710 #ifndef vm_arch_has_protected_memory
vm_arch_has_protected_memory(struct kvm_vm * vm)711 static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)
712 {
713 	return false;
714 }
715 #endif
716 
717 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);
718 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);
719 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot);
720 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
721 void vm_populate_vaddr_bitmap(struct kvm_vm *vm);
722 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
723 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
724 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
725 			    enum kvm_mem_region_type type);
726 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,
727 				 vm_vaddr_t vaddr_min,
728 				 enum kvm_mem_region_type type);
729 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);
730 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,
731 				 enum kvm_mem_region_type type);
732 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);
733 
734 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
735 	      unsigned int npages);
736 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa);
737 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva);
738 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva);
739 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa);
740 
741 #ifndef vcpu_arch_put_guest
742 #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0)
743 #endif
744 
vm_untag_gpa(struct kvm_vm * vm,vm_paddr_t gpa)745 static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa)
746 {
747 	return gpa & ~vm->gpa_tag_mask;
748 }
749 
750 void vcpu_run(struct kvm_vcpu *vcpu);
751 int _vcpu_run(struct kvm_vcpu *vcpu);
752 
__vcpu_run(struct kvm_vcpu * vcpu)753 static inline int __vcpu_run(struct kvm_vcpu *vcpu)
754 {
755 	return __vcpu_ioctl(vcpu, KVM_RUN, NULL);
756 }
757 
758 void vcpu_run_complete_io(struct kvm_vcpu *vcpu);
759 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu);
760 
vcpu_enable_cap(struct kvm_vcpu * vcpu,uint32_t cap,uint64_t arg0)761 static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap,
762 				   uint64_t arg0)
763 {
764 	struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
765 
766 	vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap);
767 }
768 
vcpu_guest_debug_set(struct kvm_vcpu * vcpu,struct kvm_guest_debug * debug)769 static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu,
770 					struct kvm_guest_debug *debug)
771 {
772 	vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug);
773 }
774 
vcpu_mp_state_get(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)775 static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu,
776 				     struct kvm_mp_state *mp_state)
777 {
778 	vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state);
779 }
vcpu_mp_state_set(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)780 static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu,
781 				     struct kvm_mp_state *mp_state)
782 {
783 	vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state);
784 }
785 
vcpu_regs_get(struct kvm_vcpu * vcpu,struct kvm_regs * regs)786 static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
787 {
788 	vcpu_ioctl(vcpu, KVM_GET_REGS, regs);
789 }
790 
vcpu_regs_set(struct kvm_vcpu * vcpu,struct kvm_regs * regs)791 static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
792 {
793 	vcpu_ioctl(vcpu, KVM_SET_REGS, regs);
794 }
vcpu_sregs_get(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)795 static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
796 {
797 	vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs);
798 
799 }
vcpu_sregs_set(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)800 static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
801 {
802 	vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
803 }
_vcpu_sregs_set(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)804 static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
805 {
806 	return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
807 }
vcpu_fpu_get(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)808 static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
809 {
810 	vcpu_ioctl(vcpu, KVM_GET_FPU, fpu);
811 }
vcpu_fpu_set(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)812 static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
813 {
814 	vcpu_ioctl(vcpu, KVM_SET_FPU, fpu);
815 }
816 
__vcpu_get_reg(struct kvm_vcpu * vcpu,uint64_t id,void * addr)817 static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
818 {
819 	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };
820 
821 	return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
822 }
__vcpu_set_reg(struct kvm_vcpu * vcpu,uint64_t id,uint64_t val)823 static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
824 {
825 	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
826 
827 	return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
828 }
vcpu_get_reg(struct kvm_vcpu * vcpu,uint64_t id)829 static inline uint64_t vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id)
830 {
831 	uint64_t val;
832 	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
833 
834 	TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id);
835 
836 	vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
837 	return val;
838 }
vcpu_set_reg(struct kvm_vcpu * vcpu,uint64_t id,uint64_t val)839 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
840 {
841 	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
842 
843 	TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id);
844 
845 	vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
846 }
847 
848 #ifdef __KVM_HAVE_VCPU_EVENTS
vcpu_events_get(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)849 static inline void vcpu_events_get(struct kvm_vcpu *vcpu,
850 				   struct kvm_vcpu_events *events)
851 {
852 	vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events);
853 }
vcpu_events_set(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)854 static inline void vcpu_events_set(struct kvm_vcpu *vcpu,
855 				   struct kvm_vcpu_events *events)
856 {
857 	vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events);
858 }
859 #endif
860 #ifdef __x86_64__
vcpu_nested_state_get(struct kvm_vcpu * vcpu,struct kvm_nested_state * state)861 static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu,
862 					 struct kvm_nested_state *state)
863 {
864 	vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state);
865 }
__vcpu_nested_state_set(struct kvm_vcpu * vcpu,struct kvm_nested_state * state)866 static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu,
867 					  struct kvm_nested_state *state)
868 {
869 	return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
870 }
871 
vcpu_nested_state_set(struct kvm_vcpu * vcpu,struct kvm_nested_state * state)872 static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu,
873 					 struct kvm_nested_state *state)
874 {
875 	vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
876 }
877 #endif
vcpu_get_stats_fd(struct kvm_vcpu * vcpu)878 static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu)
879 {
880 	int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL);
881 
882 	TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm);
883 	return fd;
884 }
885 
886 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr);
887 
kvm_has_device_attr(int dev_fd,uint32_t group,uint64_t attr)888 static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
889 {
890 	int ret = __kvm_has_device_attr(dev_fd, group, attr);
891 
892 	TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);
893 }
894 
895 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val);
896 
kvm_device_attr_get(int dev_fd,uint32_t group,uint64_t attr,void * val)897 static inline void kvm_device_attr_get(int dev_fd, uint32_t group,
898 				       uint64_t attr, void *val)
899 {
900 	int ret = __kvm_device_attr_get(dev_fd, group, attr, val);
901 
902 	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret));
903 }
904 
905 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val);
906 
kvm_device_attr_set(int dev_fd,uint32_t group,uint64_t attr,void * val)907 static inline void kvm_device_attr_set(int dev_fd, uint32_t group,
908 				       uint64_t attr, void *val)
909 {
910 	int ret = __kvm_device_attr_set(dev_fd, group, attr, val);
911 
912 	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret));
913 }
914 
__vcpu_has_device_attr(struct kvm_vcpu * vcpu,uint32_t group,uint64_t attr)915 static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
916 					 uint64_t attr)
917 {
918 	return __kvm_has_device_attr(vcpu->fd, group, attr);
919 }
920 
vcpu_has_device_attr(struct kvm_vcpu * vcpu,uint32_t group,uint64_t attr)921 static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
922 					uint64_t attr)
923 {
924 	kvm_has_device_attr(vcpu->fd, group, attr);
925 }
926 
__vcpu_device_attr_get(struct kvm_vcpu * vcpu,uint32_t group,uint64_t attr,void * val)927 static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
928 					 uint64_t attr, void *val)
929 {
930 	return __kvm_device_attr_get(vcpu->fd, group, attr, val);
931 }
932 
vcpu_device_attr_get(struct kvm_vcpu * vcpu,uint32_t group,uint64_t attr,void * val)933 static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
934 					uint64_t attr, void *val)
935 {
936 	kvm_device_attr_get(vcpu->fd, group, attr, val);
937 }
938 
__vcpu_device_attr_set(struct kvm_vcpu * vcpu,uint32_t group,uint64_t attr,void * val)939 static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
940 					 uint64_t attr, void *val)
941 {
942 	return __kvm_device_attr_set(vcpu->fd, group, attr, val);
943 }
944 
vcpu_device_attr_set(struct kvm_vcpu * vcpu,uint32_t group,uint64_t attr,void * val)945 static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
946 					uint64_t attr, void *val)
947 {
948 	kvm_device_attr_set(vcpu->fd, group, attr, val);
949 }
950 
951 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type);
952 int __kvm_create_device(struct kvm_vm *vm, uint64_t type);
953 
kvm_create_device(struct kvm_vm * vm,uint64_t type)954 static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type)
955 {
956 	int fd = __kvm_create_device(vm, type);
957 
958 	TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd));
959 	return fd;
960 }
961 
962 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu);
963 
964 /*
965  * VM VCPU Args Set
966  *
967  * Input Args:
968  *   vm - Virtual Machine
969  *   num - number of arguments
970  *   ... - arguments, each of type uint64_t
971  *
972  * Output Args: None
973  *
974  * Return: None
975  *
976  * Sets the first @num input parameters for the function at @vcpu's entry point,
977  * per the C calling convention of the architecture, to the values given as
978  * variable args. Each of the variable args is expected to be of type uint64_t.
979  * The maximum @num can be is specific to the architecture.
980  */
981 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);
982 
983 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
984 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
985 
986 #define KVM_MAX_IRQ_ROUTES		4096
987 
988 struct kvm_irq_routing *kvm_gsi_routing_create(void);
989 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
990 		uint32_t gsi, uint32_t pin);
991 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
992 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
993 
994 const char *exit_reason_str(unsigned int exit_reason);
995 
996 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
997 			     uint32_t memslot);
998 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
999 				vm_paddr_t paddr_min, uint32_t memslot,
1000 				bool protected);
1001 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm);
1002 
vm_phy_pages_alloc(struct kvm_vm * vm,size_t num,vm_paddr_t paddr_min,uint32_t memslot)1003 static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
1004 					    vm_paddr_t paddr_min, uint32_t memslot)
1005 {
1006 	/*
1007 	 * By default, allocate memory as protected for VMs that support
1008 	 * protected memory, as the majority of memory for such VMs is
1009 	 * protected, i.e. using shared memory is effectively opt-in.
1010 	 */
1011 	return __vm_phy_pages_alloc(vm, num, paddr_min, memslot,
1012 				    vm_arch_has_protected_memory(vm));
1013 }
1014 
1015 /*
1016  * ____vm_create() does KVM_CREATE_VM and little else.  __vm_create() also
1017  * loads the test binary into guest memory and creates an IRQ chip (x86 only).
1018  * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to
1019  * calculate the amount of memory needed for per-vCPU data, e.g. stacks.
1020  */
1021 struct kvm_vm *____vm_create(struct vm_shape shape);
1022 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
1023 			   uint64_t nr_extra_pages);
1024 
vm_create_barebones(void)1025 static inline struct kvm_vm *vm_create_barebones(void)
1026 {
1027 	return ____vm_create(VM_SHAPE_DEFAULT);
1028 }
1029 
vm_create_barebones_type(unsigned long type)1030 static inline struct kvm_vm *vm_create_barebones_type(unsigned long type)
1031 {
1032 	const struct vm_shape shape = {
1033 		.mode = VM_MODE_DEFAULT,
1034 		.type = type,
1035 	};
1036 
1037 	return ____vm_create(shape);
1038 }
1039 
vm_create(uint32_t nr_runnable_vcpus)1040 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
1041 {
1042 	return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0);
1043 }
1044 
1045 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
1046 				      uint64_t extra_mem_pages,
1047 				      void *guest_code, struct kvm_vcpu *vcpus[]);
1048 
vm_create_with_vcpus(uint32_t nr_vcpus,void * guest_code,struct kvm_vcpu * vcpus[])1049 static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,
1050 						  void *guest_code,
1051 						  struct kvm_vcpu *vcpus[])
1052 {
1053 	return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0,
1054 				      guest_code, vcpus);
1055 }
1056 
1057 
1058 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
1059 					       struct kvm_vcpu **vcpu,
1060 					       uint64_t extra_mem_pages,
1061 					       void *guest_code);
1062 
1063 /*
1064  * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages
1065  * additional pages of guest memory.  Returns the VM and vCPU (via out param).
1066  */
__vm_create_with_one_vcpu(struct kvm_vcpu ** vcpu,uint64_t extra_mem_pages,void * guest_code)1067 static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
1068 						       uint64_t extra_mem_pages,
1069 						       void *guest_code)
1070 {
1071 	return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu,
1072 					       extra_mem_pages, guest_code);
1073 }
1074 
vm_create_with_one_vcpu(struct kvm_vcpu ** vcpu,void * guest_code)1075 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
1076 						     void *guest_code)
1077 {
1078 	return __vm_create_with_one_vcpu(vcpu, 0, guest_code);
1079 }
1080 
vm_create_shape_with_one_vcpu(struct vm_shape shape,struct kvm_vcpu ** vcpu,void * guest_code)1081 static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape,
1082 							   struct kvm_vcpu **vcpu,
1083 							   void *guest_code)
1084 {
1085 	return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code);
1086 }
1087 
1088 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);
1089 
1090 void kvm_set_files_rlimit(uint32_t nr_vcpus);
1091 
1092 int __pin_task_to_cpu(pthread_t task, int cpu);
1093 
pin_task_to_cpu(pthread_t task,int cpu)1094 static inline void pin_task_to_cpu(pthread_t task, int cpu)
1095 {
1096 	int r;
1097 
1098 	r = __pin_task_to_cpu(task, cpu);
1099 	TEST_ASSERT(!r, "Failed to set thread affinity to pCPU '%u'", cpu);
1100 }
1101 
pin_task_to_any_cpu(pthread_t task)1102 static inline int pin_task_to_any_cpu(pthread_t task)
1103 {
1104 	int cpu = sched_getcpu();
1105 
1106 	pin_task_to_cpu(task, cpu);
1107 	return cpu;
1108 }
1109 
pin_self_to_cpu(int cpu)1110 static inline void pin_self_to_cpu(int cpu)
1111 {
1112 	pin_task_to_cpu(pthread_self(), cpu);
1113 }
1114 
pin_self_to_any_cpu(void)1115 static inline int pin_self_to_any_cpu(void)
1116 {
1117 	return pin_task_to_any_cpu(pthread_self());
1118 }
1119 
1120 void kvm_print_vcpu_pinning_help(void);
1121 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
1122 			    int nr_vcpus);
1123 
1124 unsigned long vm_compute_max_gfn(struct kvm_vm *vm);
1125 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size);
1126 unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages);
1127 unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages);
1128 static inline unsigned int
vm_adjust_num_guest_pages(enum vm_guest_mode mode,unsigned int num_guest_pages)1129 vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
1130 {
1131 	unsigned int n;
1132 	n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages));
1133 #ifdef __s390x__
1134 	/* s390 requires 1M aligned guest sizes */
1135 	n = (n + 255) & ~255;
1136 #endif
1137 	return n;
1138 }
1139 
1140 #define sync_global_to_guest(vm, g) ({				\
1141 	typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));	\
1142 	memcpy(_p, &(g), sizeof(g));				\
1143 })
1144 
1145 #define sync_global_from_guest(vm, g) ({			\
1146 	typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));	\
1147 	memcpy(&(g), _p, sizeof(g));				\
1148 })
1149 
1150 /*
1151  * Write a global value, but only in the VM's (guest's) domain.  Primarily used
1152  * for "globals" that hold per-VM values (VMs always duplicate code and global
1153  * data into their own region of physical memory), but can be used anytime it's
1154  * undesirable to change the host's copy of the global.
1155  */
1156 #define write_guest_global(vm, g, val) ({			\
1157 	typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));	\
1158 	typeof(g) _val = val;					\
1159 								\
1160 	memcpy(_p, &(_val), sizeof(g));				\
1161 })
1162 
1163 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu);
1164 
1165 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu,
1166 		    uint8_t indent);
1167 
vcpu_dump(FILE * stream,struct kvm_vcpu * vcpu,uint8_t indent)1168 static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu,
1169 			     uint8_t indent)
1170 {
1171 	vcpu_arch_dump(stream, vcpu, indent);
1172 }
1173 
1174 /*
1175  * Adds a vCPU with reasonable defaults (e.g. a stack)
1176  *
1177  * Input Args:
1178  *   vm - Virtual Machine
1179  *   vcpu_id - The id of the VCPU to add to the VM.
1180  */
1181 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
1182 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code);
1183 
vm_vcpu_add(struct kvm_vm * vm,uint32_t vcpu_id,void * guest_code)1184 static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
1185 					   void *guest_code)
1186 {
1187 	struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id);
1188 
1189 	vcpu_arch_set_entry_point(vcpu, guest_code);
1190 
1191 	return vcpu;
1192 }
1193 
1194 /* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */
1195 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id);
1196 
vm_vcpu_recreate(struct kvm_vm * vm,uint32_t vcpu_id)1197 static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm,
1198 						uint32_t vcpu_id)
1199 {
1200 	return vm_arch_vcpu_recreate(vm, vcpu_id);
1201 }
1202 
1203 void vcpu_arch_free(struct kvm_vcpu *vcpu);
1204 
1205 void virt_arch_pgd_alloc(struct kvm_vm *vm);
1206 
virt_pgd_alloc(struct kvm_vm * vm)1207 static inline void virt_pgd_alloc(struct kvm_vm *vm)
1208 {
1209 	virt_arch_pgd_alloc(vm);
1210 }
1211 
1212 /*
1213  * VM Virtual Page Map
1214  *
1215  * Input Args:
1216  *   vm - Virtual Machine
1217  *   vaddr - VM Virtual Address
1218  *   paddr - VM Physical Address
1219  *   memslot - Memory region slot for new virtual translation tables
1220  *
1221  * Output Args: None
1222  *
1223  * Return: None
1224  *
1225  * Within @vm, creates a virtual translation for the page starting
1226  * at @vaddr to the page starting at @paddr.
1227  */
1228 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr);
1229 
virt_pg_map(struct kvm_vm * vm,uint64_t vaddr,uint64_t paddr)1230 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
1231 {
1232 	virt_arch_pg_map(vm, vaddr, paddr);
1233 }
1234 
1235 
1236 /*
1237  * Address Guest Virtual to Guest Physical
1238  *
1239  * Input Args:
1240  *   vm - Virtual Machine
1241  *   gva - VM virtual address
1242  *
1243  * Output Args: None
1244  *
1245  * Return:
1246  *   Equivalent VM physical address
1247  *
1248  * Returns the VM physical address of the translated VM virtual
1249  * address given by @gva.
1250  */
1251 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva);
1252 
addr_gva2gpa(struct kvm_vm * vm,vm_vaddr_t gva)1253 static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
1254 {
1255 	return addr_arch_gva2gpa(vm, gva);
1256 }
1257 
1258 /*
1259  * Virtual Translation Tables Dump
1260  *
1261  * Input Args:
1262  *   stream - Output FILE stream
1263  *   vm     - Virtual Machine
1264  *   indent - Left margin indent amount
1265  *
1266  * Output Args: None
1267  *
1268  * Return: None
1269  *
1270  * Dumps to the FILE stream given by @stream, the contents of all the
1271  * virtual translation tables for the VM given by @vm.
1272  */
1273 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
1274 
virt_dump(FILE * stream,struct kvm_vm * vm,uint8_t indent)1275 static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1276 {
1277 	virt_arch_dump(stream, vm, indent);
1278 }
1279 
1280 
__vm_disable_nx_huge_pages(struct kvm_vm * vm)1281 static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm)
1282 {
1283 	return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0);
1284 }
1285 
1286 /*
1287  * Arch hook that is invoked via a constructor, i.e. before exeucting main(),
1288  * to allow for arch-specific setup that is common to all tests, e.g. computing
1289  * the default guest "mode".
1290  */
1291 void kvm_selftest_arch_init(void);
1292 
1293 void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus);
1294 void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm);
1295 void kvm_arch_vm_release(struct kvm_vm *vm);
1296 
1297 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr);
1298 
1299 uint32_t guest_get_vcpuid(void);
1300 
1301 bool kvm_arch_has_default_irqchip(void);
1302 
1303 #endif /* SELFTEST_KVM_UTIL_H */
1304