xref: /linux/tools/testing/selftests/kvm/lib/kvm_util.c (revision 02e5f74ef08d3e6afec438d571487d0d0cec3c48)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * tools/testing/selftests/kvm/lib/kvm_util.c
4  *
5  * Copyright (C) 2018, Google LLC.
6  */
7 #include "test_util.h"
8 #include "kvm_util.h"
9 #include "processor.h"
10 #include "ucall_common.h"
11 
12 #include <assert.h>
13 #include <sched.h>
14 #include <sys/mman.h>
15 #include <sys/resource.h>
16 #include <sys/types.h>
17 #include <sys/stat.h>
18 #include <unistd.h>
19 #include <linux/kernel.h>
20 
21 #define KVM_UTIL_MIN_PFN	2
22 
23 uint32_t guest_random_seed;
24 struct guest_random_state guest_rng;
25 static uint32_t last_guest_seed;
26 
27 static size_t vcpu_mmap_sz(void);
28 
__open_path_or_exit(const char * path,int flags,const char * enoent_help)29 int __open_path_or_exit(const char *path, int flags, const char *enoent_help)
30 {
31 	int fd;
32 
33 	fd = open(path, flags);
34 	if (fd < 0)
35 		goto error;
36 
37 	return fd;
38 
39 error:
40 	if (errno == EACCES || errno == ENOENT)
41 		ksft_exit_skip("- Cannot open '%s': %s.  %s\n",
42 			       path, strerror(errno),
43 			       errno == EACCES ? "Root required?" : enoent_help);
44 	TEST_FAIL("Failed to open '%s'", path);
45 }
46 
open_path_or_exit(const char * path,int flags)47 int open_path_or_exit(const char *path, int flags)
48 {
49 	return __open_path_or_exit(path, flags, "");
50 }
51 
52 /*
53  * Open KVM_DEV_PATH if available, otherwise exit the entire program.
54  *
55  * Input Args:
56  *   flags - The flags to pass when opening KVM_DEV_PATH.
57  *
58  * Return:
59  *   The opened file descriptor of /dev/kvm.
60  */
_open_kvm_dev_path_or_exit(int flags)61 static int _open_kvm_dev_path_or_exit(int flags)
62 {
63 	return __open_path_or_exit(KVM_DEV_PATH, flags, "Is KVM loaded and enabled?");
64 }
65 
open_kvm_dev_path_or_exit(void)66 int open_kvm_dev_path_or_exit(void)
67 {
68 	return _open_kvm_dev_path_or_exit(O_RDONLY);
69 }
70 
get_module_param(const char * module_name,const char * param,void * buffer,size_t buffer_size)71 static ssize_t get_module_param(const char *module_name, const char *param,
72 				void *buffer, size_t buffer_size)
73 {
74 	const int path_size = 128;
75 	char path[path_size];
76 	ssize_t bytes_read;
77 	int fd, r;
78 
79 	/* Verify KVM is loaded, to provide a more helpful SKIP message. */
80 	close(open_kvm_dev_path_or_exit());
81 
82 	r = snprintf(path, path_size, "/sys/module/%s/parameters/%s",
83 		     module_name, param);
84 	TEST_ASSERT(r < path_size,
85 		    "Failed to construct sysfs path in %d bytes.", path_size);
86 
87 	fd = open_path_or_exit(path, O_RDONLY);
88 
89 	bytes_read = read(fd, buffer, buffer_size);
90 	TEST_ASSERT(bytes_read > 0, "read(%s) returned %ld, wanted %ld bytes",
91 		    path, bytes_read, buffer_size);
92 
93 	r = close(fd);
94 	TEST_ASSERT(!r, "close(%s) failed", path);
95 	return bytes_read;
96 }
97 
kvm_get_module_param_integer(const char * module_name,const char * param)98 int kvm_get_module_param_integer(const char *module_name, const char *param)
99 {
100 	/*
101 	 * 16 bytes to hold a 64-bit value (1 byte per char), 1 byte for the
102 	 * NUL char, and 1 byte because the kernel sucks and inserts a newline
103 	 * at the end.
104 	 */
105 	char value[16 + 1 + 1];
106 	ssize_t r;
107 
108 	memset(value, '\0', sizeof(value));
109 
110 	r = get_module_param(module_name, param, value, sizeof(value));
111 	TEST_ASSERT(value[r - 1] == '\n',
112 		    "Expected trailing newline, got char '%c'", value[r - 1]);
113 
114 	/*
115 	 * Squash the newline, otherwise atoi_paranoid() will complain about
116 	 * trailing non-NUL characters in the string.
117 	 */
118 	value[r - 1] = '\0';
119 	return atoi_paranoid(value);
120 }
121 
kvm_get_module_param_bool(const char * module_name,const char * param)122 bool kvm_get_module_param_bool(const char *module_name, const char *param)
123 {
124 	char value;
125 	ssize_t r;
126 
127 	r = get_module_param(module_name, param, &value, sizeof(value));
128 	TEST_ASSERT_EQ(r, 1);
129 
130 	if (value == 'Y')
131 		return true;
132 	else if (value == 'N')
133 		return false;
134 
135 	TEST_FAIL("Unrecognized value '%c' for boolean module param", value);
136 }
137 
138 /*
139  * Capability
140  *
141  * Input Args:
142  *   cap - Capability
143  *
144  * Output Args: None
145  *
146  * Return:
147  *   On success, the Value corresponding to the capability (KVM_CAP_*)
148  *   specified by the value of cap.  On failure a TEST_ASSERT failure
149  *   is produced.
150  *
151  * Looks up and returns the value corresponding to the capability
152  * (KVM_CAP_*) given by cap.
153  */
kvm_check_cap(long cap)154 unsigned int kvm_check_cap(long cap)
155 {
156 	int ret;
157 	int kvm_fd;
158 
159 	kvm_fd = open_kvm_dev_path_or_exit();
160 	ret = __kvm_ioctl(kvm_fd, KVM_CHECK_EXTENSION, (void *)cap);
161 	TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_CHECK_EXTENSION, ret));
162 
163 	close(kvm_fd);
164 
165 	return (unsigned int)ret;
166 }
167 
vm_enable_dirty_ring(struct kvm_vm * vm,uint32_t ring_size)168 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
169 {
170 	if (vm_check_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL))
171 		vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL, ring_size);
172 	else
173 		vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size);
174 	vm->dirty_ring_size = ring_size;
175 }
176 
vm_open(struct kvm_vm * vm)177 static void vm_open(struct kvm_vm *vm)
178 {
179 	vm->kvm_fd = _open_kvm_dev_path_or_exit(O_RDWR);
180 
181 	TEST_REQUIRE(kvm_has_cap(KVM_CAP_IMMEDIATE_EXIT));
182 
183 	vm->fd = __kvm_ioctl(vm->kvm_fd, KVM_CREATE_VM, (void *)vm->type);
184 	TEST_ASSERT(vm->fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, vm->fd));
185 
186 	if (kvm_has_cap(KVM_CAP_BINARY_STATS_FD))
187 		vm->stats.fd = vm_get_stats_fd(vm);
188 	else
189 		vm->stats.fd = -1;
190 }
191 
vm_guest_mode_string(uint32_t i)192 const char *vm_guest_mode_string(uint32_t i)
193 {
194 	static const char * const strings[] = {
195 		[VM_MODE_P52V48_4K]	= "PA-bits:52,  VA-bits:48,  4K pages",
196 		[VM_MODE_P52V48_16K]	= "PA-bits:52,  VA-bits:48, 16K pages",
197 		[VM_MODE_P52V48_64K]	= "PA-bits:52,  VA-bits:48, 64K pages",
198 		[VM_MODE_P48V48_4K]	= "PA-bits:48,  VA-bits:48,  4K pages",
199 		[VM_MODE_P48V48_16K]	= "PA-bits:48,  VA-bits:48, 16K pages",
200 		[VM_MODE_P48V48_64K]	= "PA-bits:48,  VA-bits:48, 64K pages",
201 		[VM_MODE_P40V48_4K]	= "PA-bits:40,  VA-bits:48,  4K pages",
202 		[VM_MODE_P40V48_16K]	= "PA-bits:40,  VA-bits:48, 16K pages",
203 		[VM_MODE_P40V48_64K]	= "PA-bits:40,  VA-bits:48, 64K pages",
204 		[VM_MODE_PXXV48_4K]	= "PA-bits:ANY, VA-bits:48,  4K pages",
205 		[VM_MODE_P47V64_4K]	= "PA-bits:47,  VA-bits:64,  4K pages",
206 		[VM_MODE_P44V64_4K]	= "PA-bits:44,  VA-bits:64,  4K pages",
207 		[VM_MODE_P36V48_4K]	= "PA-bits:36,  VA-bits:48,  4K pages",
208 		[VM_MODE_P36V48_16K]	= "PA-bits:36,  VA-bits:48, 16K pages",
209 		[VM_MODE_P36V48_64K]	= "PA-bits:36,  VA-bits:48, 64K pages",
210 		[VM_MODE_P47V47_16K]	= "PA-bits:47,  VA-bits:47, 16K pages",
211 		[VM_MODE_P36V47_16K]	= "PA-bits:36,  VA-bits:47, 16K pages",
212 	};
213 	_Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,
214 		       "Missing new mode strings?");
215 
216 	TEST_ASSERT(i < NUM_VM_MODES, "Guest mode ID %d too big", i);
217 
218 	return strings[i];
219 }
220 
221 const struct vm_guest_mode_params vm_guest_mode_params[] = {
222 	[VM_MODE_P52V48_4K]	= { 52, 48,  0x1000, 12 },
223 	[VM_MODE_P52V48_16K]	= { 52, 48,  0x4000, 14 },
224 	[VM_MODE_P52V48_64K]	= { 52, 48, 0x10000, 16 },
225 	[VM_MODE_P48V48_4K]	= { 48, 48,  0x1000, 12 },
226 	[VM_MODE_P48V48_16K]	= { 48, 48,  0x4000, 14 },
227 	[VM_MODE_P48V48_64K]	= { 48, 48, 0x10000, 16 },
228 	[VM_MODE_P40V48_4K]	= { 40, 48,  0x1000, 12 },
229 	[VM_MODE_P40V48_16K]	= { 40, 48,  0x4000, 14 },
230 	[VM_MODE_P40V48_64K]	= { 40, 48, 0x10000, 16 },
231 	[VM_MODE_PXXV48_4K]	= {  0,  0,  0x1000, 12 },
232 	[VM_MODE_P47V64_4K]	= { 47, 64,  0x1000, 12 },
233 	[VM_MODE_P44V64_4K]	= { 44, 64,  0x1000, 12 },
234 	[VM_MODE_P36V48_4K]	= { 36, 48,  0x1000, 12 },
235 	[VM_MODE_P36V48_16K]	= { 36, 48,  0x4000, 14 },
236 	[VM_MODE_P36V48_64K]	= { 36, 48, 0x10000, 16 },
237 	[VM_MODE_P47V47_16K]	= { 47, 47,  0x4000, 14 },
238 	[VM_MODE_P36V47_16K]	= { 36, 47,  0x4000, 14 },
239 };
240 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
241 	       "Missing new mode params?");
242 
243 /*
244  * Initializes vm->vpages_valid to match the canonical VA space of the
245  * architecture.
246  *
247  * The default implementation is valid for architectures which split the
248  * range addressed by a single page table into a low and high region
249  * based on the MSB of the VA. On architectures with this behavior
250  * the VA region spans [0, 2^(va_bits - 1)), [-(2^(va_bits - 1), -1].
251  */
vm_vaddr_populate_bitmap(struct kvm_vm * vm)252 __weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm)
253 {
254 	sparsebit_set_num(vm->vpages_valid,
255 		0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
256 	sparsebit_set_num(vm->vpages_valid,
257 		(~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
258 		(1ULL << (vm->va_bits - 1)) >> vm->page_shift);
259 }
260 
____vm_create(struct vm_shape shape)261 struct kvm_vm *____vm_create(struct vm_shape shape)
262 {
263 	struct kvm_vm *vm;
264 
265 	vm = calloc(1, sizeof(*vm));
266 	TEST_ASSERT(vm != NULL, "Insufficient Memory");
267 
268 	INIT_LIST_HEAD(&vm->vcpus);
269 	vm->regions.gpa_tree = RB_ROOT;
270 	vm->regions.hva_tree = RB_ROOT;
271 	hash_init(vm->regions.slot_hash);
272 
273 	vm->mode = shape.mode;
274 	vm->type = shape.type;
275 
276 	vm->pa_bits = vm_guest_mode_params[vm->mode].pa_bits;
277 	vm->va_bits = vm_guest_mode_params[vm->mode].va_bits;
278 	vm->page_size = vm_guest_mode_params[vm->mode].page_size;
279 	vm->page_shift = vm_guest_mode_params[vm->mode].page_shift;
280 
281 	/* Setup mode specific traits. */
282 	switch (vm->mode) {
283 	case VM_MODE_P52V48_4K:
284 		vm->pgtable_levels = 4;
285 		break;
286 	case VM_MODE_P52V48_64K:
287 		vm->pgtable_levels = 3;
288 		break;
289 	case VM_MODE_P48V48_4K:
290 		vm->pgtable_levels = 4;
291 		break;
292 	case VM_MODE_P48V48_64K:
293 		vm->pgtable_levels = 3;
294 		break;
295 	case VM_MODE_P40V48_4K:
296 	case VM_MODE_P36V48_4K:
297 		vm->pgtable_levels = 4;
298 		break;
299 	case VM_MODE_P40V48_64K:
300 	case VM_MODE_P36V48_64K:
301 		vm->pgtable_levels = 3;
302 		break;
303 	case VM_MODE_P52V48_16K:
304 	case VM_MODE_P48V48_16K:
305 	case VM_MODE_P40V48_16K:
306 	case VM_MODE_P36V48_16K:
307 		vm->pgtable_levels = 4;
308 		break;
309 	case VM_MODE_P47V47_16K:
310 	case VM_MODE_P36V47_16K:
311 		vm->pgtable_levels = 3;
312 		break;
313 	case VM_MODE_PXXV48_4K:
314 #ifdef __x86_64__
315 		kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);
316 		kvm_init_vm_address_properties(vm);
317 		/*
318 		 * Ignore KVM support for 5-level paging (vm->va_bits == 57),
319 		 * it doesn't take effect unless a CR4.LA57 is set, which it
320 		 * isn't for this mode (48-bit virtual address space).
321 		 */
322 		TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,
323 			    "Linear address width (%d bits) not supported",
324 			    vm->va_bits);
325 		pr_debug("Guest physical address width detected: %d\n",
326 			 vm->pa_bits);
327 		vm->pgtable_levels = 4;
328 		vm->va_bits = 48;
329 #else
330 		TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");
331 #endif
332 		break;
333 	case VM_MODE_P47V64_4K:
334 		vm->pgtable_levels = 5;
335 		break;
336 	case VM_MODE_P44V64_4K:
337 		vm->pgtable_levels = 5;
338 		break;
339 	default:
340 		TEST_FAIL("Unknown guest mode: 0x%x", vm->mode);
341 	}
342 
343 #ifdef __aarch64__
344 	TEST_ASSERT(!vm->type, "ARM doesn't support test-provided types");
345 	if (vm->pa_bits != 40)
346 		vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
347 #endif
348 
349 	vm_open(vm);
350 
351 	/* Limit to VA-bit canonical virtual addresses. */
352 	vm->vpages_valid = sparsebit_alloc();
353 	vm_vaddr_populate_bitmap(vm);
354 
355 	/* Limit physical addresses to PA-bits. */
356 	vm->max_gfn = vm_compute_max_gfn(vm);
357 
358 	/* Allocate and setup memory for guest. */
359 	vm->vpages_mapped = sparsebit_alloc();
360 
361 	return vm;
362 }
363 
vm_nr_pages_required(enum vm_guest_mode mode,uint32_t nr_runnable_vcpus,uint64_t extra_mem_pages)364 static uint64_t vm_nr_pages_required(enum vm_guest_mode mode,
365 				     uint32_t nr_runnable_vcpus,
366 				     uint64_t extra_mem_pages)
367 {
368 	uint64_t page_size = vm_guest_mode_params[mode].page_size;
369 	uint64_t nr_pages;
370 
371 	TEST_ASSERT(nr_runnable_vcpus,
372 		    "Use vm_create_barebones() for VMs that _never_ have vCPUs");
373 
374 	TEST_ASSERT(nr_runnable_vcpus <= kvm_check_cap(KVM_CAP_MAX_VCPUS),
375 		    "nr_vcpus = %d too large for host, max-vcpus = %d",
376 		    nr_runnable_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS));
377 
378 	/*
379 	 * Arbitrarily allocate 512 pages (2mb when page size is 4kb) for the
380 	 * test code and other per-VM assets that will be loaded into memslot0.
381 	 */
382 	nr_pages = 512;
383 
384 	/* Account for the per-vCPU stacks on behalf of the test. */
385 	nr_pages += nr_runnable_vcpus * DEFAULT_STACK_PGS;
386 
387 	/*
388 	 * Account for the number of pages needed for the page tables.  The
389 	 * maximum page table size for a memory region will be when the
390 	 * smallest page size is used. Considering each page contains x page
391 	 * table descriptors, the total extra size for page tables (for extra
392 	 * N pages) will be: N/x+N/x^2+N/x^3+... which is definitely smaller
393 	 * than N/x*2.
394 	 */
395 	nr_pages += (nr_pages + extra_mem_pages) / PTES_PER_MIN_PAGE * 2;
396 
397 	/* Account for the number of pages needed by ucall. */
398 	nr_pages += ucall_nr_pages_required(page_size);
399 
400 	return vm_adjust_num_guest_pages(mode, nr_pages);
401 }
402 
kvm_set_files_rlimit(uint32_t nr_vcpus)403 void kvm_set_files_rlimit(uint32_t nr_vcpus)
404 {
405 	/*
406 	 * Each vCPU will open two file descriptors: the vCPU itself and the
407 	 * vCPU's binary stats file descriptor.  Add an arbitrary amount of
408 	 * buffer for all other files a test may open.
409 	 */
410 	int nr_fds_wanted = nr_vcpus * 2 + 100;
411 	struct rlimit rl;
412 
413 	/*
414 	 * Check that we're allowed to open nr_fds_wanted file descriptors and
415 	 * try raising the limits if needed.
416 	 */
417 	TEST_ASSERT(!getrlimit(RLIMIT_NOFILE, &rl), "getrlimit() failed!");
418 
419 	if (rl.rlim_cur < nr_fds_wanted) {
420 		rl.rlim_cur = nr_fds_wanted;
421 		if (rl.rlim_max < nr_fds_wanted) {
422 			int old_rlim_max = rl.rlim_max;
423 
424 			rl.rlim_max = nr_fds_wanted;
425 			__TEST_REQUIRE(setrlimit(RLIMIT_NOFILE, &rl) >= 0,
426 				       "RLIMIT_NOFILE hard limit is too low (%d, wanted %d)",
427 				       old_rlim_max, nr_fds_wanted);
428 		} else {
429 			TEST_ASSERT(!setrlimit(RLIMIT_NOFILE, &rl), "setrlimit() failed!");
430 		}
431 	}
432 
433 }
434 
is_guest_memfd_required(struct vm_shape shape)435 static bool is_guest_memfd_required(struct vm_shape shape)
436 {
437 #ifdef __x86_64__
438 	return shape.type == KVM_X86_SNP_VM;
439 #else
440 	return false;
441 #endif
442 }
443 
__vm_create(struct vm_shape shape,uint32_t nr_runnable_vcpus,uint64_t nr_extra_pages)444 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
445 			   uint64_t nr_extra_pages)
446 {
447 	uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,
448 						 nr_extra_pages);
449 	struct userspace_mem_region *slot0;
450 	struct kvm_vm *vm;
451 	int i, flags;
452 
453 	kvm_set_files_rlimit(nr_runnable_vcpus);
454 
455 	pr_debug("%s: mode='%s' type='%d', pages='%ld'\n", __func__,
456 		 vm_guest_mode_string(shape.mode), shape.type, nr_pages);
457 
458 	vm = ____vm_create(shape);
459 
460 	/*
461 	 * Force GUEST_MEMFD for the primary memory region if necessary, e.g.
462 	 * for CoCo VMs that require GUEST_MEMFD backed private memory.
463 	 */
464 	flags = 0;
465 	if (is_guest_memfd_required(shape))
466 		flags |= KVM_MEM_GUEST_MEMFD;
467 
468 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags);
469 	for (i = 0; i < NR_MEM_REGIONS; i++)
470 		vm->memslots[i] = 0;
471 
472 	kvm_vm_elf_load(vm, program_invocation_name);
473 
474 	/*
475 	 * TODO: Add proper defines to protect the library's memslots, and then
476 	 * carve out memslot1 for the ucall MMIO address.  KVM treats writes to
477 	 * read-only memslots as MMIO, and creating a read-only memslot for the
478 	 * MMIO region would prevent silently clobbering the MMIO region.
479 	 */
480 	slot0 = memslot2region(vm, 0);
481 	ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);
482 
483 	if (guest_random_seed != last_guest_seed) {
484 		pr_info("Random seed: 0x%x\n", guest_random_seed);
485 		last_guest_seed = guest_random_seed;
486 	}
487 	guest_rng = new_guest_random_state(guest_random_seed);
488 	sync_global_to_guest(vm, guest_rng);
489 
490 	kvm_arch_vm_post_create(vm, nr_runnable_vcpus);
491 
492 	return vm;
493 }
494 
495 /*
496  * VM Create with customized parameters
497  *
498  * Input Args:
499  *   mode - VM Mode (e.g. VM_MODE_P52V48_4K)
500  *   nr_vcpus - VCPU count
501  *   extra_mem_pages - Non-slot0 physical memory total size
502  *   guest_code - Guest entry point
503  *   vcpuids - VCPU IDs
504  *
505  * Output Args: None
506  *
507  * Return:
508  *   Pointer to opaque structure that describes the created VM.
509  *
510  * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
511  * extra_mem_pages is only used to calculate the maximum page table size,
512  * no real memory allocation for non-slot0 memory in this function.
513  */
__vm_create_with_vcpus(struct vm_shape shape,uint32_t nr_vcpus,uint64_t extra_mem_pages,void * guest_code,struct kvm_vcpu * vcpus[])514 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
515 				      uint64_t extra_mem_pages,
516 				      void *guest_code, struct kvm_vcpu *vcpus[])
517 {
518 	struct kvm_vm *vm;
519 	int i;
520 
521 	TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array");
522 
523 	vm = __vm_create(shape, nr_vcpus, extra_mem_pages);
524 
525 	for (i = 0; i < nr_vcpus; ++i)
526 		vcpus[i] = vm_vcpu_add(vm, i, guest_code);
527 
528 	kvm_arch_vm_finalize_vcpus(vm);
529 	return vm;
530 }
531 
__vm_create_shape_with_one_vcpu(struct vm_shape shape,struct kvm_vcpu ** vcpu,uint64_t extra_mem_pages,void * guest_code)532 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
533 					       struct kvm_vcpu **vcpu,
534 					       uint64_t extra_mem_pages,
535 					       void *guest_code)
536 {
537 	struct kvm_vcpu *vcpus[1];
538 	struct kvm_vm *vm;
539 
540 	vm = __vm_create_with_vcpus(shape, 1, extra_mem_pages, guest_code, vcpus);
541 
542 	*vcpu = vcpus[0];
543 	return vm;
544 }
545 
546 /*
547  * VM Restart
548  *
549  * Input Args:
550  *   vm - VM that has been released before
551  *
552  * Output Args: None
553  *
554  * Reopens the file descriptors associated to the VM and reinstates the
555  * global state, such as the irqchip and the memory regions that are mapped
556  * into the guest.
557  */
kvm_vm_restart(struct kvm_vm * vmp)558 void kvm_vm_restart(struct kvm_vm *vmp)
559 {
560 	int ctr;
561 	struct userspace_mem_region *region;
562 
563 	vm_open(vmp);
564 	if (vmp->has_irqchip)
565 		vm_create_irqchip(vmp);
566 
567 	hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {
568 		int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION2, &region->region);
569 
570 		TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
571 			    "  rc: %i errno: %i\n"
572 			    "  slot: %u flags: 0x%x\n"
573 			    "  guest_phys_addr: 0x%llx size: 0x%llx",
574 			    ret, errno, region->region.slot,
575 			    region->region.flags,
576 			    region->region.guest_phys_addr,
577 			    region->region.memory_size);
578 	}
579 }
580 
vm_arch_vcpu_recreate(struct kvm_vm * vm,uint32_t vcpu_id)581 __weak struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm,
582 					      uint32_t vcpu_id)
583 {
584 	return __vm_vcpu_add(vm, vcpu_id);
585 }
586 
vm_recreate_with_one_vcpu(struct kvm_vm * vm)587 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm)
588 {
589 	kvm_vm_restart(vm);
590 
591 	return vm_vcpu_recreate(vm, 0);
592 }
593 
__pin_task_to_cpu(pthread_t task,int cpu)594 int __pin_task_to_cpu(pthread_t task, int cpu)
595 {
596 	cpu_set_t cpuset;
597 
598 	CPU_ZERO(&cpuset);
599 	CPU_SET(cpu, &cpuset);
600 
601 	return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset);
602 }
603 
parse_pcpu(const char * cpu_str,const cpu_set_t * allowed_mask)604 static uint32_t parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask)
605 {
606 	uint32_t pcpu = atoi_non_negative("CPU number", cpu_str);
607 
608 	TEST_ASSERT(CPU_ISSET(pcpu, allowed_mask),
609 		    "Not allowed to run on pCPU '%d', check cgroups?", pcpu);
610 	return pcpu;
611 }
612 
kvm_print_vcpu_pinning_help(void)613 void kvm_print_vcpu_pinning_help(void)
614 {
615 	const char *name = program_invocation_name;
616 
617 	printf(" -c: Pin tasks to physical CPUs.  Takes a list of comma separated\n"
618 	       "     values (target pCPU), one for each vCPU, plus an optional\n"
619 	       "     entry for the main application task (specified via entry\n"
620 	       "     <nr_vcpus + 1>).  If used, entries must be provided for all\n"
621 	       "     vCPUs, i.e. pinning vCPUs is all or nothing.\n\n"
622 	       "     E.g. to create 3 vCPUs, pin vCPU0=>pCPU22, vCPU1=>pCPU23,\n"
623 	       "     vCPU2=>pCPU24, and pin the application task to pCPU50:\n\n"
624 	       "         %s -v 3 -c 22,23,24,50\n\n"
625 	       "     To leave the application task unpinned, drop the final entry:\n\n"
626 	       "         %s -v 3 -c 22,23,24\n\n"
627 	       "     (default: no pinning)\n", name, name);
628 }
629 
kvm_parse_vcpu_pinning(const char * pcpus_string,uint32_t vcpu_to_pcpu[],int nr_vcpus)630 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
631 			    int nr_vcpus)
632 {
633 	cpu_set_t allowed_mask;
634 	char *cpu, *cpu_list;
635 	char delim[2] = ",";
636 	int i, r;
637 
638 	cpu_list = strdup(pcpus_string);
639 	TEST_ASSERT(cpu_list, "strdup() allocation failed.");
640 
641 	r = sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask);
642 	TEST_ASSERT(!r, "sched_getaffinity() failed");
643 
644 	cpu = strtok(cpu_list, delim);
645 
646 	/* 1. Get all pcpus for vcpus. */
647 	for (i = 0; i < nr_vcpus; i++) {
648 		TEST_ASSERT(cpu, "pCPU not provided for vCPU '%d'", i);
649 		vcpu_to_pcpu[i] = parse_pcpu(cpu, &allowed_mask);
650 		cpu = strtok(NULL, delim);
651 	}
652 
653 	/* 2. Check if the main worker needs to be pinned. */
654 	if (cpu) {
655 		pin_self_to_cpu(parse_pcpu(cpu, &allowed_mask));
656 		cpu = strtok(NULL, delim);
657 	}
658 
659 	TEST_ASSERT(!cpu, "pCPU list contains trailing garbage characters '%s'", cpu);
660 	free(cpu_list);
661 }
662 
663 /*
664  * Userspace Memory Region Find
665  *
666  * Input Args:
667  *   vm - Virtual Machine
668  *   start - Starting VM physical address
669  *   end - Ending VM physical address, inclusive.
670  *
671  * Output Args: None
672  *
673  * Return:
674  *   Pointer to overlapping region, NULL if no such region.
675  *
676  * Searches for a region with any physical memory that overlaps with
677  * any portion of the guest physical addresses from start to end
678  * inclusive.  If multiple overlapping regions exist, a pointer to any
679  * of the regions is returned.  Null is returned only when no overlapping
680  * region exists.
681  */
682 static struct userspace_mem_region *
userspace_mem_region_find(struct kvm_vm * vm,uint64_t start,uint64_t end)683 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
684 {
685 	struct rb_node *node;
686 
687 	for (node = vm->regions.gpa_tree.rb_node; node; ) {
688 		struct userspace_mem_region *region =
689 			container_of(node, struct userspace_mem_region, gpa_node);
690 		uint64_t existing_start = region->region.guest_phys_addr;
691 		uint64_t existing_end = region->region.guest_phys_addr
692 			+ region->region.memory_size - 1;
693 		if (start <= existing_end && end >= existing_start)
694 			return region;
695 
696 		if (start < existing_start)
697 			node = node->rb_left;
698 		else
699 			node = node->rb_right;
700 	}
701 
702 	return NULL;
703 }
704 
kvm_stats_release(struct kvm_binary_stats * stats)705 static void kvm_stats_release(struct kvm_binary_stats *stats)
706 {
707 	int ret;
708 
709 	if (stats->fd < 0)
710 		return;
711 
712 	if (stats->desc) {
713 		free(stats->desc);
714 		stats->desc = NULL;
715 	}
716 
717 	ret = close(stats->fd);
718 	TEST_ASSERT(!ret,  __KVM_SYSCALL_ERROR("close()", ret));
719 	stats->fd = -1;
720 }
721 
vcpu_arch_free(struct kvm_vcpu * vcpu)722 __weak void vcpu_arch_free(struct kvm_vcpu *vcpu)
723 {
724 
725 }
726 
727 /*
728  * VM VCPU Remove
729  *
730  * Input Args:
731  *   vcpu - VCPU to remove
732  *
733  * Output Args: None
734  *
735  * Return: None, TEST_ASSERT failures for all error conditions
736  *
737  * Removes a vCPU from a VM and frees its resources.
738  */
vm_vcpu_rm(struct kvm_vm * vm,struct kvm_vcpu * vcpu)739 static void vm_vcpu_rm(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
740 {
741 	int ret;
742 
743 	if (vcpu->dirty_gfns) {
744 		kvm_munmap(vcpu->dirty_gfns, vm->dirty_ring_size);
745 		vcpu->dirty_gfns = NULL;
746 	}
747 
748 	kvm_munmap(vcpu->run, vcpu_mmap_sz());
749 
750 	ret = close(vcpu->fd);
751 	TEST_ASSERT(!ret,  __KVM_SYSCALL_ERROR("close()", ret));
752 
753 	kvm_stats_release(&vcpu->stats);
754 
755 	list_del(&vcpu->list);
756 
757 	vcpu_arch_free(vcpu);
758 	free(vcpu);
759 }
760 
kvm_vm_release(struct kvm_vm * vmp)761 void kvm_vm_release(struct kvm_vm *vmp)
762 {
763 	struct kvm_vcpu *vcpu, *tmp;
764 	int ret;
765 
766 	list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)
767 		vm_vcpu_rm(vmp, vcpu);
768 
769 	ret = close(vmp->fd);
770 	TEST_ASSERT(!ret,  __KVM_SYSCALL_ERROR("close()", ret));
771 
772 	ret = close(vmp->kvm_fd);
773 	TEST_ASSERT(!ret,  __KVM_SYSCALL_ERROR("close()", ret));
774 
775 	/* Free cached stats metadata and close FD */
776 	kvm_stats_release(&vmp->stats);
777 
778 	kvm_arch_vm_release(vmp);
779 }
780 
__vm_mem_region_delete(struct kvm_vm * vm,struct userspace_mem_region * region)781 static void __vm_mem_region_delete(struct kvm_vm *vm,
782 				   struct userspace_mem_region *region)
783 {
784 	rb_erase(&region->gpa_node, &vm->regions.gpa_tree);
785 	rb_erase(&region->hva_node, &vm->regions.hva_tree);
786 	hash_del(&region->slot_node);
787 
788 	sparsebit_free(&region->unused_phy_pages);
789 	sparsebit_free(&region->protected_phy_pages);
790 	kvm_munmap(region->mmap_start, region->mmap_size);
791 	if (region->fd >= 0) {
792 		/* There's an extra map when using shared memory. */
793 		kvm_munmap(region->mmap_alias, region->mmap_size);
794 		close(region->fd);
795 	}
796 	if (region->region.guest_memfd >= 0)
797 		close(region->region.guest_memfd);
798 
799 	free(region);
800 }
801 
802 /*
803  * Destroys and frees the VM pointed to by vmp.
804  */
kvm_vm_free(struct kvm_vm * vmp)805 void kvm_vm_free(struct kvm_vm *vmp)
806 {
807 	int ctr;
808 	struct hlist_node *node;
809 	struct userspace_mem_region *region;
810 
811 	if (vmp == NULL)
812 		return;
813 
814 	/* Free userspace_mem_regions. */
815 	hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)
816 		__vm_mem_region_delete(vmp, region);
817 
818 	/* Free sparsebit arrays. */
819 	sparsebit_free(&vmp->vpages_valid);
820 	sparsebit_free(&vmp->vpages_mapped);
821 
822 	kvm_vm_release(vmp);
823 
824 	/* Free the structure describing the VM. */
825 	free(vmp);
826 }
827 
kvm_memfd_alloc(size_t size,bool hugepages)828 int kvm_memfd_alloc(size_t size, bool hugepages)
829 {
830 	int memfd_flags = MFD_CLOEXEC;
831 	int fd, r;
832 
833 	if (hugepages)
834 		memfd_flags |= MFD_HUGETLB;
835 
836 	fd = memfd_create("kvm_selftest", memfd_flags);
837 	TEST_ASSERT(fd != -1, __KVM_SYSCALL_ERROR("memfd_create()", fd));
838 
839 	r = ftruncate(fd, size);
840 	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("ftruncate()", r));
841 
842 	r = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0, size);
843 	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
844 
845 	return fd;
846 }
847 
vm_userspace_mem_region_gpa_insert(struct rb_root * gpa_tree,struct userspace_mem_region * region)848 static void vm_userspace_mem_region_gpa_insert(struct rb_root *gpa_tree,
849 					       struct userspace_mem_region *region)
850 {
851 	struct rb_node **cur, *parent;
852 
853 	for (cur = &gpa_tree->rb_node, parent = NULL; *cur; ) {
854 		struct userspace_mem_region *cregion;
855 
856 		cregion = container_of(*cur, typeof(*cregion), gpa_node);
857 		parent = *cur;
858 		if (region->region.guest_phys_addr <
859 		    cregion->region.guest_phys_addr)
860 			cur = &(*cur)->rb_left;
861 		else {
862 			TEST_ASSERT(region->region.guest_phys_addr !=
863 				    cregion->region.guest_phys_addr,
864 				    "Duplicate GPA in region tree");
865 
866 			cur = &(*cur)->rb_right;
867 		}
868 	}
869 
870 	rb_link_node(&region->gpa_node, parent, cur);
871 	rb_insert_color(&region->gpa_node, gpa_tree);
872 }
873 
vm_userspace_mem_region_hva_insert(struct rb_root * hva_tree,struct userspace_mem_region * region)874 static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,
875 					       struct userspace_mem_region *region)
876 {
877 	struct rb_node **cur, *parent;
878 
879 	for (cur = &hva_tree->rb_node, parent = NULL; *cur; ) {
880 		struct userspace_mem_region *cregion;
881 
882 		cregion = container_of(*cur, typeof(*cregion), hva_node);
883 		parent = *cur;
884 		if (region->host_mem < cregion->host_mem)
885 			cur = &(*cur)->rb_left;
886 		else {
887 			TEST_ASSERT(region->host_mem !=
888 				    cregion->host_mem,
889 				    "Duplicate HVA in region tree");
890 
891 			cur = &(*cur)->rb_right;
892 		}
893 	}
894 
895 	rb_link_node(&region->hva_node, parent, cur);
896 	rb_insert_color(&region->hva_node, hva_tree);
897 }
898 
899 
__vm_set_user_memory_region(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva)900 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
901 				uint64_t gpa, uint64_t size, void *hva)
902 {
903 	struct kvm_userspace_memory_region region = {
904 		.slot = slot,
905 		.flags = flags,
906 		.guest_phys_addr = gpa,
907 		.memory_size = size,
908 		.userspace_addr = (uintptr_t)hva,
909 	};
910 
911 	return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, &region);
912 }
913 
vm_set_user_memory_region(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva)914 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
915 			       uint64_t gpa, uint64_t size, void *hva)
916 {
917 	int ret = __vm_set_user_memory_region(vm, slot, flags, gpa, size, hva);
918 
919 	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed, errno = %d (%s)",
920 		    errno, strerror(errno));
921 }
922 
923 #define TEST_REQUIRE_SET_USER_MEMORY_REGION2()			\
924 	__TEST_REQUIRE(kvm_has_cap(KVM_CAP_USER_MEMORY2),	\
925 		       "KVM selftests now require KVM_SET_USER_MEMORY_REGION2 (introduced in v6.8)")
926 
__vm_set_user_memory_region2(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva,uint32_t guest_memfd,uint64_t guest_memfd_offset)927 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
928 				 uint64_t gpa, uint64_t size, void *hva,
929 				 uint32_t guest_memfd, uint64_t guest_memfd_offset)
930 {
931 	struct kvm_userspace_memory_region2 region = {
932 		.slot = slot,
933 		.flags = flags,
934 		.guest_phys_addr = gpa,
935 		.memory_size = size,
936 		.userspace_addr = (uintptr_t)hva,
937 		.guest_memfd = guest_memfd,
938 		.guest_memfd_offset = guest_memfd_offset,
939 	};
940 
941 	TEST_REQUIRE_SET_USER_MEMORY_REGION2();
942 
943 	return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION2, &region);
944 }
945 
vm_set_user_memory_region2(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva,uint32_t guest_memfd,uint64_t guest_memfd_offset)946 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
947 				uint64_t gpa, uint64_t size, void *hva,
948 				uint32_t guest_memfd, uint64_t guest_memfd_offset)
949 {
950 	int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva,
951 					       guest_memfd, guest_memfd_offset);
952 
953 	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)",
954 		    errno, strerror(errno));
955 }
956 
957 
958 /* FIXME: This thing needs to be ripped apart and rewritten. */
vm_mem_add(struct kvm_vm * vm,enum vm_mem_backing_src_type src_type,uint64_t guest_paddr,uint32_t slot,uint64_t npages,uint32_t flags,int guest_memfd,uint64_t guest_memfd_offset)959 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
960 		uint64_t guest_paddr, uint32_t slot, uint64_t npages,
961 		uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset)
962 {
963 	int ret;
964 	struct userspace_mem_region *region;
965 	size_t backing_src_pagesz = get_backing_src_pagesz(src_type);
966 	size_t mem_size = npages * vm->page_size;
967 	size_t alignment;
968 
969 	TEST_REQUIRE_SET_USER_MEMORY_REGION2();
970 
971 	TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,
972 		"Number of guest pages is not compatible with the host. "
973 		"Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));
974 
975 	TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
976 		"address not on a page boundary.\n"
977 		"  guest_paddr: 0x%lx vm->page_size: 0x%x",
978 		guest_paddr, vm->page_size);
979 	TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
980 		<= vm->max_gfn, "Physical range beyond maximum "
981 		"supported physical address,\n"
982 		"  guest_paddr: 0x%lx npages: 0x%lx\n"
983 		"  vm->max_gfn: 0x%lx vm->page_size: 0x%x",
984 		guest_paddr, npages, vm->max_gfn, vm->page_size);
985 
986 	/*
987 	 * Confirm a mem region with an overlapping address doesn't
988 	 * already exist.
989 	 */
990 	region = (struct userspace_mem_region *) userspace_mem_region_find(
991 		vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
992 	if (region != NULL)
993 		TEST_FAIL("overlapping userspace_mem_region already "
994 			"exists\n"
995 			"  requested guest_paddr: 0x%lx npages: 0x%lx "
996 			"page_size: 0x%x\n"
997 			"  existing guest_paddr: 0x%lx size: 0x%lx",
998 			guest_paddr, npages, vm->page_size,
999 			(uint64_t) region->region.guest_phys_addr,
1000 			(uint64_t) region->region.memory_size);
1001 
1002 	/* Confirm no region with the requested slot already exists. */
1003 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
1004 			       slot) {
1005 		if (region->region.slot != slot)
1006 			continue;
1007 
1008 		TEST_FAIL("A mem region with the requested slot "
1009 			"already exists.\n"
1010 			"  requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
1011 			"  existing slot: %u paddr: 0x%lx size: 0x%lx",
1012 			slot, guest_paddr, npages,
1013 			region->region.slot,
1014 			(uint64_t) region->region.guest_phys_addr,
1015 			(uint64_t) region->region.memory_size);
1016 	}
1017 
1018 	/* Allocate and initialize new mem region structure. */
1019 	region = calloc(1, sizeof(*region));
1020 	TEST_ASSERT(region != NULL, "Insufficient Memory");
1021 	region->mmap_size = mem_size;
1022 
1023 #ifdef __s390x__
1024 	/* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
1025 	alignment = 0x100000;
1026 #else
1027 	alignment = 1;
1028 #endif
1029 
1030 	/*
1031 	 * When using THP mmap is not guaranteed to returned a hugepage aligned
1032 	 * address so we have to pad the mmap. Padding is not needed for HugeTLB
1033 	 * because mmap will always return an address aligned to the HugeTLB
1034 	 * page size.
1035 	 */
1036 	if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
1037 		alignment = max(backing_src_pagesz, alignment);
1038 
1039 	TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, backing_src_pagesz));
1040 
1041 	/* Add enough memory to align up if necessary */
1042 	if (alignment > 1)
1043 		region->mmap_size += alignment;
1044 
1045 	region->fd = -1;
1046 	if (backing_src_is_shared(src_type))
1047 		region->fd = kvm_memfd_alloc(region->mmap_size,
1048 					     src_type == VM_MEM_SRC_SHARED_HUGETLB);
1049 
1050 	region->mmap_start = kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE,
1051 				      vm_mem_backing_src_alias(src_type)->flag,
1052 				      region->fd);
1053 
1054 	TEST_ASSERT(!is_backing_src_hugetlb(src_type) ||
1055 		    region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz),
1056 		    "mmap_start %p is not aligned to HugeTLB page size 0x%lx",
1057 		    region->mmap_start, backing_src_pagesz);
1058 
1059 	/* Align host address */
1060 	region->host_mem = align_ptr_up(region->mmap_start, alignment);
1061 
1062 	/* As needed perform madvise */
1063 	if ((src_type == VM_MEM_SRC_ANONYMOUS ||
1064 	     src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
1065 		ret = madvise(region->host_mem, mem_size,
1066 			      src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
1067 		TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
1068 			    region->host_mem, mem_size,
1069 			    vm_mem_backing_src_alias(src_type)->name);
1070 	}
1071 
1072 	region->backing_src_type = src_type;
1073 
1074 	if (flags & KVM_MEM_GUEST_MEMFD) {
1075 		if (guest_memfd < 0) {
1076 			uint32_t guest_memfd_flags = 0;
1077 			TEST_ASSERT(!guest_memfd_offset,
1078 				    "Offset must be zero when creating new guest_memfd");
1079 			guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags);
1080 		} else {
1081 			/*
1082 			 * Install a unique fd for each memslot so that the fd
1083 			 * can be closed when the region is deleted without
1084 			 * needing to track if the fd is owned by the framework
1085 			 * or by the caller.
1086 			 */
1087 			guest_memfd = dup(guest_memfd);
1088 			TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd));
1089 		}
1090 
1091 		region->region.guest_memfd = guest_memfd;
1092 		region->region.guest_memfd_offset = guest_memfd_offset;
1093 	} else {
1094 		region->region.guest_memfd = -1;
1095 	}
1096 
1097 	region->unused_phy_pages = sparsebit_alloc();
1098 	if (vm_arch_has_protected_memory(vm))
1099 		region->protected_phy_pages = sparsebit_alloc();
1100 	sparsebit_set_num(region->unused_phy_pages,
1101 		guest_paddr >> vm->page_shift, npages);
1102 	region->region.slot = slot;
1103 	region->region.flags = flags;
1104 	region->region.guest_phys_addr = guest_paddr;
1105 	region->region.memory_size = npages * vm->page_size;
1106 	region->region.userspace_addr = (uintptr_t) region->host_mem;
1107 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1108 	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
1109 		"  rc: %i errno: %i\n"
1110 		"  slot: %u flags: 0x%x\n"
1111 		"  guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d",
1112 		ret, errno, slot, flags,
1113 		guest_paddr, (uint64_t) region->region.memory_size,
1114 		region->region.guest_memfd);
1115 
1116 	/* Add to quick lookup data structures */
1117 	vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);
1118 	vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);
1119 	hash_add(vm->regions.slot_hash, &region->slot_node, slot);
1120 
1121 	/* If shared memory, create an alias. */
1122 	if (region->fd >= 0) {
1123 		region->mmap_alias = kvm_mmap(region->mmap_size,
1124 					      PROT_READ | PROT_WRITE,
1125 					      vm_mem_backing_src_alias(src_type)->flag,
1126 					      region->fd);
1127 
1128 		/* Align host alias address */
1129 		region->host_alias = align_ptr_up(region->mmap_alias, alignment);
1130 	}
1131 }
1132 
vm_userspace_mem_region_add(struct kvm_vm * vm,enum vm_mem_backing_src_type src_type,uint64_t guest_paddr,uint32_t slot,uint64_t npages,uint32_t flags)1133 void vm_userspace_mem_region_add(struct kvm_vm *vm,
1134 				 enum vm_mem_backing_src_type src_type,
1135 				 uint64_t guest_paddr, uint32_t slot,
1136 				 uint64_t npages, uint32_t flags)
1137 {
1138 	vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0);
1139 }
1140 
1141 /*
1142  * Memslot to region
1143  *
1144  * Input Args:
1145  *   vm - Virtual Machine
1146  *   memslot - KVM memory slot ID
1147  *
1148  * Output Args: None
1149  *
1150  * Return:
1151  *   Pointer to memory region structure that describe memory region
1152  *   using kvm memory slot ID given by memslot.  TEST_ASSERT failure
1153  *   on error (e.g. currently no memory region using memslot as a KVM
1154  *   memory slot ID).
1155  */
1156 struct userspace_mem_region *
memslot2region(struct kvm_vm * vm,uint32_t memslot)1157 memslot2region(struct kvm_vm *vm, uint32_t memslot)
1158 {
1159 	struct userspace_mem_region *region;
1160 
1161 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
1162 			       memslot)
1163 		if (region->region.slot == memslot)
1164 			return region;
1165 
1166 	fprintf(stderr, "No mem region with the requested slot found,\n"
1167 		"  requested slot: %u\n", memslot);
1168 	fputs("---- vm dump ----\n", stderr);
1169 	vm_dump(stderr, vm, 2);
1170 	TEST_FAIL("Mem region not found");
1171 	return NULL;
1172 }
1173 
1174 /*
1175  * VM Memory Region Flags Set
1176  *
1177  * Input Args:
1178  *   vm - Virtual Machine
1179  *   flags - Starting guest physical address
1180  *
1181  * Output Args: None
1182  *
1183  * Return: None
1184  *
1185  * Sets the flags of the memory region specified by the value of slot,
1186  * to the values given by flags.
1187  */
vm_mem_region_set_flags(struct kvm_vm * vm,uint32_t slot,uint32_t flags)1188 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
1189 {
1190 	int ret;
1191 	struct userspace_mem_region *region;
1192 
1193 	region = memslot2region(vm, slot);
1194 
1195 	region->region.flags = flags;
1196 
1197 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1198 
1199 	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
1200 		"  rc: %i errno: %i slot: %u flags: 0x%x",
1201 		ret, errno, slot, flags);
1202 }
1203 
1204 /*
1205  * VM Memory Region Move
1206  *
1207  * Input Args:
1208  *   vm - Virtual Machine
1209  *   slot - Slot of the memory region to move
1210  *   new_gpa - Starting guest physical address
1211  *
1212  * Output Args: None
1213  *
1214  * Return: None
1215  *
1216  * Change the gpa of a memory region.
1217  */
vm_mem_region_move(struct kvm_vm * vm,uint32_t slot,uint64_t new_gpa)1218 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
1219 {
1220 	struct userspace_mem_region *region;
1221 	int ret;
1222 
1223 	region = memslot2region(vm, slot);
1224 
1225 	region->region.guest_phys_addr = new_gpa;
1226 
1227 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1228 
1229 	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed\n"
1230 		    "ret: %i errno: %i slot: %u new_gpa: 0x%lx",
1231 		    ret, errno, slot, new_gpa);
1232 }
1233 
1234 /*
1235  * VM Memory Region Delete
1236  *
1237  * Input Args:
1238  *   vm - Virtual Machine
1239  *   slot - Slot of the memory region to delete
1240  *
1241  * Output Args: None
1242  *
1243  * Return: None
1244  *
1245  * Delete a memory region.
1246  */
vm_mem_region_delete(struct kvm_vm * vm,uint32_t slot)1247 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
1248 {
1249 	struct userspace_mem_region *region = memslot2region(vm, slot);
1250 
1251 	region->region.memory_size = 0;
1252 	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1253 
1254 	__vm_mem_region_delete(vm, region);
1255 }
1256 
vm_guest_mem_fallocate(struct kvm_vm * vm,uint64_t base,uint64_t size,bool punch_hole)1257 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
1258 			    bool punch_hole)
1259 {
1260 	const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0);
1261 	struct userspace_mem_region *region;
1262 	uint64_t end = base + size;
1263 	uint64_t gpa, len;
1264 	off_t fd_offset;
1265 	int ret;
1266 
1267 	for (gpa = base; gpa < end; gpa += len) {
1268 		uint64_t offset;
1269 
1270 		region = userspace_mem_region_find(vm, gpa, gpa);
1271 		TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,
1272 			    "Private memory region not found for GPA 0x%lx", gpa);
1273 
1274 		offset = gpa - region->region.guest_phys_addr;
1275 		fd_offset = region->region.guest_memfd_offset + offset;
1276 		len = min_t(uint64_t, end - gpa, region->region.memory_size - offset);
1277 
1278 		ret = fallocate(region->region.guest_memfd, mode, fd_offset, len);
1279 		TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx",
1280 			    punch_hole ? "punch hole" : "allocate", gpa, len,
1281 			    region->region.guest_memfd, mode, fd_offset);
1282 	}
1283 }
1284 
1285 /* Returns the size of a vCPU's kvm_run structure. */
vcpu_mmap_sz(void)1286 static size_t vcpu_mmap_sz(void)
1287 {
1288 	int dev_fd, ret;
1289 
1290 	dev_fd = open_kvm_dev_path_or_exit();
1291 
1292 	ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
1293 	TEST_ASSERT(ret >= 0 && ret >= sizeof(struct kvm_run),
1294 		    KVM_IOCTL_ERROR(KVM_GET_VCPU_MMAP_SIZE, ret));
1295 
1296 	close(dev_fd);
1297 
1298 	return ret;
1299 }
1300 
vcpu_exists(struct kvm_vm * vm,uint32_t vcpu_id)1301 static bool vcpu_exists(struct kvm_vm *vm, uint32_t vcpu_id)
1302 {
1303 	struct kvm_vcpu *vcpu;
1304 
1305 	list_for_each_entry(vcpu, &vm->vcpus, list) {
1306 		if (vcpu->id == vcpu_id)
1307 			return true;
1308 	}
1309 
1310 	return false;
1311 }
1312 
1313 /*
1314  * Adds a virtual CPU to the VM specified by vm with the ID given by vcpu_id.
1315  * No additional vCPU setup is done.  Returns the vCPU.
1316  */
__vm_vcpu_add(struct kvm_vm * vm,uint32_t vcpu_id)1317 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
1318 {
1319 	struct kvm_vcpu *vcpu;
1320 
1321 	/* Confirm a vcpu with the specified id doesn't already exist. */
1322 	TEST_ASSERT(!vcpu_exists(vm, vcpu_id), "vCPU%d already exists", vcpu_id);
1323 
1324 	/* Allocate and initialize new vcpu structure. */
1325 	vcpu = calloc(1, sizeof(*vcpu));
1326 	TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
1327 
1328 	vcpu->vm = vm;
1329 	vcpu->id = vcpu_id;
1330 	vcpu->fd = __vm_ioctl(vm, KVM_CREATE_VCPU, (void *)(unsigned long)vcpu_id);
1331 	TEST_ASSERT_VM_VCPU_IOCTL(vcpu->fd >= 0, KVM_CREATE_VCPU, vcpu->fd, vm);
1332 
1333 	TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->run), "vcpu mmap size "
1334 		"smaller than expected, vcpu_mmap_sz: %zi expected_min: %zi",
1335 		vcpu_mmap_sz(), sizeof(*vcpu->run));
1336 	vcpu->run = kvm_mmap(vcpu_mmap_sz(), PROT_READ | PROT_WRITE,
1337 			     MAP_SHARED, vcpu->fd);
1338 
1339 	if (kvm_has_cap(KVM_CAP_BINARY_STATS_FD))
1340 		vcpu->stats.fd = vcpu_get_stats_fd(vcpu);
1341 	else
1342 		vcpu->stats.fd = -1;
1343 
1344 	/* Add to linked-list of VCPUs. */
1345 	list_add(&vcpu->list, &vm->vcpus);
1346 
1347 	return vcpu;
1348 }
1349 
1350 /*
1351  * VM Virtual Address Unused Gap
1352  *
1353  * Input Args:
1354  *   vm - Virtual Machine
1355  *   sz - Size (bytes)
1356  *   vaddr_min - Minimum Virtual Address
1357  *
1358  * Output Args: None
1359  *
1360  * Return:
1361  *   Lowest virtual address at or below vaddr_min, with at least
1362  *   sz unused bytes.  TEST_ASSERT failure if no area of at least
1363  *   size sz is available.
1364  *
1365  * Within the VM specified by vm, locates the lowest starting virtual
1366  * address >= vaddr_min, that has at least sz unallocated bytes.  A
1367  * TEST_ASSERT failure occurs for invalid input or no area of at least
1368  * sz unallocated bytes >= vaddr_min is available.
1369  */
vm_vaddr_unused_gap(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min)1370 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
1371 			       vm_vaddr_t vaddr_min)
1372 {
1373 	uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
1374 
1375 	/* Determine lowest permitted virtual page index. */
1376 	uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
1377 	if ((pgidx_start * vm->page_size) < vaddr_min)
1378 		goto no_va_found;
1379 
1380 	/* Loop over section with enough valid virtual page indexes. */
1381 	if (!sparsebit_is_set_num(vm->vpages_valid,
1382 		pgidx_start, pages))
1383 		pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
1384 			pgidx_start, pages);
1385 	do {
1386 		/*
1387 		 * Are there enough unused virtual pages available at
1388 		 * the currently proposed starting virtual page index.
1389 		 * If not, adjust proposed starting index to next
1390 		 * possible.
1391 		 */
1392 		if (sparsebit_is_clear_num(vm->vpages_mapped,
1393 			pgidx_start, pages))
1394 			goto va_found;
1395 		pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
1396 			pgidx_start, pages);
1397 		if (pgidx_start == 0)
1398 			goto no_va_found;
1399 
1400 		/*
1401 		 * If needed, adjust proposed starting virtual address,
1402 		 * to next range of valid virtual addresses.
1403 		 */
1404 		if (!sparsebit_is_set_num(vm->vpages_valid,
1405 			pgidx_start, pages)) {
1406 			pgidx_start = sparsebit_next_set_num(
1407 				vm->vpages_valid, pgidx_start, pages);
1408 			if (pgidx_start == 0)
1409 				goto no_va_found;
1410 		}
1411 	} while (pgidx_start != 0);
1412 
1413 no_va_found:
1414 	TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);
1415 
1416 	/* NOT REACHED */
1417 	return -1;
1418 
1419 va_found:
1420 	TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
1421 		pgidx_start, pages),
1422 		"Unexpected, invalid virtual page index range,\n"
1423 		"  pgidx_start: 0x%lx\n"
1424 		"  pages: 0x%lx",
1425 		pgidx_start, pages);
1426 	TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
1427 		pgidx_start, pages),
1428 		"Unexpected, pages already mapped,\n"
1429 		"  pgidx_start: 0x%lx\n"
1430 		"  pages: 0x%lx",
1431 		pgidx_start, pages);
1432 
1433 	return pgidx_start * vm->page_size;
1434 }
1435 
____vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type,bool protected)1436 static vm_vaddr_t ____vm_vaddr_alloc(struct kvm_vm *vm, size_t sz,
1437 				     vm_vaddr_t vaddr_min,
1438 				     enum kvm_mem_region_type type,
1439 				     bool protected)
1440 {
1441 	uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
1442 
1443 	virt_pgd_alloc(vm);
1444 	vm_paddr_t paddr = __vm_phy_pages_alloc(vm, pages,
1445 						KVM_UTIL_MIN_PFN * vm->page_size,
1446 						vm->memslots[type], protected);
1447 
1448 	/*
1449 	 * Find an unused range of virtual page addresses of at least
1450 	 * pages in length.
1451 	 */
1452 	vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
1453 
1454 	/* Map the virtual pages. */
1455 	for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
1456 		pages--, vaddr += vm->page_size, paddr += vm->page_size) {
1457 
1458 		virt_pg_map(vm, vaddr, paddr);
1459 
1460 		sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1461 	}
1462 
1463 	return vaddr_start;
1464 }
1465 
__vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type)1466 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1467 			    enum kvm_mem_region_type type)
1468 {
1469 	return ____vm_vaddr_alloc(vm, sz, vaddr_min, type,
1470 				  vm_arch_has_protected_memory(vm));
1471 }
1472 
vm_vaddr_alloc_shared(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type)1473 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,
1474 				 vm_vaddr_t vaddr_min,
1475 				 enum kvm_mem_region_type type)
1476 {
1477 	return ____vm_vaddr_alloc(vm, sz, vaddr_min, type, false);
1478 }
1479 
1480 /*
1481  * VM Virtual Address Allocate
1482  *
1483  * Input Args:
1484  *   vm - Virtual Machine
1485  *   sz - Size in bytes
1486  *   vaddr_min - Minimum starting virtual address
1487  *
1488  * Output Args: None
1489  *
1490  * Return:
1491  *   Starting guest virtual address
1492  *
1493  * Allocates at least sz bytes within the virtual address space of the vm
1494  * given by vm.  The allocated bytes are mapped to a virtual address >=
1495  * the address given by vaddr_min.  Note that each allocation uses a
1496  * a unique set of pages, with the minimum real allocation being at least
1497  * a page. The allocated physical space comes from the TEST_DATA memory region.
1498  */
vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min)1499 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
1500 {
1501 	return __vm_vaddr_alloc(vm, sz, vaddr_min, MEM_REGION_TEST_DATA);
1502 }
1503 
1504 /*
1505  * VM Virtual Address Allocate Pages
1506  *
1507  * Input Args:
1508  *   vm - Virtual Machine
1509  *
1510  * Output Args: None
1511  *
1512  * Return:
1513  *   Starting guest virtual address
1514  *
1515  * Allocates at least N system pages worth of bytes within the virtual address
1516  * space of the vm.
1517  */
vm_vaddr_alloc_pages(struct kvm_vm * vm,int nr_pages)1518 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages)
1519 {
1520 	return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);
1521 }
1522 
__vm_vaddr_alloc_page(struct kvm_vm * vm,enum kvm_mem_region_type type)1523 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)
1524 {
1525 	return __vm_vaddr_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);
1526 }
1527 
1528 /*
1529  * VM Virtual Address Allocate Page
1530  *
1531  * Input Args:
1532  *   vm - Virtual Machine
1533  *
1534  * Output Args: None
1535  *
1536  * Return:
1537  *   Starting guest virtual address
1538  *
1539  * Allocates at least one system page worth of bytes within the virtual address
1540  * space of the vm.
1541  */
vm_vaddr_alloc_page(struct kvm_vm * vm)1542 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm)
1543 {
1544 	return vm_vaddr_alloc_pages(vm, 1);
1545 }
1546 
1547 /*
1548  * Map a range of VM virtual address to the VM's physical address
1549  *
1550  * Input Args:
1551  *   vm - Virtual Machine
1552  *   vaddr - Virtuall address to map
1553  *   paddr - VM Physical Address
1554  *   npages - The number of pages to map
1555  *
1556  * Output Args: None
1557  *
1558  * Return: None
1559  *
1560  * Within the VM given by @vm, creates a virtual translation for
1561  * @npages starting at @vaddr to the page range starting at @paddr.
1562  */
virt_map(struct kvm_vm * vm,uint64_t vaddr,uint64_t paddr,unsigned int npages)1563 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
1564 	      unsigned int npages)
1565 {
1566 	size_t page_size = vm->page_size;
1567 	size_t size = npages * page_size;
1568 
1569 	TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
1570 	TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
1571 
1572 	while (npages--) {
1573 		virt_pg_map(vm, vaddr, paddr);
1574 		sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1575 
1576 		vaddr += page_size;
1577 		paddr += page_size;
1578 	}
1579 }
1580 
1581 /*
1582  * Address VM Physical to Host Virtual
1583  *
1584  * Input Args:
1585  *   vm - Virtual Machine
1586  *   gpa - VM physical address
1587  *
1588  * Output Args: None
1589  *
1590  * Return:
1591  *   Equivalent host virtual address
1592  *
1593  * Locates the memory region containing the VM physical address given
1594  * by gpa, within the VM given by vm.  When found, the host virtual
1595  * address providing the memory to the vm physical address is returned.
1596  * A TEST_ASSERT failure occurs if no region containing gpa exists.
1597  */
addr_gpa2hva(struct kvm_vm * vm,vm_paddr_t gpa)1598 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
1599 {
1600 	struct userspace_mem_region *region;
1601 
1602 	gpa = vm_untag_gpa(vm, gpa);
1603 
1604 	region = userspace_mem_region_find(vm, gpa, gpa);
1605 	if (!region) {
1606 		TEST_FAIL("No vm physical memory at 0x%lx", gpa);
1607 		return NULL;
1608 	}
1609 
1610 	return (void *)((uintptr_t)region->host_mem
1611 		+ (gpa - region->region.guest_phys_addr));
1612 }
1613 
1614 /*
1615  * Address Host Virtual to VM Physical
1616  *
1617  * Input Args:
1618  *   vm - Virtual Machine
1619  *   hva - Host virtual address
1620  *
1621  * Output Args: None
1622  *
1623  * Return:
1624  *   Equivalent VM physical address
1625  *
1626  * Locates the memory region containing the host virtual address given
1627  * by hva, within the VM given by vm.  When found, the equivalent
1628  * VM physical address is returned. A TEST_ASSERT failure occurs if no
1629  * region containing hva exists.
1630  */
addr_hva2gpa(struct kvm_vm * vm,void * hva)1631 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1632 {
1633 	struct rb_node *node;
1634 
1635 	for (node = vm->regions.hva_tree.rb_node; node; ) {
1636 		struct userspace_mem_region *region =
1637 			container_of(node, struct userspace_mem_region, hva_node);
1638 
1639 		if (hva >= region->host_mem) {
1640 			if (hva <= (region->host_mem
1641 				+ region->region.memory_size - 1))
1642 				return (vm_paddr_t)((uintptr_t)
1643 					region->region.guest_phys_addr
1644 					+ (hva - (uintptr_t)region->host_mem));
1645 
1646 			node = node->rb_right;
1647 		} else
1648 			node = node->rb_left;
1649 	}
1650 
1651 	TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);
1652 	return -1;
1653 }
1654 
1655 /*
1656  * Address VM physical to Host Virtual *alias*.
1657  *
1658  * Input Args:
1659  *   vm - Virtual Machine
1660  *   gpa - VM physical address
1661  *
1662  * Output Args: None
1663  *
1664  * Return:
1665  *   Equivalent address within the host virtual *alias* area, or NULL
1666  *   (without failing the test) if the guest memory is not shared (so
1667  *   no alias exists).
1668  *
1669  * Create a writable, shared virtual=>physical alias for the specific GPA.
1670  * The primary use case is to allow the host selftest to manipulate guest
1671  * memory without mapping said memory in the guest's address space. And, for
1672  * userfaultfd-based demand paging, to do so without triggering userfaults.
1673  */
addr_gpa2alias(struct kvm_vm * vm,vm_paddr_t gpa)1674 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa)
1675 {
1676 	struct userspace_mem_region *region;
1677 	uintptr_t offset;
1678 
1679 	region = userspace_mem_region_find(vm, gpa, gpa);
1680 	if (!region)
1681 		return NULL;
1682 
1683 	if (!region->host_alias)
1684 		return NULL;
1685 
1686 	offset = gpa - region->region.guest_phys_addr;
1687 	return (void *) ((uintptr_t) region->host_alias + offset);
1688 }
1689 
1690 /* Create an interrupt controller chip for the specified VM. */
vm_create_irqchip(struct kvm_vm * vm)1691 void vm_create_irqchip(struct kvm_vm *vm)
1692 {
1693 	int r;
1694 
1695 	/*
1696 	 * Allocate a fully in-kernel IRQ chip by default, but fall back to a
1697 	 * split model (x86 only) if that fails (KVM x86 allows compiling out
1698 	 * support for KVM_CREATE_IRQCHIP).
1699 	 */
1700 	r = __vm_ioctl(vm, KVM_CREATE_IRQCHIP, NULL);
1701 	if (r && errno == ENOTTY && kvm_has_cap(KVM_CAP_SPLIT_IRQCHIP))
1702 		vm_enable_cap(vm, KVM_CAP_SPLIT_IRQCHIP, 24);
1703 	else
1704 		TEST_ASSERT_VM_VCPU_IOCTL(!r, KVM_CREATE_IRQCHIP, r, vm);
1705 
1706 	vm->has_irqchip = true;
1707 }
1708 
_vcpu_run(struct kvm_vcpu * vcpu)1709 int _vcpu_run(struct kvm_vcpu *vcpu)
1710 {
1711 	int rc;
1712 
1713 	do {
1714 		rc = __vcpu_run(vcpu);
1715 	} while (rc == -1 && errno == EINTR);
1716 
1717 	if (!rc)
1718 		assert_on_unhandled_exception(vcpu);
1719 
1720 	return rc;
1721 }
1722 
1723 /*
1724  * Invoke KVM_RUN on a vCPU until KVM returns something other than -EINTR.
1725  * Assert if the KVM returns an error (other than -EINTR).
1726  */
vcpu_run(struct kvm_vcpu * vcpu)1727 void vcpu_run(struct kvm_vcpu *vcpu)
1728 {
1729 	int ret = _vcpu_run(vcpu);
1730 
1731 	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_RUN, ret));
1732 }
1733 
vcpu_run_complete_io(struct kvm_vcpu * vcpu)1734 void vcpu_run_complete_io(struct kvm_vcpu *vcpu)
1735 {
1736 	int ret;
1737 
1738 	vcpu->run->immediate_exit = 1;
1739 	ret = __vcpu_run(vcpu);
1740 	vcpu->run->immediate_exit = 0;
1741 
1742 	TEST_ASSERT(ret == -1 && errno == EINTR,
1743 		    "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1744 		    ret, errno);
1745 }
1746 
1747 /*
1748  * Get the list of guest registers which are supported for
1749  * KVM_GET_ONE_REG/KVM_SET_ONE_REG ioctls.  Returns a kvm_reg_list pointer,
1750  * it is the caller's responsibility to free the list.
1751  */
vcpu_get_reg_list(struct kvm_vcpu * vcpu)1752 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu)
1753 {
1754 	struct kvm_reg_list reg_list_n = { .n = 0 }, *reg_list;
1755 	int ret;
1756 
1757 	ret = __vcpu_ioctl(vcpu, KVM_GET_REG_LIST, &reg_list_n);
1758 	TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");
1759 
1760 	reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));
1761 	reg_list->n = reg_list_n.n;
1762 	vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list);
1763 	return reg_list;
1764 }
1765 
vcpu_map_dirty_ring(struct kvm_vcpu * vcpu)1766 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)
1767 {
1768 	uint32_t page_size = getpagesize();
1769 	uint32_t size = vcpu->vm->dirty_ring_size;
1770 
1771 	TEST_ASSERT(size > 0, "Should enable dirty ring first");
1772 
1773 	if (!vcpu->dirty_gfns) {
1774 		void *addr;
1775 
1776 		addr = mmap(NULL, size, PROT_READ, MAP_PRIVATE, vcpu->fd,
1777 			    page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1778 		TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");
1779 
1780 		addr = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_PRIVATE, vcpu->fd,
1781 			    page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1782 		TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");
1783 
1784 		addr = __kvm_mmap(size, PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd,
1785 				  page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1786 
1787 		vcpu->dirty_gfns = addr;
1788 		vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
1789 	}
1790 
1791 	return vcpu->dirty_gfns;
1792 }
1793 
1794 /*
1795  * Device Ioctl
1796  */
1797 
__kvm_has_device_attr(int dev_fd,uint32_t group,uint64_t attr)1798 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
1799 {
1800 	struct kvm_device_attr attribute = {
1801 		.group = group,
1802 		.attr = attr,
1803 		.flags = 0,
1804 	};
1805 
1806 	return ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute);
1807 }
1808 
__kvm_test_create_device(struct kvm_vm * vm,uint64_t type)1809 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type)
1810 {
1811 	struct kvm_create_device create_dev = {
1812 		.type = type,
1813 		.flags = KVM_CREATE_DEVICE_TEST,
1814 	};
1815 
1816 	return __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);
1817 }
1818 
__kvm_create_device(struct kvm_vm * vm,uint64_t type)1819 int __kvm_create_device(struct kvm_vm *vm, uint64_t type)
1820 {
1821 	struct kvm_create_device create_dev = {
1822 		.type = type,
1823 		.fd = -1,
1824 		.flags = 0,
1825 	};
1826 	int err;
1827 
1828 	err = __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);
1829 	TEST_ASSERT(err <= 0, "KVM_CREATE_DEVICE shouldn't return a positive value");
1830 	return err ? : create_dev.fd;
1831 }
1832 
__kvm_device_attr_get(int dev_fd,uint32_t group,uint64_t attr,void * val)1833 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val)
1834 {
1835 	struct kvm_device_attr kvmattr = {
1836 		.group = group,
1837 		.attr = attr,
1838 		.flags = 0,
1839 		.addr = (uintptr_t)val,
1840 	};
1841 
1842 	return __kvm_ioctl(dev_fd, KVM_GET_DEVICE_ATTR, &kvmattr);
1843 }
1844 
__kvm_device_attr_set(int dev_fd,uint32_t group,uint64_t attr,void * val)1845 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val)
1846 {
1847 	struct kvm_device_attr kvmattr = {
1848 		.group = group,
1849 		.attr = attr,
1850 		.flags = 0,
1851 		.addr = (uintptr_t)val,
1852 	};
1853 
1854 	return __kvm_ioctl(dev_fd, KVM_SET_DEVICE_ATTR, &kvmattr);
1855 }
1856 
1857 /*
1858  * IRQ related functions.
1859  */
1860 
_kvm_irq_line(struct kvm_vm * vm,uint32_t irq,int level)1861 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
1862 {
1863 	struct kvm_irq_level irq_level = {
1864 		.irq    = irq,
1865 		.level  = level,
1866 	};
1867 
1868 	return __vm_ioctl(vm, KVM_IRQ_LINE, &irq_level);
1869 }
1870 
kvm_irq_line(struct kvm_vm * vm,uint32_t irq,int level)1871 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
1872 {
1873 	int ret = _kvm_irq_line(vm, irq, level);
1874 
1875 	TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret));
1876 }
1877 
kvm_gsi_routing_create(void)1878 struct kvm_irq_routing *kvm_gsi_routing_create(void)
1879 {
1880 	struct kvm_irq_routing *routing;
1881 	size_t size;
1882 
1883 	size = sizeof(struct kvm_irq_routing);
1884 	/* Allocate space for the max number of entries: this wastes 196 KBs. */
1885 	size += KVM_MAX_IRQ_ROUTES * sizeof(struct kvm_irq_routing_entry);
1886 	routing = calloc(1, size);
1887 	assert(routing);
1888 
1889 	return routing;
1890 }
1891 
kvm_gsi_routing_irqchip_add(struct kvm_irq_routing * routing,uint32_t gsi,uint32_t pin)1892 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
1893 		uint32_t gsi, uint32_t pin)
1894 {
1895 	int i;
1896 
1897 	assert(routing);
1898 	assert(routing->nr < KVM_MAX_IRQ_ROUTES);
1899 
1900 	i = routing->nr;
1901 	routing->entries[i].gsi = gsi;
1902 	routing->entries[i].type = KVM_IRQ_ROUTING_IRQCHIP;
1903 	routing->entries[i].flags = 0;
1904 	routing->entries[i].u.irqchip.irqchip = 0;
1905 	routing->entries[i].u.irqchip.pin = pin;
1906 	routing->nr++;
1907 }
1908 
_kvm_gsi_routing_write(struct kvm_vm * vm,struct kvm_irq_routing * routing)1909 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
1910 {
1911 	int ret;
1912 
1913 	assert(routing);
1914 	ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing);
1915 	free(routing);
1916 
1917 	return ret;
1918 }
1919 
kvm_gsi_routing_write(struct kvm_vm * vm,struct kvm_irq_routing * routing)1920 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
1921 {
1922 	int ret;
1923 
1924 	ret = _kvm_gsi_routing_write(vm, routing);
1925 	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_GSI_ROUTING, ret));
1926 }
1927 
1928 /*
1929  * VM Dump
1930  *
1931  * Input Args:
1932  *   vm - Virtual Machine
1933  *   indent - Left margin indent amount
1934  *
1935  * Output Args:
1936  *   stream - Output FILE stream
1937  *
1938  * Return: None
1939  *
1940  * Dumps the current state of the VM given by vm, to the FILE stream
1941  * given by stream.
1942  */
vm_dump(FILE * stream,struct kvm_vm * vm,uint8_t indent)1943 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1944 {
1945 	int ctr;
1946 	struct userspace_mem_region *region;
1947 	struct kvm_vcpu *vcpu;
1948 
1949 	fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1950 	fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1951 	fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1952 	fprintf(stream, "%*sMem Regions:\n", indent, "");
1953 	hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {
1954 		fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1955 			"host_virt: %p\n", indent + 2, "",
1956 			(uint64_t) region->region.guest_phys_addr,
1957 			(uint64_t) region->region.memory_size,
1958 			region->host_mem);
1959 		fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1960 		sparsebit_dump(stream, region->unused_phy_pages, 0);
1961 		if (region->protected_phy_pages) {
1962 			fprintf(stream, "%*sprotected_phy_pages: ", indent + 2, "");
1963 			sparsebit_dump(stream, region->protected_phy_pages, 0);
1964 		}
1965 	}
1966 	fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1967 	sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1968 	fprintf(stream, "%*spgd_created: %u\n", indent, "",
1969 		vm->pgd_created);
1970 	if (vm->pgd_created) {
1971 		fprintf(stream, "%*sVirtual Translation Tables:\n",
1972 			indent + 2, "");
1973 		virt_dump(stream, vm, indent + 4);
1974 	}
1975 	fprintf(stream, "%*sVCPUs:\n", indent, "");
1976 
1977 	list_for_each_entry(vcpu, &vm->vcpus, list)
1978 		vcpu_dump(stream, vcpu, indent + 2);
1979 }
1980 
1981 #define KVM_EXIT_STRING(x) {KVM_EXIT_##x, #x}
1982 
1983 /* Known KVM exit reasons */
1984 static struct exit_reason {
1985 	unsigned int reason;
1986 	const char *name;
1987 } exit_reasons_known[] = {
1988 	KVM_EXIT_STRING(UNKNOWN),
1989 	KVM_EXIT_STRING(EXCEPTION),
1990 	KVM_EXIT_STRING(IO),
1991 	KVM_EXIT_STRING(HYPERCALL),
1992 	KVM_EXIT_STRING(DEBUG),
1993 	KVM_EXIT_STRING(HLT),
1994 	KVM_EXIT_STRING(MMIO),
1995 	KVM_EXIT_STRING(IRQ_WINDOW_OPEN),
1996 	KVM_EXIT_STRING(SHUTDOWN),
1997 	KVM_EXIT_STRING(FAIL_ENTRY),
1998 	KVM_EXIT_STRING(INTR),
1999 	KVM_EXIT_STRING(SET_TPR),
2000 	KVM_EXIT_STRING(TPR_ACCESS),
2001 	KVM_EXIT_STRING(S390_SIEIC),
2002 	KVM_EXIT_STRING(S390_RESET),
2003 	KVM_EXIT_STRING(DCR),
2004 	KVM_EXIT_STRING(NMI),
2005 	KVM_EXIT_STRING(INTERNAL_ERROR),
2006 	KVM_EXIT_STRING(OSI),
2007 	KVM_EXIT_STRING(PAPR_HCALL),
2008 	KVM_EXIT_STRING(S390_UCONTROL),
2009 	KVM_EXIT_STRING(WATCHDOG),
2010 	KVM_EXIT_STRING(S390_TSCH),
2011 	KVM_EXIT_STRING(EPR),
2012 	KVM_EXIT_STRING(SYSTEM_EVENT),
2013 	KVM_EXIT_STRING(S390_STSI),
2014 	KVM_EXIT_STRING(IOAPIC_EOI),
2015 	KVM_EXIT_STRING(HYPERV),
2016 	KVM_EXIT_STRING(ARM_NISV),
2017 	KVM_EXIT_STRING(X86_RDMSR),
2018 	KVM_EXIT_STRING(X86_WRMSR),
2019 	KVM_EXIT_STRING(DIRTY_RING_FULL),
2020 	KVM_EXIT_STRING(AP_RESET_HOLD),
2021 	KVM_EXIT_STRING(X86_BUS_LOCK),
2022 	KVM_EXIT_STRING(XEN),
2023 	KVM_EXIT_STRING(RISCV_SBI),
2024 	KVM_EXIT_STRING(RISCV_CSR),
2025 	KVM_EXIT_STRING(NOTIFY),
2026 	KVM_EXIT_STRING(LOONGARCH_IOCSR),
2027 	KVM_EXIT_STRING(MEMORY_FAULT),
2028 };
2029 
2030 /*
2031  * Exit Reason String
2032  *
2033  * Input Args:
2034  *   exit_reason - Exit reason
2035  *
2036  * Output Args: None
2037  *
2038  * Return:
2039  *   Constant string pointer describing the exit reason.
2040  *
2041  * Locates and returns a constant string that describes the KVM exit
2042  * reason given by exit_reason.  If no such string is found, a constant
2043  * string of "Unknown" is returned.
2044  */
exit_reason_str(unsigned int exit_reason)2045 const char *exit_reason_str(unsigned int exit_reason)
2046 {
2047 	unsigned int n1;
2048 
2049 	for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
2050 		if (exit_reason == exit_reasons_known[n1].reason)
2051 			return exit_reasons_known[n1].name;
2052 	}
2053 
2054 	return "Unknown";
2055 }
2056 
2057 /*
2058  * Physical Contiguous Page Allocator
2059  *
2060  * Input Args:
2061  *   vm - Virtual Machine
2062  *   num - number of pages
2063  *   paddr_min - Physical address minimum
2064  *   memslot - Memory region to allocate page from
2065  *   protected - True if the pages will be used as protected/private memory
2066  *
2067  * Output Args: None
2068  *
2069  * Return:
2070  *   Starting physical address
2071  *
2072  * Within the VM specified by vm, locates a range of available physical
2073  * pages at or above paddr_min. If found, the pages are marked as in use
2074  * and their base address is returned. A TEST_ASSERT failure occurs if
2075  * not enough pages are available at or above paddr_min.
2076  */
__vm_phy_pages_alloc(struct kvm_vm * vm,size_t num,vm_paddr_t paddr_min,uint32_t memslot,bool protected)2077 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
2078 				vm_paddr_t paddr_min, uint32_t memslot,
2079 				bool protected)
2080 {
2081 	struct userspace_mem_region *region;
2082 	sparsebit_idx_t pg, base;
2083 
2084 	TEST_ASSERT(num > 0, "Must allocate at least one page");
2085 
2086 	TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
2087 		"not divisible by page size.\n"
2088 		"  paddr_min: 0x%lx page_size: 0x%x",
2089 		paddr_min, vm->page_size);
2090 
2091 	region = memslot2region(vm, memslot);
2092 	TEST_ASSERT(!protected || region->protected_phy_pages,
2093 		    "Region doesn't support protected memory");
2094 
2095 	base = pg = paddr_min >> vm->page_shift;
2096 	do {
2097 		for (; pg < base + num; ++pg) {
2098 			if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
2099 				base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
2100 				break;
2101 			}
2102 		}
2103 	} while (pg && pg != base + num);
2104 
2105 	if (pg == 0) {
2106 		fprintf(stderr, "No guest physical page available, "
2107 			"paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
2108 			paddr_min, vm->page_size, memslot);
2109 		fputs("---- vm dump ----\n", stderr);
2110 		vm_dump(stderr, vm, 2);
2111 		abort();
2112 	}
2113 
2114 	for (pg = base; pg < base + num; ++pg) {
2115 		sparsebit_clear(region->unused_phy_pages, pg);
2116 		if (protected)
2117 			sparsebit_set(region->protected_phy_pages, pg);
2118 	}
2119 
2120 	return base * vm->page_size;
2121 }
2122 
vm_phy_page_alloc(struct kvm_vm * vm,vm_paddr_t paddr_min,uint32_t memslot)2123 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
2124 			     uint32_t memslot)
2125 {
2126 	return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
2127 }
2128 
vm_alloc_page_table(struct kvm_vm * vm)2129 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)
2130 {
2131 	return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,
2132 				 vm->memslots[MEM_REGION_PT]);
2133 }
2134 
2135 /*
2136  * Address Guest Virtual to Host Virtual
2137  *
2138  * Input Args:
2139  *   vm - Virtual Machine
2140  *   gva - VM virtual address
2141  *
2142  * Output Args: None
2143  *
2144  * Return:
2145  *   Equivalent host virtual address
2146  */
addr_gva2hva(struct kvm_vm * vm,vm_vaddr_t gva)2147 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
2148 {
2149 	return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
2150 }
2151 
vm_compute_max_gfn(struct kvm_vm * vm)2152 unsigned long __weak vm_compute_max_gfn(struct kvm_vm *vm)
2153 {
2154 	return ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
2155 }
2156 
vm_calc_num_pages(unsigned int num_pages,unsigned int page_shift,unsigned int new_page_shift,bool ceil)2157 static unsigned int vm_calc_num_pages(unsigned int num_pages,
2158 				      unsigned int page_shift,
2159 				      unsigned int new_page_shift,
2160 				      bool ceil)
2161 {
2162 	unsigned int n = 1 << (new_page_shift - page_shift);
2163 
2164 	if (page_shift >= new_page_shift)
2165 		return num_pages * (1 << (page_shift - new_page_shift));
2166 
2167 	return num_pages / n + !!(ceil && num_pages % n);
2168 }
2169 
getpageshift(void)2170 static inline int getpageshift(void)
2171 {
2172 	return __builtin_ffs(getpagesize()) - 1;
2173 }
2174 
2175 unsigned int
vm_num_host_pages(enum vm_guest_mode mode,unsigned int num_guest_pages)2176 vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
2177 {
2178 	return vm_calc_num_pages(num_guest_pages,
2179 				 vm_guest_mode_params[mode].page_shift,
2180 				 getpageshift(), true);
2181 }
2182 
2183 unsigned int
vm_num_guest_pages(enum vm_guest_mode mode,unsigned int num_host_pages)2184 vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages)
2185 {
2186 	return vm_calc_num_pages(num_host_pages, getpageshift(),
2187 				 vm_guest_mode_params[mode].page_shift, false);
2188 }
2189 
vm_calc_num_guest_pages(enum vm_guest_mode mode,size_t size)2190 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size)
2191 {
2192 	unsigned int n;
2193 	n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size);
2194 	return vm_adjust_num_guest_pages(mode, n);
2195 }
2196 
2197 /*
2198  * Read binary stats descriptors
2199  *
2200  * Input Args:
2201  *   stats_fd - the file descriptor for the binary stats file from which to read
2202  *   header - the binary stats metadata header corresponding to the given FD
2203  *
2204  * Output Args: None
2205  *
2206  * Return:
2207  *   A pointer to a newly allocated series of stat descriptors.
2208  *   Caller is responsible for freeing the returned kvm_stats_desc.
2209  *
2210  * Read the stats descriptors from the binary stats interface.
2211  */
read_stats_descriptors(int stats_fd,struct kvm_stats_header * header)2212 struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
2213 					      struct kvm_stats_header *header)
2214 {
2215 	struct kvm_stats_desc *stats_desc;
2216 	ssize_t desc_size, total_size, ret;
2217 
2218 	desc_size = get_stats_descriptor_size(header);
2219 	total_size = header->num_desc * desc_size;
2220 
2221 	stats_desc = calloc(header->num_desc, desc_size);
2222 	TEST_ASSERT(stats_desc, "Allocate memory for stats descriptors");
2223 
2224 	ret = pread(stats_fd, stats_desc, total_size, header->desc_offset);
2225 	TEST_ASSERT(ret == total_size, "Read KVM stats descriptors");
2226 
2227 	return stats_desc;
2228 }
2229 
2230 /*
2231  * Read stat data for a particular stat
2232  *
2233  * Input Args:
2234  *   stats_fd - the file descriptor for the binary stats file from which to read
2235  *   header - the binary stats metadata header corresponding to the given FD
2236  *   desc - the binary stat metadata for the particular stat to be read
2237  *   max_elements - the maximum number of 8-byte values to read into data
2238  *
2239  * Output Args:
2240  *   data - the buffer into which stat data should be read
2241  *
2242  * Read the data values of a specified stat from the binary stats interface.
2243  */
read_stat_data(int stats_fd,struct kvm_stats_header * header,struct kvm_stats_desc * desc,uint64_t * data,size_t max_elements)2244 void read_stat_data(int stats_fd, struct kvm_stats_header *header,
2245 		    struct kvm_stats_desc *desc, uint64_t *data,
2246 		    size_t max_elements)
2247 {
2248 	size_t nr_elements = min_t(ssize_t, desc->size, max_elements);
2249 	size_t size = nr_elements * sizeof(*data);
2250 	ssize_t ret;
2251 
2252 	TEST_ASSERT(desc->size, "No elements in stat '%s'", desc->name);
2253 	TEST_ASSERT(max_elements, "Zero elements requested for stat '%s'", desc->name);
2254 
2255 	ret = pread(stats_fd, data, size,
2256 		    header->data_offset + desc->offset);
2257 
2258 	TEST_ASSERT(ret >= 0, "pread() failed on stat '%s', errno: %i (%s)",
2259 		    desc->name, errno, strerror(errno));
2260 	TEST_ASSERT(ret == size,
2261 		    "pread() on stat '%s' read %ld bytes, wanted %lu bytes",
2262 		    desc->name, size, ret);
2263 }
2264 
kvm_get_stat(struct kvm_binary_stats * stats,const char * name,uint64_t * data,size_t max_elements)2265 void kvm_get_stat(struct kvm_binary_stats *stats, const char *name,
2266 		  uint64_t *data, size_t max_elements)
2267 {
2268 	struct kvm_stats_desc *desc;
2269 	size_t size_desc;
2270 	int i;
2271 
2272 	if (!stats->desc) {
2273 		read_stats_header(stats->fd, &stats->header);
2274 		stats->desc = read_stats_descriptors(stats->fd, &stats->header);
2275 	}
2276 
2277 	size_desc = get_stats_descriptor_size(&stats->header);
2278 
2279 	for (i = 0; i < stats->header.num_desc; ++i) {
2280 		desc = (void *)stats->desc + (i * size_desc);
2281 
2282 		if (strcmp(desc->name, name))
2283 			continue;
2284 
2285 		read_stat_data(stats->fd, &stats->header, desc, data, max_elements);
2286 		return;
2287 	}
2288 
2289 	TEST_FAIL("Unable to find stat '%s'", name);
2290 }
2291 
kvm_arch_vm_post_create(struct kvm_vm * vm,unsigned int nr_vcpus)2292 __weak void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus)
2293 {
2294 }
2295 
kvm_arch_vm_finalize_vcpus(struct kvm_vm * vm)2296 __weak void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm)
2297 {
2298 }
2299 
kvm_arch_vm_release(struct kvm_vm * vm)2300 __weak void kvm_arch_vm_release(struct kvm_vm *vm)
2301 {
2302 }
2303 
kvm_selftest_arch_init(void)2304 __weak void kvm_selftest_arch_init(void)
2305 {
2306 }
2307 
kvm_selftest_init(void)2308 void __attribute((constructor)) kvm_selftest_init(void)
2309 {
2310 	/* Tell stdout not to buffer its content. */
2311 	setbuf(stdout, NULL);
2312 
2313 	guest_random_seed = last_guest_seed = random();
2314 	pr_info("Random seed: 0x%x\n", guest_random_seed);
2315 
2316 	kvm_selftest_arch_init();
2317 }
2318 
vm_is_gpa_protected(struct kvm_vm * vm,vm_paddr_t paddr)2319 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr)
2320 {
2321 	sparsebit_idx_t pg = 0;
2322 	struct userspace_mem_region *region;
2323 
2324 	if (!vm_arch_has_protected_memory(vm))
2325 		return false;
2326 
2327 	region = userspace_mem_region_find(vm, paddr, paddr);
2328 	TEST_ASSERT(region, "No vm physical memory at 0x%lx", paddr);
2329 
2330 	pg = paddr >> vm->page_shift;
2331 	return sparsebit_is_set(region->protected_phy_pages, pg);
2332 }
2333 
kvm_arch_has_default_irqchip(void)2334 __weak bool kvm_arch_has_default_irqchip(void)
2335 {
2336 	return false;
2337 }
2338