xref: /linux/tools/testing/selftests/kvm/lib/kvm_util.c (revision 3efc57369a0ce8f76bf0804f7e673982384e4ac9)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * tools/testing/selftests/kvm/lib/kvm_util.c
4  *
5  * Copyright (C) 2018, Google LLC.
6  */
7 #include "test_util.h"
8 #include "kvm_util.h"
9 #include "processor.h"
10 #include "ucall_common.h"
11 
12 #include <assert.h>
13 #include <sched.h>
14 #include <sys/mman.h>
15 #include <sys/types.h>
16 #include <sys/stat.h>
17 #include <unistd.h>
18 #include <linux/kernel.h>
19 
20 #define KVM_UTIL_MIN_PFN	2
21 
22 uint32_t guest_random_seed;
23 struct guest_random_state guest_rng;
24 static uint32_t last_guest_seed;
25 
26 static int vcpu_mmap_sz(void);
27 
open_path_or_exit(const char * path,int flags)28 int open_path_or_exit(const char *path, int flags)
29 {
30 	int fd;
31 
32 	fd = open(path, flags);
33 	__TEST_REQUIRE(fd >= 0 || errno != ENOENT, "Cannot open %s: %s", path, strerror(errno));
34 	TEST_ASSERT(fd >= 0, "Failed to open '%s'", path);
35 
36 	return fd;
37 }
38 
39 /*
40  * Open KVM_DEV_PATH if available, otherwise exit the entire program.
41  *
42  * Input Args:
43  *   flags - The flags to pass when opening KVM_DEV_PATH.
44  *
45  * Return:
46  *   The opened file descriptor of /dev/kvm.
47  */
_open_kvm_dev_path_or_exit(int flags)48 static int _open_kvm_dev_path_or_exit(int flags)
49 {
50 	return open_path_or_exit(KVM_DEV_PATH, flags);
51 }
52 
open_kvm_dev_path_or_exit(void)53 int open_kvm_dev_path_or_exit(void)
54 {
55 	return _open_kvm_dev_path_or_exit(O_RDONLY);
56 }
57 
get_module_param(const char * module_name,const char * param,void * buffer,size_t buffer_size)58 static ssize_t get_module_param(const char *module_name, const char *param,
59 				void *buffer, size_t buffer_size)
60 {
61 	const int path_size = 128;
62 	char path[path_size];
63 	ssize_t bytes_read;
64 	int fd, r;
65 
66 	r = snprintf(path, path_size, "/sys/module/%s/parameters/%s",
67 		     module_name, param);
68 	TEST_ASSERT(r < path_size,
69 		    "Failed to construct sysfs path in %d bytes.", path_size);
70 
71 	fd = open_path_or_exit(path, O_RDONLY);
72 
73 	bytes_read = read(fd, buffer, buffer_size);
74 	TEST_ASSERT(bytes_read > 0, "read(%s) returned %ld, wanted %ld bytes",
75 		    path, bytes_read, buffer_size);
76 
77 	r = close(fd);
78 	TEST_ASSERT(!r, "close(%s) failed", path);
79 	return bytes_read;
80 }
81 
get_module_param_integer(const char * module_name,const char * param)82 static int get_module_param_integer(const char *module_name, const char *param)
83 {
84 	/*
85 	 * 16 bytes to hold a 64-bit value (1 byte per char), 1 byte for the
86 	 * NUL char, and 1 byte because the kernel sucks and inserts a newline
87 	 * at the end.
88 	 */
89 	char value[16 + 1 + 1];
90 	ssize_t r;
91 
92 	memset(value, '\0', sizeof(value));
93 
94 	r = get_module_param(module_name, param, value, sizeof(value));
95 	TEST_ASSERT(value[r - 1] == '\n',
96 		    "Expected trailing newline, got char '%c'", value[r - 1]);
97 
98 	/*
99 	 * Squash the newline, otherwise atoi_paranoid() will complain about
100 	 * trailing non-NUL characters in the string.
101 	 */
102 	value[r - 1] = '\0';
103 	return atoi_paranoid(value);
104 }
105 
get_module_param_bool(const char * module_name,const char * param)106 static bool get_module_param_bool(const char *module_name, const char *param)
107 {
108 	char value;
109 	ssize_t r;
110 
111 	r = get_module_param(module_name, param, &value, sizeof(value));
112 	TEST_ASSERT_EQ(r, 1);
113 
114 	if (value == 'Y')
115 		return true;
116 	else if (value == 'N')
117 		return false;
118 
119 	TEST_FAIL("Unrecognized value '%c' for boolean module param", value);
120 }
121 
get_kvm_param_bool(const char * param)122 bool get_kvm_param_bool(const char *param)
123 {
124 	return get_module_param_bool("kvm", param);
125 }
126 
get_kvm_intel_param_bool(const char * param)127 bool get_kvm_intel_param_bool(const char *param)
128 {
129 	return get_module_param_bool("kvm_intel", param);
130 }
131 
get_kvm_amd_param_bool(const char * param)132 bool get_kvm_amd_param_bool(const char *param)
133 {
134 	return get_module_param_bool("kvm_amd", param);
135 }
136 
get_kvm_param_integer(const char * param)137 int get_kvm_param_integer(const char *param)
138 {
139 	return get_module_param_integer("kvm", param);
140 }
141 
get_kvm_intel_param_integer(const char * param)142 int get_kvm_intel_param_integer(const char *param)
143 {
144 	return get_module_param_integer("kvm_intel", param);
145 }
146 
get_kvm_amd_param_integer(const char * param)147 int get_kvm_amd_param_integer(const char *param)
148 {
149 	return get_module_param_integer("kvm_amd", param);
150 }
151 
152 /*
153  * Capability
154  *
155  * Input Args:
156  *   cap - Capability
157  *
158  * Output Args: None
159  *
160  * Return:
161  *   On success, the Value corresponding to the capability (KVM_CAP_*)
162  *   specified by the value of cap.  On failure a TEST_ASSERT failure
163  *   is produced.
164  *
165  * Looks up and returns the value corresponding to the capability
166  * (KVM_CAP_*) given by cap.
167  */
kvm_check_cap(long cap)168 unsigned int kvm_check_cap(long cap)
169 {
170 	int ret;
171 	int kvm_fd;
172 
173 	kvm_fd = open_kvm_dev_path_or_exit();
174 	ret = __kvm_ioctl(kvm_fd, KVM_CHECK_EXTENSION, (void *)cap);
175 	TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_CHECK_EXTENSION, ret));
176 
177 	close(kvm_fd);
178 
179 	return (unsigned int)ret;
180 }
181 
vm_enable_dirty_ring(struct kvm_vm * vm,uint32_t ring_size)182 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
183 {
184 	if (vm_check_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL))
185 		vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL, ring_size);
186 	else
187 		vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size);
188 	vm->dirty_ring_size = ring_size;
189 }
190 
vm_open(struct kvm_vm * vm)191 static void vm_open(struct kvm_vm *vm)
192 {
193 	vm->kvm_fd = _open_kvm_dev_path_or_exit(O_RDWR);
194 
195 	TEST_REQUIRE(kvm_has_cap(KVM_CAP_IMMEDIATE_EXIT));
196 
197 	vm->fd = __kvm_ioctl(vm->kvm_fd, KVM_CREATE_VM, (void *)vm->type);
198 	TEST_ASSERT(vm->fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, vm->fd));
199 }
200 
vm_guest_mode_string(uint32_t i)201 const char *vm_guest_mode_string(uint32_t i)
202 {
203 	static const char * const strings[] = {
204 		[VM_MODE_P52V48_4K]	= "PA-bits:52,  VA-bits:48,  4K pages",
205 		[VM_MODE_P52V48_16K]	= "PA-bits:52,  VA-bits:48, 16K pages",
206 		[VM_MODE_P52V48_64K]	= "PA-bits:52,  VA-bits:48, 64K pages",
207 		[VM_MODE_P48V48_4K]	= "PA-bits:48,  VA-bits:48,  4K pages",
208 		[VM_MODE_P48V48_16K]	= "PA-bits:48,  VA-bits:48, 16K pages",
209 		[VM_MODE_P48V48_64K]	= "PA-bits:48,  VA-bits:48, 64K pages",
210 		[VM_MODE_P40V48_4K]	= "PA-bits:40,  VA-bits:48,  4K pages",
211 		[VM_MODE_P40V48_16K]	= "PA-bits:40,  VA-bits:48, 16K pages",
212 		[VM_MODE_P40V48_64K]	= "PA-bits:40,  VA-bits:48, 64K pages",
213 		[VM_MODE_PXXV48_4K]	= "PA-bits:ANY, VA-bits:48,  4K pages",
214 		[VM_MODE_P47V64_4K]	= "PA-bits:47,  VA-bits:64,  4K pages",
215 		[VM_MODE_P44V64_4K]	= "PA-bits:44,  VA-bits:64,  4K pages",
216 		[VM_MODE_P36V48_4K]	= "PA-bits:36,  VA-bits:48,  4K pages",
217 		[VM_MODE_P36V48_16K]	= "PA-bits:36,  VA-bits:48, 16K pages",
218 		[VM_MODE_P36V48_64K]	= "PA-bits:36,  VA-bits:48, 64K pages",
219 		[VM_MODE_P36V47_16K]	= "PA-bits:36,  VA-bits:47, 16K pages",
220 	};
221 	_Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,
222 		       "Missing new mode strings?");
223 
224 	TEST_ASSERT(i < NUM_VM_MODES, "Guest mode ID %d too big", i);
225 
226 	return strings[i];
227 }
228 
229 const struct vm_guest_mode_params vm_guest_mode_params[] = {
230 	[VM_MODE_P52V48_4K]	= { 52, 48,  0x1000, 12 },
231 	[VM_MODE_P52V48_16K]	= { 52, 48,  0x4000, 14 },
232 	[VM_MODE_P52V48_64K]	= { 52, 48, 0x10000, 16 },
233 	[VM_MODE_P48V48_4K]	= { 48, 48,  0x1000, 12 },
234 	[VM_MODE_P48V48_16K]	= { 48, 48,  0x4000, 14 },
235 	[VM_MODE_P48V48_64K]	= { 48, 48, 0x10000, 16 },
236 	[VM_MODE_P40V48_4K]	= { 40, 48,  0x1000, 12 },
237 	[VM_MODE_P40V48_16K]	= { 40, 48,  0x4000, 14 },
238 	[VM_MODE_P40V48_64K]	= { 40, 48, 0x10000, 16 },
239 	[VM_MODE_PXXV48_4K]	= {  0,  0,  0x1000, 12 },
240 	[VM_MODE_P47V64_4K]	= { 47, 64,  0x1000, 12 },
241 	[VM_MODE_P44V64_4K]	= { 44, 64,  0x1000, 12 },
242 	[VM_MODE_P36V48_4K]	= { 36, 48,  0x1000, 12 },
243 	[VM_MODE_P36V48_16K]	= { 36, 48,  0x4000, 14 },
244 	[VM_MODE_P36V48_64K]	= { 36, 48, 0x10000, 16 },
245 	[VM_MODE_P36V47_16K]	= { 36, 47,  0x4000, 14 },
246 };
247 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
248 	       "Missing new mode params?");
249 
250 /*
251  * Initializes vm->vpages_valid to match the canonical VA space of the
252  * architecture.
253  *
254  * The default implementation is valid for architectures which split the
255  * range addressed by a single page table into a low and high region
256  * based on the MSB of the VA. On architectures with this behavior
257  * the VA region spans [0, 2^(va_bits - 1)), [-(2^(va_bits - 1), -1].
258  */
vm_vaddr_populate_bitmap(struct kvm_vm * vm)259 __weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm)
260 {
261 	sparsebit_set_num(vm->vpages_valid,
262 		0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
263 	sparsebit_set_num(vm->vpages_valid,
264 		(~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
265 		(1ULL << (vm->va_bits - 1)) >> vm->page_shift);
266 }
267 
____vm_create(struct vm_shape shape)268 struct kvm_vm *____vm_create(struct vm_shape shape)
269 {
270 	struct kvm_vm *vm;
271 
272 	vm = calloc(1, sizeof(*vm));
273 	TEST_ASSERT(vm != NULL, "Insufficient Memory");
274 
275 	INIT_LIST_HEAD(&vm->vcpus);
276 	vm->regions.gpa_tree = RB_ROOT;
277 	vm->regions.hva_tree = RB_ROOT;
278 	hash_init(vm->regions.slot_hash);
279 
280 	vm->mode = shape.mode;
281 	vm->type = shape.type;
282 
283 	vm->pa_bits = vm_guest_mode_params[vm->mode].pa_bits;
284 	vm->va_bits = vm_guest_mode_params[vm->mode].va_bits;
285 	vm->page_size = vm_guest_mode_params[vm->mode].page_size;
286 	vm->page_shift = vm_guest_mode_params[vm->mode].page_shift;
287 
288 	/* Setup mode specific traits. */
289 	switch (vm->mode) {
290 	case VM_MODE_P52V48_4K:
291 		vm->pgtable_levels = 4;
292 		break;
293 	case VM_MODE_P52V48_64K:
294 		vm->pgtable_levels = 3;
295 		break;
296 	case VM_MODE_P48V48_4K:
297 		vm->pgtable_levels = 4;
298 		break;
299 	case VM_MODE_P48V48_64K:
300 		vm->pgtable_levels = 3;
301 		break;
302 	case VM_MODE_P40V48_4K:
303 	case VM_MODE_P36V48_4K:
304 		vm->pgtable_levels = 4;
305 		break;
306 	case VM_MODE_P40V48_64K:
307 	case VM_MODE_P36V48_64K:
308 		vm->pgtable_levels = 3;
309 		break;
310 	case VM_MODE_P52V48_16K:
311 	case VM_MODE_P48V48_16K:
312 	case VM_MODE_P40V48_16K:
313 	case VM_MODE_P36V48_16K:
314 		vm->pgtable_levels = 4;
315 		break;
316 	case VM_MODE_P36V47_16K:
317 		vm->pgtable_levels = 3;
318 		break;
319 	case VM_MODE_PXXV48_4K:
320 #ifdef __x86_64__
321 		kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);
322 		kvm_init_vm_address_properties(vm);
323 		/*
324 		 * Ignore KVM support for 5-level paging (vm->va_bits == 57),
325 		 * it doesn't take effect unless a CR4.LA57 is set, which it
326 		 * isn't for this mode (48-bit virtual address space).
327 		 */
328 		TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,
329 			    "Linear address width (%d bits) not supported",
330 			    vm->va_bits);
331 		pr_debug("Guest physical address width detected: %d\n",
332 			 vm->pa_bits);
333 		vm->pgtable_levels = 4;
334 		vm->va_bits = 48;
335 #else
336 		TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");
337 #endif
338 		break;
339 	case VM_MODE_P47V64_4K:
340 		vm->pgtable_levels = 5;
341 		break;
342 	case VM_MODE_P44V64_4K:
343 		vm->pgtable_levels = 5;
344 		break;
345 	default:
346 		TEST_FAIL("Unknown guest mode: 0x%x", vm->mode);
347 	}
348 
349 #ifdef __aarch64__
350 	TEST_ASSERT(!vm->type, "ARM doesn't support test-provided types");
351 	if (vm->pa_bits != 40)
352 		vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
353 #endif
354 
355 	vm_open(vm);
356 
357 	/* Limit to VA-bit canonical virtual addresses. */
358 	vm->vpages_valid = sparsebit_alloc();
359 	vm_vaddr_populate_bitmap(vm);
360 
361 	/* Limit physical addresses to PA-bits. */
362 	vm->max_gfn = vm_compute_max_gfn(vm);
363 
364 	/* Allocate and setup memory for guest. */
365 	vm->vpages_mapped = sparsebit_alloc();
366 
367 	return vm;
368 }
369 
vm_nr_pages_required(enum vm_guest_mode mode,uint32_t nr_runnable_vcpus,uint64_t extra_mem_pages)370 static uint64_t vm_nr_pages_required(enum vm_guest_mode mode,
371 				     uint32_t nr_runnable_vcpus,
372 				     uint64_t extra_mem_pages)
373 {
374 	uint64_t page_size = vm_guest_mode_params[mode].page_size;
375 	uint64_t nr_pages;
376 
377 	TEST_ASSERT(nr_runnable_vcpus,
378 		    "Use vm_create_barebones() for VMs that _never_ have vCPUs");
379 
380 	TEST_ASSERT(nr_runnable_vcpus <= kvm_check_cap(KVM_CAP_MAX_VCPUS),
381 		    "nr_vcpus = %d too large for host, max-vcpus = %d",
382 		    nr_runnable_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS));
383 
384 	/*
385 	 * Arbitrarily allocate 512 pages (2mb when page size is 4kb) for the
386 	 * test code and other per-VM assets that will be loaded into memslot0.
387 	 */
388 	nr_pages = 512;
389 
390 	/* Account for the per-vCPU stacks on behalf of the test. */
391 	nr_pages += nr_runnable_vcpus * DEFAULT_STACK_PGS;
392 
393 	/*
394 	 * Account for the number of pages needed for the page tables.  The
395 	 * maximum page table size for a memory region will be when the
396 	 * smallest page size is used. Considering each page contains x page
397 	 * table descriptors, the total extra size for page tables (for extra
398 	 * N pages) will be: N/x+N/x^2+N/x^3+... which is definitely smaller
399 	 * than N/x*2.
400 	 */
401 	nr_pages += (nr_pages + extra_mem_pages) / PTES_PER_MIN_PAGE * 2;
402 
403 	/* Account for the number of pages needed by ucall. */
404 	nr_pages += ucall_nr_pages_required(page_size);
405 
406 	return vm_adjust_num_guest_pages(mode, nr_pages);
407 }
408 
__vm_create(struct vm_shape shape,uint32_t nr_runnable_vcpus,uint64_t nr_extra_pages)409 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
410 			   uint64_t nr_extra_pages)
411 {
412 	uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,
413 						 nr_extra_pages);
414 	struct userspace_mem_region *slot0;
415 	struct kvm_vm *vm;
416 	int i;
417 
418 	pr_debug("%s: mode='%s' type='%d', pages='%ld'\n", __func__,
419 		 vm_guest_mode_string(shape.mode), shape.type, nr_pages);
420 
421 	vm = ____vm_create(shape);
422 
423 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, 0);
424 	for (i = 0; i < NR_MEM_REGIONS; i++)
425 		vm->memslots[i] = 0;
426 
427 	kvm_vm_elf_load(vm, program_invocation_name);
428 
429 	/*
430 	 * TODO: Add proper defines to protect the library's memslots, and then
431 	 * carve out memslot1 for the ucall MMIO address.  KVM treats writes to
432 	 * read-only memslots as MMIO, and creating a read-only memslot for the
433 	 * MMIO region would prevent silently clobbering the MMIO region.
434 	 */
435 	slot0 = memslot2region(vm, 0);
436 	ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);
437 
438 	if (guest_random_seed != last_guest_seed) {
439 		pr_info("Random seed: 0x%x\n", guest_random_seed);
440 		last_guest_seed = guest_random_seed;
441 	}
442 	guest_rng = new_guest_random_state(guest_random_seed);
443 	sync_global_to_guest(vm, guest_rng);
444 
445 	kvm_arch_vm_post_create(vm);
446 
447 	return vm;
448 }
449 
450 /*
451  * VM Create with customized parameters
452  *
453  * Input Args:
454  *   mode - VM Mode (e.g. VM_MODE_P52V48_4K)
455  *   nr_vcpus - VCPU count
456  *   extra_mem_pages - Non-slot0 physical memory total size
457  *   guest_code - Guest entry point
458  *   vcpuids - VCPU IDs
459  *
460  * Output Args: None
461  *
462  * Return:
463  *   Pointer to opaque structure that describes the created VM.
464  *
465  * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
466  * extra_mem_pages is only used to calculate the maximum page table size,
467  * no real memory allocation for non-slot0 memory in this function.
468  */
__vm_create_with_vcpus(struct vm_shape shape,uint32_t nr_vcpus,uint64_t extra_mem_pages,void * guest_code,struct kvm_vcpu * vcpus[])469 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
470 				      uint64_t extra_mem_pages,
471 				      void *guest_code, struct kvm_vcpu *vcpus[])
472 {
473 	struct kvm_vm *vm;
474 	int i;
475 
476 	TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array");
477 
478 	vm = __vm_create(shape, nr_vcpus, extra_mem_pages);
479 
480 	for (i = 0; i < nr_vcpus; ++i)
481 		vcpus[i] = vm_vcpu_add(vm, i, guest_code);
482 
483 	return vm;
484 }
485 
__vm_create_shape_with_one_vcpu(struct vm_shape shape,struct kvm_vcpu ** vcpu,uint64_t extra_mem_pages,void * guest_code)486 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
487 					       struct kvm_vcpu **vcpu,
488 					       uint64_t extra_mem_pages,
489 					       void *guest_code)
490 {
491 	struct kvm_vcpu *vcpus[1];
492 	struct kvm_vm *vm;
493 
494 	vm = __vm_create_with_vcpus(shape, 1, extra_mem_pages, guest_code, vcpus);
495 
496 	*vcpu = vcpus[0];
497 	return vm;
498 }
499 
500 /*
501  * VM Restart
502  *
503  * Input Args:
504  *   vm - VM that has been released before
505  *
506  * Output Args: None
507  *
508  * Reopens the file descriptors associated to the VM and reinstates the
509  * global state, such as the irqchip and the memory regions that are mapped
510  * into the guest.
511  */
kvm_vm_restart(struct kvm_vm * vmp)512 void kvm_vm_restart(struct kvm_vm *vmp)
513 {
514 	int ctr;
515 	struct userspace_mem_region *region;
516 
517 	vm_open(vmp);
518 	if (vmp->has_irqchip)
519 		vm_create_irqchip(vmp);
520 
521 	hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {
522 		int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION2, &region->region);
523 
524 		TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
525 			    "  rc: %i errno: %i\n"
526 			    "  slot: %u flags: 0x%x\n"
527 			    "  guest_phys_addr: 0x%llx size: 0x%llx",
528 			    ret, errno, region->region.slot,
529 			    region->region.flags,
530 			    region->region.guest_phys_addr,
531 			    region->region.memory_size);
532 	}
533 }
534 
vm_arch_vcpu_recreate(struct kvm_vm * vm,uint32_t vcpu_id)535 __weak struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm,
536 					      uint32_t vcpu_id)
537 {
538 	return __vm_vcpu_add(vm, vcpu_id);
539 }
540 
vm_recreate_with_one_vcpu(struct kvm_vm * vm)541 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm)
542 {
543 	kvm_vm_restart(vm);
544 
545 	return vm_vcpu_recreate(vm, 0);
546 }
547 
kvm_pin_this_task_to_pcpu(uint32_t pcpu)548 void kvm_pin_this_task_to_pcpu(uint32_t pcpu)
549 {
550 	cpu_set_t mask;
551 	int r;
552 
553 	CPU_ZERO(&mask);
554 	CPU_SET(pcpu, &mask);
555 	r = sched_setaffinity(0, sizeof(mask), &mask);
556 	TEST_ASSERT(!r, "sched_setaffinity() failed for pCPU '%u'.", pcpu);
557 }
558 
parse_pcpu(const char * cpu_str,const cpu_set_t * allowed_mask)559 static uint32_t parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask)
560 {
561 	uint32_t pcpu = atoi_non_negative("CPU number", cpu_str);
562 
563 	TEST_ASSERT(CPU_ISSET(pcpu, allowed_mask),
564 		    "Not allowed to run on pCPU '%d', check cgroups?", pcpu);
565 	return pcpu;
566 }
567 
kvm_print_vcpu_pinning_help(void)568 void kvm_print_vcpu_pinning_help(void)
569 {
570 	const char *name = program_invocation_name;
571 
572 	printf(" -c: Pin tasks to physical CPUs.  Takes a list of comma separated\n"
573 	       "     values (target pCPU), one for each vCPU, plus an optional\n"
574 	       "     entry for the main application task (specified via entry\n"
575 	       "     <nr_vcpus + 1>).  If used, entries must be provided for all\n"
576 	       "     vCPUs, i.e. pinning vCPUs is all or nothing.\n\n"
577 	       "     E.g. to create 3 vCPUs, pin vCPU0=>pCPU22, vCPU1=>pCPU23,\n"
578 	       "     vCPU2=>pCPU24, and pin the application task to pCPU50:\n\n"
579 	       "         %s -v 3 -c 22,23,24,50\n\n"
580 	       "     To leave the application task unpinned, drop the final entry:\n\n"
581 	       "         %s -v 3 -c 22,23,24\n\n"
582 	       "     (default: no pinning)\n", name, name);
583 }
584 
kvm_parse_vcpu_pinning(const char * pcpus_string,uint32_t vcpu_to_pcpu[],int nr_vcpus)585 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
586 			    int nr_vcpus)
587 {
588 	cpu_set_t allowed_mask;
589 	char *cpu, *cpu_list;
590 	char delim[2] = ",";
591 	int i, r;
592 
593 	cpu_list = strdup(pcpus_string);
594 	TEST_ASSERT(cpu_list, "strdup() allocation failed.");
595 
596 	r = sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask);
597 	TEST_ASSERT(!r, "sched_getaffinity() failed");
598 
599 	cpu = strtok(cpu_list, delim);
600 
601 	/* 1. Get all pcpus for vcpus. */
602 	for (i = 0; i < nr_vcpus; i++) {
603 		TEST_ASSERT(cpu, "pCPU not provided for vCPU '%d'", i);
604 		vcpu_to_pcpu[i] = parse_pcpu(cpu, &allowed_mask);
605 		cpu = strtok(NULL, delim);
606 	}
607 
608 	/* 2. Check if the main worker needs to be pinned. */
609 	if (cpu) {
610 		kvm_pin_this_task_to_pcpu(parse_pcpu(cpu, &allowed_mask));
611 		cpu = strtok(NULL, delim);
612 	}
613 
614 	TEST_ASSERT(!cpu, "pCPU list contains trailing garbage characters '%s'", cpu);
615 	free(cpu_list);
616 }
617 
618 /*
619  * Userspace Memory Region Find
620  *
621  * Input Args:
622  *   vm - Virtual Machine
623  *   start - Starting VM physical address
624  *   end - Ending VM physical address, inclusive.
625  *
626  * Output Args: None
627  *
628  * Return:
629  *   Pointer to overlapping region, NULL if no such region.
630  *
631  * Searches for a region with any physical memory that overlaps with
632  * any portion of the guest physical addresses from start to end
633  * inclusive.  If multiple overlapping regions exist, a pointer to any
634  * of the regions is returned.  Null is returned only when no overlapping
635  * region exists.
636  */
637 static struct userspace_mem_region *
userspace_mem_region_find(struct kvm_vm * vm,uint64_t start,uint64_t end)638 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
639 {
640 	struct rb_node *node;
641 
642 	for (node = vm->regions.gpa_tree.rb_node; node; ) {
643 		struct userspace_mem_region *region =
644 			container_of(node, struct userspace_mem_region, gpa_node);
645 		uint64_t existing_start = region->region.guest_phys_addr;
646 		uint64_t existing_end = region->region.guest_phys_addr
647 			+ region->region.memory_size - 1;
648 		if (start <= existing_end && end >= existing_start)
649 			return region;
650 
651 		if (start < existing_start)
652 			node = node->rb_left;
653 		else
654 			node = node->rb_right;
655 	}
656 
657 	return NULL;
658 }
659 
vcpu_arch_free(struct kvm_vcpu * vcpu)660 __weak void vcpu_arch_free(struct kvm_vcpu *vcpu)
661 {
662 
663 }
664 
665 /*
666  * VM VCPU Remove
667  *
668  * Input Args:
669  *   vcpu - VCPU to remove
670  *
671  * Output Args: None
672  *
673  * Return: None, TEST_ASSERT failures for all error conditions
674  *
675  * Removes a vCPU from a VM and frees its resources.
676  */
vm_vcpu_rm(struct kvm_vm * vm,struct kvm_vcpu * vcpu)677 static void vm_vcpu_rm(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
678 {
679 	int ret;
680 
681 	if (vcpu->dirty_gfns) {
682 		ret = munmap(vcpu->dirty_gfns, vm->dirty_ring_size);
683 		TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
684 		vcpu->dirty_gfns = NULL;
685 	}
686 
687 	ret = munmap(vcpu->run, vcpu_mmap_sz());
688 	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
689 
690 	ret = close(vcpu->fd);
691 	TEST_ASSERT(!ret,  __KVM_SYSCALL_ERROR("close()", ret));
692 
693 	list_del(&vcpu->list);
694 
695 	vcpu_arch_free(vcpu);
696 	free(vcpu);
697 }
698 
kvm_vm_release(struct kvm_vm * vmp)699 void kvm_vm_release(struct kvm_vm *vmp)
700 {
701 	struct kvm_vcpu *vcpu, *tmp;
702 	int ret;
703 
704 	list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)
705 		vm_vcpu_rm(vmp, vcpu);
706 
707 	ret = close(vmp->fd);
708 	TEST_ASSERT(!ret,  __KVM_SYSCALL_ERROR("close()", ret));
709 
710 	ret = close(vmp->kvm_fd);
711 	TEST_ASSERT(!ret,  __KVM_SYSCALL_ERROR("close()", ret));
712 }
713 
__vm_mem_region_delete(struct kvm_vm * vm,struct userspace_mem_region * region)714 static void __vm_mem_region_delete(struct kvm_vm *vm,
715 				   struct userspace_mem_region *region)
716 {
717 	int ret;
718 
719 	rb_erase(&region->gpa_node, &vm->regions.gpa_tree);
720 	rb_erase(&region->hva_node, &vm->regions.hva_tree);
721 	hash_del(&region->slot_node);
722 
723 	region->region.memory_size = 0;
724 	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
725 
726 	sparsebit_free(&region->unused_phy_pages);
727 	sparsebit_free(&region->protected_phy_pages);
728 	ret = munmap(region->mmap_start, region->mmap_size);
729 	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
730 	if (region->fd >= 0) {
731 		/* There's an extra map when using shared memory. */
732 		ret = munmap(region->mmap_alias, region->mmap_size);
733 		TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
734 		close(region->fd);
735 	}
736 	if (region->region.guest_memfd >= 0)
737 		close(region->region.guest_memfd);
738 
739 	free(region);
740 }
741 
742 /*
743  * Destroys and frees the VM pointed to by vmp.
744  */
kvm_vm_free(struct kvm_vm * vmp)745 void kvm_vm_free(struct kvm_vm *vmp)
746 {
747 	int ctr;
748 	struct hlist_node *node;
749 	struct userspace_mem_region *region;
750 
751 	if (vmp == NULL)
752 		return;
753 
754 	/* Free cached stats metadata and close FD */
755 	if (vmp->stats_fd) {
756 		free(vmp->stats_desc);
757 		close(vmp->stats_fd);
758 	}
759 
760 	/* Free userspace_mem_regions. */
761 	hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)
762 		__vm_mem_region_delete(vmp, region);
763 
764 	/* Free sparsebit arrays. */
765 	sparsebit_free(&vmp->vpages_valid);
766 	sparsebit_free(&vmp->vpages_mapped);
767 
768 	kvm_vm_release(vmp);
769 
770 	/* Free the structure describing the VM. */
771 	free(vmp);
772 }
773 
kvm_memfd_alloc(size_t size,bool hugepages)774 int kvm_memfd_alloc(size_t size, bool hugepages)
775 {
776 	int memfd_flags = MFD_CLOEXEC;
777 	int fd, r;
778 
779 	if (hugepages)
780 		memfd_flags |= MFD_HUGETLB;
781 
782 	fd = memfd_create("kvm_selftest", memfd_flags);
783 	TEST_ASSERT(fd != -1, __KVM_SYSCALL_ERROR("memfd_create()", fd));
784 
785 	r = ftruncate(fd, size);
786 	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("ftruncate()", r));
787 
788 	r = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0, size);
789 	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
790 
791 	return fd;
792 }
793 
vm_userspace_mem_region_gpa_insert(struct rb_root * gpa_tree,struct userspace_mem_region * region)794 static void vm_userspace_mem_region_gpa_insert(struct rb_root *gpa_tree,
795 					       struct userspace_mem_region *region)
796 {
797 	struct rb_node **cur, *parent;
798 
799 	for (cur = &gpa_tree->rb_node, parent = NULL; *cur; ) {
800 		struct userspace_mem_region *cregion;
801 
802 		cregion = container_of(*cur, typeof(*cregion), gpa_node);
803 		parent = *cur;
804 		if (region->region.guest_phys_addr <
805 		    cregion->region.guest_phys_addr)
806 			cur = &(*cur)->rb_left;
807 		else {
808 			TEST_ASSERT(region->region.guest_phys_addr !=
809 				    cregion->region.guest_phys_addr,
810 				    "Duplicate GPA in region tree");
811 
812 			cur = &(*cur)->rb_right;
813 		}
814 	}
815 
816 	rb_link_node(&region->gpa_node, parent, cur);
817 	rb_insert_color(&region->gpa_node, gpa_tree);
818 }
819 
vm_userspace_mem_region_hva_insert(struct rb_root * hva_tree,struct userspace_mem_region * region)820 static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,
821 					       struct userspace_mem_region *region)
822 {
823 	struct rb_node **cur, *parent;
824 
825 	for (cur = &hva_tree->rb_node, parent = NULL; *cur; ) {
826 		struct userspace_mem_region *cregion;
827 
828 		cregion = container_of(*cur, typeof(*cregion), hva_node);
829 		parent = *cur;
830 		if (region->host_mem < cregion->host_mem)
831 			cur = &(*cur)->rb_left;
832 		else {
833 			TEST_ASSERT(region->host_mem !=
834 				    cregion->host_mem,
835 				    "Duplicate HVA in region tree");
836 
837 			cur = &(*cur)->rb_right;
838 		}
839 	}
840 
841 	rb_link_node(&region->hva_node, parent, cur);
842 	rb_insert_color(&region->hva_node, hva_tree);
843 }
844 
845 
__vm_set_user_memory_region(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva)846 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
847 				uint64_t gpa, uint64_t size, void *hva)
848 {
849 	struct kvm_userspace_memory_region region = {
850 		.slot = slot,
851 		.flags = flags,
852 		.guest_phys_addr = gpa,
853 		.memory_size = size,
854 		.userspace_addr = (uintptr_t)hva,
855 	};
856 
857 	return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, &region);
858 }
859 
vm_set_user_memory_region(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva)860 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
861 			       uint64_t gpa, uint64_t size, void *hva)
862 {
863 	int ret = __vm_set_user_memory_region(vm, slot, flags, gpa, size, hva);
864 
865 	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed, errno = %d (%s)",
866 		    errno, strerror(errno));
867 }
868 
869 #define TEST_REQUIRE_SET_USER_MEMORY_REGION2()			\
870 	__TEST_REQUIRE(kvm_has_cap(KVM_CAP_USER_MEMORY2),	\
871 		       "KVM selftests now require KVM_SET_USER_MEMORY_REGION2 (introduced in v6.8)")
872 
__vm_set_user_memory_region2(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva,uint32_t guest_memfd,uint64_t guest_memfd_offset)873 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
874 				 uint64_t gpa, uint64_t size, void *hva,
875 				 uint32_t guest_memfd, uint64_t guest_memfd_offset)
876 {
877 	struct kvm_userspace_memory_region2 region = {
878 		.slot = slot,
879 		.flags = flags,
880 		.guest_phys_addr = gpa,
881 		.memory_size = size,
882 		.userspace_addr = (uintptr_t)hva,
883 		.guest_memfd = guest_memfd,
884 		.guest_memfd_offset = guest_memfd_offset,
885 	};
886 
887 	TEST_REQUIRE_SET_USER_MEMORY_REGION2();
888 
889 	return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION2, &region);
890 }
891 
vm_set_user_memory_region2(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva,uint32_t guest_memfd,uint64_t guest_memfd_offset)892 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
893 				uint64_t gpa, uint64_t size, void *hva,
894 				uint32_t guest_memfd, uint64_t guest_memfd_offset)
895 {
896 	int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva,
897 					       guest_memfd, guest_memfd_offset);
898 
899 	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)",
900 		    errno, strerror(errno));
901 }
902 
903 
904 /* FIXME: This thing needs to be ripped apart and rewritten. */
vm_mem_add(struct kvm_vm * vm,enum vm_mem_backing_src_type src_type,uint64_t guest_paddr,uint32_t slot,uint64_t npages,uint32_t flags,int guest_memfd,uint64_t guest_memfd_offset)905 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
906 		uint64_t guest_paddr, uint32_t slot, uint64_t npages,
907 		uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset)
908 {
909 	int ret;
910 	struct userspace_mem_region *region;
911 	size_t backing_src_pagesz = get_backing_src_pagesz(src_type);
912 	size_t mem_size = npages * vm->page_size;
913 	size_t alignment;
914 
915 	TEST_REQUIRE_SET_USER_MEMORY_REGION2();
916 
917 	TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,
918 		"Number of guest pages is not compatible with the host. "
919 		"Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));
920 
921 	TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
922 		"address not on a page boundary.\n"
923 		"  guest_paddr: 0x%lx vm->page_size: 0x%x",
924 		guest_paddr, vm->page_size);
925 	TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
926 		<= vm->max_gfn, "Physical range beyond maximum "
927 		"supported physical address,\n"
928 		"  guest_paddr: 0x%lx npages: 0x%lx\n"
929 		"  vm->max_gfn: 0x%lx vm->page_size: 0x%x",
930 		guest_paddr, npages, vm->max_gfn, vm->page_size);
931 
932 	/*
933 	 * Confirm a mem region with an overlapping address doesn't
934 	 * already exist.
935 	 */
936 	region = (struct userspace_mem_region *) userspace_mem_region_find(
937 		vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
938 	if (region != NULL)
939 		TEST_FAIL("overlapping userspace_mem_region already "
940 			"exists\n"
941 			"  requested guest_paddr: 0x%lx npages: 0x%lx "
942 			"page_size: 0x%x\n"
943 			"  existing guest_paddr: 0x%lx size: 0x%lx",
944 			guest_paddr, npages, vm->page_size,
945 			(uint64_t) region->region.guest_phys_addr,
946 			(uint64_t) region->region.memory_size);
947 
948 	/* Confirm no region with the requested slot already exists. */
949 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
950 			       slot) {
951 		if (region->region.slot != slot)
952 			continue;
953 
954 		TEST_FAIL("A mem region with the requested slot "
955 			"already exists.\n"
956 			"  requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
957 			"  existing slot: %u paddr: 0x%lx size: 0x%lx",
958 			slot, guest_paddr, npages,
959 			region->region.slot,
960 			(uint64_t) region->region.guest_phys_addr,
961 			(uint64_t) region->region.memory_size);
962 	}
963 
964 	/* Allocate and initialize new mem region structure. */
965 	region = calloc(1, sizeof(*region));
966 	TEST_ASSERT(region != NULL, "Insufficient Memory");
967 	region->mmap_size = mem_size;
968 
969 #ifdef __s390x__
970 	/* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
971 	alignment = 0x100000;
972 #else
973 	alignment = 1;
974 #endif
975 
976 	/*
977 	 * When using THP mmap is not guaranteed to returned a hugepage aligned
978 	 * address so we have to pad the mmap. Padding is not needed for HugeTLB
979 	 * because mmap will always return an address aligned to the HugeTLB
980 	 * page size.
981 	 */
982 	if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
983 		alignment = max(backing_src_pagesz, alignment);
984 
985 	TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, backing_src_pagesz));
986 
987 	/* Add enough memory to align up if necessary */
988 	if (alignment > 1)
989 		region->mmap_size += alignment;
990 
991 	region->fd = -1;
992 	if (backing_src_is_shared(src_type))
993 		region->fd = kvm_memfd_alloc(region->mmap_size,
994 					     src_type == VM_MEM_SRC_SHARED_HUGETLB);
995 
996 	region->mmap_start = mmap(NULL, region->mmap_size,
997 				  PROT_READ | PROT_WRITE,
998 				  vm_mem_backing_src_alias(src_type)->flag,
999 				  region->fd, 0);
1000 	TEST_ASSERT(region->mmap_start != MAP_FAILED,
1001 		    __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED));
1002 
1003 	TEST_ASSERT(!is_backing_src_hugetlb(src_type) ||
1004 		    region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz),
1005 		    "mmap_start %p is not aligned to HugeTLB page size 0x%lx",
1006 		    region->mmap_start, backing_src_pagesz);
1007 
1008 	/* Align host address */
1009 	region->host_mem = align_ptr_up(region->mmap_start, alignment);
1010 
1011 	/* As needed perform madvise */
1012 	if ((src_type == VM_MEM_SRC_ANONYMOUS ||
1013 	     src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
1014 		ret = madvise(region->host_mem, mem_size,
1015 			      src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
1016 		TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
1017 			    region->host_mem, mem_size,
1018 			    vm_mem_backing_src_alias(src_type)->name);
1019 	}
1020 
1021 	region->backing_src_type = src_type;
1022 
1023 	if (flags & KVM_MEM_GUEST_MEMFD) {
1024 		if (guest_memfd < 0) {
1025 			uint32_t guest_memfd_flags = 0;
1026 			TEST_ASSERT(!guest_memfd_offset,
1027 				    "Offset must be zero when creating new guest_memfd");
1028 			guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags);
1029 		} else {
1030 			/*
1031 			 * Install a unique fd for each memslot so that the fd
1032 			 * can be closed when the region is deleted without
1033 			 * needing to track if the fd is owned by the framework
1034 			 * or by the caller.
1035 			 */
1036 			guest_memfd = dup(guest_memfd);
1037 			TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd));
1038 		}
1039 
1040 		region->region.guest_memfd = guest_memfd;
1041 		region->region.guest_memfd_offset = guest_memfd_offset;
1042 	} else {
1043 		region->region.guest_memfd = -1;
1044 	}
1045 
1046 	region->unused_phy_pages = sparsebit_alloc();
1047 	if (vm_arch_has_protected_memory(vm))
1048 		region->protected_phy_pages = sparsebit_alloc();
1049 	sparsebit_set_num(region->unused_phy_pages,
1050 		guest_paddr >> vm->page_shift, npages);
1051 	region->region.slot = slot;
1052 	region->region.flags = flags;
1053 	region->region.guest_phys_addr = guest_paddr;
1054 	region->region.memory_size = npages * vm->page_size;
1055 	region->region.userspace_addr = (uintptr_t) region->host_mem;
1056 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1057 	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
1058 		"  rc: %i errno: %i\n"
1059 		"  slot: %u flags: 0x%x\n"
1060 		"  guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d",
1061 		ret, errno, slot, flags,
1062 		guest_paddr, (uint64_t) region->region.memory_size,
1063 		region->region.guest_memfd);
1064 
1065 	/* Add to quick lookup data structures */
1066 	vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);
1067 	vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);
1068 	hash_add(vm->regions.slot_hash, &region->slot_node, slot);
1069 
1070 	/* If shared memory, create an alias. */
1071 	if (region->fd >= 0) {
1072 		region->mmap_alias = mmap(NULL, region->mmap_size,
1073 					  PROT_READ | PROT_WRITE,
1074 					  vm_mem_backing_src_alias(src_type)->flag,
1075 					  region->fd, 0);
1076 		TEST_ASSERT(region->mmap_alias != MAP_FAILED,
1077 			    __KVM_SYSCALL_ERROR("mmap()",  (int)(unsigned long)MAP_FAILED));
1078 
1079 		/* Align host alias address */
1080 		region->host_alias = align_ptr_up(region->mmap_alias, alignment);
1081 	}
1082 }
1083 
vm_userspace_mem_region_add(struct kvm_vm * vm,enum vm_mem_backing_src_type src_type,uint64_t guest_paddr,uint32_t slot,uint64_t npages,uint32_t flags)1084 void vm_userspace_mem_region_add(struct kvm_vm *vm,
1085 				 enum vm_mem_backing_src_type src_type,
1086 				 uint64_t guest_paddr, uint32_t slot,
1087 				 uint64_t npages, uint32_t flags)
1088 {
1089 	vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0);
1090 }
1091 
1092 /*
1093  * Memslot to region
1094  *
1095  * Input Args:
1096  *   vm - Virtual Machine
1097  *   memslot - KVM memory slot ID
1098  *
1099  * Output Args: None
1100  *
1101  * Return:
1102  *   Pointer to memory region structure that describe memory region
1103  *   using kvm memory slot ID given by memslot.  TEST_ASSERT failure
1104  *   on error (e.g. currently no memory region using memslot as a KVM
1105  *   memory slot ID).
1106  */
1107 struct userspace_mem_region *
memslot2region(struct kvm_vm * vm,uint32_t memslot)1108 memslot2region(struct kvm_vm *vm, uint32_t memslot)
1109 {
1110 	struct userspace_mem_region *region;
1111 
1112 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
1113 			       memslot)
1114 		if (region->region.slot == memslot)
1115 			return region;
1116 
1117 	fprintf(stderr, "No mem region with the requested slot found,\n"
1118 		"  requested slot: %u\n", memslot);
1119 	fputs("---- vm dump ----\n", stderr);
1120 	vm_dump(stderr, vm, 2);
1121 	TEST_FAIL("Mem region not found");
1122 	return NULL;
1123 }
1124 
1125 /*
1126  * VM Memory Region Flags Set
1127  *
1128  * Input Args:
1129  *   vm - Virtual Machine
1130  *   flags - Starting guest physical address
1131  *
1132  * Output Args: None
1133  *
1134  * Return: None
1135  *
1136  * Sets the flags of the memory region specified by the value of slot,
1137  * to the values given by flags.
1138  */
vm_mem_region_set_flags(struct kvm_vm * vm,uint32_t slot,uint32_t flags)1139 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
1140 {
1141 	int ret;
1142 	struct userspace_mem_region *region;
1143 
1144 	region = memslot2region(vm, slot);
1145 
1146 	region->region.flags = flags;
1147 
1148 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1149 
1150 	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
1151 		"  rc: %i errno: %i slot: %u flags: 0x%x",
1152 		ret, errno, slot, flags);
1153 }
1154 
1155 /*
1156  * VM Memory Region Move
1157  *
1158  * Input Args:
1159  *   vm - Virtual Machine
1160  *   slot - Slot of the memory region to move
1161  *   new_gpa - Starting guest physical address
1162  *
1163  * Output Args: None
1164  *
1165  * Return: None
1166  *
1167  * Change the gpa of a memory region.
1168  */
vm_mem_region_move(struct kvm_vm * vm,uint32_t slot,uint64_t new_gpa)1169 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
1170 {
1171 	struct userspace_mem_region *region;
1172 	int ret;
1173 
1174 	region = memslot2region(vm, slot);
1175 
1176 	region->region.guest_phys_addr = new_gpa;
1177 
1178 	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1179 
1180 	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed\n"
1181 		    "ret: %i errno: %i slot: %u new_gpa: 0x%lx",
1182 		    ret, errno, slot, new_gpa);
1183 }
1184 
1185 /*
1186  * VM Memory Region Delete
1187  *
1188  * Input Args:
1189  *   vm - Virtual Machine
1190  *   slot - Slot of the memory region to delete
1191  *
1192  * Output Args: None
1193  *
1194  * Return: None
1195  *
1196  * Delete a memory region.
1197  */
vm_mem_region_delete(struct kvm_vm * vm,uint32_t slot)1198 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
1199 {
1200 	__vm_mem_region_delete(vm, memslot2region(vm, slot));
1201 }
1202 
vm_guest_mem_fallocate(struct kvm_vm * vm,uint64_t base,uint64_t size,bool punch_hole)1203 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
1204 			    bool punch_hole)
1205 {
1206 	const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0);
1207 	struct userspace_mem_region *region;
1208 	uint64_t end = base + size;
1209 	uint64_t gpa, len;
1210 	off_t fd_offset;
1211 	int ret;
1212 
1213 	for (gpa = base; gpa < end; gpa += len) {
1214 		uint64_t offset;
1215 
1216 		region = userspace_mem_region_find(vm, gpa, gpa);
1217 		TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,
1218 			    "Private memory region not found for GPA 0x%lx", gpa);
1219 
1220 		offset = gpa - region->region.guest_phys_addr;
1221 		fd_offset = region->region.guest_memfd_offset + offset;
1222 		len = min_t(uint64_t, end - gpa, region->region.memory_size - offset);
1223 
1224 		ret = fallocate(region->region.guest_memfd, mode, fd_offset, len);
1225 		TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx",
1226 			    punch_hole ? "punch hole" : "allocate", gpa, len,
1227 			    region->region.guest_memfd, mode, fd_offset);
1228 	}
1229 }
1230 
1231 /* Returns the size of a vCPU's kvm_run structure. */
vcpu_mmap_sz(void)1232 static int vcpu_mmap_sz(void)
1233 {
1234 	int dev_fd, ret;
1235 
1236 	dev_fd = open_kvm_dev_path_or_exit();
1237 
1238 	ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
1239 	TEST_ASSERT(ret >= sizeof(struct kvm_run),
1240 		    KVM_IOCTL_ERROR(KVM_GET_VCPU_MMAP_SIZE, ret));
1241 
1242 	close(dev_fd);
1243 
1244 	return ret;
1245 }
1246 
vcpu_exists(struct kvm_vm * vm,uint32_t vcpu_id)1247 static bool vcpu_exists(struct kvm_vm *vm, uint32_t vcpu_id)
1248 {
1249 	struct kvm_vcpu *vcpu;
1250 
1251 	list_for_each_entry(vcpu, &vm->vcpus, list) {
1252 		if (vcpu->id == vcpu_id)
1253 			return true;
1254 	}
1255 
1256 	return false;
1257 }
1258 
1259 /*
1260  * Adds a virtual CPU to the VM specified by vm with the ID given by vcpu_id.
1261  * No additional vCPU setup is done.  Returns the vCPU.
1262  */
__vm_vcpu_add(struct kvm_vm * vm,uint32_t vcpu_id)1263 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
1264 {
1265 	struct kvm_vcpu *vcpu;
1266 
1267 	/* Confirm a vcpu with the specified id doesn't already exist. */
1268 	TEST_ASSERT(!vcpu_exists(vm, vcpu_id), "vCPU%d already exists", vcpu_id);
1269 
1270 	/* Allocate and initialize new vcpu structure. */
1271 	vcpu = calloc(1, sizeof(*vcpu));
1272 	TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
1273 
1274 	vcpu->vm = vm;
1275 	vcpu->id = vcpu_id;
1276 	vcpu->fd = __vm_ioctl(vm, KVM_CREATE_VCPU, (void *)(unsigned long)vcpu_id);
1277 	TEST_ASSERT_VM_VCPU_IOCTL(vcpu->fd >= 0, KVM_CREATE_VCPU, vcpu->fd, vm);
1278 
1279 	TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->run), "vcpu mmap size "
1280 		"smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
1281 		vcpu_mmap_sz(), sizeof(*vcpu->run));
1282 	vcpu->run = (struct kvm_run *) mmap(NULL, vcpu_mmap_sz(),
1283 		PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);
1284 	TEST_ASSERT(vcpu->run != MAP_FAILED,
1285 		    __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED));
1286 
1287 	/* Add to linked-list of VCPUs. */
1288 	list_add(&vcpu->list, &vm->vcpus);
1289 
1290 	return vcpu;
1291 }
1292 
1293 /*
1294  * VM Virtual Address Unused Gap
1295  *
1296  * Input Args:
1297  *   vm - Virtual Machine
1298  *   sz - Size (bytes)
1299  *   vaddr_min - Minimum Virtual Address
1300  *
1301  * Output Args: None
1302  *
1303  * Return:
1304  *   Lowest virtual address at or below vaddr_min, with at least
1305  *   sz unused bytes.  TEST_ASSERT failure if no area of at least
1306  *   size sz is available.
1307  *
1308  * Within the VM specified by vm, locates the lowest starting virtual
1309  * address >= vaddr_min, that has at least sz unallocated bytes.  A
1310  * TEST_ASSERT failure occurs for invalid input or no area of at least
1311  * sz unallocated bytes >= vaddr_min is available.
1312  */
vm_vaddr_unused_gap(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min)1313 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
1314 			       vm_vaddr_t vaddr_min)
1315 {
1316 	uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
1317 
1318 	/* Determine lowest permitted virtual page index. */
1319 	uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
1320 	if ((pgidx_start * vm->page_size) < vaddr_min)
1321 		goto no_va_found;
1322 
1323 	/* Loop over section with enough valid virtual page indexes. */
1324 	if (!sparsebit_is_set_num(vm->vpages_valid,
1325 		pgidx_start, pages))
1326 		pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
1327 			pgidx_start, pages);
1328 	do {
1329 		/*
1330 		 * Are there enough unused virtual pages available at
1331 		 * the currently proposed starting virtual page index.
1332 		 * If not, adjust proposed starting index to next
1333 		 * possible.
1334 		 */
1335 		if (sparsebit_is_clear_num(vm->vpages_mapped,
1336 			pgidx_start, pages))
1337 			goto va_found;
1338 		pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
1339 			pgidx_start, pages);
1340 		if (pgidx_start == 0)
1341 			goto no_va_found;
1342 
1343 		/*
1344 		 * If needed, adjust proposed starting virtual address,
1345 		 * to next range of valid virtual addresses.
1346 		 */
1347 		if (!sparsebit_is_set_num(vm->vpages_valid,
1348 			pgidx_start, pages)) {
1349 			pgidx_start = sparsebit_next_set_num(
1350 				vm->vpages_valid, pgidx_start, pages);
1351 			if (pgidx_start == 0)
1352 				goto no_va_found;
1353 		}
1354 	} while (pgidx_start != 0);
1355 
1356 no_va_found:
1357 	TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);
1358 
1359 	/* NOT REACHED */
1360 	return -1;
1361 
1362 va_found:
1363 	TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
1364 		pgidx_start, pages),
1365 		"Unexpected, invalid virtual page index range,\n"
1366 		"  pgidx_start: 0x%lx\n"
1367 		"  pages: 0x%lx",
1368 		pgidx_start, pages);
1369 	TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
1370 		pgidx_start, pages),
1371 		"Unexpected, pages already mapped,\n"
1372 		"  pgidx_start: 0x%lx\n"
1373 		"  pages: 0x%lx",
1374 		pgidx_start, pages);
1375 
1376 	return pgidx_start * vm->page_size;
1377 }
1378 
____vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type,bool protected)1379 static vm_vaddr_t ____vm_vaddr_alloc(struct kvm_vm *vm, size_t sz,
1380 				     vm_vaddr_t vaddr_min,
1381 				     enum kvm_mem_region_type type,
1382 				     bool protected)
1383 {
1384 	uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
1385 
1386 	virt_pgd_alloc(vm);
1387 	vm_paddr_t paddr = __vm_phy_pages_alloc(vm, pages,
1388 						KVM_UTIL_MIN_PFN * vm->page_size,
1389 						vm->memslots[type], protected);
1390 
1391 	/*
1392 	 * Find an unused range of virtual page addresses of at least
1393 	 * pages in length.
1394 	 */
1395 	vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
1396 
1397 	/* Map the virtual pages. */
1398 	for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
1399 		pages--, vaddr += vm->page_size, paddr += vm->page_size) {
1400 
1401 		virt_pg_map(vm, vaddr, paddr);
1402 
1403 		sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1404 	}
1405 
1406 	return vaddr_start;
1407 }
1408 
__vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type)1409 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1410 			    enum kvm_mem_region_type type)
1411 {
1412 	return ____vm_vaddr_alloc(vm, sz, vaddr_min, type,
1413 				  vm_arch_has_protected_memory(vm));
1414 }
1415 
vm_vaddr_alloc_shared(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type)1416 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,
1417 				 vm_vaddr_t vaddr_min,
1418 				 enum kvm_mem_region_type type)
1419 {
1420 	return ____vm_vaddr_alloc(vm, sz, vaddr_min, type, false);
1421 }
1422 
1423 /*
1424  * VM Virtual Address Allocate
1425  *
1426  * Input Args:
1427  *   vm - Virtual Machine
1428  *   sz - Size in bytes
1429  *   vaddr_min - Minimum starting virtual address
1430  *
1431  * Output Args: None
1432  *
1433  * Return:
1434  *   Starting guest virtual address
1435  *
1436  * Allocates at least sz bytes within the virtual address space of the vm
1437  * given by vm.  The allocated bytes are mapped to a virtual address >=
1438  * the address given by vaddr_min.  Note that each allocation uses a
1439  * a unique set of pages, with the minimum real allocation being at least
1440  * a page. The allocated physical space comes from the TEST_DATA memory region.
1441  */
vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min)1442 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
1443 {
1444 	return __vm_vaddr_alloc(vm, sz, vaddr_min, MEM_REGION_TEST_DATA);
1445 }
1446 
1447 /*
1448  * VM Virtual Address Allocate Pages
1449  *
1450  * Input Args:
1451  *   vm - Virtual Machine
1452  *
1453  * Output Args: None
1454  *
1455  * Return:
1456  *   Starting guest virtual address
1457  *
1458  * Allocates at least N system pages worth of bytes within the virtual address
1459  * space of the vm.
1460  */
vm_vaddr_alloc_pages(struct kvm_vm * vm,int nr_pages)1461 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages)
1462 {
1463 	return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);
1464 }
1465 
__vm_vaddr_alloc_page(struct kvm_vm * vm,enum kvm_mem_region_type type)1466 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)
1467 {
1468 	return __vm_vaddr_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);
1469 }
1470 
1471 /*
1472  * VM Virtual Address Allocate Page
1473  *
1474  * Input Args:
1475  *   vm - Virtual Machine
1476  *
1477  * Output Args: None
1478  *
1479  * Return:
1480  *   Starting guest virtual address
1481  *
1482  * Allocates at least one system page worth of bytes within the virtual address
1483  * space of the vm.
1484  */
vm_vaddr_alloc_page(struct kvm_vm * vm)1485 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm)
1486 {
1487 	return vm_vaddr_alloc_pages(vm, 1);
1488 }
1489 
1490 /*
1491  * Map a range of VM virtual address to the VM's physical address
1492  *
1493  * Input Args:
1494  *   vm - Virtual Machine
1495  *   vaddr - Virtuall address to map
1496  *   paddr - VM Physical Address
1497  *   npages - The number of pages to map
1498  *
1499  * Output Args: None
1500  *
1501  * Return: None
1502  *
1503  * Within the VM given by @vm, creates a virtual translation for
1504  * @npages starting at @vaddr to the page range starting at @paddr.
1505  */
virt_map(struct kvm_vm * vm,uint64_t vaddr,uint64_t paddr,unsigned int npages)1506 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
1507 	      unsigned int npages)
1508 {
1509 	size_t page_size = vm->page_size;
1510 	size_t size = npages * page_size;
1511 
1512 	TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
1513 	TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
1514 
1515 	while (npages--) {
1516 		virt_pg_map(vm, vaddr, paddr);
1517 		sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1518 
1519 		vaddr += page_size;
1520 		paddr += page_size;
1521 	}
1522 }
1523 
1524 /*
1525  * Address VM Physical to Host Virtual
1526  *
1527  * Input Args:
1528  *   vm - Virtual Machine
1529  *   gpa - VM physical address
1530  *
1531  * Output Args: None
1532  *
1533  * Return:
1534  *   Equivalent host virtual address
1535  *
1536  * Locates the memory region containing the VM physical address given
1537  * by gpa, within the VM given by vm.  When found, the host virtual
1538  * address providing the memory to the vm physical address is returned.
1539  * A TEST_ASSERT failure occurs if no region containing gpa exists.
1540  */
addr_gpa2hva(struct kvm_vm * vm,vm_paddr_t gpa)1541 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
1542 {
1543 	struct userspace_mem_region *region;
1544 
1545 	gpa = vm_untag_gpa(vm, gpa);
1546 
1547 	region = userspace_mem_region_find(vm, gpa, gpa);
1548 	if (!region) {
1549 		TEST_FAIL("No vm physical memory at 0x%lx", gpa);
1550 		return NULL;
1551 	}
1552 
1553 	return (void *)((uintptr_t)region->host_mem
1554 		+ (gpa - region->region.guest_phys_addr));
1555 }
1556 
1557 /*
1558  * Address Host Virtual to VM Physical
1559  *
1560  * Input Args:
1561  *   vm - Virtual Machine
1562  *   hva - Host virtual address
1563  *
1564  * Output Args: None
1565  *
1566  * Return:
1567  *   Equivalent VM physical address
1568  *
1569  * Locates the memory region containing the host virtual address given
1570  * by hva, within the VM given by vm.  When found, the equivalent
1571  * VM physical address is returned. A TEST_ASSERT failure occurs if no
1572  * region containing hva exists.
1573  */
addr_hva2gpa(struct kvm_vm * vm,void * hva)1574 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1575 {
1576 	struct rb_node *node;
1577 
1578 	for (node = vm->regions.hva_tree.rb_node; node; ) {
1579 		struct userspace_mem_region *region =
1580 			container_of(node, struct userspace_mem_region, hva_node);
1581 
1582 		if (hva >= region->host_mem) {
1583 			if (hva <= (region->host_mem
1584 				+ region->region.memory_size - 1))
1585 				return (vm_paddr_t)((uintptr_t)
1586 					region->region.guest_phys_addr
1587 					+ (hva - (uintptr_t)region->host_mem));
1588 
1589 			node = node->rb_right;
1590 		} else
1591 			node = node->rb_left;
1592 	}
1593 
1594 	TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);
1595 	return -1;
1596 }
1597 
1598 /*
1599  * Address VM physical to Host Virtual *alias*.
1600  *
1601  * Input Args:
1602  *   vm - Virtual Machine
1603  *   gpa - VM physical address
1604  *
1605  * Output Args: None
1606  *
1607  * Return:
1608  *   Equivalent address within the host virtual *alias* area, or NULL
1609  *   (without failing the test) if the guest memory is not shared (so
1610  *   no alias exists).
1611  *
1612  * Create a writable, shared virtual=>physical alias for the specific GPA.
1613  * The primary use case is to allow the host selftest to manipulate guest
1614  * memory without mapping said memory in the guest's address space. And, for
1615  * userfaultfd-based demand paging, to do so without triggering userfaults.
1616  */
addr_gpa2alias(struct kvm_vm * vm,vm_paddr_t gpa)1617 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa)
1618 {
1619 	struct userspace_mem_region *region;
1620 	uintptr_t offset;
1621 
1622 	region = userspace_mem_region_find(vm, gpa, gpa);
1623 	if (!region)
1624 		return NULL;
1625 
1626 	if (!region->host_alias)
1627 		return NULL;
1628 
1629 	offset = gpa - region->region.guest_phys_addr;
1630 	return (void *) ((uintptr_t) region->host_alias + offset);
1631 }
1632 
1633 /* Create an interrupt controller chip for the specified VM. */
vm_create_irqchip(struct kvm_vm * vm)1634 void vm_create_irqchip(struct kvm_vm *vm)
1635 {
1636 	vm_ioctl(vm, KVM_CREATE_IRQCHIP, NULL);
1637 
1638 	vm->has_irqchip = true;
1639 }
1640 
_vcpu_run(struct kvm_vcpu * vcpu)1641 int _vcpu_run(struct kvm_vcpu *vcpu)
1642 {
1643 	int rc;
1644 
1645 	do {
1646 		rc = __vcpu_run(vcpu);
1647 	} while (rc == -1 && errno == EINTR);
1648 
1649 	assert_on_unhandled_exception(vcpu);
1650 
1651 	return rc;
1652 }
1653 
1654 /*
1655  * Invoke KVM_RUN on a vCPU until KVM returns something other than -EINTR.
1656  * Assert if the KVM returns an error (other than -EINTR).
1657  */
vcpu_run(struct kvm_vcpu * vcpu)1658 void vcpu_run(struct kvm_vcpu *vcpu)
1659 {
1660 	int ret = _vcpu_run(vcpu);
1661 
1662 	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_RUN, ret));
1663 }
1664 
vcpu_run_complete_io(struct kvm_vcpu * vcpu)1665 void vcpu_run_complete_io(struct kvm_vcpu *vcpu)
1666 {
1667 	int ret;
1668 
1669 	vcpu->run->immediate_exit = 1;
1670 	ret = __vcpu_run(vcpu);
1671 	vcpu->run->immediate_exit = 0;
1672 
1673 	TEST_ASSERT(ret == -1 && errno == EINTR,
1674 		    "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1675 		    ret, errno);
1676 }
1677 
1678 /*
1679  * Get the list of guest registers which are supported for
1680  * KVM_GET_ONE_REG/KVM_SET_ONE_REG ioctls.  Returns a kvm_reg_list pointer,
1681  * it is the caller's responsibility to free the list.
1682  */
vcpu_get_reg_list(struct kvm_vcpu * vcpu)1683 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu)
1684 {
1685 	struct kvm_reg_list reg_list_n = { .n = 0 }, *reg_list;
1686 	int ret;
1687 
1688 	ret = __vcpu_ioctl(vcpu, KVM_GET_REG_LIST, &reg_list_n);
1689 	TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");
1690 
1691 	reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));
1692 	reg_list->n = reg_list_n.n;
1693 	vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list);
1694 	return reg_list;
1695 }
1696 
vcpu_map_dirty_ring(struct kvm_vcpu * vcpu)1697 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)
1698 {
1699 	uint32_t page_size = getpagesize();
1700 	uint32_t size = vcpu->vm->dirty_ring_size;
1701 
1702 	TEST_ASSERT(size > 0, "Should enable dirty ring first");
1703 
1704 	if (!vcpu->dirty_gfns) {
1705 		void *addr;
1706 
1707 		addr = mmap(NULL, size, PROT_READ, MAP_PRIVATE, vcpu->fd,
1708 			    page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1709 		TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");
1710 
1711 		addr = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_PRIVATE, vcpu->fd,
1712 			    page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1713 		TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");
1714 
1715 		addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd,
1716 			    page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1717 		TEST_ASSERT(addr != MAP_FAILED, "Dirty ring map failed");
1718 
1719 		vcpu->dirty_gfns = addr;
1720 		vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
1721 	}
1722 
1723 	return vcpu->dirty_gfns;
1724 }
1725 
1726 /*
1727  * Device Ioctl
1728  */
1729 
__kvm_has_device_attr(int dev_fd,uint32_t group,uint64_t attr)1730 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
1731 {
1732 	struct kvm_device_attr attribute = {
1733 		.group = group,
1734 		.attr = attr,
1735 		.flags = 0,
1736 	};
1737 
1738 	return ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute);
1739 }
1740 
__kvm_test_create_device(struct kvm_vm * vm,uint64_t type)1741 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type)
1742 {
1743 	struct kvm_create_device create_dev = {
1744 		.type = type,
1745 		.flags = KVM_CREATE_DEVICE_TEST,
1746 	};
1747 
1748 	return __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);
1749 }
1750 
__kvm_create_device(struct kvm_vm * vm,uint64_t type)1751 int __kvm_create_device(struct kvm_vm *vm, uint64_t type)
1752 {
1753 	struct kvm_create_device create_dev = {
1754 		.type = type,
1755 		.fd = -1,
1756 		.flags = 0,
1757 	};
1758 	int err;
1759 
1760 	err = __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);
1761 	TEST_ASSERT(err <= 0, "KVM_CREATE_DEVICE shouldn't return a positive value");
1762 	return err ? : create_dev.fd;
1763 }
1764 
__kvm_device_attr_get(int dev_fd,uint32_t group,uint64_t attr,void * val)1765 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val)
1766 {
1767 	struct kvm_device_attr kvmattr = {
1768 		.group = group,
1769 		.attr = attr,
1770 		.flags = 0,
1771 		.addr = (uintptr_t)val,
1772 	};
1773 
1774 	return __kvm_ioctl(dev_fd, KVM_GET_DEVICE_ATTR, &kvmattr);
1775 }
1776 
__kvm_device_attr_set(int dev_fd,uint32_t group,uint64_t attr,void * val)1777 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val)
1778 {
1779 	struct kvm_device_attr kvmattr = {
1780 		.group = group,
1781 		.attr = attr,
1782 		.flags = 0,
1783 		.addr = (uintptr_t)val,
1784 	};
1785 
1786 	return __kvm_ioctl(dev_fd, KVM_SET_DEVICE_ATTR, &kvmattr);
1787 }
1788 
1789 /*
1790  * IRQ related functions.
1791  */
1792 
_kvm_irq_line(struct kvm_vm * vm,uint32_t irq,int level)1793 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
1794 {
1795 	struct kvm_irq_level irq_level = {
1796 		.irq    = irq,
1797 		.level  = level,
1798 	};
1799 
1800 	return __vm_ioctl(vm, KVM_IRQ_LINE, &irq_level);
1801 }
1802 
kvm_irq_line(struct kvm_vm * vm,uint32_t irq,int level)1803 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
1804 {
1805 	int ret = _kvm_irq_line(vm, irq, level);
1806 
1807 	TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret));
1808 }
1809 
kvm_gsi_routing_create(void)1810 struct kvm_irq_routing *kvm_gsi_routing_create(void)
1811 {
1812 	struct kvm_irq_routing *routing;
1813 	size_t size;
1814 
1815 	size = sizeof(struct kvm_irq_routing);
1816 	/* Allocate space for the max number of entries: this wastes 196 KBs. */
1817 	size += KVM_MAX_IRQ_ROUTES * sizeof(struct kvm_irq_routing_entry);
1818 	routing = calloc(1, size);
1819 	assert(routing);
1820 
1821 	return routing;
1822 }
1823 
kvm_gsi_routing_irqchip_add(struct kvm_irq_routing * routing,uint32_t gsi,uint32_t pin)1824 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
1825 		uint32_t gsi, uint32_t pin)
1826 {
1827 	int i;
1828 
1829 	assert(routing);
1830 	assert(routing->nr < KVM_MAX_IRQ_ROUTES);
1831 
1832 	i = routing->nr;
1833 	routing->entries[i].gsi = gsi;
1834 	routing->entries[i].type = KVM_IRQ_ROUTING_IRQCHIP;
1835 	routing->entries[i].flags = 0;
1836 	routing->entries[i].u.irqchip.irqchip = 0;
1837 	routing->entries[i].u.irqchip.pin = pin;
1838 	routing->nr++;
1839 }
1840 
_kvm_gsi_routing_write(struct kvm_vm * vm,struct kvm_irq_routing * routing)1841 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
1842 {
1843 	int ret;
1844 
1845 	assert(routing);
1846 	ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing);
1847 	free(routing);
1848 
1849 	return ret;
1850 }
1851 
kvm_gsi_routing_write(struct kvm_vm * vm,struct kvm_irq_routing * routing)1852 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
1853 {
1854 	int ret;
1855 
1856 	ret = _kvm_gsi_routing_write(vm, routing);
1857 	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_GSI_ROUTING, ret));
1858 }
1859 
1860 /*
1861  * VM Dump
1862  *
1863  * Input Args:
1864  *   vm - Virtual Machine
1865  *   indent - Left margin indent amount
1866  *
1867  * Output Args:
1868  *   stream - Output FILE stream
1869  *
1870  * Return: None
1871  *
1872  * Dumps the current state of the VM given by vm, to the FILE stream
1873  * given by stream.
1874  */
vm_dump(FILE * stream,struct kvm_vm * vm,uint8_t indent)1875 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1876 {
1877 	int ctr;
1878 	struct userspace_mem_region *region;
1879 	struct kvm_vcpu *vcpu;
1880 
1881 	fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1882 	fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1883 	fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1884 	fprintf(stream, "%*sMem Regions:\n", indent, "");
1885 	hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {
1886 		fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1887 			"host_virt: %p\n", indent + 2, "",
1888 			(uint64_t) region->region.guest_phys_addr,
1889 			(uint64_t) region->region.memory_size,
1890 			region->host_mem);
1891 		fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1892 		sparsebit_dump(stream, region->unused_phy_pages, 0);
1893 		if (region->protected_phy_pages) {
1894 			fprintf(stream, "%*sprotected_phy_pages: ", indent + 2, "");
1895 			sparsebit_dump(stream, region->protected_phy_pages, 0);
1896 		}
1897 	}
1898 	fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1899 	sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1900 	fprintf(stream, "%*spgd_created: %u\n", indent, "",
1901 		vm->pgd_created);
1902 	if (vm->pgd_created) {
1903 		fprintf(stream, "%*sVirtual Translation Tables:\n",
1904 			indent + 2, "");
1905 		virt_dump(stream, vm, indent + 4);
1906 	}
1907 	fprintf(stream, "%*sVCPUs:\n", indent, "");
1908 
1909 	list_for_each_entry(vcpu, &vm->vcpus, list)
1910 		vcpu_dump(stream, vcpu, indent + 2);
1911 }
1912 
1913 #define KVM_EXIT_STRING(x) {KVM_EXIT_##x, #x}
1914 
1915 /* Known KVM exit reasons */
1916 static struct exit_reason {
1917 	unsigned int reason;
1918 	const char *name;
1919 } exit_reasons_known[] = {
1920 	KVM_EXIT_STRING(UNKNOWN),
1921 	KVM_EXIT_STRING(EXCEPTION),
1922 	KVM_EXIT_STRING(IO),
1923 	KVM_EXIT_STRING(HYPERCALL),
1924 	KVM_EXIT_STRING(DEBUG),
1925 	KVM_EXIT_STRING(HLT),
1926 	KVM_EXIT_STRING(MMIO),
1927 	KVM_EXIT_STRING(IRQ_WINDOW_OPEN),
1928 	KVM_EXIT_STRING(SHUTDOWN),
1929 	KVM_EXIT_STRING(FAIL_ENTRY),
1930 	KVM_EXIT_STRING(INTR),
1931 	KVM_EXIT_STRING(SET_TPR),
1932 	KVM_EXIT_STRING(TPR_ACCESS),
1933 	KVM_EXIT_STRING(S390_SIEIC),
1934 	KVM_EXIT_STRING(S390_RESET),
1935 	KVM_EXIT_STRING(DCR),
1936 	KVM_EXIT_STRING(NMI),
1937 	KVM_EXIT_STRING(INTERNAL_ERROR),
1938 	KVM_EXIT_STRING(OSI),
1939 	KVM_EXIT_STRING(PAPR_HCALL),
1940 	KVM_EXIT_STRING(S390_UCONTROL),
1941 	KVM_EXIT_STRING(WATCHDOG),
1942 	KVM_EXIT_STRING(S390_TSCH),
1943 	KVM_EXIT_STRING(EPR),
1944 	KVM_EXIT_STRING(SYSTEM_EVENT),
1945 	KVM_EXIT_STRING(S390_STSI),
1946 	KVM_EXIT_STRING(IOAPIC_EOI),
1947 	KVM_EXIT_STRING(HYPERV),
1948 	KVM_EXIT_STRING(ARM_NISV),
1949 	KVM_EXIT_STRING(X86_RDMSR),
1950 	KVM_EXIT_STRING(X86_WRMSR),
1951 	KVM_EXIT_STRING(DIRTY_RING_FULL),
1952 	KVM_EXIT_STRING(AP_RESET_HOLD),
1953 	KVM_EXIT_STRING(X86_BUS_LOCK),
1954 	KVM_EXIT_STRING(XEN),
1955 	KVM_EXIT_STRING(RISCV_SBI),
1956 	KVM_EXIT_STRING(RISCV_CSR),
1957 	KVM_EXIT_STRING(NOTIFY),
1958 #ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1959 	KVM_EXIT_STRING(MEMORY_NOT_PRESENT),
1960 #endif
1961 };
1962 
1963 /*
1964  * Exit Reason String
1965  *
1966  * Input Args:
1967  *   exit_reason - Exit reason
1968  *
1969  * Output Args: None
1970  *
1971  * Return:
1972  *   Constant string pointer describing the exit reason.
1973  *
1974  * Locates and returns a constant string that describes the KVM exit
1975  * reason given by exit_reason.  If no such string is found, a constant
1976  * string of "Unknown" is returned.
1977  */
exit_reason_str(unsigned int exit_reason)1978 const char *exit_reason_str(unsigned int exit_reason)
1979 {
1980 	unsigned int n1;
1981 
1982 	for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
1983 		if (exit_reason == exit_reasons_known[n1].reason)
1984 			return exit_reasons_known[n1].name;
1985 	}
1986 
1987 	return "Unknown";
1988 }
1989 
1990 /*
1991  * Physical Contiguous Page Allocator
1992  *
1993  * Input Args:
1994  *   vm - Virtual Machine
1995  *   num - number of pages
1996  *   paddr_min - Physical address minimum
1997  *   memslot - Memory region to allocate page from
1998  *   protected - True if the pages will be used as protected/private memory
1999  *
2000  * Output Args: None
2001  *
2002  * Return:
2003  *   Starting physical address
2004  *
2005  * Within the VM specified by vm, locates a range of available physical
2006  * pages at or above paddr_min. If found, the pages are marked as in use
2007  * and their base address is returned. A TEST_ASSERT failure occurs if
2008  * not enough pages are available at or above paddr_min.
2009  */
__vm_phy_pages_alloc(struct kvm_vm * vm,size_t num,vm_paddr_t paddr_min,uint32_t memslot,bool protected)2010 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
2011 				vm_paddr_t paddr_min, uint32_t memslot,
2012 				bool protected)
2013 {
2014 	struct userspace_mem_region *region;
2015 	sparsebit_idx_t pg, base;
2016 
2017 	TEST_ASSERT(num > 0, "Must allocate at least one page");
2018 
2019 	TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
2020 		"not divisible by page size.\n"
2021 		"  paddr_min: 0x%lx page_size: 0x%x",
2022 		paddr_min, vm->page_size);
2023 
2024 	region = memslot2region(vm, memslot);
2025 	TEST_ASSERT(!protected || region->protected_phy_pages,
2026 		    "Region doesn't support protected memory");
2027 
2028 	base = pg = paddr_min >> vm->page_shift;
2029 	do {
2030 		for (; pg < base + num; ++pg) {
2031 			if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
2032 				base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
2033 				break;
2034 			}
2035 		}
2036 	} while (pg && pg != base + num);
2037 
2038 	if (pg == 0) {
2039 		fprintf(stderr, "No guest physical page available, "
2040 			"paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
2041 			paddr_min, vm->page_size, memslot);
2042 		fputs("---- vm dump ----\n", stderr);
2043 		vm_dump(stderr, vm, 2);
2044 		abort();
2045 	}
2046 
2047 	for (pg = base; pg < base + num; ++pg) {
2048 		sparsebit_clear(region->unused_phy_pages, pg);
2049 		if (protected)
2050 			sparsebit_set(region->protected_phy_pages, pg);
2051 	}
2052 
2053 	return base * vm->page_size;
2054 }
2055 
vm_phy_page_alloc(struct kvm_vm * vm,vm_paddr_t paddr_min,uint32_t memslot)2056 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
2057 			     uint32_t memslot)
2058 {
2059 	return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
2060 }
2061 
vm_alloc_page_table(struct kvm_vm * vm)2062 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)
2063 {
2064 	return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,
2065 				 vm->memslots[MEM_REGION_PT]);
2066 }
2067 
2068 /*
2069  * Address Guest Virtual to Host Virtual
2070  *
2071  * Input Args:
2072  *   vm - Virtual Machine
2073  *   gva - VM virtual address
2074  *
2075  * Output Args: None
2076  *
2077  * Return:
2078  *   Equivalent host virtual address
2079  */
addr_gva2hva(struct kvm_vm * vm,vm_vaddr_t gva)2080 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
2081 {
2082 	return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
2083 }
2084 
vm_compute_max_gfn(struct kvm_vm * vm)2085 unsigned long __weak vm_compute_max_gfn(struct kvm_vm *vm)
2086 {
2087 	return ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
2088 }
2089 
vm_calc_num_pages(unsigned int num_pages,unsigned int page_shift,unsigned int new_page_shift,bool ceil)2090 static unsigned int vm_calc_num_pages(unsigned int num_pages,
2091 				      unsigned int page_shift,
2092 				      unsigned int new_page_shift,
2093 				      bool ceil)
2094 {
2095 	unsigned int n = 1 << (new_page_shift - page_shift);
2096 
2097 	if (page_shift >= new_page_shift)
2098 		return num_pages * (1 << (page_shift - new_page_shift));
2099 
2100 	return num_pages / n + !!(ceil && num_pages % n);
2101 }
2102 
getpageshift(void)2103 static inline int getpageshift(void)
2104 {
2105 	return __builtin_ffs(getpagesize()) - 1;
2106 }
2107 
2108 unsigned int
vm_num_host_pages(enum vm_guest_mode mode,unsigned int num_guest_pages)2109 vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
2110 {
2111 	return vm_calc_num_pages(num_guest_pages,
2112 				 vm_guest_mode_params[mode].page_shift,
2113 				 getpageshift(), true);
2114 }
2115 
2116 unsigned int
vm_num_guest_pages(enum vm_guest_mode mode,unsigned int num_host_pages)2117 vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages)
2118 {
2119 	return vm_calc_num_pages(num_host_pages, getpageshift(),
2120 				 vm_guest_mode_params[mode].page_shift, false);
2121 }
2122 
vm_calc_num_guest_pages(enum vm_guest_mode mode,size_t size)2123 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size)
2124 {
2125 	unsigned int n;
2126 	n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size);
2127 	return vm_adjust_num_guest_pages(mode, n);
2128 }
2129 
2130 /*
2131  * Read binary stats descriptors
2132  *
2133  * Input Args:
2134  *   stats_fd - the file descriptor for the binary stats file from which to read
2135  *   header - the binary stats metadata header corresponding to the given FD
2136  *
2137  * Output Args: None
2138  *
2139  * Return:
2140  *   A pointer to a newly allocated series of stat descriptors.
2141  *   Caller is responsible for freeing the returned kvm_stats_desc.
2142  *
2143  * Read the stats descriptors from the binary stats interface.
2144  */
read_stats_descriptors(int stats_fd,struct kvm_stats_header * header)2145 struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
2146 					      struct kvm_stats_header *header)
2147 {
2148 	struct kvm_stats_desc *stats_desc;
2149 	ssize_t desc_size, total_size, ret;
2150 
2151 	desc_size = get_stats_descriptor_size(header);
2152 	total_size = header->num_desc * desc_size;
2153 
2154 	stats_desc = calloc(header->num_desc, desc_size);
2155 	TEST_ASSERT(stats_desc, "Allocate memory for stats descriptors");
2156 
2157 	ret = pread(stats_fd, stats_desc, total_size, header->desc_offset);
2158 	TEST_ASSERT(ret == total_size, "Read KVM stats descriptors");
2159 
2160 	return stats_desc;
2161 }
2162 
2163 /*
2164  * Read stat data for a particular stat
2165  *
2166  * Input Args:
2167  *   stats_fd - the file descriptor for the binary stats file from which to read
2168  *   header - the binary stats metadata header corresponding to the given FD
2169  *   desc - the binary stat metadata for the particular stat to be read
2170  *   max_elements - the maximum number of 8-byte values to read into data
2171  *
2172  * Output Args:
2173  *   data - the buffer into which stat data should be read
2174  *
2175  * Read the data values of a specified stat from the binary stats interface.
2176  */
read_stat_data(int stats_fd,struct kvm_stats_header * header,struct kvm_stats_desc * desc,uint64_t * data,size_t max_elements)2177 void read_stat_data(int stats_fd, struct kvm_stats_header *header,
2178 		    struct kvm_stats_desc *desc, uint64_t *data,
2179 		    size_t max_elements)
2180 {
2181 	size_t nr_elements = min_t(ssize_t, desc->size, max_elements);
2182 	size_t size = nr_elements * sizeof(*data);
2183 	ssize_t ret;
2184 
2185 	TEST_ASSERT(desc->size, "No elements in stat '%s'", desc->name);
2186 	TEST_ASSERT(max_elements, "Zero elements requested for stat '%s'", desc->name);
2187 
2188 	ret = pread(stats_fd, data, size,
2189 		    header->data_offset + desc->offset);
2190 
2191 	TEST_ASSERT(ret >= 0, "pread() failed on stat '%s', errno: %i (%s)",
2192 		    desc->name, errno, strerror(errno));
2193 	TEST_ASSERT(ret == size,
2194 		    "pread() on stat '%s' read %ld bytes, wanted %lu bytes",
2195 		    desc->name, size, ret);
2196 }
2197 
2198 /*
2199  * Read the data of the named stat
2200  *
2201  * Input Args:
2202  *   vm - the VM for which the stat should be read
2203  *   stat_name - the name of the stat to read
2204  *   max_elements - the maximum number of 8-byte values to read into data
2205  *
2206  * Output Args:
2207  *   data - the buffer into which stat data should be read
2208  *
2209  * Read the data values of a specified stat from the binary stats interface.
2210  */
__vm_get_stat(struct kvm_vm * vm,const char * stat_name,uint64_t * data,size_t max_elements)2211 void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data,
2212 		   size_t max_elements)
2213 {
2214 	struct kvm_stats_desc *desc;
2215 	size_t size_desc;
2216 	int i;
2217 
2218 	if (!vm->stats_fd) {
2219 		vm->stats_fd = vm_get_stats_fd(vm);
2220 		read_stats_header(vm->stats_fd, &vm->stats_header);
2221 		vm->stats_desc = read_stats_descriptors(vm->stats_fd,
2222 							&vm->stats_header);
2223 	}
2224 
2225 	size_desc = get_stats_descriptor_size(&vm->stats_header);
2226 
2227 	for (i = 0; i < vm->stats_header.num_desc; ++i) {
2228 		desc = (void *)vm->stats_desc + (i * size_desc);
2229 
2230 		if (strcmp(desc->name, stat_name))
2231 			continue;
2232 
2233 		read_stat_data(vm->stats_fd, &vm->stats_header, desc,
2234 			       data, max_elements);
2235 
2236 		break;
2237 	}
2238 }
2239 
kvm_arch_vm_post_create(struct kvm_vm * vm)2240 __weak void kvm_arch_vm_post_create(struct kvm_vm *vm)
2241 {
2242 }
2243 
kvm_selftest_arch_init(void)2244 __weak void kvm_selftest_arch_init(void)
2245 {
2246 }
2247 
kvm_selftest_init(void)2248 void __attribute((constructor)) kvm_selftest_init(void)
2249 {
2250 	/* Tell stdout not to buffer its content. */
2251 	setbuf(stdout, NULL);
2252 
2253 	guest_random_seed = last_guest_seed = random();
2254 	pr_info("Random seed: 0x%x\n", guest_random_seed);
2255 
2256 	kvm_selftest_arch_init();
2257 }
2258 
vm_is_gpa_protected(struct kvm_vm * vm,vm_paddr_t paddr)2259 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr)
2260 {
2261 	sparsebit_idx_t pg = 0;
2262 	struct userspace_mem_region *region;
2263 
2264 	if (!vm_arch_has_protected_memory(vm))
2265 		return false;
2266 
2267 	region = userspace_mem_region_find(vm, paddr, paddr);
2268 	TEST_ASSERT(region, "No vm physical memory at 0x%lx", paddr);
2269 
2270 	pg = paddr >> vm->page_shift;
2271 	return sparsebit_is_set(region->protected_phy_pages, pg);
2272 }
2273