1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * tools/testing/selftests/kvm/lib/kvm_util.c
4 *
5 * Copyright (C) 2018, Google LLC.
6 */
7 #include "test_util.h"
8 #include "kvm_util.h"
9 #include "processor.h"
10 #include "ucall_common.h"
11
12 #include <assert.h>
13 #include <sched.h>
14 #include <sys/mman.h>
15 #include <sys/resource.h>
16 #include <sys/types.h>
17 #include <sys/stat.h>
18 #include <unistd.h>
19 #include <linux/kernel.h>
20
21 #define KVM_UTIL_MIN_PFN 2
22
23 uint32_t guest_random_seed;
24 struct guest_random_state guest_rng;
25 static uint32_t last_guest_seed;
26
27 static size_t vcpu_mmap_sz(void);
28
__open_path_or_exit(const char * path,int flags,const char * enoent_help)29 int __open_path_or_exit(const char *path, int flags, const char *enoent_help)
30 {
31 int fd;
32
33 fd = open(path, flags);
34 if (fd < 0)
35 goto error;
36
37 return fd;
38
39 error:
40 if (errno == EACCES || errno == ENOENT)
41 ksft_exit_skip("- Cannot open '%s': %s. %s\n",
42 path, strerror(errno),
43 errno == EACCES ? "Root required?" : enoent_help);
44 TEST_FAIL("Failed to open '%s'", path);
45 }
46
open_path_or_exit(const char * path,int flags)47 int open_path_or_exit(const char *path, int flags)
48 {
49 return __open_path_or_exit(path, flags, "");
50 }
51
52 /*
53 * Open KVM_DEV_PATH if available, otherwise exit the entire program.
54 *
55 * Input Args:
56 * flags - The flags to pass when opening KVM_DEV_PATH.
57 *
58 * Return:
59 * The opened file descriptor of /dev/kvm.
60 */
_open_kvm_dev_path_or_exit(int flags)61 static int _open_kvm_dev_path_or_exit(int flags)
62 {
63 return __open_path_or_exit(KVM_DEV_PATH, flags, "Is KVM loaded and enabled?");
64 }
65
open_kvm_dev_path_or_exit(void)66 int open_kvm_dev_path_or_exit(void)
67 {
68 return _open_kvm_dev_path_or_exit(O_RDONLY);
69 }
70
get_module_param(const char * module_name,const char * param,void * buffer,size_t buffer_size)71 static ssize_t get_module_param(const char *module_name, const char *param,
72 void *buffer, size_t buffer_size)
73 {
74 const int path_size = 128;
75 char path[path_size];
76 ssize_t bytes_read;
77 int fd, r;
78
79 /* Verify KVM is loaded, to provide a more helpful SKIP message. */
80 close(open_kvm_dev_path_or_exit());
81
82 r = snprintf(path, path_size, "/sys/module/%s/parameters/%s",
83 module_name, param);
84 TEST_ASSERT(r < path_size,
85 "Failed to construct sysfs path in %d bytes.", path_size);
86
87 fd = open_path_or_exit(path, O_RDONLY);
88
89 bytes_read = read(fd, buffer, buffer_size);
90 TEST_ASSERT(bytes_read > 0, "read(%s) returned %ld, wanted %ld bytes",
91 path, bytes_read, buffer_size);
92
93 r = close(fd);
94 TEST_ASSERT(!r, "close(%s) failed", path);
95 return bytes_read;
96 }
97
kvm_get_module_param_integer(const char * module_name,const char * param)98 int kvm_get_module_param_integer(const char *module_name, const char *param)
99 {
100 /*
101 * 16 bytes to hold a 64-bit value (1 byte per char), 1 byte for the
102 * NUL char, and 1 byte because the kernel sucks and inserts a newline
103 * at the end.
104 */
105 char value[16 + 1 + 1];
106 ssize_t r;
107
108 memset(value, '\0', sizeof(value));
109
110 r = get_module_param(module_name, param, value, sizeof(value));
111 TEST_ASSERT(value[r - 1] == '\n',
112 "Expected trailing newline, got char '%c'", value[r - 1]);
113
114 /*
115 * Squash the newline, otherwise atoi_paranoid() will complain about
116 * trailing non-NUL characters in the string.
117 */
118 value[r - 1] = '\0';
119 return atoi_paranoid(value);
120 }
121
kvm_get_module_param_bool(const char * module_name,const char * param)122 bool kvm_get_module_param_bool(const char *module_name, const char *param)
123 {
124 char value;
125 ssize_t r;
126
127 r = get_module_param(module_name, param, &value, sizeof(value));
128 TEST_ASSERT_EQ(r, 1);
129
130 if (value == 'Y')
131 return true;
132 else if (value == 'N')
133 return false;
134
135 TEST_FAIL("Unrecognized value '%c' for boolean module param", value);
136 }
137
138 /*
139 * Capability
140 *
141 * Input Args:
142 * cap - Capability
143 *
144 * Output Args: None
145 *
146 * Return:
147 * On success, the Value corresponding to the capability (KVM_CAP_*)
148 * specified by the value of cap. On failure a TEST_ASSERT failure
149 * is produced.
150 *
151 * Looks up and returns the value corresponding to the capability
152 * (KVM_CAP_*) given by cap.
153 */
kvm_check_cap(long cap)154 unsigned int kvm_check_cap(long cap)
155 {
156 int ret;
157 int kvm_fd;
158
159 kvm_fd = open_kvm_dev_path_or_exit();
160 ret = __kvm_ioctl(kvm_fd, KVM_CHECK_EXTENSION, (void *)cap);
161 TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_CHECK_EXTENSION, ret));
162
163 close(kvm_fd);
164
165 return (unsigned int)ret;
166 }
167
vm_enable_dirty_ring(struct kvm_vm * vm,uint32_t ring_size)168 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
169 {
170 if (vm_check_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL))
171 vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL, ring_size);
172 else
173 vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size);
174 vm->dirty_ring_size = ring_size;
175 }
176
vm_open(struct kvm_vm * vm)177 static void vm_open(struct kvm_vm *vm)
178 {
179 vm->kvm_fd = _open_kvm_dev_path_or_exit(O_RDWR);
180
181 TEST_REQUIRE(kvm_has_cap(KVM_CAP_IMMEDIATE_EXIT));
182
183 vm->fd = __kvm_ioctl(vm->kvm_fd, KVM_CREATE_VM, (void *)vm->type);
184 TEST_ASSERT(vm->fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, vm->fd));
185
186 if (kvm_has_cap(KVM_CAP_BINARY_STATS_FD))
187 vm->stats.fd = vm_get_stats_fd(vm);
188 else
189 vm->stats.fd = -1;
190 }
191
vm_guest_mode_string(uint32_t i)192 const char *vm_guest_mode_string(uint32_t i)
193 {
194 static const char * const strings[] = {
195 [VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",
196 [VM_MODE_P52V48_16K] = "PA-bits:52, VA-bits:48, 16K pages",
197 [VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages",
198 [VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages",
199 [VM_MODE_P48V48_16K] = "PA-bits:48, VA-bits:48, 16K pages",
200 [VM_MODE_P48V48_64K] = "PA-bits:48, VA-bits:48, 64K pages",
201 [VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",
202 [VM_MODE_P40V48_16K] = "PA-bits:40, VA-bits:48, 16K pages",
203 [VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",
204 [VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",
205 [VM_MODE_P47V64_4K] = "PA-bits:47, VA-bits:64, 4K pages",
206 [VM_MODE_P44V64_4K] = "PA-bits:44, VA-bits:64, 4K pages",
207 [VM_MODE_P36V48_4K] = "PA-bits:36, VA-bits:48, 4K pages",
208 [VM_MODE_P36V48_16K] = "PA-bits:36, VA-bits:48, 16K pages",
209 [VM_MODE_P36V48_64K] = "PA-bits:36, VA-bits:48, 64K pages",
210 [VM_MODE_P47V47_16K] = "PA-bits:47, VA-bits:47, 16K pages",
211 [VM_MODE_P36V47_16K] = "PA-bits:36, VA-bits:47, 16K pages",
212 };
213 _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,
214 "Missing new mode strings?");
215
216 TEST_ASSERT(i < NUM_VM_MODES, "Guest mode ID %d too big", i);
217
218 return strings[i];
219 }
220
221 const struct vm_guest_mode_params vm_guest_mode_params[] = {
222 [VM_MODE_P52V48_4K] = { 52, 48, 0x1000, 12 },
223 [VM_MODE_P52V48_16K] = { 52, 48, 0x4000, 14 },
224 [VM_MODE_P52V48_64K] = { 52, 48, 0x10000, 16 },
225 [VM_MODE_P48V48_4K] = { 48, 48, 0x1000, 12 },
226 [VM_MODE_P48V48_16K] = { 48, 48, 0x4000, 14 },
227 [VM_MODE_P48V48_64K] = { 48, 48, 0x10000, 16 },
228 [VM_MODE_P40V48_4K] = { 40, 48, 0x1000, 12 },
229 [VM_MODE_P40V48_16K] = { 40, 48, 0x4000, 14 },
230 [VM_MODE_P40V48_64K] = { 40, 48, 0x10000, 16 },
231 [VM_MODE_PXXV48_4K] = { 0, 0, 0x1000, 12 },
232 [VM_MODE_P47V64_4K] = { 47, 64, 0x1000, 12 },
233 [VM_MODE_P44V64_4K] = { 44, 64, 0x1000, 12 },
234 [VM_MODE_P36V48_4K] = { 36, 48, 0x1000, 12 },
235 [VM_MODE_P36V48_16K] = { 36, 48, 0x4000, 14 },
236 [VM_MODE_P36V48_64K] = { 36, 48, 0x10000, 16 },
237 [VM_MODE_P47V47_16K] = { 47, 47, 0x4000, 14 },
238 [VM_MODE_P36V47_16K] = { 36, 47, 0x4000, 14 },
239 };
240 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
241 "Missing new mode params?");
242
243 /*
244 * Initializes vm->vpages_valid to match the canonical VA space of the
245 * architecture.
246 *
247 * The default implementation is valid for architectures which split the
248 * range addressed by a single page table into a low and high region
249 * based on the MSB of the VA. On architectures with this behavior
250 * the VA region spans [0, 2^(va_bits - 1)), [-(2^(va_bits - 1), -1].
251 */
vm_vaddr_populate_bitmap(struct kvm_vm * vm)252 __weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm)
253 {
254 sparsebit_set_num(vm->vpages_valid,
255 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
256 sparsebit_set_num(vm->vpages_valid,
257 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
258 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
259 }
260
____vm_create(struct vm_shape shape)261 struct kvm_vm *____vm_create(struct vm_shape shape)
262 {
263 struct kvm_vm *vm;
264
265 vm = calloc(1, sizeof(*vm));
266 TEST_ASSERT(vm != NULL, "Insufficient Memory");
267
268 INIT_LIST_HEAD(&vm->vcpus);
269 vm->regions.gpa_tree = RB_ROOT;
270 vm->regions.hva_tree = RB_ROOT;
271 hash_init(vm->regions.slot_hash);
272
273 vm->mode = shape.mode;
274 vm->type = shape.type;
275
276 vm->pa_bits = vm_guest_mode_params[vm->mode].pa_bits;
277 vm->va_bits = vm_guest_mode_params[vm->mode].va_bits;
278 vm->page_size = vm_guest_mode_params[vm->mode].page_size;
279 vm->page_shift = vm_guest_mode_params[vm->mode].page_shift;
280
281 /* Setup mode specific traits. */
282 switch (vm->mode) {
283 case VM_MODE_P52V48_4K:
284 vm->pgtable_levels = 4;
285 break;
286 case VM_MODE_P52V48_64K:
287 vm->pgtable_levels = 3;
288 break;
289 case VM_MODE_P48V48_4K:
290 vm->pgtable_levels = 4;
291 break;
292 case VM_MODE_P48V48_64K:
293 vm->pgtable_levels = 3;
294 break;
295 case VM_MODE_P40V48_4K:
296 case VM_MODE_P36V48_4K:
297 vm->pgtable_levels = 4;
298 break;
299 case VM_MODE_P40V48_64K:
300 case VM_MODE_P36V48_64K:
301 vm->pgtable_levels = 3;
302 break;
303 case VM_MODE_P52V48_16K:
304 case VM_MODE_P48V48_16K:
305 case VM_MODE_P40V48_16K:
306 case VM_MODE_P36V48_16K:
307 vm->pgtable_levels = 4;
308 break;
309 case VM_MODE_P47V47_16K:
310 case VM_MODE_P36V47_16K:
311 vm->pgtable_levels = 3;
312 break;
313 case VM_MODE_PXXV48_4K:
314 #ifdef __x86_64__
315 kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);
316 kvm_init_vm_address_properties(vm);
317 /*
318 * Ignore KVM support for 5-level paging (vm->va_bits == 57),
319 * it doesn't take effect unless a CR4.LA57 is set, which it
320 * isn't for this mode (48-bit virtual address space).
321 */
322 TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,
323 "Linear address width (%d bits) not supported",
324 vm->va_bits);
325 pr_debug("Guest physical address width detected: %d\n",
326 vm->pa_bits);
327 vm->pgtable_levels = 4;
328 vm->va_bits = 48;
329 #else
330 TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");
331 #endif
332 break;
333 case VM_MODE_P47V64_4K:
334 vm->pgtable_levels = 5;
335 break;
336 case VM_MODE_P44V64_4K:
337 vm->pgtable_levels = 5;
338 break;
339 default:
340 TEST_FAIL("Unknown guest mode: 0x%x", vm->mode);
341 }
342
343 #ifdef __aarch64__
344 TEST_ASSERT(!vm->type, "ARM doesn't support test-provided types");
345 if (vm->pa_bits != 40)
346 vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
347 #endif
348
349 vm_open(vm);
350
351 /* Limit to VA-bit canonical virtual addresses. */
352 vm->vpages_valid = sparsebit_alloc();
353 vm_vaddr_populate_bitmap(vm);
354
355 /* Limit physical addresses to PA-bits. */
356 vm->max_gfn = vm_compute_max_gfn(vm);
357
358 /* Allocate and setup memory for guest. */
359 vm->vpages_mapped = sparsebit_alloc();
360
361 return vm;
362 }
363
vm_nr_pages_required(enum vm_guest_mode mode,uint32_t nr_runnable_vcpus,uint64_t extra_mem_pages)364 static uint64_t vm_nr_pages_required(enum vm_guest_mode mode,
365 uint32_t nr_runnable_vcpus,
366 uint64_t extra_mem_pages)
367 {
368 uint64_t page_size = vm_guest_mode_params[mode].page_size;
369 uint64_t nr_pages;
370
371 TEST_ASSERT(nr_runnable_vcpus,
372 "Use vm_create_barebones() for VMs that _never_ have vCPUs");
373
374 TEST_ASSERT(nr_runnable_vcpus <= kvm_check_cap(KVM_CAP_MAX_VCPUS),
375 "nr_vcpus = %d too large for host, max-vcpus = %d",
376 nr_runnable_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS));
377
378 /*
379 * Arbitrarily allocate 512 pages (2mb when page size is 4kb) for the
380 * test code and other per-VM assets that will be loaded into memslot0.
381 */
382 nr_pages = 512;
383
384 /* Account for the per-vCPU stacks on behalf of the test. */
385 nr_pages += nr_runnable_vcpus * DEFAULT_STACK_PGS;
386
387 /*
388 * Account for the number of pages needed for the page tables. The
389 * maximum page table size for a memory region will be when the
390 * smallest page size is used. Considering each page contains x page
391 * table descriptors, the total extra size for page tables (for extra
392 * N pages) will be: N/x+N/x^2+N/x^3+... which is definitely smaller
393 * than N/x*2.
394 */
395 nr_pages += (nr_pages + extra_mem_pages) / PTES_PER_MIN_PAGE * 2;
396
397 /* Account for the number of pages needed by ucall. */
398 nr_pages += ucall_nr_pages_required(page_size);
399
400 return vm_adjust_num_guest_pages(mode, nr_pages);
401 }
402
kvm_set_files_rlimit(uint32_t nr_vcpus)403 void kvm_set_files_rlimit(uint32_t nr_vcpus)
404 {
405 /*
406 * Each vCPU will open two file descriptors: the vCPU itself and the
407 * vCPU's binary stats file descriptor. Add an arbitrary amount of
408 * buffer for all other files a test may open.
409 */
410 int nr_fds_wanted = nr_vcpus * 2 + 100;
411 struct rlimit rl;
412
413 /*
414 * Check that we're allowed to open nr_fds_wanted file descriptors and
415 * try raising the limits if needed.
416 */
417 TEST_ASSERT(!getrlimit(RLIMIT_NOFILE, &rl), "getrlimit() failed!");
418
419 if (rl.rlim_cur < nr_fds_wanted) {
420 rl.rlim_cur = nr_fds_wanted;
421 if (rl.rlim_max < nr_fds_wanted) {
422 int old_rlim_max = rl.rlim_max;
423
424 rl.rlim_max = nr_fds_wanted;
425 __TEST_REQUIRE(setrlimit(RLIMIT_NOFILE, &rl) >= 0,
426 "RLIMIT_NOFILE hard limit is too low (%d, wanted %d)",
427 old_rlim_max, nr_fds_wanted);
428 } else {
429 TEST_ASSERT(!setrlimit(RLIMIT_NOFILE, &rl), "setrlimit() failed!");
430 }
431 }
432
433 }
434
is_guest_memfd_required(struct vm_shape shape)435 static bool is_guest_memfd_required(struct vm_shape shape)
436 {
437 #ifdef __x86_64__
438 return shape.type == KVM_X86_SNP_VM;
439 #else
440 return false;
441 #endif
442 }
443
__vm_create(struct vm_shape shape,uint32_t nr_runnable_vcpus,uint64_t nr_extra_pages)444 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
445 uint64_t nr_extra_pages)
446 {
447 uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,
448 nr_extra_pages);
449 struct userspace_mem_region *slot0;
450 struct kvm_vm *vm;
451 int i, flags;
452
453 kvm_set_files_rlimit(nr_runnable_vcpus);
454
455 pr_debug("%s: mode='%s' type='%d', pages='%ld'\n", __func__,
456 vm_guest_mode_string(shape.mode), shape.type, nr_pages);
457
458 vm = ____vm_create(shape);
459
460 /*
461 * Force GUEST_MEMFD for the primary memory region if necessary, e.g.
462 * for CoCo VMs that require GUEST_MEMFD backed private memory.
463 */
464 flags = 0;
465 if (is_guest_memfd_required(shape))
466 flags |= KVM_MEM_GUEST_MEMFD;
467
468 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags);
469 for (i = 0; i < NR_MEM_REGIONS; i++)
470 vm->memslots[i] = 0;
471
472 kvm_vm_elf_load(vm, program_invocation_name);
473
474 /*
475 * TODO: Add proper defines to protect the library's memslots, and then
476 * carve out memslot1 for the ucall MMIO address. KVM treats writes to
477 * read-only memslots as MMIO, and creating a read-only memslot for the
478 * MMIO region would prevent silently clobbering the MMIO region.
479 */
480 slot0 = memslot2region(vm, 0);
481 ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);
482
483 if (guest_random_seed != last_guest_seed) {
484 pr_info("Random seed: 0x%x\n", guest_random_seed);
485 last_guest_seed = guest_random_seed;
486 }
487 guest_rng = new_guest_random_state(guest_random_seed);
488 sync_global_to_guest(vm, guest_rng);
489
490 kvm_arch_vm_post_create(vm, nr_runnable_vcpus);
491
492 return vm;
493 }
494
495 /*
496 * VM Create with customized parameters
497 *
498 * Input Args:
499 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)
500 * nr_vcpus - VCPU count
501 * extra_mem_pages - Non-slot0 physical memory total size
502 * guest_code - Guest entry point
503 * vcpuids - VCPU IDs
504 *
505 * Output Args: None
506 *
507 * Return:
508 * Pointer to opaque structure that describes the created VM.
509 *
510 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
511 * extra_mem_pages is only used to calculate the maximum page table size,
512 * no real memory allocation for non-slot0 memory in this function.
513 */
__vm_create_with_vcpus(struct vm_shape shape,uint32_t nr_vcpus,uint64_t extra_mem_pages,void * guest_code,struct kvm_vcpu * vcpus[])514 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
515 uint64_t extra_mem_pages,
516 void *guest_code, struct kvm_vcpu *vcpus[])
517 {
518 struct kvm_vm *vm;
519 int i;
520
521 TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array");
522
523 vm = __vm_create(shape, nr_vcpus, extra_mem_pages);
524
525 for (i = 0; i < nr_vcpus; ++i)
526 vcpus[i] = vm_vcpu_add(vm, i, guest_code);
527
528 kvm_arch_vm_finalize_vcpus(vm);
529 return vm;
530 }
531
__vm_create_shape_with_one_vcpu(struct vm_shape shape,struct kvm_vcpu ** vcpu,uint64_t extra_mem_pages,void * guest_code)532 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
533 struct kvm_vcpu **vcpu,
534 uint64_t extra_mem_pages,
535 void *guest_code)
536 {
537 struct kvm_vcpu *vcpus[1];
538 struct kvm_vm *vm;
539
540 vm = __vm_create_with_vcpus(shape, 1, extra_mem_pages, guest_code, vcpus);
541
542 *vcpu = vcpus[0];
543 return vm;
544 }
545
546 /*
547 * VM Restart
548 *
549 * Input Args:
550 * vm - VM that has been released before
551 *
552 * Output Args: None
553 *
554 * Reopens the file descriptors associated to the VM and reinstates the
555 * global state, such as the irqchip and the memory regions that are mapped
556 * into the guest.
557 */
kvm_vm_restart(struct kvm_vm * vmp)558 void kvm_vm_restart(struct kvm_vm *vmp)
559 {
560 int ctr;
561 struct userspace_mem_region *region;
562
563 vm_open(vmp);
564 if (vmp->has_irqchip)
565 vm_create_irqchip(vmp);
566
567 hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {
568 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION2, ®ion->region);
569
570 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
571 " rc: %i errno: %i\n"
572 " slot: %u flags: 0x%x\n"
573 " guest_phys_addr: 0x%llx size: 0x%llx",
574 ret, errno, region->region.slot,
575 region->region.flags,
576 region->region.guest_phys_addr,
577 region->region.memory_size);
578 }
579 }
580
vm_arch_vcpu_recreate(struct kvm_vm * vm,uint32_t vcpu_id)581 __weak struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm,
582 uint32_t vcpu_id)
583 {
584 return __vm_vcpu_add(vm, vcpu_id);
585 }
586
vm_recreate_with_one_vcpu(struct kvm_vm * vm)587 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm)
588 {
589 kvm_vm_restart(vm);
590
591 return vm_vcpu_recreate(vm, 0);
592 }
593
__pin_task_to_cpu(pthread_t task,int cpu)594 int __pin_task_to_cpu(pthread_t task, int cpu)
595 {
596 cpu_set_t cpuset;
597
598 CPU_ZERO(&cpuset);
599 CPU_SET(cpu, &cpuset);
600
601 return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset);
602 }
603
parse_pcpu(const char * cpu_str,const cpu_set_t * allowed_mask)604 static uint32_t parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask)
605 {
606 uint32_t pcpu = atoi_non_negative("CPU number", cpu_str);
607
608 TEST_ASSERT(CPU_ISSET(pcpu, allowed_mask),
609 "Not allowed to run on pCPU '%d', check cgroups?", pcpu);
610 return pcpu;
611 }
612
kvm_print_vcpu_pinning_help(void)613 void kvm_print_vcpu_pinning_help(void)
614 {
615 const char *name = program_invocation_name;
616
617 printf(" -c: Pin tasks to physical CPUs. Takes a list of comma separated\n"
618 " values (target pCPU), one for each vCPU, plus an optional\n"
619 " entry for the main application task (specified via entry\n"
620 " <nr_vcpus + 1>). If used, entries must be provided for all\n"
621 " vCPUs, i.e. pinning vCPUs is all or nothing.\n\n"
622 " E.g. to create 3 vCPUs, pin vCPU0=>pCPU22, vCPU1=>pCPU23,\n"
623 " vCPU2=>pCPU24, and pin the application task to pCPU50:\n\n"
624 " %s -v 3 -c 22,23,24,50\n\n"
625 " To leave the application task unpinned, drop the final entry:\n\n"
626 " %s -v 3 -c 22,23,24\n\n"
627 " (default: no pinning)\n", name, name);
628 }
629
kvm_parse_vcpu_pinning(const char * pcpus_string,uint32_t vcpu_to_pcpu[],int nr_vcpus)630 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
631 int nr_vcpus)
632 {
633 cpu_set_t allowed_mask;
634 char *cpu, *cpu_list;
635 char delim[2] = ",";
636 int i, r;
637
638 cpu_list = strdup(pcpus_string);
639 TEST_ASSERT(cpu_list, "strdup() allocation failed.");
640
641 r = sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask);
642 TEST_ASSERT(!r, "sched_getaffinity() failed");
643
644 cpu = strtok(cpu_list, delim);
645
646 /* 1. Get all pcpus for vcpus. */
647 for (i = 0; i < nr_vcpus; i++) {
648 TEST_ASSERT(cpu, "pCPU not provided for vCPU '%d'", i);
649 vcpu_to_pcpu[i] = parse_pcpu(cpu, &allowed_mask);
650 cpu = strtok(NULL, delim);
651 }
652
653 /* 2. Check if the main worker needs to be pinned. */
654 if (cpu) {
655 pin_self_to_cpu(parse_pcpu(cpu, &allowed_mask));
656 cpu = strtok(NULL, delim);
657 }
658
659 TEST_ASSERT(!cpu, "pCPU list contains trailing garbage characters '%s'", cpu);
660 free(cpu_list);
661 }
662
663 /*
664 * Userspace Memory Region Find
665 *
666 * Input Args:
667 * vm - Virtual Machine
668 * start - Starting VM physical address
669 * end - Ending VM physical address, inclusive.
670 *
671 * Output Args: None
672 *
673 * Return:
674 * Pointer to overlapping region, NULL if no such region.
675 *
676 * Searches for a region with any physical memory that overlaps with
677 * any portion of the guest physical addresses from start to end
678 * inclusive. If multiple overlapping regions exist, a pointer to any
679 * of the regions is returned. Null is returned only when no overlapping
680 * region exists.
681 */
682 static struct userspace_mem_region *
userspace_mem_region_find(struct kvm_vm * vm,uint64_t start,uint64_t end)683 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
684 {
685 struct rb_node *node;
686
687 for (node = vm->regions.gpa_tree.rb_node; node; ) {
688 struct userspace_mem_region *region =
689 container_of(node, struct userspace_mem_region, gpa_node);
690 uint64_t existing_start = region->region.guest_phys_addr;
691 uint64_t existing_end = region->region.guest_phys_addr
692 + region->region.memory_size - 1;
693 if (start <= existing_end && end >= existing_start)
694 return region;
695
696 if (start < existing_start)
697 node = node->rb_left;
698 else
699 node = node->rb_right;
700 }
701
702 return NULL;
703 }
704
kvm_stats_release(struct kvm_binary_stats * stats)705 static void kvm_stats_release(struct kvm_binary_stats *stats)
706 {
707 int ret;
708
709 if (stats->fd < 0)
710 return;
711
712 if (stats->desc) {
713 free(stats->desc);
714 stats->desc = NULL;
715 }
716
717 ret = close(stats->fd);
718 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));
719 stats->fd = -1;
720 }
721
vcpu_arch_free(struct kvm_vcpu * vcpu)722 __weak void vcpu_arch_free(struct kvm_vcpu *vcpu)
723 {
724
725 }
726
727 /*
728 * VM VCPU Remove
729 *
730 * Input Args:
731 * vcpu - VCPU to remove
732 *
733 * Output Args: None
734 *
735 * Return: None, TEST_ASSERT failures for all error conditions
736 *
737 * Removes a vCPU from a VM and frees its resources.
738 */
vm_vcpu_rm(struct kvm_vm * vm,struct kvm_vcpu * vcpu)739 static void vm_vcpu_rm(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
740 {
741 int ret;
742
743 if (vcpu->dirty_gfns) {
744 kvm_munmap(vcpu->dirty_gfns, vm->dirty_ring_size);
745 vcpu->dirty_gfns = NULL;
746 }
747
748 kvm_munmap(vcpu->run, vcpu_mmap_sz());
749
750 ret = close(vcpu->fd);
751 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));
752
753 kvm_stats_release(&vcpu->stats);
754
755 list_del(&vcpu->list);
756
757 vcpu_arch_free(vcpu);
758 free(vcpu);
759 }
760
kvm_vm_release(struct kvm_vm * vmp)761 void kvm_vm_release(struct kvm_vm *vmp)
762 {
763 struct kvm_vcpu *vcpu, *tmp;
764 int ret;
765
766 list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)
767 vm_vcpu_rm(vmp, vcpu);
768
769 ret = close(vmp->fd);
770 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));
771
772 ret = close(vmp->kvm_fd);
773 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));
774
775 /* Free cached stats metadata and close FD */
776 kvm_stats_release(&vmp->stats);
777
778 kvm_arch_vm_release(vmp);
779 }
780
__vm_mem_region_delete(struct kvm_vm * vm,struct userspace_mem_region * region)781 static void __vm_mem_region_delete(struct kvm_vm *vm,
782 struct userspace_mem_region *region)
783 {
784 rb_erase(®ion->gpa_node, &vm->regions.gpa_tree);
785 rb_erase(®ion->hva_node, &vm->regions.hva_tree);
786 hash_del(®ion->slot_node);
787
788 sparsebit_free(®ion->unused_phy_pages);
789 sparsebit_free(®ion->protected_phy_pages);
790 kvm_munmap(region->mmap_start, region->mmap_size);
791 if (region->fd >= 0) {
792 /* There's an extra map when using shared memory. */
793 kvm_munmap(region->mmap_alias, region->mmap_size);
794 close(region->fd);
795 }
796 if (region->region.guest_memfd >= 0)
797 close(region->region.guest_memfd);
798
799 free(region);
800 }
801
802 /*
803 * Destroys and frees the VM pointed to by vmp.
804 */
kvm_vm_free(struct kvm_vm * vmp)805 void kvm_vm_free(struct kvm_vm *vmp)
806 {
807 int ctr;
808 struct hlist_node *node;
809 struct userspace_mem_region *region;
810
811 if (vmp == NULL)
812 return;
813
814 /* Free userspace_mem_regions. */
815 hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)
816 __vm_mem_region_delete(vmp, region);
817
818 /* Free sparsebit arrays. */
819 sparsebit_free(&vmp->vpages_valid);
820 sparsebit_free(&vmp->vpages_mapped);
821
822 kvm_vm_release(vmp);
823
824 /* Free the structure describing the VM. */
825 free(vmp);
826 }
827
kvm_memfd_alloc(size_t size,bool hugepages)828 int kvm_memfd_alloc(size_t size, bool hugepages)
829 {
830 int memfd_flags = MFD_CLOEXEC;
831 int fd, r;
832
833 if (hugepages)
834 memfd_flags |= MFD_HUGETLB;
835
836 fd = memfd_create("kvm_selftest", memfd_flags);
837 TEST_ASSERT(fd != -1, __KVM_SYSCALL_ERROR("memfd_create()", fd));
838
839 r = ftruncate(fd, size);
840 TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("ftruncate()", r));
841
842 r = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0, size);
843 TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
844
845 return fd;
846 }
847
vm_userspace_mem_region_gpa_insert(struct rb_root * gpa_tree,struct userspace_mem_region * region)848 static void vm_userspace_mem_region_gpa_insert(struct rb_root *gpa_tree,
849 struct userspace_mem_region *region)
850 {
851 struct rb_node **cur, *parent;
852
853 for (cur = &gpa_tree->rb_node, parent = NULL; *cur; ) {
854 struct userspace_mem_region *cregion;
855
856 cregion = container_of(*cur, typeof(*cregion), gpa_node);
857 parent = *cur;
858 if (region->region.guest_phys_addr <
859 cregion->region.guest_phys_addr)
860 cur = &(*cur)->rb_left;
861 else {
862 TEST_ASSERT(region->region.guest_phys_addr !=
863 cregion->region.guest_phys_addr,
864 "Duplicate GPA in region tree");
865
866 cur = &(*cur)->rb_right;
867 }
868 }
869
870 rb_link_node(®ion->gpa_node, parent, cur);
871 rb_insert_color(®ion->gpa_node, gpa_tree);
872 }
873
vm_userspace_mem_region_hva_insert(struct rb_root * hva_tree,struct userspace_mem_region * region)874 static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,
875 struct userspace_mem_region *region)
876 {
877 struct rb_node **cur, *parent;
878
879 for (cur = &hva_tree->rb_node, parent = NULL; *cur; ) {
880 struct userspace_mem_region *cregion;
881
882 cregion = container_of(*cur, typeof(*cregion), hva_node);
883 parent = *cur;
884 if (region->host_mem < cregion->host_mem)
885 cur = &(*cur)->rb_left;
886 else {
887 TEST_ASSERT(region->host_mem !=
888 cregion->host_mem,
889 "Duplicate HVA in region tree");
890
891 cur = &(*cur)->rb_right;
892 }
893 }
894
895 rb_link_node(®ion->hva_node, parent, cur);
896 rb_insert_color(®ion->hva_node, hva_tree);
897 }
898
899
__vm_set_user_memory_region(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva)900 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
901 uint64_t gpa, uint64_t size, void *hva)
902 {
903 struct kvm_userspace_memory_region region = {
904 .slot = slot,
905 .flags = flags,
906 .guest_phys_addr = gpa,
907 .memory_size = size,
908 .userspace_addr = (uintptr_t)hva,
909 };
910
911 return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion);
912 }
913
vm_set_user_memory_region(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva)914 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
915 uint64_t gpa, uint64_t size, void *hva)
916 {
917 int ret = __vm_set_user_memory_region(vm, slot, flags, gpa, size, hva);
918
919 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed, errno = %d (%s)",
920 errno, strerror(errno));
921 }
922
923 #define TEST_REQUIRE_SET_USER_MEMORY_REGION2() \
924 __TEST_REQUIRE(kvm_has_cap(KVM_CAP_USER_MEMORY2), \
925 "KVM selftests now require KVM_SET_USER_MEMORY_REGION2 (introduced in v6.8)")
926
__vm_set_user_memory_region2(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva,uint32_t guest_memfd,uint64_t guest_memfd_offset)927 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
928 uint64_t gpa, uint64_t size, void *hva,
929 uint32_t guest_memfd, uint64_t guest_memfd_offset)
930 {
931 struct kvm_userspace_memory_region2 region = {
932 .slot = slot,
933 .flags = flags,
934 .guest_phys_addr = gpa,
935 .memory_size = size,
936 .userspace_addr = (uintptr_t)hva,
937 .guest_memfd = guest_memfd,
938 .guest_memfd_offset = guest_memfd_offset,
939 };
940
941 TEST_REQUIRE_SET_USER_MEMORY_REGION2();
942
943 return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION2, ®ion);
944 }
945
vm_set_user_memory_region2(struct kvm_vm * vm,uint32_t slot,uint32_t flags,uint64_t gpa,uint64_t size,void * hva,uint32_t guest_memfd,uint64_t guest_memfd_offset)946 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
947 uint64_t gpa, uint64_t size, void *hva,
948 uint32_t guest_memfd, uint64_t guest_memfd_offset)
949 {
950 int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva,
951 guest_memfd, guest_memfd_offset);
952
953 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)",
954 errno, strerror(errno));
955 }
956
957
958 /* FIXME: This thing needs to be ripped apart and rewritten. */
vm_mem_add(struct kvm_vm * vm,enum vm_mem_backing_src_type src_type,uint64_t guest_paddr,uint32_t slot,uint64_t npages,uint32_t flags,int guest_memfd,uint64_t guest_memfd_offset)959 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
960 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
961 uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset)
962 {
963 int ret;
964 struct userspace_mem_region *region;
965 size_t backing_src_pagesz = get_backing_src_pagesz(src_type);
966 size_t mem_size = npages * vm->page_size;
967 size_t alignment;
968
969 TEST_REQUIRE_SET_USER_MEMORY_REGION2();
970
971 TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,
972 "Number of guest pages is not compatible with the host. "
973 "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));
974
975 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
976 "address not on a page boundary.\n"
977 " guest_paddr: 0x%lx vm->page_size: 0x%x",
978 guest_paddr, vm->page_size);
979 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
980 <= vm->max_gfn, "Physical range beyond maximum "
981 "supported physical address,\n"
982 " guest_paddr: 0x%lx npages: 0x%lx\n"
983 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
984 guest_paddr, npages, vm->max_gfn, vm->page_size);
985
986 /*
987 * Confirm a mem region with an overlapping address doesn't
988 * already exist.
989 */
990 region = (struct userspace_mem_region *) userspace_mem_region_find(
991 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
992 if (region != NULL)
993 TEST_FAIL("overlapping userspace_mem_region already "
994 "exists\n"
995 " requested guest_paddr: 0x%lx npages: 0x%lx "
996 "page_size: 0x%x\n"
997 " existing guest_paddr: 0x%lx size: 0x%lx",
998 guest_paddr, npages, vm->page_size,
999 (uint64_t) region->region.guest_phys_addr,
1000 (uint64_t) region->region.memory_size);
1001
1002 /* Confirm no region with the requested slot already exists. */
1003 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
1004 slot) {
1005 if (region->region.slot != slot)
1006 continue;
1007
1008 TEST_FAIL("A mem region with the requested slot "
1009 "already exists.\n"
1010 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
1011 " existing slot: %u paddr: 0x%lx size: 0x%lx",
1012 slot, guest_paddr, npages,
1013 region->region.slot,
1014 (uint64_t) region->region.guest_phys_addr,
1015 (uint64_t) region->region.memory_size);
1016 }
1017
1018 /* Allocate and initialize new mem region structure. */
1019 region = calloc(1, sizeof(*region));
1020 TEST_ASSERT(region != NULL, "Insufficient Memory");
1021 region->mmap_size = mem_size;
1022
1023 #ifdef __s390x__
1024 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
1025 alignment = 0x100000;
1026 #else
1027 alignment = 1;
1028 #endif
1029
1030 /*
1031 * When using THP mmap is not guaranteed to returned a hugepage aligned
1032 * address so we have to pad the mmap. Padding is not needed for HugeTLB
1033 * because mmap will always return an address aligned to the HugeTLB
1034 * page size.
1035 */
1036 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
1037 alignment = max(backing_src_pagesz, alignment);
1038
1039 TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, backing_src_pagesz));
1040
1041 /* Add enough memory to align up if necessary */
1042 if (alignment > 1)
1043 region->mmap_size += alignment;
1044
1045 region->fd = -1;
1046 if (backing_src_is_shared(src_type))
1047 region->fd = kvm_memfd_alloc(region->mmap_size,
1048 src_type == VM_MEM_SRC_SHARED_HUGETLB);
1049
1050 region->mmap_start = kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE,
1051 vm_mem_backing_src_alias(src_type)->flag,
1052 region->fd);
1053
1054 TEST_ASSERT(!is_backing_src_hugetlb(src_type) ||
1055 region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz),
1056 "mmap_start %p is not aligned to HugeTLB page size 0x%lx",
1057 region->mmap_start, backing_src_pagesz);
1058
1059 /* Align host address */
1060 region->host_mem = align_ptr_up(region->mmap_start, alignment);
1061
1062 /* As needed perform madvise */
1063 if ((src_type == VM_MEM_SRC_ANONYMOUS ||
1064 src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
1065 ret = madvise(region->host_mem, mem_size,
1066 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
1067 TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
1068 region->host_mem, mem_size,
1069 vm_mem_backing_src_alias(src_type)->name);
1070 }
1071
1072 region->backing_src_type = src_type;
1073
1074 if (flags & KVM_MEM_GUEST_MEMFD) {
1075 if (guest_memfd < 0) {
1076 uint32_t guest_memfd_flags = 0;
1077 TEST_ASSERT(!guest_memfd_offset,
1078 "Offset must be zero when creating new guest_memfd");
1079 guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags);
1080 } else {
1081 /*
1082 * Install a unique fd for each memslot so that the fd
1083 * can be closed when the region is deleted without
1084 * needing to track if the fd is owned by the framework
1085 * or by the caller.
1086 */
1087 guest_memfd = dup(guest_memfd);
1088 TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd));
1089 }
1090
1091 region->region.guest_memfd = guest_memfd;
1092 region->region.guest_memfd_offset = guest_memfd_offset;
1093 } else {
1094 region->region.guest_memfd = -1;
1095 }
1096
1097 region->unused_phy_pages = sparsebit_alloc();
1098 if (vm_arch_has_protected_memory(vm))
1099 region->protected_phy_pages = sparsebit_alloc();
1100 sparsebit_set_num(region->unused_phy_pages,
1101 guest_paddr >> vm->page_shift, npages);
1102 region->region.slot = slot;
1103 region->region.flags = flags;
1104 region->region.guest_phys_addr = guest_paddr;
1105 region->region.memory_size = npages * vm->page_size;
1106 region->region.userspace_addr = (uintptr_t) region->host_mem;
1107 ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);
1108 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
1109 " rc: %i errno: %i\n"
1110 " slot: %u flags: 0x%x\n"
1111 " guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d",
1112 ret, errno, slot, flags,
1113 guest_paddr, (uint64_t) region->region.memory_size,
1114 region->region.guest_memfd);
1115
1116 /* Add to quick lookup data structures */
1117 vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);
1118 vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);
1119 hash_add(vm->regions.slot_hash, ®ion->slot_node, slot);
1120
1121 /* If shared memory, create an alias. */
1122 if (region->fd >= 0) {
1123 region->mmap_alias = kvm_mmap(region->mmap_size,
1124 PROT_READ | PROT_WRITE,
1125 vm_mem_backing_src_alias(src_type)->flag,
1126 region->fd);
1127
1128 /* Align host alias address */
1129 region->host_alias = align_ptr_up(region->mmap_alias, alignment);
1130 }
1131 }
1132
vm_userspace_mem_region_add(struct kvm_vm * vm,enum vm_mem_backing_src_type src_type,uint64_t guest_paddr,uint32_t slot,uint64_t npages,uint32_t flags)1133 void vm_userspace_mem_region_add(struct kvm_vm *vm,
1134 enum vm_mem_backing_src_type src_type,
1135 uint64_t guest_paddr, uint32_t slot,
1136 uint64_t npages, uint32_t flags)
1137 {
1138 vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0);
1139 }
1140
1141 /*
1142 * Memslot to region
1143 *
1144 * Input Args:
1145 * vm - Virtual Machine
1146 * memslot - KVM memory slot ID
1147 *
1148 * Output Args: None
1149 *
1150 * Return:
1151 * Pointer to memory region structure that describe memory region
1152 * using kvm memory slot ID given by memslot. TEST_ASSERT failure
1153 * on error (e.g. currently no memory region using memslot as a KVM
1154 * memory slot ID).
1155 */
1156 struct userspace_mem_region *
memslot2region(struct kvm_vm * vm,uint32_t memslot)1157 memslot2region(struct kvm_vm *vm, uint32_t memslot)
1158 {
1159 struct userspace_mem_region *region;
1160
1161 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
1162 memslot)
1163 if (region->region.slot == memslot)
1164 return region;
1165
1166 fprintf(stderr, "No mem region with the requested slot found,\n"
1167 " requested slot: %u\n", memslot);
1168 fputs("---- vm dump ----\n", stderr);
1169 vm_dump(stderr, vm, 2);
1170 TEST_FAIL("Mem region not found");
1171 return NULL;
1172 }
1173
1174 /*
1175 * VM Memory Region Flags Set
1176 *
1177 * Input Args:
1178 * vm - Virtual Machine
1179 * flags - Starting guest physical address
1180 *
1181 * Output Args: None
1182 *
1183 * Return: None
1184 *
1185 * Sets the flags of the memory region specified by the value of slot,
1186 * to the values given by flags.
1187 */
vm_mem_region_set_flags(struct kvm_vm * vm,uint32_t slot,uint32_t flags)1188 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
1189 {
1190 int ret;
1191 struct userspace_mem_region *region;
1192
1193 region = memslot2region(vm, slot);
1194
1195 region->region.flags = flags;
1196
1197 ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);
1198
1199 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
1200 " rc: %i errno: %i slot: %u flags: 0x%x",
1201 ret, errno, slot, flags);
1202 }
1203
1204 /*
1205 * VM Memory Region Move
1206 *
1207 * Input Args:
1208 * vm - Virtual Machine
1209 * slot - Slot of the memory region to move
1210 * new_gpa - Starting guest physical address
1211 *
1212 * Output Args: None
1213 *
1214 * Return: None
1215 *
1216 * Change the gpa of a memory region.
1217 */
vm_mem_region_move(struct kvm_vm * vm,uint32_t slot,uint64_t new_gpa)1218 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
1219 {
1220 struct userspace_mem_region *region;
1221 int ret;
1222
1223 region = memslot2region(vm, slot);
1224
1225 region->region.guest_phys_addr = new_gpa;
1226
1227 ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);
1228
1229 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed\n"
1230 "ret: %i errno: %i slot: %u new_gpa: 0x%lx",
1231 ret, errno, slot, new_gpa);
1232 }
1233
1234 /*
1235 * VM Memory Region Delete
1236 *
1237 * Input Args:
1238 * vm - Virtual Machine
1239 * slot - Slot of the memory region to delete
1240 *
1241 * Output Args: None
1242 *
1243 * Return: None
1244 *
1245 * Delete a memory region.
1246 */
vm_mem_region_delete(struct kvm_vm * vm,uint32_t slot)1247 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
1248 {
1249 struct userspace_mem_region *region = memslot2region(vm, slot);
1250
1251 region->region.memory_size = 0;
1252 vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);
1253
1254 __vm_mem_region_delete(vm, region);
1255 }
1256
vm_guest_mem_fallocate(struct kvm_vm * vm,uint64_t base,uint64_t size,bool punch_hole)1257 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
1258 bool punch_hole)
1259 {
1260 const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0);
1261 struct userspace_mem_region *region;
1262 uint64_t end = base + size;
1263 uint64_t gpa, len;
1264 off_t fd_offset;
1265 int ret;
1266
1267 for (gpa = base; gpa < end; gpa += len) {
1268 uint64_t offset;
1269
1270 region = userspace_mem_region_find(vm, gpa, gpa);
1271 TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,
1272 "Private memory region not found for GPA 0x%lx", gpa);
1273
1274 offset = gpa - region->region.guest_phys_addr;
1275 fd_offset = region->region.guest_memfd_offset + offset;
1276 len = min_t(uint64_t, end - gpa, region->region.memory_size - offset);
1277
1278 ret = fallocate(region->region.guest_memfd, mode, fd_offset, len);
1279 TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx",
1280 punch_hole ? "punch hole" : "allocate", gpa, len,
1281 region->region.guest_memfd, mode, fd_offset);
1282 }
1283 }
1284
1285 /* Returns the size of a vCPU's kvm_run structure. */
vcpu_mmap_sz(void)1286 static size_t vcpu_mmap_sz(void)
1287 {
1288 int dev_fd, ret;
1289
1290 dev_fd = open_kvm_dev_path_or_exit();
1291
1292 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
1293 TEST_ASSERT(ret >= 0 && ret >= sizeof(struct kvm_run),
1294 KVM_IOCTL_ERROR(KVM_GET_VCPU_MMAP_SIZE, ret));
1295
1296 close(dev_fd);
1297
1298 return ret;
1299 }
1300
vcpu_exists(struct kvm_vm * vm,uint32_t vcpu_id)1301 static bool vcpu_exists(struct kvm_vm *vm, uint32_t vcpu_id)
1302 {
1303 struct kvm_vcpu *vcpu;
1304
1305 list_for_each_entry(vcpu, &vm->vcpus, list) {
1306 if (vcpu->id == vcpu_id)
1307 return true;
1308 }
1309
1310 return false;
1311 }
1312
1313 /*
1314 * Adds a virtual CPU to the VM specified by vm with the ID given by vcpu_id.
1315 * No additional vCPU setup is done. Returns the vCPU.
1316 */
__vm_vcpu_add(struct kvm_vm * vm,uint32_t vcpu_id)1317 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
1318 {
1319 struct kvm_vcpu *vcpu;
1320
1321 /* Confirm a vcpu with the specified id doesn't already exist. */
1322 TEST_ASSERT(!vcpu_exists(vm, vcpu_id), "vCPU%d already exists", vcpu_id);
1323
1324 /* Allocate and initialize new vcpu structure. */
1325 vcpu = calloc(1, sizeof(*vcpu));
1326 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
1327
1328 vcpu->vm = vm;
1329 vcpu->id = vcpu_id;
1330 vcpu->fd = __vm_ioctl(vm, KVM_CREATE_VCPU, (void *)(unsigned long)vcpu_id);
1331 TEST_ASSERT_VM_VCPU_IOCTL(vcpu->fd >= 0, KVM_CREATE_VCPU, vcpu->fd, vm);
1332
1333 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->run), "vcpu mmap size "
1334 "smaller than expected, vcpu_mmap_sz: %zi expected_min: %zi",
1335 vcpu_mmap_sz(), sizeof(*vcpu->run));
1336 vcpu->run = kvm_mmap(vcpu_mmap_sz(), PROT_READ | PROT_WRITE,
1337 MAP_SHARED, vcpu->fd);
1338
1339 if (kvm_has_cap(KVM_CAP_BINARY_STATS_FD))
1340 vcpu->stats.fd = vcpu_get_stats_fd(vcpu);
1341 else
1342 vcpu->stats.fd = -1;
1343
1344 /* Add to linked-list of VCPUs. */
1345 list_add(&vcpu->list, &vm->vcpus);
1346
1347 return vcpu;
1348 }
1349
1350 /*
1351 * VM Virtual Address Unused Gap
1352 *
1353 * Input Args:
1354 * vm - Virtual Machine
1355 * sz - Size (bytes)
1356 * vaddr_min - Minimum Virtual Address
1357 *
1358 * Output Args: None
1359 *
1360 * Return:
1361 * Lowest virtual address at or below vaddr_min, with at least
1362 * sz unused bytes. TEST_ASSERT failure if no area of at least
1363 * size sz is available.
1364 *
1365 * Within the VM specified by vm, locates the lowest starting virtual
1366 * address >= vaddr_min, that has at least sz unallocated bytes. A
1367 * TEST_ASSERT failure occurs for invalid input or no area of at least
1368 * sz unallocated bytes >= vaddr_min is available.
1369 */
vm_vaddr_unused_gap(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min)1370 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
1371 vm_vaddr_t vaddr_min)
1372 {
1373 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
1374
1375 /* Determine lowest permitted virtual page index. */
1376 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
1377 if ((pgidx_start * vm->page_size) < vaddr_min)
1378 goto no_va_found;
1379
1380 /* Loop over section with enough valid virtual page indexes. */
1381 if (!sparsebit_is_set_num(vm->vpages_valid,
1382 pgidx_start, pages))
1383 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
1384 pgidx_start, pages);
1385 do {
1386 /*
1387 * Are there enough unused virtual pages available at
1388 * the currently proposed starting virtual page index.
1389 * If not, adjust proposed starting index to next
1390 * possible.
1391 */
1392 if (sparsebit_is_clear_num(vm->vpages_mapped,
1393 pgidx_start, pages))
1394 goto va_found;
1395 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
1396 pgidx_start, pages);
1397 if (pgidx_start == 0)
1398 goto no_va_found;
1399
1400 /*
1401 * If needed, adjust proposed starting virtual address,
1402 * to next range of valid virtual addresses.
1403 */
1404 if (!sparsebit_is_set_num(vm->vpages_valid,
1405 pgidx_start, pages)) {
1406 pgidx_start = sparsebit_next_set_num(
1407 vm->vpages_valid, pgidx_start, pages);
1408 if (pgidx_start == 0)
1409 goto no_va_found;
1410 }
1411 } while (pgidx_start != 0);
1412
1413 no_va_found:
1414 TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);
1415
1416 /* NOT REACHED */
1417 return -1;
1418
1419 va_found:
1420 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
1421 pgidx_start, pages),
1422 "Unexpected, invalid virtual page index range,\n"
1423 " pgidx_start: 0x%lx\n"
1424 " pages: 0x%lx",
1425 pgidx_start, pages);
1426 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
1427 pgidx_start, pages),
1428 "Unexpected, pages already mapped,\n"
1429 " pgidx_start: 0x%lx\n"
1430 " pages: 0x%lx",
1431 pgidx_start, pages);
1432
1433 return pgidx_start * vm->page_size;
1434 }
1435
____vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type,bool protected)1436 static vm_vaddr_t ____vm_vaddr_alloc(struct kvm_vm *vm, size_t sz,
1437 vm_vaddr_t vaddr_min,
1438 enum kvm_mem_region_type type,
1439 bool protected)
1440 {
1441 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
1442
1443 virt_pgd_alloc(vm);
1444 vm_paddr_t paddr = __vm_phy_pages_alloc(vm, pages,
1445 KVM_UTIL_MIN_PFN * vm->page_size,
1446 vm->memslots[type], protected);
1447
1448 /*
1449 * Find an unused range of virtual page addresses of at least
1450 * pages in length.
1451 */
1452 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
1453
1454 /* Map the virtual pages. */
1455 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
1456 pages--, vaddr += vm->page_size, paddr += vm->page_size) {
1457
1458 virt_pg_map(vm, vaddr, paddr);
1459
1460 sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1461 }
1462
1463 return vaddr_start;
1464 }
1465
__vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type)1466 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1467 enum kvm_mem_region_type type)
1468 {
1469 return ____vm_vaddr_alloc(vm, sz, vaddr_min, type,
1470 vm_arch_has_protected_memory(vm));
1471 }
1472
vm_vaddr_alloc_shared(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min,enum kvm_mem_region_type type)1473 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,
1474 vm_vaddr_t vaddr_min,
1475 enum kvm_mem_region_type type)
1476 {
1477 return ____vm_vaddr_alloc(vm, sz, vaddr_min, type, false);
1478 }
1479
1480 /*
1481 * VM Virtual Address Allocate
1482 *
1483 * Input Args:
1484 * vm - Virtual Machine
1485 * sz - Size in bytes
1486 * vaddr_min - Minimum starting virtual address
1487 *
1488 * Output Args: None
1489 *
1490 * Return:
1491 * Starting guest virtual address
1492 *
1493 * Allocates at least sz bytes within the virtual address space of the vm
1494 * given by vm. The allocated bytes are mapped to a virtual address >=
1495 * the address given by vaddr_min. Note that each allocation uses a
1496 * a unique set of pages, with the minimum real allocation being at least
1497 * a page. The allocated physical space comes from the TEST_DATA memory region.
1498 */
vm_vaddr_alloc(struct kvm_vm * vm,size_t sz,vm_vaddr_t vaddr_min)1499 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
1500 {
1501 return __vm_vaddr_alloc(vm, sz, vaddr_min, MEM_REGION_TEST_DATA);
1502 }
1503
1504 /*
1505 * VM Virtual Address Allocate Pages
1506 *
1507 * Input Args:
1508 * vm - Virtual Machine
1509 *
1510 * Output Args: None
1511 *
1512 * Return:
1513 * Starting guest virtual address
1514 *
1515 * Allocates at least N system pages worth of bytes within the virtual address
1516 * space of the vm.
1517 */
vm_vaddr_alloc_pages(struct kvm_vm * vm,int nr_pages)1518 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages)
1519 {
1520 return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);
1521 }
1522
__vm_vaddr_alloc_page(struct kvm_vm * vm,enum kvm_mem_region_type type)1523 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)
1524 {
1525 return __vm_vaddr_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);
1526 }
1527
1528 /*
1529 * VM Virtual Address Allocate Page
1530 *
1531 * Input Args:
1532 * vm - Virtual Machine
1533 *
1534 * Output Args: None
1535 *
1536 * Return:
1537 * Starting guest virtual address
1538 *
1539 * Allocates at least one system page worth of bytes within the virtual address
1540 * space of the vm.
1541 */
vm_vaddr_alloc_page(struct kvm_vm * vm)1542 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm)
1543 {
1544 return vm_vaddr_alloc_pages(vm, 1);
1545 }
1546
1547 /*
1548 * Map a range of VM virtual address to the VM's physical address
1549 *
1550 * Input Args:
1551 * vm - Virtual Machine
1552 * vaddr - Virtuall address to map
1553 * paddr - VM Physical Address
1554 * npages - The number of pages to map
1555 *
1556 * Output Args: None
1557 *
1558 * Return: None
1559 *
1560 * Within the VM given by @vm, creates a virtual translation for
1561 * @npages starting at @vaddr to the page range starting at @paddr.
1562 */
virt_map(struct kvm_vm * vm,uint64_t vaddr,uint64_t paddr,unsigned int npages)1563 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
1564 unsigned int npages)
1565 {
1566 size_t page_size = vm->page_size;
1567 size_t size = npages * page_size;
1568
1569 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
1570 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
1571
1572 while (npages--) {
1573 virt_pg_map(vm, vaddr, paddr);
1574 sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1575
1576 vaddr += page_size;
1577 paddr += page_size;
1578 }
1579 }
1580
1581 /*
1582 * Address VM Physical to Host Virtual
1583 *
1584 * Input Args:
1585 * vm - Virtual Machine
1586 * gpa - VM physical address
1587 *
1588 * Output Args: None
1589 *
1590 * Return:
1591 * Equivalent host virtual address
1592 *
1593 * Locates the memory region containing the VM physical address given
1594 * by gpa, within the VM given by vm. When found, the host virtual
1595 * address providing the memory to the vm physical address is returned.
1596 * A TEST_ASSERT failure occurs if no region containing gpa exists.
1597 */
addr_gpa2hva(struct kvm_vm * vm,vm_paddr_t gpa)1598 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
1599 {
1600 struct userspace_mem_region *region;
1601
1602 gpa = vm_untag_gpa(vm, gpa);
1603
1604 region = userspace_mem_region_find(vm, gpa, gpa);
1605 if (!region) {
1606 TEST_FAIL("No vm physical memory at 0x%lx", gpa);
1607 return NULL;
1608 }
1609
1610 return (void *)((uintptr_t)region->host_mem
1611 + (gpa - region->region.guest_phys_addr));
1612 }
1613
1614 /*
1615 * Address Host Virtual to VM Physical
1616 *
1617 * Input Args:
1618 * vm - Virtual Machine
1619 * hva - Host virtual address
1620 *
1621 * Output Args: None
1622 *
1623 * Return:
1624 * Equivalent VM physical address
1625 *
1626 * Locates the memory region containing the host virtual address given
1627 * by hva, within the VM given by vm. When found, the equivalent
1628 * VM physical address is returned. A TEST_ASSERT failure occurs if no
1629 * region containing hva exists.
1630 */
addr_hva2gpa(struct kvm_vm * vm,void * hva)1631 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1632 {
1633 struct rb_node *node;
1634
1635 for (node = vm->regions.hva_tree.rb_node; node; ) {
1636 struct userspace_mem_region *region =
1637 container_of(node, struct userspace_mem_region, hva_node);
1638
1639 if (hva >= region->host_mem) {
1640 if (hva <= (region->host_mem
1641 + region->region.memory_size - 1))
1642 return (vm_paddr_t)((uintptr_t)
1643 region->region.guest_phys_addr
1644 + (hva - (uintptr_t)region->host_mem));
1645
1646 node = node->rb_right;
1647 } else
1648 node = node->rb_left;
1649 }
1650
1651 TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);
1652 return -1;
1653 }
1654
1655 /*
1656 * Address VM physical to Host Virtual *alias*.
1657 *
1658 * Input Args:
1659 * vm - Virtual Machine
1660 * gpa - VM physical address
1661 *
1662 * Output Args: None
1663 *
1664 * Return:
1665 * Equivalent address within the host virtual *alias* area, or NULL
1666 * (without failing the test) if the guest memory is not shared (so
1667 * no alias exists).
1668 *
1669 * Create a writable, shared virtual=>physical alias for the specific GPA.
1670 * The primary use case is to allow the host selftest to manipulate guest
1671 * memory without mapping said memory in the guest's address space. And, for
1672 * userfaultfd-based demand paging, to do so without triggering userfaults.
1673 */
addr_gpa2alias(struct kvm_vm * vm,vm_paddr_t gpa)1674 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa)
1675 {
1676 struct userspace_mem_region *region;
1677 uintptr_t offset;
1678
1679 region = userspace_mem_region_find(vm, gpa, gpa);
1680 if (!region)
1681 return NULL;
1682
1683 if (!region->host_alias)
1684 return NULL;
1685
1686 offset = gpa - region->region.guest_phys_addr;
1687 return (void *) ((uintptr_t) region->host_alias + offset);
1688 }
1689
1690 /* Create an interrupt controller chip for the specified VM. */
vm_create_irqchip(struct kvm_vm * vm)1691 void vm_create_irqchip(struct kvm_vm *vm)
1692 {
1693 int r;
1694
1695 /*
1696 * Allocate a fully in-kernel IRQ chip by default, but fall back to a
1697 * split model (x86 only) if that fails (KVM x86 allows compiling out
1698 * support for KVM_CREATE_IRQCHIP).
1699 */
1700 r = __vm_ioctl(vm, KVM_CREATE_IRQCHIP, NULL);
1701 if (r && errno == ENOTTY && kvm_has_cap(KVM_CAP_SPLIT_IRQCHIP))
1702 vm_enable_cap(vm, KVM_CAP_SPLIT_IRQCHIP, 24);
1703 else
1704 TEST_ASSERT_VM_VCPU_IOCTL(!r, KVM_CREATE_IRQCHIP, r, vm);
1705
1706 vm->has_irqchip = true;
1707 }
1708
_vcpu_run(struct kvm_vcpu * vcpu)1709 int _vcpu_run(struct kvm_vcpu *vcpu)
1710 {
1711 int rc;
1712
1713 do {
1714 rc = __vcpu_run(vcpu);
1715 } while (rc == -1 && errno == EINTR);
1716
1717 if (!rc)
1718 assert_on_unhandled_exception(vcpu);
1719
1720 return rc;
1721 }
1722
1723 /*
1724 * Invoke KVM_RUN on a vCPU until KVM returns something other than -EINTR.
1725 * Assert if the KVM returns an error (other than -EINTR).
1726 */
vcpu_run(struct kvm_vcpu * vcpu)1727 void vcpu_run(struct kvm_vcpu *vcpu)
1728 {
1729 int ret = _vcpu_run(vcpu);
1730
1731 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_RUN, ret));
1732 }
1733
vcpu_run_complete_io(struct kvm_vcpu * vcpu)1734 void vcpu_run_complete_io(struct kvm_vcpu *vcpu)
1735 {
1736 int ret;
1737
1738 vcpu->run->immediate_exit = 1;
1739 ret = __vcpu_run(vcpu);
1740 vcpu->run->immediate_exit = 0;
1741
1742 TEST_ASSERT(ret == -1 && errno == EINTR,
1743 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1744 ret, errno);
1745 }
1746
1747 /*
1748 * Get the list of guest registers which are supported for
1749 * KVM_GET_ONE_REG/KVM_SET_ONE_REG ioctls. Returns a kvm_reg_list pointer,
1750 * it is the caller's responsibility to free the list.
1751 */
vcpu_get_reg_list(struct kvm_vcpu * vcpu)1752 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu)
1753 {
1754 struct kvm_reg_list reg_list_n = { .n = 0 }, *reg_list;
1755 int ret;
1756
1757 ret = __vcpu_ioctl(vcpu, KVM_GET_REG_LIST, ®_list_n);
1758 TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");
1759
1760 reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));
1761 reg_list->n = reg_list_n.n;
1762 vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list);
1763 return reg_list;
1764 }
1765
vcpu_map_dirty_ring(struct kvm_vcpu * vcpu)1766 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)
1767 {
1768 uint32_t page_size = getpagesize();
1769 uint32_t size = vcpu->vm->dirty_ring_size;
1770
1771 TEST_ASSERT(size > 0, "Should enable dirty ring first");
1772
1773 if (!vcpu->dirty_gfns) {
1774 void *addr;
1775
1776 addr = mmap(NULL, size, PROT_READ, MAP_PRIVATE, vcpu->fd,
1777 page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1778 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");
1779
1780 addr = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_PRIVATE, vcpu->fd,
1781 page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1782 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");
1783
1784 addr = __kvm_mmap(size, PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd,
1785 page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1786
1787 vcpu->dirty_gfns = addr;
1788 vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
1789 }
1790
1791 return vcpu->dirty_gfns;
1792 }
1793
1794 /*
1795 * Device Ioctl
1796 */
1797
__kvm_has_device_attr(int dev_fd,uint32_t group,uint64_t attr)1798 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
1799 {
1800 struct kvm_device_attr attribute = {
1801 .group = group,
1802 .attr = attr,
1803 .flags = 0,
1804 };
1805
1806 return ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute);
1807 }
1808
__kvm_test_create_device(struct kvm_vm * vm,uint64_t type)1809 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type)
1810 {
1811 struct kvm_create_device create_dev = {
1812 .type = type,
1813 .flags = KVM_CREATE_DEVICE_TEST,
1814 };
1815
1816 return __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);
1817 }
1818
__kvm_create_device(struct kvm_vm * vm,uint64_t type)1819 int __kvm_create_device(struct kvm_vm *vm, uint64_t type)
1820 {
1821 struct kvm_create_device create_dev = {
1822 .type = type,
1823 .fd = -1,
1824 .flags = 0,
1825 };
1826 int err;
1827
1828 err = __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);
1829 TEST_ASSERT(err <= 0, "KVM_CREATE_DEVICE shouldn't return a positive value");
1830 return err ? : create_dev.fd;
1831 }
1832
__kvm_device_attr_get(int dev_fd,uint32_t group,uint64_t attr,void * val)1833 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val)
1834 {
1835 struct kvm_device_attr kvmattr = {
1836 .group = group,
1837 .attr = attr,
1838 .flags = 0,
1839 .addr = (uintptr_t)val,
1840 };
1841
1842 return __kvm_ioctl(dev_fd, KVM_GET_DEVICE_ATTR, &kvmattr);
1843 }
1844
__kvm_device_attr_set(int dev_fd,uint32_t group,uint64_t attr,void * val)1845 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val)
1846 {
1847 struct kvm_device_attr kvmattr = {
1848 .group = group,
1849 .attr = attr,
1850 .flags = 0,
1851 .addr = (uintptr_t)val,
1852 };
1853
1854 return __kvm_ioctl(dev_fd, KVM_SET_DEVICE_ATTR, &kvmattr);
1855 }
1856
1857 /*
1858 * IRQ related functions.
1859 */
1860
_kvm_irq_line(struct kvm_vm * vm,uint32_t irq,int level)1861 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
1862 {
1863 struct kvm_irq_level irq_level = {
1864 .irq = irq,
1865 .level = level,
1866 };
1867
1868 return __vm_ioctl(vm, KVM_IRQ_LINE, &irq_level);
1869 }
1870
kvm_irq_line(struct kvm_vm * vm,uint32_t irq,int level)1871 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
1872 {
1873 int ret = _kvm_irq_line(vm, irq, level);
1874
1875 TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret));
1876 }
1877
kvm_gsi_routing_create(void)1878 struct kvm_irq_routing *kvm_gsi_routing_create(void)
1879 {
1880 struct kvm_irq_routing *routing;
1881 size_t size;
1882
1883 size = sizeof(struct kvm_irq_routing);
1884 /* Allocate space for the max number of entries: this wastes 196 KBs. */
1885 size += KVM_MAX_IRQ_ROUTES * sizeof(struct kvm_irq_routing_entry);
1886 routing = calloc(1, size);
1887 assert(routing);
1888
1889 return routing;
1890 }
1891
kvm_gsi_routing_irqchip_add(struct kvm_irq_routing * routing,uint32_t gsi,uint32_t pin)1892 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
1893 uint32_t gsi, uint32_t pin)
1894 {
1895 int i;
1896
1897 assert(routing);
1898 assert(routing->nr < KVM_MAX_IRQ_ROUTES);
1899
1900 i = routing->nr;
1901 routing->entries[i].gsi = gsi;
1902 routing->entries[i].type = KVM_IRQ_ROUTING_IRQCHIP;
1903 routing->entries[i].flags = 0;
1904 routing->entries[i].u.irqchip.irqchip = 0;
1905 routing->entries[i].u.irqchip.pin = pin;
1906 routing->nr++;
1907 }
1908
_kvm_gsi_routing_write(struct kvm_vm * vm,struct kvm_irq_routing * routing)1909 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
1910 {
1911 int ret;
1912
1913 assert(routing);
1914 ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing);
1915 free(routing);
1916
1917 return ret;
1918 }
1919
kvm_gsi_routing_write(struct kvm_vm * vm,struct kvm_irq_routing * routing)1920 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
1921 {
1922 int ret;
1923
1924 ret = _kvm_gsi_routing_write(vm, routing);
1925 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_GSI_ROUTING, ret));
1926 }
1927
1928 /*
1929 * VM Dump
1930 *
1931 * Input Args:
1932 * vm - Virtual Machine
1933 * indent - Left margin indent amount
1934 *
1935 * Output Args:
1936 * stream - Output FILE stream
1937 *
1938 * Return: None
1939 *
1940 * Dumps the current state of the VM given by vm, to the FILE stream
1941 * given by stream.
1942 */
vm_dump(FILE * stream,struct kvm_vm * vm,uint8_t indent)1943 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1944 {
1945 int ctr;
1946 struct userspace_mem_region *region;
1947 struct kvm_vcpu *vcpu;
1948
1949 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1950 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1951 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1952 fprintf(stream, "%*sMem Regions:\n", indent, "");
1953 hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {
1954 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1955 "host_virt: %p\n", indent + 2, "",
1956 (uint64_t) region->region.guest_phys_addr,
1957 (uint64_t) region->region.memory_size,
1958 region->host_mem);
1959 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1960 sparsebit_dump(stream, region->unused_phy_pages, 0);
1961 if (region->protected_phy_pages) {
1962 fprintf(stream, "%*sprotected_phy_pages: ", indent + 2, "");
1963 sparsebit_dump(stream, region->protected_phy_pages, 0);
1964 }
1965 }
1966 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1967 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1968 fprintf(stream, "%*spgd_created: %u\n", indent, "",
1969 vm->pgd_created);
1970 if (vm->pgd_created) {
1971 fprintf(stream, "%*sVirtual Translation Tables:\n",
1972 indent + 2, "");
1973 virt_dump(stream, vm, indent + 4);
1974 }
1975 fprintf(stream, "%*sVCPUs:\n", indent, "");
1976
1977 list_for_each_entry(vcpu, &vm->vcpus, list)
1978 vcpu_dump(stream, vcpu, indent + 2);
1979 }
1980
1981 #define KVM_EXIT_STRING(x) {KVM_EXIT_##x, #x}
1982
1983 /* Known KVM exit reasons */
1984 static struct exit_reason {
1985 unsigned int reason;
1986 const char *name;
1987 } exit_reasons_known[] = {
1988 KVM_EXIT_STRING(UNKNOWN),
1989 KVM_EXIT_STRING(EXCEPTION),
1990 KVM_EXIT_STRING(IO),
1991 KVM_EXIT_STRING(HYPERCALL),
1992 KVM_EXIT_STRING(DEBUG),
1993 KVM_EXIT_STRING(HLT),
1994 KVM_EXIT_STRING(MMIO),
1995 KVM_EXIT_STRING(IRQ_WINDOW_OPEN),
1996 KVM_EXIT_STRING(SHUTDOWN),
1997 KVM_EXIT_STRING(FAIL_ENTRY),
1998 KVM_EXIT_STRING(INTR),
1999 KVM_EXIT_STRING(SET_TPR),
2000 KVM_EXIT_STRING(TPR_ACCESS),
2001 KVM_EXIT_STRING(S390_SIEIC),
2002 KVM_EXIT_STRING(S390_RESET),
2003 KVM_EXIT_STRING(DCR),
2004 KVM_EXIT_STRING(NMI),
2005 KVM_EXIT_STRING(INTERNAL_ERROR),
2006 KVM_EXIT_STRING(OSI),
2007 KVM_EXIT_STRING(PAPR_HCALL),
2008 KVM_EXIT_STRING(S390_UCONTROL),
2009 KVM_EXIT_STRING(WATCHDOG),
2010 KVM_EXIT_STRING(S390_TSCH),
2011 KVM_EXIT_STRING(EPR),
2012 KVM_EXIT_STRING(SYSTEM_EVENT),
2013 KVM_EXIT_STRING(S390_STSI),
2014 KVM_EXIT_STRING(IOAPIC_EOI),
2015 KVM_EXIT_STRING(HYPERV),
2016 KVM_EXIT_STRING(ARM_NISV),
2017 KVM_EXIT_STRING(X86_RDMSR),
2018 KVM_EXIT_STRING(X86_WRMSR),
2019 KVM_EXIT_STRING(DIRTY_RING_FULL),
2020 KVM_EXIT_STRING(AP_RESET_HOLD),
2021 KVM_EXIT_STRING(X86_BUS_LOCK),
2022 KVM_EXIT_STRING(XEN),
2023 KVM_EXIT_STRING(RISCV_SBI),
2024 KVM_EXIT_STRING(RISCV_CSR),
2025 KVM_EXIT_STRING(NOTIFY),
2026 KVM_EXIT_STRING(LOONGARCH_IOCSR),
2027 KVM_EXIT_STRING(MEMORY_FAULT),
2028 };
2029
2030 /*
2031 * Exit Reason String
2032 *
2033 * Input Args:
2034 * exit_reason - Exit reason
2035 *
2036 * Output Args: None
2037 *
2038 * Return:
2039 * Constant string pointer describing the exit reason.
2040 *
2041 * Locates and returns a constant string that describes the KVM exit
2042 * reason given by exit_reason. If no such string is found, a constant
2043 * string of "Unknown" is returned.
2044 */
exit_reason_str(unsigned int exit_reason)2045 const char *exit_reason_str(unsigned int exit_reason)
2046 {
2047 unsigned int n1;
2048
2049 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
2050 if (exit_reason == exit_reasons_known[n1].reason)
2051 return exit_reasons_known[n1].name;
2052 }
2053
2054 return "Unknown";
2055 }
2056
2057 /*
2058 * Physical Contiguous Page Allocator
2059 *
2060 * Input Args:
2061 * vm - Virtual Machine
2062 * num - number of pages
2063 * paddr_min - Physical address minimum
2064 * memslot - Memory region to allocate page from
2065 * protected - True if the pages will be used as protected/private memory
2066 *
2067 * Output Args: None
2068 *
2069 * Return:
2070 * Starting physical address
2071 *
2072 * Within the VM specified by vm, locates a range of available physical
2073 * pages at or above paddr_min. If found, the pages are marked as in use
2074 * and their base address is returned. A TEST_ASSERT failure occurs if
2075 * not enough pages are available at or above paddr_min.
2076 */
__vm_phy_pages_alloc(struct kvm_vm * vm,size_t num,vm_paddr_t paddr_min,uint32_t memslot,bool protected)2077 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
2078 vm_paddr_t paddr_min, uint32_t memslot,
2079 bool protected)
2080 {
2081 struct userspace_mem_region *region;
2082 sparsebit_idx_t pg, base;
2083
2084 TEST_ASSERT(num > 0, "Must allocate at least one page");
2085
2086 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
2087 "not divisible by page size.\n"
2088 " paddr_min: 0x%lx page_size: 0x%x",
2089 paddr_min, vm->page_size);
2090
2091 region = memslot2region(vm, memslot);
2092 TEST_ASSERT(!protected || region->protected_phy_pages,
2093 "Region doesn't support protected memory");
2094
2095 base = pg = paddr_min >> vm->page_shift;
2096 do {
2097 for (; pg < base + num; ++pg) {
2098 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
2099 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
2100 break;
2101 }
2102 }
2103 } while (pg && pg != base + num);
2104
2105 if (pg == 0) {
2106 fprintf(stderr, "No guest physical page available, "
2107 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
2108 paddr_min, vm->page_size, memslot);
2109 fputs("---- vm dump ----\n", stderr);
2110 vm_dump(stderr, vm, 2);
2111 abort();
2112 }
2113
2114 for (pg = base; pg < base + num; ++pg) {
2115 sparsebit_clear(region->unused_phy_pages, pg);
2116 if (protected)
2117 sparsebit_set(region->protected_phy_pages, pg);
2118 }
2119
2120 return base * vm->page_size;
2121 }
2122
vm_phy_page_alloc(struct kvm_vm * vm,vm_paddr_t paddr_min,uint32_t memslot)2123 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
2124 uint32_t memslot)
2125 {
2126 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
2127 }
2128
vm_alloc_page_table(struct kvm_vm * vm)2129 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)
2130 {
2131 return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,
2132 vm->memslots[MEM_REGION_PT]);
2133 }
2134
2135 /*
2136 * Address Guest Virtual to Host Virtual
2137 *
2138 * Input Args:
2139 * vm - Virtual Machine
2140 * gva - VM virtual address
2141 *
2142 * Output Args: None
2143 *
2144 * Return:
2145 * Equivalent host virtual address
2146 */
addr_gva2hva(struct kvm_vm * vm,vm_vaddr_t gva)2147 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
2148 {
2149 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
2150 }
2151
vm_compute_max_gfn(struct kvm_vm * vm)2152 unsigned long __weak vm_compute_max_gfn(struct kvm_vm *vm)
2153 {
2154 return ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
2155 }
2156
vm_calc_num_pages(unsigned int num_pages,unsigned int page_shift,unsigned int new_page_shift,bool ceil)2157 static unsigned int vm_calc_num_pages(unsigned int num_pages,
2158 unsigned int page_shift,
2159 unsigned int new_page_shift,
2160 bool ceil)
2161 {
2162 unsigned int n = 1 << (new_page_shift - page_shift);
2163
2164 if (page_shift >= new_page_shift)
2165 return num_pages * (1 << (page_shift - new_page_shift));
2166
2167 return num_pages / n + !!(ceil && num_pages % n);
2168 }
2169
getpageshift(void)2170 static inline int getpageshift(void)
2171 {
2172 return __builtin_ffs(getpagesize()) - 1;
2173 }
2174
2175 unsigned int
vm_num_host_pages(enum vm_guest_mode mode,unsigned int num_guest_pages)2176 vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
2177 {
2178 return vm_calc_num_pages(num_guest_pages,
2179 vm_guest_mode_params[mode].page_shift,
2180 getpageshift(), true);
2181 }
2182
2183 unsigned int
vm_num_guest_pages(enum vm_guest_mode mode,unsigned int num_host_pages)2184 vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages)
2185 {
2186 return vm_calc_num_pages(num_host_pages, getpageshift(),
2187 vm_guest_mode_params[mode].page_shift, false);
2188 }
2189
vm_calc_num_guest_pages(enum vm_guest_mode mode,size_t size)2190 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size)
2191 {
2192 unsigned int n;
2193 n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size);
2194 return vm_adjust_num_guest_pages(mode, n);
2195 }
2196
2197 /*
2198 * Read binary stats descriptors
2199 *
2200 * Input Args:
2201 * stats_fd - the file descriptor for the binary stats file from which to read
2202 * header - the binary stats metadata header corresponding to the given FD
2203 *
2204 * Output Args: None
2205 *
2206 * Return:
2207 * A pointer to a newly allocated series of stat descriptors.
2208 * Caller is responsible for freeing the returned kvm_stats_desc.
2209 *
2210 * Read the stats descriptors from the binary stats interface.
2211 */
read_stats_descriptors(int stats_fd,struct kvm_stats_header * header)2212 struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
2213 struct kvm_stats_header *header)
2214 {
2215 struct kvm_stats_desc *stats_desc;
2216 ssize_t desc_size, total_size, ret;
2217
2218 desc_size = get_stats_descriptor_size(header);
2219 total_size = header->num_desc * desc_size;
2220
2221 stats_desc = calloc(header->num_desc, desc_size);
2222 TEST_ASSERT(stats_desc, "Allocate memory for stats descriptors");
2223
2224 ret = pread(stats_fd, stats_desc, total_size, header->desc_offset);
2225 TEST_ASSERT(ret == total_size, "Read KVM stats descriptors");
2226
2227 return stats_desc;
2228 }
2229
2230 /*
2231 * Read stat data for a particular stat
2232 *
2233 * Input Args:
2234 * stats_fd - the file descriptor for the binary stats file from which to read
2235 * header - the binary stats metadata header corresponding to the given FD
2236 * desc - the binary stat metadata for the particular stat to be read
2237 * max_elements - the maximum number of 8-byte values to read into data
2238 *
2239 * Output Args:
2240 * data - the buffer into which stat data should be read
2241 *
2242 * Read the data values of a specified stat from the binary stats interface.
2243 */
read_stat_data(int stats_fd,struct kvm_stats_header * header,struct kvm_stats_desc * desc,uint64_t * data,size_t max_elements)2244 void read_stat_data(int stats_fd, struct kvm_stats_header *header,
2245 struct kvm_stats_desc *desc, uint64_t *data,
2246 size_t max_elements)
2247 {
2248 size_t nr_elements = min_t(ssize_t, desc->size, max_elements);
2249 size_t size = nr_elements * sizeof(*data);
2250 ssize_t ret;
2251
2252 TEST_ASSERT(desc->size, "No elements in stat '%s'", desc->name);
2253 TEST_ASSERT(max_elements, "Zero elements requested for stat '%s'", desc->name);
2254
2255 ret = pread(stats_fd, data, size,
2256 header->data_offset + desc->offset);
2257
2258 TEST_ASSERT(ret >= 0, "pread() failed on stat '%s', errno: %i (%s)",
2259 desc->name, errno, strerror(errno));
2260 TEST_ASSERT(ret == size,
2261 "pread() on stat '%s' read %ld bytes, wanted %lu bytes",
2262 desc->name, size, ret);
2263 }
2264
kvm_get_stat(struct kvm_binary_stats * stats,const char * name,uint64_t * data,size_t max_elements)2265 void kvm_get_stat(struct kvm_binary_stats *stats, const char *name,
2266 uint64_t *data, size_t max_elements)
2267 {
2268 struct kvm_stats_desc *desc;
2269 size_t size_desc;
2270 int i;
2271
2272 if (!stats->desc) {
2273 read_stats_header(stats->fd, &stats->header);
2274 stats->desc = read_stats_descriptors(stats->fd, &stats->header);
2275 }
2276
2277 size_desc = get_stats_descriptor_size(&stats->header);
2278
2279 for (i = 0; i < stats->header.num_desc; ++i) {
2280 desc = (void *)stats->desc + (i * size_desc);
2281
2282 if (strcmp(desc->name, name))
2283 continue;
2284
2285 read_stat_data(stats->fd, &stats->header, desc, data, max_elements);
2286 return;
2287 }
2288
2289 TEST_FAIL("Unable to find stat '%s'", name);
2290 }
2291
kvm_arch_vm_post_create(struct kvm_vm * vm,unsigned int nr_vcpus)2292 __weak void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus)
2293 {
2294 }
2295
kvm_arch_vm_finalize_vcpus(struct kvm_vm * vm)2296 __weak void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm)
2297 {
2298 }
2299
kvm_arch_vm_release(struct kvm_vm * vm)2300 __weak void kvm_arch_vm_release(struct kvm_vm *vm)
2301 {
2302 }
2303
kvm_selftest_arch_init(void)2304 __weak void kvm_selftest_arch_init(void)
2305 {
2306 }
2307
kvm_selftest_init(void)2308 void __attribute((constructor)) kvm_selftest_init(void)
2309 {
2310 /* Tell stdout not to buffer its content. */
2311 setbuf(stdout, NULL);
2312
2313 guest_random_seed = last_guest_seed = random();
2314 pr_info("Random seed: 0x%x\n", guest_random_seed);
2315
2316 kvm_selftest_arch_init();
2317 }
2318
vm_is_gpa_protected(struct kvm_vm * vm,vm_paddr_t paddr)2319 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr)
2320 {
2321 sparsebit_idx_t pg = 0;
2322 struct userspace_mem_region *region;
2323
2324 if (!vm_arch_has_protected_memory(vm))
2325 return false;
2326
2327 region = userspace_mem_region_find(vm, paddr, paddr);
2328 TEST_ASSERT(region, "No vm physical memory at 0x%lx", paddr);
2329
2330 pg = paddr >> vm->page_shift;
2331 return sparsebit_is_set(region->protected_phy_pages, pg);
2332 }
2333
kvm_arch_has_default_irqchip(void)2334 __weak bool kvm_arch_has_default_irqchip(void)
2335 {
2336 return false;
2337 }
2338