1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KVM page table test 4 * 5 * Copyright (C) 2021, Huawei, Inc. 6 * 7 * Make sure that THP has been enabled or enough HUGETLB pages with specific 8 * page size have been pre-allocated on your system, if you are planning to 9 * use hugepages to back the guest memory for testing. 10 */ 11 #include <stdio.h> 12 #include <stdlib.h> 13 #include <time.h> 14 #include <pthread.h> 15 #include <semaphore.h> 16 17 #include "test_util.h" 18 #include "kvm_util.h" 19 #include "processor.h" 20 #include "guest_modes.h" 21 #include "ucall_common.h" 22 23 #define TEST_MEM_SLOT_INDEX 1 24 25 /* Default size(1GB) of the memory for testing */ 26 #define DEFAULT_TEST_MEM_SIZE (1 << 30) 27 28 /* Default guest test virtual memory offset */ 29 #define DEFAULT_GUEST_TEST_MEM 0xc0000000 30 31 /* Different guest memory accessing stages */ 32 enum test_stage { 33 KVM_BEFORE_MAPPINGS, 34 KVM_CREATE_MAPPINGS, 35 KVM_UPDATE_MAPPINGS, 36 KVM_ADJUST_MAPPINGS, 37 NUM_TEST_STAGES, 38 }; 39 40 static const char * const test_stage_string[] = { 41 "KVM_BEFORE_MAPPINGS", 42 "KVM_CREATE_MAPPINGS", 43 "KVM_UPDATE_MAPPINGS", 44 "KVM_ADJUST_MAPPINGS", 45 }; 46 47 struct test_args { 48 struct kvm_vm *vm; 49 u64 guest_test_virt_mem; 50 u64 host_page_size; 51 u64 host_num_pages; 52 u64 large_page_size; 53 u64 large_num_pages; 54 u64 host_pages_per_lpage; 55 enum vm_mem_backing_src_type src_type; 56 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; 57 }; 58 59 /* 60 * Guest variables. Use addr_gva2hva() if these variables need 61 * to be changed in host. 62 */ 63 static enum test_stage guest_test_stage; 64 65 /* Host variables */ 66 static u32 nr_vcpus = 1; 67 static struct test_args test_args; 68 static enum test_stage *current_stage; 69 static bool host_quit; 70 71 /* Whether the test stage is updated, or completed */ 72 static sem_t test_stage_updated; 73 static sem_t test_stage_completed; 74 75 /* 76 * Guest physical memory offset of the testing memory slot. 77 * This will be set to the topmost valid physical address minus 78 * the test memory size. 79 */ 80 static u64 guest_test_phys_mem; 81 82 /* 83 * Guest virtual memory offset of the testing memory slot. 84 * Must not conflict with identity mapped test code. 85 */ 86 static u64 guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM; 87 88 static void guest_code(bool do_write) 89 { 90 struct test_args *p = &test_args; 91 enum test_stage *current_stage = &guest_test_stage; 92 u64 addr; 93 int i, j; 94 95 while (true) { 96 addr = p->guest_test_virt_mem; 97 98 switch (READ_ONCE(*current_stage)) { 99 /* 100 * All vCPU threads will be started in this stage, 101 * where guest code of each vCPU will do nothing. 102 */ 103 case KVM_BEFORE_MAPPINGS: 104 break; 105 106 /* 107 * Before dirty logging, vCPUs concurrently access the first 108 * 8 bytes of each page (host page/large page) within the same 109 * memory region with different accessing types (read/write). 110 * Then KVM will create normal page mappings or huge block 111 * mappings for them. 112 */ 113 case KVM_CREATE_MAPPINGS: 114 for (i = 0; i < p->large_num_pages; i++) { 115 if (do_write) 116 *(u64 *)addr = 0x0123456789ABCDEF; 117 else 118 READ_ONCE(*(u64 *)addr); 119 120 addr += p->large_page_size; 121 } 122 break; 123 124 /* 125 * During dirty logging, KVM will only update attributes of the 126 * normal page mappings from RO to RW if memory backing src type 127 * is anonymous. In other cases, KVM will split the huge block 128 * mappings into normal page mappings if memory backing src type 129 * is THP or HUGETLB. 130 */ 131 case KVM_UPDATE_MAPPINGS: 132 if (p->src_type == VM_MEM_SRC_ANONYMOUS) { 133 for (i = 0; i < p->host_num_pages; i++) { 134 *(u64 *)addr = 0x0123456789ABCDEF; 135 addr += p->host_page_size; 136 } 137 break; 138 } 139 140 for (i = 0; i < p->large_num_pages; i++) { 141 /* 142 * Write to the first host page in each large 143 * page region, and triger break of large pages. 144 */ 145 *(u64 *)addr = 0x0123456789ABCDEF; 146 147 /* 148 * Access the middle host pages in each large 149 * page region. Since dirty logging is enabled, 150 * this will create new mappings at the smallest 151 * granularity. 152 */ 153 addr += p->large_page_size / 2; 154 for (j = 0; j < p->host_pages_per_lpage / 2; j++) { 155 READ_ONCE(*(u64 *)addr); 156 addr += p->host_page_size; 157 } 158 } 159 break; 160 161 /* 162 * After dirty logging is stopped, vCPUs concurrently read 163 * from every single host page. Then KVM will coalesce the 164 * split page mappings back to block mappings. And a TLB 165 * conflict abort could occur here if TLB entries of the 166 * page mappings are not fully invalidated. 167 */ 168 case KVM_ADJUST_MAPPINGS: 169 for (i = 0; i < p->host_num_pages; i++) { 170 READ_ONCE(*(u64 *)addr); 171 addr += p->host_page_size; 172 } 173 break; 174 175 default: 176 GUEST_ASSERT(0); 177 } 178 179 GUEST_SYNC(1); 180 } 181 } 182 183 static void *vcpu_worker(void *data) 184 { 185 struct kvm_vcpu *vcpu = data; 186 bool do_write = !(vcpu->id % 2); 187 struct timespec start; 188 struct timespec ts_diff; 189 enum test_stage stage; 190 int ret; 191 192 vcpu_args_set(vcpu, 1, do_write); 193 194 while (!READ_ONCE(host_quit)) { 195 ret = sem_wait(&test_stage_updated); 196 TEST_ASSERT(ret == 0, "Error in sem_wait"); 197 198 if (READ_ONCE(host_quit)) 199 return NULL; 200 201 clock_gettime(CLOCK_MONOTONIC, &start); 202 ret = _vcpu_run(vcpu); 203 ts_diff = timespec_elapsed(start); 204 205 TEST_ASSERT(ret == 0, "vcpu_run failed: %d", ret); 206 TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC, 207 "Invalid guest sync status: exit_reason=%s", 208 exit_reason_str(vcpu->run->exit_reason)); 209 210 pr_debug("Got sync event from vCPU %d\n", vcpu->id); 211 stage = READ_ONCE(*current_stage); 212 213 /* 214 * Here we can know the execution time of every 215 * single vcpu running in different test stages. 216 */ 217 pr_debug("vCPU %d has completed stage %s\n" 218 "execution time is: %ld.%.9lds\n\n", 219 vcpu->id, test_stage_string[stage], 220 ts_diff.tv_sec, ts_diff.tv_nsec); 221 222 ret = sem_post(&test_stage_completed); 223 TEST_ASSERT(ret == 0, "Error in sem_post"); 224 } 225 226 return NULL; 227 } 228 229 struct test_params { 230 u64 phys_offset; 231 u64 test_mem_size; 232 enum vm_mem_backing_src_type src_type; 233 bool misalign_slot_gpa; 234 }; 235 236 static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg) 237 { 238 int ret; 239 struct test_params *p = arg; 240 enum vm_mem_backing_src_type src_type = p->src_type; 241 u64 large_page_size = get_backing_src_pagesz(src_type); 242 u64 guest_page_size = vm_guest_mode_params[mode].page_size; 243 u64 host_page_size = getpagesize(); 244 u64 test_mem_size = p->test_mem_size; 245 u64 guest_num_pages; 246 u64 alignment; 247 void *host_test_mem; 248 struct userspace_mem_region *region; 249 struct kvm_vm *vm; 250 251 /* Align up the test memory size */ 252 alignment = max(large_page_size, guest_page_size); 253 test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1); 254 255 /* Create a VM with enough guest pages */ 256 guest_num_pages = test_mem_size / guest_page_size; 257 vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages, 258 guest_code, test_args.vcpus); 259 260 /* Align down GPA of the testing memslot */ 261 if (!p->phys_offset) 262 guest_test_phys_mem = (vm->max_gfn - guest_num_pages) * 263 guest_page_size; 264 else 265 guest_test_phys_mem = p->phys_offset; 266 guest_test_phys_mem = align_down(guest_test_phys_mem, alignment); 267 268 /* Set up the shared data structure test_args */ 269 test_args.vm = vm; 270 test_args.guest_test_virt_mem = guest_test_virt_mem; 271 test_args.host_page_size = host_page_size; 272 test_args.host_num_pages = test_mem_size / host_page_size; 273 test_args.large_page_size = large_page_size; 274 test_args.large_num_pages = test_mem_size / large_page_size; 275 test_args.host_pages_per_lpage = large_page_size / host_page_size; 276 test_args.src_type = src_type; 277 278 /* Add an extra memory slot with specified backing src type */ 279 vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem, 280 TEST_MEM_SLOT_INDEX, guest_num_pages, 0); 281 region = memslot2region(vm, TEST_MEM_SLOT_INDEX); 282 host_test_mem = region->host_mem; 283 284 if (p->misalign_slot_gpa) { 285 TEST_ASSERT(is_backing_src_hugetlb(src_type), 286 "Memslot GPA misalignment requires hugetlb backing"); 287 TEST_ASSERT(guest_num_pages > 1, 288 "Need at least two guest pages to misalign memslot GPA"); 289 290 guest_test_phys_mem += guest_page_size; 291 vm_mem_region_move(vm, TEST_MEM_SLOT_INDEX, guest_test_phys_mem); 292 } 293 294 /* Do mapping(GVA->GPA) for the testing memory slot */ 295 virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages); 296 297 /* Export shared structure test_args to guest */ 298 sync_global_to_guest(vm, test_args); 299 300 ret = sem_init(&test_stage_updated, 0, 0); 301 TEST_ASSERT(ret == 0, "Error in sem_init"); 302 303 ret = sem_init(&test_stage_completed, 0, 0); 304 TEST_ASSERT(ret == 0, "Error in sem_init"); 305 306 current_stage = addr_gva2hva(vm, (gva_t)(&guest_test_stage)); 307 *current_stage = NUM_TEST_STAGES; 308 309 pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); 310 pr_info("Testing memory backing src type: %s\n", 311 vm_mem_backing_src_alias(src_type)->name); 312 pr_info("Testing memory backing src granularity: 0x%lx\n", 313 large_page_size); 314 pr_info("Testing memory size(aligned): 0x%lx\n", test_mem_size); 315 pr_info("Guest physical test memory offset: 0x%lx\n", 316 guest_test_phys_mem); 317 pr_info("Host virtual test memory offset: 0x%lx\n", 318 (u64)host_test_mem); 319 pr_info("Number of testing vCPUs: %d\n", nr_vcpus); 320 321 return vm; 322 } 323 324 static void vcpus_complete_new_stage(enum test_stage stage) 325 { 326 int ret; 327 int vcpus; 328 329 /* Wake up all the vcpus to run new test stage */ 330 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) { 331 ret = sem_post(&test_stage_updated); 332 TEST_ASSERT(ret == 0, "Error in sem_post"); 333 } 334 pr_debug("All vcpus have been notified to continue\n"); 335 336 /* Wait for all the vcpus to complete new test stage */ 337 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) { 338 ret = sem_wait(&test_stage_completed); 339 TEST_ASSERT(ret == 0, "Error in sem_wait"); 340 341 pr_debug("%d vcpus have completed stage %s\n", 342 vcpus + 1, test_stage_string[stage]); 343 } 344 345 pr_debug("All vcpus have completed stage %s\n", 346 test_stage_string[stage]); 347 } 348 349 static void run_test(enum vm_guest_mode mode, void *arg) 350 { 351 pthread_t *vcpu_threads; 352 struct kvm_vm *vm; 353 struct timespec start; 354 struct timespec ts_diff; 355 int ret, i; 356 357 /* Create VM with vCPUs and make some pre-initialization */ 358 vm = pre_init_before_test(mode, arg); 359 360 vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads)); 361 TEST_ASSERT(vcpu_threads, "Memory allocation failed"); 362 363 host_quit = false; 364 *current_stage = KVM_BEFORE_MAPPINGS; 365 366 for (i = 0; i < nr_vcpus; i++) 367 pthread_create(&vcpu_threads[i], NULL, vcpu_worker, 368 test_args.vcpus[i]); 369 370 vcpus_complete_new_stage(*current_stage); 371 pr_info("Started all vCPUs successfully\n"); 372 373 /* Test the stage of KVM creating mappings */ 374 *current_stage = KVM_CREATE_MAPPINGS; 375 376 clock_gettime(CLOCK_MONOTONIC, &start); 377 vcpus_complete_new_stage(*current_stage); 378 ts_diff = timespec_elapsed(start); 379 380 pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n", 381 ts_diff.tv_sec, ts_diff.tv_nsec); 382 383 /* Test the stage of KVM updating mappings */ 384 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 385 KVM_MEM_LOG_DIRTY_PAGES); 386 387 *current_stage = KVM_UPDATE_MAPPINGS; 388 389 clock_gettime(CLOCK_MONOTONIC, &start); 390 vcpus_complete_new_stage(*current_stage); 391 ts_diff = timespec_elapsed(start); 392 393 pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n", 394 ts_diff.tv_sec, ts_diff.tv_nsec); 395 396 /* Test the stage of KVM adjusting mappings */ 397 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0); 398 399 *current_stage = KVM_ADJUST_MAPPINGS; 400 401 clock_gettime(CLOCK_MONOTONIC, &start); 402 vcpus_complete_new_stage(*current_stage); 403 ts_diff = timespec_elapsed(start); 404 405 pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n", 406 ts_diff.tv_sec, ts_diff.tv_nsec); 407 408 /* Tell the vcpu thread to quit */ 409 host_quit = true; 410 for (i = 0; i < nr_vcpus; i++) { 411 ret = sem_post(&test_stage_updated); 412 TEST_ASSERT(ret == 0, "Error in sem_post"); 413 } 414 415 for (i = 0; i < nr_vcpus; i++) 416 pthread_join(vcpu_threads[i], NULL); 417 418 ret = sem_destroy(&test_stage_updated); 419 TEST_ASSERT(ret == 0, "Error in sem_destroy"); 420 421 ret = sem_destroy(&test_stage_completed); 422 TEST_ASSERT(ret == 0, "Error in sem_destroy"); 423 424 free(vcpu_threads); 425 kvm_vm_free(vm); 426 } 427 428 static void help(char *name) 429 { 430 puts(""); 431 printf("usage: %s [-h] [-p offset] [-m mode] [-b mem-size]\n", name); 432 printf(" [-v vcpus] [-s mem-type] [-u]\n"); 433 puts(""); 434 printf(" -p: specify guest physical test memory offset\n" 435 " Warning: a low offset can conflict with the loaded test code.\n"); 436 guest_modes_help(); 437 printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n" 438 " (default: 1G)\n"); 439 printf(" -v: specify the number of vCPUs to run\n" 440 " (default: 1)\n"); 441 backing_src_help("-s"); 442 printf(" -u: move the test memslot GPA by one guest page after creating\n" 443 " the memslot, forcing a hugetlb HVA/GPA offset mismatch\n"); 444 puts(""); 445 } 446 447 int main(int argc, char *argv[]) 448 { 449 int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); 450 struct test_params p = { 451 .test_mem_size = DEFAULT_TEST_MEM_SIZE, 452 .src_type = DEFAULT_VM_MEM_SRC, 453 }; 454 int opt; 455 456 guest_modes_append_default(); 457 458 while ((opt = getopt(argc, argv, "hp:m:b:v:s:u")) != -1) { 459 switch (opt) { 460 case 'p': 461 p.phys_offset = strtoull(optarg, NULL, 0); 462 break; 463 case 'm': 464 guest_modes_cmdline(optarg); 465 break; 466 case 'b': 467 p.test_mem_size = parse_size(optarg); 468 break; 469 case 'v': 470 nr_vcpus = atoi_positive("Number of vCPUs", optarg); 471 TEST_ASSERT(nr_vcpus <= max_vcpus, 472 "Invalid number of vcpus, must be between 1 and %d", max_vcpus); 473 break; 474 case 's': 475 p.src_type = parse_backing_src_type(optarg); 476 break; 477 case 'u': 478 p.misalign_slot_gpa = true; 479 break; 480 case 'h': 481 default: 482 help(argv[0]); 483 exit(0); 484 } 485 } 486 487 for_each_guest_mode(run_test, &p); 488 489 return 0; 490 } 491