1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * access_tracking_perf_test 4 * 5 * Copyright (C) 2021, Google, Inc. 6 * 7 * This test measures the performance effects of KVM's access tracking. 8 * Access tracking is driven by the MMU notifiers test_young, clear_young, and 9 * clear_flush_young. These notifiers do not have a direct userspace API, 10 * however the clear_young notifier can be triggered by marking a pages as idle 11 * in /sys/kernel/mm/page_idle/bitmap. This test leverages that mechanism to 12 * enable access tracking on guest memory. 13 * 14 * To measure performance this test runs a VM with a configurable number of 15 * vCPUs that each touch every page in disjoint regions of memory. Performance 16 * is measured in the time it takes all vCPUs to finish touching their 17 * predefined region. 18 * 19 * Note that a deterministic correctness test of access tracking is not possible 20 * by using page_idle as it exists today. This is for a few reasons: 21 * 22 * 1. page_idle only issues clear_young notifiers, which lack a TLB flush. This 23 * means subsequent guest accesses are not guaranteed to see page table 24 * updates made by KVM until some time in the future. 25 * 26 * 2. page_idle only operates on LRU pages. Newly allocated pages are not 27 * immediately allocated to LRU lists. Instead they are held in a "pagevec", 28 * which is drained to LRU lists some time in the future. There is no 29 * userspace API to force this drain to occur. 30 * 31 * These limitations are worked around in this test by using a large enough 32 * region of memory for each vCPU such that the number of translations cached in 33 * the TLB and the number of pages held in pagevecs are a small fraction of the 34 * overall workload. And if either of those conditions are not true (for example 35 * in nesting, where TLB size is unlimited) this test will print a warning 36 * rather than silently passing. 37 */ 38 #include <inttypes.h> 39 #include <limits.h> 40 #include <pthread.h> 41 #include <sys/mman.h> 42 #include <sys/types.h> 43 #include <sys/stat.h> 44 45 #include "kvm_util.h" 46 #include "test_util.h" 47 #include "memstress.h" 48 #include "guest_modes.h" 49 #include "processor.h" 50 51 /* Global variable used to synchronize all of the vCPU threads. */ 52 static int iteration; 53 54 /* Defines what vCPU threads should do during a given iteration. */ 55 static enum { 56 /* Run the vCPU to access all its memory. */ 57 ITERATION_ACCESS_MEMORY, 58 /* Mark the vCPU's memory idle in page_idle. */ 59 ITERATION_MARK_IDLE, 60 } iteration_work; 61 62 /* The iteration that was last completed by each vCPU. */ 63 static int vcpu_last_completed_iteration[KVM_MAX_VCPUS]; 64 65 /* Whether to overlap the regions of memory vCPUs access. */ 66 static bool overlap_memory_access; 67 68 /* 69 * If the test should only warn if there are too many idle pages (i.e., it is 70 * expected). 71 * -1: Not yet set. 72 * 0: We do not expect too many idle pages, so FAIL if too many idle pages. 73 * 1: Having too many idle pages is expected, so merely print a warning if 74 * too many idle pages are found. 75 */ 76 static int idle_pages_warn_only = -1; 77 78 struct test_params { 79 /* The backing source for the region of memory. */ 80 enum vm_mem_backing_src_type backing_src; 81 82 /* The amount of memory to allocate for each vCPU. */ 83 uint64_t vcpu_memory_bytes; 84 85 /* The number of vCPUs to create in the VM. */ 86 int nr_vcpus; 87 }; 88 89 static uint64_t pread_uint64(int fd, const char *filename, uint64_t index) 90 { 91 uint64_t value; 92 off_t offset = index * sizeof(value); 93 94 TEST_ASSERT(pread(fd, &value, sizeof(value), offset) == sizeof(value), 95 "pread from %s offset 0x%" PRIx64 " failed!", 96 filename, offset); 97 98 return value; 99 100 } 101 102 #define PAGEMAP_PRESENT (1ULL << 63) 103 #define PAGEMAP_PFN_MASK ((1ULL << 55) - 1) 104 105 static uint64_t lookup_pfn(int pagemap_fd, struct kvm_vm *vm, uint64_t gva) 106 { 107 uint64_t hva = (uint64_t) addr_gva2hva(vm, gva); 108 uint64_t entry; 109 uint64_t pfn; 110 111 entry = pread_uint64(pagemap_fd, "pagemap", hva / getpagesize()); 112 if (!(entry & PAGEMAP_PRESENT)) 113 return 0; 114 115 pfn = entry & PAGEMAP_PFN_MASK; 116 __TEST_REQUIRE(pfn, "Looking up PFNs requires CAP_SYS_ADMIN"); 117 118 return pfn; 119 } 120 121 static bool is_page_idle(int page_idle_fd, uint64_t pfn) 122 { 123 uint64_t bits = pread_uint64(page_idle_fd, "page_idle", pfn / 64); 124 125 return !!((bits >> (pfn % 64)) & 1); 126 } 127 128 static void mark_page_idle(int page_idle_fd, uint64_t pfn) 129 { 130 uint64_t bits = 1ULL << (pfn % 64); 131 132 TEST_ASSERT(pwrite(page_idle_fd, &bits, 8, 8 * (pfn / 64)) == 8, 133 "Set page_idle bits for PFN 0x%" PRIx64, pfn); 134 } 135 136 static void mark_vcpu_memory_idle(struct kvm_vm *vm, 137 struct memstress_vcpu_args *vcpu_args) 138 { 139 int vcpu_idx = vcpu_args->vcpu_idx; 140 uint64_t base_gva = vcpu_args->gva; 141 uint64_t pages = vcpu_args->pages; 142 uint64_t page; 143 uint64_t still_idle = 0; 144 uint64_t no_pfn = 0; 145 int page_idle_fd; 146 int pagemap_fd; 147 148 /* If vCPUs are using an overlapping region, let vCPU 0 mark it idle. */ 149 if (overlap_memory_access && vcpu_idx) 150 return; 151 152 page_idle_fd = open("/sys/kernel/mm/page_idle/bitmap", O_RDWR); 153 TEST_ASSERT(page_idle_fd > 0, "Failed to open page_idle."); 154 155 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 156 TEST_ASSERT(pagemap_fd > 0, "Failed to open pagemap."); 157 158 for (page = 0; page < pages; page++) { 159 uint64_t gva = base_gva + page * memstress_args.guest_page_size; 160 uint64_t pfn = lookup_pfn(pagemap_fd, vm, gva); 161 162 if (!pfn) { 163 no_pfn++; 164 continue; 165 } 166 167 if (is_page_idle(page_idle_fd, pfn)) { 168 still_idle++; 169 continue; 170 } 171 172 mark_page_idle(page_idle_fd, pfn); 173 } 174 175 /* 176 * Assumption: Less than 1% of pages are going to be swapped out from 177 * under us during this test. 178 */ 179 TEST_ASSERT(no_pfn < pages / 100, 180 "vCPU %d: No PFN for %" PRIu64 " out of %" PRIu64 " pages.", 181 vcpu_idx, no_pfn, pages); 182 183 /* 184 * Check that at least 90% of memory has been marked idle (the rest 185 * might not be marked idle because the pages have not yet made it to an 186 * LRU list or the translations are still cached in the TLB). 90% is 187 * arbitrary; high enough that we ensure most memory access went through 188 * access tracking but low enough as to not make the test too brittle 189 * over time and across architectures. 190 */ 191 if (still_idle >= pages / 10) { 192 TEST_ASSERT(idle_pages_warn_only, 193 "vCPU%d: Too many pages still idle (%lu out of %lu)", 194 vcpu_idx, still_idle, pages); 195 196 printf("WARNING: vCPU%d: Too many pages still idle (%lu out of %lu), " 197 "this will affect performance results.\n", 198 vcpu_idx, still_idle, pages); 199 } 200 201 close(page_idle_fd); 202 close(pagemap_fd); 203 } 204 205 static void assert_ucall(struct kvm_vcpu *vcpu, uint64_t expected_ucall) 206 { 207 struct ucall uc; 208 uint64_t actual_ucall = get_ucall(vcpu, &uc); 209 210 TEST_ASSERT(expected_ucall == actual_ucall, 211 "Guest exited unexpectedly (expected ucall %" PRIu64 212 ", got %" PRIu64 ")", 213 expected_ucall, actual_ucall); 214 } 215 216 static bool spin_wait_for_next_iteration(int *current_iteration) 217 { 218 int last_iteration = *current_iteration; 219 220 do { 221 if (READ_ONCE(memstress_args.stop_vcpus)) 222 return false; 223 224 *current_iteration = READ_ONCE(iteration); 225 } while (last_iteration == *current_iteration); 226 227 return true; 228 } 229 230 static void vcpu_thread_main(struct memstress_vcpu_args *vcpu_args) 231 { 232 struct kvm_vcpu *vcpu = vcpu_args->vcpu; 233 struct kvm_vm *vm = memstress_args.vm; 234 int vcpu_idx = vcpu_args->vcpu_idx; 235 int current_iteration = 0; 236 237 while (spin_wait_for_next_iteration(¤t_iteration)) { 238 switch (READ_ONCE(iteration_work)) { 239 case ITERATION_ACCESS_MEMORY: 240 vcpu_run(vcpu); 241 assert_ucall(vcpu, UCALL_SYNC); 242 break; 243 case ITERATION_MARK_IDLE: 244 mark_vcpu_memory_idle(vm, vcpu_args); 245 break; 246 } 247 248 vcpu_last_completed_iteration[vcpu_idx] = current_iteration; 249 } 250 } 251 252 static void spin_wait_for_vcpu(int vcpu_idx, int target_iteration) 253 { 254 while (READ_ONCE(vcpu_last_completed_iteration[vcpu_idx]) != 255 target_iteration) { 256 continue; 257 } 258 } 259 260 /* The type of memory accesses to perform in the VM. */ 261 enum access_type { 262 ACCESS_READ, 263 ACCESS_WRITE, 264 }; 265 266 static void run_iteration(struct kvm_vm *vm, int nr_vcpus, const char *description) 267 { 268 struct timespec ts_start; 269 struct timespec ts_elapsed; 270 int next_iteration, i; 271 272 /* Kick off the vCPUs by incrementing iteration. */ 273 next_iteration = ++iteration; 274 275 clock_gettime(CLOCK_MONOTONIC, &ts_start); 276 277 /* Wait for all vCPUs to finish the iteration. */ 278 for (i = 0; i < nr_vcpus; i++) 279 spin_wait_for_vcpu(i, next_iteration); 280 281 ts_elapsed = timespec_elapsed(ts_start); 282 pr_info("%-30s: %ld.%09lds\n", 283 description, ts_elapsed.tv_sec, ts_elapsed.tv_nsec); 284 } 285 286 static void access_memory(struct kvm_vm *vm, int nr_vcpus, 287 enum access_type access, const char *description) 288 { 289 memstress_set_write_percent(vm, (access == ACCESS_READ) ? 0 : 100); 290 iteration_work = ITERATION_ACCESS_MEMORY; 291 run_iteration(vm, nr_vcpus, description); 292 } 293 294 static void mark_memory_idle(struct kvm_vm *vm, int nr_vcpus) 295 { 296 /* 297 * Even though this parallelizes the work across vCPUs, this is still a 298 * very slow operation because page_idle forces the test to mark one pfn 299 * at a time and the clear_young notifier serializes on the KVM MMU 300 * lock. 301 */ 302 pr_debug("Marking VM memory idle (slow)...\n"); 303 iteration_work = ITERATION_MARK_IDLE; 304 run_iteration(vm, nr_vcpus, "Mark memory idle"); 305 } 306 307 static void run_test(enum vm_guest_mode mode, void *arg) 308 { 309 struct test_params *params = arg; 310 struct kvm_vm *vm; 311 int nr_vcpus = params->nr_vcpus; 312 313 vm = memstress_create_vm(mode, nr_vcpus, params->vcpu_memory_bytes, 1, 314 params->backing_src, !overlap_memory_access); 315 316 memstress_start_vcpu_threads(nr_vcpus, vcpu_thread_main); 317 318 pr_info("\n"); 319 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Populating memory"); 320 321 /* As a control, read and write to the populated memory first. */ 322 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Writing to populated memory"); 323 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from populated memory"); 324 325 /* Repeat on memory that has been marked as idle. */ 326 mark_memory_idle(vm, nr_vcpus); 327 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Writing to idle memory"); 328 mark_memory_idle(vm, nr_vcpus); 329 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from idle memory"); 330 331 memstress_join_vcpu_threads(nr_vcpus); 332 memstress_destroy_vm(vm); 333 } 334 335 static int access_tracking_unreliable(void) 336 { 337 #ifdef __x86_64__ 338 /* 339 * When running nested, the TLB size may be effectively unlimited (for 340 * example, this is the case when running on KVM L0), and KVM doesn't 341 * explicitly flush the TLB when aging SPTEs. As a result, more pages 342 * are cached and the guest won't see the "idle" bit cleared. 343 */ 344 if (this_cpu_has(X86_FEATURE_HYPERVISOR)) { 345 puts("Skipping idle page count sanity check, because the test is run nested"); 346 return 1; 347 } 348 #endif 349 /* 350 * When NUMA balancing is enabled, guest memory will be unmapped to get 351 * NUMA faults, dropping the Accessed bits. 352 */ 353 if (is_numa_balancing_enabled()) { 354 puts("Skipping idle page count sanity check, because NUMA balancing is enabled"); 355 return 1; 356 } 357 358 return 0; 359 } 360 361 static void help(char *name) 362 { 363 puts(""); 364 printf("usage: %s [-h] [-m mode] [-b vcpu_bytes] [-v vcpus] [-o] [-s mem_type]\n", 365 name); 366 puts(""); 367 printf(" -h: Display this help message."); 368 guest_modes_help(); 369 printf(" -b: specify the size of the memory region which should be\n" 370 " dirtied by each vCPU. e.g. 10M or 3G.\n" 371 " (default: 1G)\n"); 372 printf(" -v: specify the number of vCPUs to run.\n"); 373 printf(" -o: Overlap guest memory accesses instead of partitioning\n" 374 " them into a separate region of memory for each vCPU.\n"); 375 printf(" -w: Control whether the test warns or fails if more than 10%%\n" 376 " of pages are still seen as idle/old after accessing guest\n" 377 " memory. >0 == warn only, 0 == fail, <0 == auto. For auto\n" 378 " mode, the test fails by default, but switches to warn only\n" 379 " if NUMA balancing is enabled or the test detects it's running\n" 380 " in a VM.\n"); 381 backing_src_help("-s"); 382 puts(""); 383 exit(0); 384 } 385 386 int main(int argc, char *argv[]) 387 { 388 struct test_params params = { 389 .backing_src = DEFAULT_VM_MEM_SRC, 390 .vcpu_memory_bytes = DEFAULT_PER_VCPU_MEM_SIZE, 391 .nr_vcpus = 1, 392 }; 393 int page_idle_fd; 394 int opt; 395 396 guest_modes_append_default(); 397 398 while ((opt = getopt(argc, argv, "hm:b:v:os:w:")) != -1) { 399 switch (opt) { 400 case 'm': 401 guest_modes_cmdline(optarg); 402 break; 403 case 'b': 404 params.vcpu_memory_bytes = parse_size(optarg); 405 break; 406 case 'v': 407 params.nr_vcpus = atoi_positive("Number of vCPUs", optarg); 408 break; 409 case 'o': 410 overlap_memory_access = true; 411 break; 412 case 's': 413 params.backing_src = parse_backing_src_type(optarg); 414 break; 415 case 'w': 416 idle_pages_warn_only = 417 atoi_non_negative("Idle pages warning", 418 optarg); 419 break; 420 case 'h': 421 default: 422 help(argv[0]); 423 break; 424 } 425 } 426 427 page_idle_fd = open("/sys/kernel/mm/page_idle/bitmap", O_RDWR); 428 __TEST_REQUIRE(page_idle_fd >= 0, 429 "CONFIG_IDLE_PAGE_TRACKING is not enabled"); 430 close(page_idle_fd); 431 432 if (idle_pages_warn_only == -1) 433 idle_pages_warn_only = access_tracking_unreliable(); 434 435 for_each_guest_mode(run_test, ¶ms); 436 437 return 0; 438 } 439