1 // SPDX-License-Identifier: GPL-2.0 2 #include <stdio.h> 3 #include <stdlib.h> 4 #include <pthread.h> 5 #include <semaphore.h> 6 #include <sys/types.h> 7 #include <signal.h> 8 #include <errno.h> 9 #include <linux/bitmap.h> 10 #include <linux/bitops.h> 11 #include <linux/atomic.h> 12 #include <linux/sizes.h> 13 14 #include "kvm_util.h" 15 #include "test_util.h" 16 #include "guest_modes.h" 17 #include "processor.h" 18 #include "ucall_common.h" 19 20 static bool mprotect_ro_done; 21 static bool all_vcpus_hit_ro_fault; 22 23 static void guest_code(u64 start_gpa, u64 end_gpa, u64 stride) 24 { 25 gpa_t gpa; 26 int i; 27 28 for (i = 0; i < 2; i++) { 29 for (gpa = start_gpa; gpa < end_gpa; gpa += stride) 30 vcpu_arch_put_guest(*((volatile u64 *)gpa), gpa); 31 GUEST_SYNC(i); 32 } 33 34 for (gpa = start_gpa; gpa < end_gpa; gpa += stride) 35 *((volatile u64 *)gpa); 36 GUEST_SYNC(2); 37 38 /* 39 * Write to the region while mprotect(PROT_READ) is underway. Keep 40 * looping until the memory is guaranteed to be read-only and a fault 41 * has occurred, otherwise vCPUs may complete their writes and advance 42 * to the next stage prematurely. 43 * 44 * For architectures that support skipping the faulting instruction, 45 * generate the store via inline assembly to ensure the exact length 46 * of the instruction is known and stable (vcpu_arch_put_guest() on 47 * fixed-length architectures should work, but the cost of paranoia 48 * is low in this case). For x86, hand-code the exact opcode so that 49 * there is no room for variability in the generated instruction. 50 */ 51 do { 52 for (gpa = start_gpa; gpa < end_gpa; gpa += stride) 53 #ifdef __x86_64__ 54 asm volatile(".byte 0x48,0x89,0x00" :: "a"(gpa) : "memory"); /* mov %rax, (%rax) */ 55 #elif defined(__aarch64__) 56 asm volatile("str %0, [%0]" :: "r" (gpa) : "memory"); 57 #else 58 vcpu_arch_put_guest(*((volatile u64 *)gpa), gpa); 59 #endif 60 } while (!READ_ONCE(mprotect_ro_done) || !READ_ONCE(all_vcpus_hit_ro_fault)); 61 62 /* 63 * Only architectures that write the entire range can explicitly sync, 64 * as other architectures will be stuck on the write fault. 65 */ 66 #if defined(__x86_64__) || defined(__aarch64__) 67 GUEST_SYNC(3); 68 #endif 69 70 for (gpa = start_gpa; gpa < end_gpa; gpa += stride) 71 vcpu_arch_put_guest(*((volatile u64 *)gpa), gpa); 72 GUEST_SYNC(4); 73 74 GUEST_ASSERT(0); 75 } 76 77 struct vcpu_info { 78 struct kvm_vcpu *vcpu; 79 u64 start_gpa; 80 u64 end_gpa; 81 }; 82 83 static int nr_vcpus; 84 static atomic_t rendezvous; 85 static atomic_t nr_ro_faults; 86 87 static void rendezvous_with_boss(void) 88 { 89 int orig = atomic_read(&rendezvous); 90 91 if (orig > 0) { 92 atomic_dec_and_test(&rendezvous); 93 while (atomic_read(&rendezvous) > 0) 94 cpu_relax(); 95 } else { 96 atomic_inc(&rendezvous); 97 while (atomic_read(&rendezvous) < 0) 98 cpu_relax(); 99 } 100 } 101 102 static void assert_sync_stage(struct kvm_vcpu *vcpu, int stage) 103 { 104 struct ucall uc; 105 106 TEST_ASSERT_EQ(get_ucall(vcpu, &uc), UCALL_SYNC); 107 TEST_ASSERT_EQ(uc.args[1], stage); 108 } 109 110 static void run_vcpu(struct kvm_vcpu *vcpu, int stage) 111 { 112 vcpu_run(vcpu); 113 assert_sync_stage(vcpu, stage); 114 } 115 116 static void *vcpu_worker(void *data) 117 { 118 struct kvm_sregs __maybe_unused sregs; 119 struct vcpu_info *info = data; 120 struct kvm_vcpu *vcpu = info->vcpu; 121 struct kvm_vm *vm = vcpu->vm; 122 int r; 123 124 vcpu_args_set(vcpu, 3, info->start_gpa, info->end_gpa, vm->page_size); 125 126 rendezvous_with_boss(); 127 128 /* Stage 0, write all of guest memory. */ 129 run_vcpu(vcpu, 0); 130 rendezvous_with_boss(); 131 #ifdef __x86_64__ 132 vcpu_sregs_get(vcpu, &sregs); 133 /* Toggle CR0.WP to trigger a MMU context reset. */ 134 sregs.cr0 ^= X86_CR0_WP; 135 vcpu_sregs_set(vcpu, &sregs); 136 #endif 137 rendezvous_with_boss(); 138 139 /* Stage 1, re-write all of guest memory. */ 140 run_vcpu(vcpu, 1); 141 rendezvous_with_boss(); 142 143 /* Stage 2, read all of guest memory, which is now read-only. */ 144 run_vcpu(vcpu, 2); 145 146 /* 147 * Stage 3, write guest memory and verify KVM returns -EFAULT for once 148 * the mprotect(PROT_READ) lands. Only architectures that support 149 * validating *all* of guest memory sync for this stage, as vCPUs will 150 * be stuck on the faulting instruction for other architectures. Go to 151 * stage 3 without a rendezvous 152 */ 153 r = _vcpu_run(vcpu); 154 TEST_ASSERT(r == -1 && errno == EFAULT, 155 "Expected EFAULT on write to RO memory, got r = %d, errno = %d", r, errno); 156 157 atomic_inc(&nr_ro_faults); 158 if (atomic_read(&nr_ro_faults) == nr_vcpus) 159 WRITE_AND_SYNC_TO_GUEST(vm, all_vcpus_hit_ro_fault, true); 160 161 #if defined(__x86_64__) || defined(__aarch64__) 162 /* 163 * Verify *all* writes from the guest hit EFAULT due to the VMA now 164 * being read-only. x86 and arm64 only at this time as skipping the 165 * instruction that hits the EFAULT requires advancing the program 166 * counter, which is arch specific and relies on inline assembly. 167 */ 168 #ifdef __x86_64__ 169 vcpu->run->kvm_valid_regs = KVM_SYNC_X86_REGS; 170 #endif 171 for (;;) { 172 r = _vcpu_run(vcpu); 173 if (!r) 174 break; 175 TEST_ASSERT_EQ(errno, EFAULT); 176 #if defined(__x86_64__) 177 WRITE_ONCE(vcpu->run->kvm_dirty_regs, KVM_SYNC_X86_REGS); 178 vcpu->run->s.regs.regs.rip += 3; 179 #elif defined(__aarch64__) 180 vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.pc), 181 vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pc)) + 4); 182 #endif 183 184 } 185 assert_sync_stage(vcpu, 3); 186 #endif /* __x86_64__ || __aarch64__ */ 187 rendezvous_with_boss(); 188 189 /* 190 * Stage 4. Run to completion, waiting for mprotect(PROT_WRITE) to 191 * make the memory writable again. 192 */ 193 do { 194 r = _vcpu_run(vcpu); 195 } while (r && errno == EFAULT); 196 TEST_ASSERT_EQ(r, 0); 197 assert_sync_stage(vcpu, 4); 198 rendezvous_with_boss(); 199 200 return NULL; 201 } 202 203 static pthread_t *spawn_workers(struct kvm_vm *vm, struct kvm_vcpu **vcpus, 204 u64 start_gpa, u64 end_gpa) 205 { 206 struct vcpu_info *info; 207 gpa_t gpa, nr_bytes; 208 pthread_t *threads; 209 int i; 210 211 threads = malloc(nr_vcpus * sizeof(*threads)); 212 TEST_ASSERT(threads, "Failed to allocate vCPU threads"); 213 214 info = malloc(nr_vcpus * sizeof(*info)); 215 TEST_ASSERT(info, "Failed to allocate vCPU gpa ranges"); 216 217 nr_bytes = ((end_gpa - start_gpa) / nr_vcpus) & 218 ~((u64)vm->page_size - 1); 219 TEST_ASSERT(nr_bytes, "C'mon, no way you have %d CPUs", nr_vcpus); 220 221 for (i = 0, gpa = start_gpa; i < nr_vcpus; i++, gpa += nr_bytes) { 222 info[i].vcpu = vcpus[i]; 223 info[i].start_gpa = gpa; 224 info[i].end_gpa = gpa + nr_bytes; 225 kvm_pthread_create(&threads[i], NULL, vcpu_worker, &info[i]); 226 } 227 return threads; 228 } 229 230 static void rendezvous_with_vcpus(struct timespec *time, const char *name) 231 { 232 int i, rendezvoused; 233 234 pr_info("Waiting for vCPUs to finish %s...\n", name); 235 236 rendezvoused = atomic_read(&rendezvous); 237 for (i = 0; abs(rendezvoused) != 1; i++) { 238 usleep(100); 239 if (!(i & 0x3f)) 240 pr_info("\r%d vCPUs haven't rendezvoused...", 241 abs(rendezvoused) - 1); 242 rendezvoused = atomic_read(&rendezvous); 243 } 244 245 clock_gettime(CLOCK_MONOTONIC, time); 246 247 /* Release the vCPUs after getting the time of the previous action. */ 248 pr_info("\rAll vCPUs finished %s, releasing...\n", name); 249 if (rendezvoused > 0) 250 atomic_set(&rendezvous, -nr_vcpus - 1); 251 else 252 atomic_set(&rendezvous, nr_vcpus + 1); 253 } 254 255 static void calc_default_nr_vcpus(void) 256 { 257 cpu_set_t possible_mask; 258 kvm_sched_getaffinity(0, sizeof(possible_mask), &possible_mask); 259 260 nr_vcpus = CPU_COUNT(&possible_mask); 261 TEST_ASSERT(nr_vcpus > 0, "Uh, no CPUs?"); 262 if (nr_vcpus >= 2) 263 nr_vcpus = nr_vcpus * 3/4; 264 } 265 266 int main(int argc, char *argv[]) 267 { 268 /* 269 * Skip the first 4gb and slot0. slot0 maps <1gb and is used to back 270 * the guest's code, stack, and page tables. Because selftests creates 271 * an IRQCHIP, a.k.a. a local APIC, KVM creates an internal memslot 272 * just below the 4gb boundary. This test could create memory at 273 * 1gb-3gb,but it's simpler to skip straight to 4gb. 274 */ 275 const u64 start_gpa = SZ_4G; 276 const int first_slot = 1; 277 278 struct timespec time_start, time_run1, time_reset, time_run2, time_ro, time_rw; 279 u64 max_gpa, gpa, slot_size, max_mem, i; 280 int max_slots, slot, opt, fd; 281 bool hugepages = false; 282 struct kvm_vcpu **vcpus; 283 pthread_t *threads; 284 struct kvm_vm *vm; 285 void *mem; 286 287 /* 288 * Default to 2gb so that maxing out systems with MAXPHADDR=46, which 289 * are quite common for x86, requires changing only max_mem (KVM allows 290 * 32k memslots, 32k * 2gb == ~64tb of guest memory). 291 */ 292 slot_size = SZ_2G; 293 294 max_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS); 295 TEST_ASSERT(max_slots > first_slot, "KVM is broken"); 296 297 /* All KVM MMUs should be able to survive a 128gb guest. */ 298 max_mem = 128ull * SZ_1G; 299 300 calc_default_nr_vcpus(); 301 302 while ((opt = getopt(argc, argv, "c:h:m:s:H")) != -1) { 303 switch (opt) { 304 case 'c': 305 nr_vcpus = atoi_positive("Number of vCPUs", optarg); 306 break; 307 case 'm': 308 max_mem = 1ull * atoi_positive("Memory size", optarg) * SZ_1G; 309 break; 310 case 's': 311 slot_size = 1ull * atoi_positive("Slot size", optarg) * SZ_1G; 312 break; 313 case 'H': 314 hugepages = true; 315 break; 316 case 'h': 317 default: 318 printf("usage: %s [-c nr_vcpus] [-m max_mem_in_gb] [-s slot_size_in_gb] [-H]\n", argv[0]); 319 exit(1); 320 } 321 } 322 323 vcpus = malloc(nr_vcpus * sizeof(*vcpus)); 324 TEST_ASSERT(vcpus, "Failed to allocate vCPU array"); 325 326 vm = __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 327 #ifdef __x86_64__ 328 max_mem / SZ_1G, 329 #else 330 max_mem / vm_guest_mode_params[VM_MODE_DEFAULT].page_size, 331 #endif 332 guest_code, vcpus); 333 334 max_gpa = vm->max_gfn << vm->page_shift; 335 TEST_ASSERT(max_gpa > (4 * slot_size), "MAXPHYADDR <4gb "); 336 337 fd = kvm_memfd_alloc(slot_size, hugepages); 338 mem = kvm_mmap(slot_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd); 339 340 TEST_ASSERT(!madvise(mem, slot_size, MADV_NOHUGEPAGE), "madvise() failed"); 341 342 /* Pre-fault the memory to avoid taking mmap_sem on guest page faults. */ 343 for (i = 0; i < slot_size; i += vm->page_size) 344 ((u8 *)mem)[i] = 0xaa; 345 346 gpa = 0; 347 for (slot = first_slot; slot < max_slots; slot++) { 348 gpa = start_gpa + ((slot - first_slot) * slot_size); 349 if (gpa + slot_size > max_gpa) 350 break; 351 352 if ((gpa - start_gpa) >= max_mem) 353 break; 354 355 vm_set_user_memory_region(vm, slot, 0, gpa, slot_size, mem); 356 357 #ifdef __x86_64__ 358 /* Identity map memory in the guest using 1gb pages. */ 359 virt_map_level(vm, gpa, gpa, slot_size, PG_LEVEL_1G); 360 #else 361 virt_map(vm, gpa, gpa, slot_size >> vm->page_shift); 362 #endif 363 } 364 365 atomic_set(&rendezvous, nr_vcpus + 1); 366 threads = spawn_workers(vm, vcpus, start_gpa, gpa); 367 368 free(vcpus); 369 vcpus = NULL; 370 371 pr_info("Running with %lugb of guest memory and %u vCPUs\n", 372 (gpa - start_gpa) / SZ_1G, nr_vcpus); 373 374 rendezvous_with_vcpus(&time_start, "spawning"); 375 rendezvous_with_vcpus(&time_run1, "run 1"); 376 rendezvous_with_vcpus(&time_reset, "reset"); 377 rendezvous_with_vcpus(&time_run2, "run 2"); 378 379 mprotect(mem, slot_size, PROT_READ); 380 WRITE_AND_SYNC_TO_GUEST(vm, mprotect_ro_done, true); 381 382 rendezvous_with_vcpus(&time_ro, "mprotect RO"); 383 mprotect(mem, slot_size, PROT_READ | PROT_WRITE); 384 rendezvous_with_vcpus(&time_rw, "mprotect RW"); 385 386 time_rw = timespec_sub(time_rw, time_ro); 387 time_ro = timespec_sub(time_ro, time_run2); 388 time_run2 = timespec_sub(time_run2, time_reset); 389 time_reset = timespec_sub(time_reset, time_run1); 390 time_run1 = timespec_sub(time_run1, time_start); 391 392 pr_info("run1 = %ld.%.9lds, reset = %ld.%.9lds, run2 = %ld.%.9lds, " 393 "ro = %ld.%.9lds, rw = %ld.%.9lds\n", 394 time_run1.tv_sec, time_run1.tv_nsec, 395 time_reset.tv_sec, time_reset.tv_nsec, 396 time_run2.tv_sec, time_run2.tv_nsec, 397 time_ro.tv_sec, time_ro.tv_nsec, 398 time_rw.tv_sec, time_rw.tv_nsec); 399 400 /* 401 * Delete even numbered slots (arbitrary) and unmap the first half of 402 * the backing (also arbitrary) to verify KVM correctly drops all 403 * references to the removed regions. 404 */ 405 for (slot = (slot - 1) & ~1ull; slot >= first_slot; slot -= 2) 406 vm_set_user_memory_region(vm, slot, 0, 0, 0, NULL); 407 408 kvm_munmap(mem, slot_size / 2); 409 410 /* Sanity check that the vCPUs actually ran. */ 411 for (i = 0; i < nr_vcpus; i++) 412 kvm_pthread_join(threads[i], NULL); 413 414 /* 415 * Deliberately exit without deleting the remaining memslots or closing 416 * kvm_fd to test cleanup via mmu_notifier.release. 417 */ 418 } 419