1 // SPDX-License-Identifier: GPL-2.0 2 #define _GNU_SOURCE /* for program_invocation_short_name */ 3 #include <fcntl.h> 4 #include <pthread.h> 5 #include <sched.h> 6 #include <semaphore.h> 7 #include <signal.h> 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <string.h> 11 #include <sys/ioctl.h> 12 #include <sys/mman.h> 13 14 #include <linux/compiler.h> 15 16 #include <test_util.h> 17 #include <kvm_util.h> 18 #include <processor.h> 19 20 /* 21 * s390x needs at least 1MB alignment, and the x86_64 MOVE/DELETE tests need a 22 * 2MB sized and aligned region so that the initial region corresponds to 23 * exactly one large page. 24 */ 25 #define MEM_REGION_SIZE 0x200000 26 27 #ifdef __x86_64__ 28 /* 29 * Somewhat arbitrary location and slot, intended to not overlap anything. 30 */ 31 #define MEM_REGION_GPA 0xc0000000 32 #define MEM_REGION_SLOT 10 33 34 static const uint64_t MMIO_VAL = 0xbeefull; 35 36 extern const uint64_t final_rip_start; 37 extern const uint64_t final_rip_end; 38 39 static sem_t vcpu_ready; 40 41 static inline uint64_t guest_spin_on_val(uint64_t spin_val) 42 { 43 uint64_t val; 44 45 do { 46 val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA)); 47 } while (val == spin_val); 48 49 GUEST_SYNC(0); 50 return val; 51 } 52 53 static void *vcpu_worker(void *data) 54 { 55 struct kvm_vcpu *vcpu = data; 56 struct kvm_run *run = vcpu->run; 57 struct ucall uc; 58 uint64_t cmd; 59 60 /* 61 * Loop until the guest is done. Re-enter the guest on all MMIO exits, 62 * which will occur if the guest attempts to access a memslot after it 63 * has been deleted or while it is being moved . 64 */ 65 while (1) { 66 vcpu_run(vcpu); 67 68 if (run->exit_reason == KVM_EXIT_IO) { 69 cmd = get_ucall(vcpu, &uc); 70 if (cmd != UCALL_SYNC) 71 break; 72 73 sem_post(&vcpu_ready); 74 continue; 75 } 76 77 if (run->exit_reason != KVM_EXIT_MMIO) 78 break; 79 80 TEST_ASSERT(!run->mmio.is_write, "Unexpected exit mmio write"); 81 TEST_ASSERT(run->mmio.len == 8, 82 "Unexpected exit mmio size = %u", run->mmio.len); 83 84 TEST_ASSERT(run->mmio.phys_addr == MEM_REGION_GPA, 85 "Unexpected exit mmio address = 0x%llx", 86 run->mmio.phys_addr); 87 memcpy(run->mmio.data, &MMIO_VAL, 8); 88 } 89 90 if (run->exit_reason == KVM_EXIT_IO && cmd == UCALL_ABORT) 91 REPORT_GUEST_ASSERT_1(uc, "val = %lu"); 92 93 return NULL; 94 } 95 96 static void wait_for_vcpu(void) 97 { 98 struct timespec ts; 99 100 TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts), 101 "clock_gettime() failed: %d\n", errno); 102 103 ts.tv_sec += 2; 104 TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts), 105 "sem_timedwait() failed: %d\n", errno); 106 107 /* Wait for the vCPU thread to reenter the guest. */ 108 usleep(100000); 109 } 110 111 static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread, 112 void *guest_code) 113 { 114 struct kvm_vm *vm; 115 uint64_t *hva; 116 uint64_t gpa; 117 118 vm = vm_create_with_one_vcpu(vcpu, guest_code); 119 120 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP, 121 MEM_REGION_GPA, MEM_REGION_SLOT, 122 MEM_REGION_SIZE / getpagesize(), 0); 123 124 /* 125 * Allocate and map two pages so that the GPA accessed by guest_code() 126 * stays valid across the memslot move. 127 */ 128 gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT); 129 TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); 130 131 virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2); 132 133 /* Ditto for the host mapping so that both pages can be zeroed. */ 134 hva = addr_gpa2hva(vm, MEM_REGION_GPA); 135 memset(hva, 0, 2 * 4096); 136 137 pthread_create(vcpu_thread, NULL, vcpu_worker, *vcpu); 138 139 /* Ensure the guest thread is spun up. */ 140 wait_for_vcpu(); 141 142 return vm; 143 } 144 145 146 static void guest_code_move_memory_region(void) 147 { 148 uint64_t val; 149 150 GUEST_SYNC(0); 151 152 /* 153 * Spin until the memory region starts getting moved to a 154 * misaligned address. 155 * Every region move may or may not trigger MMIO, as the 156 * window where the memslot is invalid is usually quite small. 157 */ 158 val = guest_spin_on_val(0); 159 GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val); 160 161 /* Spin until the misaligning memory region move completes. */ 162 val = guest_spin_on_val(MMIO_VAL); 163 GUEST_ASSERT_1(val == 1 || val == 0, val); 164 165 /* Spin until the memory region starts to get re-aligned. */ 166 val = guest_spin_on_val(0); 167 GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val); 168 169 /* Spin until the re-aligning memory region move completes. */ 170 val = guest_spin_on_val(MMIO_VAL); 171 GUEST_ASSERT_1(val == 1, val); 172 173 GUEST_DONE(); 174 } 175 176 static void test_move_memory_region(void) 177 { 178 pthread_t vcpu_thread; 179 struct kvm_vcpu *vcpu; 180 struct kvm_vm *vm; 181 uint64_t *hva; 182 183 vm = spawn_vm(&vcpu, &vcpu_thread, guest_code_move_memory_region); 184 185 hva = addr_gpa2hva(vm, MEM_REGION_GPA); 186 187 /* 188 * Shift the region's base GPA. The guest should not see "2" as the 189 * hva->gpa translation is misaligned, i.e. the guest is accessing a 190 * different host pfn. 191 */ 192 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA - 4096); 193 WRITE_ONCE(*hva, 2); 194 195 /* 196 * The guest _might_ see an invalid memslot and trigger MMIO, but it's 197 * a tiny window. Spin and defer the sync until the memslot is 198 * restored and guest behavior is once again deterministic. 199 */ 200 usleep(100000); 201 202 /* 203 * Note, value in memory needs to be changed *before* restoring the 204 * memslot, else the guest could race the update and see "2". 205 */ 206 WRITE_ONCE(*hva, 1); 207 208 /* Restore the original base, the guest should see "1". */ 209 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA); 210 wait_for_vcpu(); 211 /* Defered sync from when the memslot was misaligned (above). */ 212 wait_for_vcpu(); 213 214 pthread_join(vcpu_thread, NULL); 215 216 kvm_vm_free(vm); 217 } 218 219 static void guest_code_delete_memory_region(void) 220 { 221 uint64_t val; 222 223 GUEST_SYNC(0); 224 225 /* Spin until the memory region is deleted. */ 226 val = guest_spin_on_val(0); 227 GUEST_ASSERT_1(val == MMIO_VAL, val); 228 229 /* Spin until the memory region is recreated. */ 230 val = guest_spin_on_val(MMIO_VAL); 231 GUEST_ASSERT_1(val == 0, val); 232 233 /* Spin until the memory region is deleted. */ 234 val = guest_spin_on_val(0); 235 GUEST_ASSERT_1(val == MMIO_VAL, val); 236 237 asm("1:\n\t" 238 ".pushsection .rodata\n\t" 239 ".global final_rip_start\n\t" 240 "final_rip_start: .quad 1b\n\t" 241 ".popsection"); 242 243 /* Spin indefinitely (until the code memslot is deleted). */ 244 guest_spin_on_val(MMIO_VAL); 245 246 asm("1:\n\t" 247 ".pushsection .rodata\n\t" 248 ".global final_rip_end\n\t" 249 "final_rip_end: .quad 1b\n\t" 250 ".popsection"); 251 252 GUEST_ASSERT_1(0, 0); 253 } 254 255 static void test_delete_memory_region(void) 256 { 257 pthread_t vcpu_thread; 258 struct kvm_vcpu *vcpu; 259 struct kvm_regs regs; 260 struct kvm_run *run; 261 struct kvm_vm *vm; 262 263 vm = spawn_vm(&vcpu, &vcpu_thread, guest_code_delete_memory_region); 264 265 /* Delete the memory region, the guest should not die. */ 266 vm_mem_region_delete(vm, MEM_REGION_SLOT); 267 wait_for_vcpu(); 268 269 /* Recreate the memory region. The guest should see "0". */ 270 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP, 271 MEM_REGION_GPA, MEM_REGION_SLOT, 272 MEM_REGION_SIZE / getpagesize(), 0); 273 wait_for_vcpu(); 274 275 /* Delete the region again so that there's only one memslot left. */ 276 vm_mem_region_delete(vm, MEM_REGION_SLOT); 277 wait_for_vcpu(); 278 279 /* 280 * Delete the primary memslot. This should cause an emulation error or 281 * shutdown due to the page tables getting nuked. 282 */ 283 vm_mem_region_delete(vm, 0); 284 285 pthread_join(vcpu_thread, NULL); 286 287 run = vcpu->run; 288 289 TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN || 290 run->exit_reason == KVM_EXIT_INTERNAL_ERROR, 291 "Unexpected exit reason = %d", run->exit_reason); 292 293 vcpu_regs_get(vcpu, ®s); 294 295 /* 296 * On AMD, after KVM_EXIT_SHUTDOWN the VMCB has been reinitialized already, 297 * so the instruction pointer would point to the reset vector. 298 */ 299 if (run->exit_reason == KVM_EXIT_INTERNAL_ERROR) 300 TEST_ASSERT(regs.rip >= final_rip_start && 301 regs.rip < final_rip_end, 302 "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx\n", 303 final_rip_start, final_rip_end, regs.rip); 304 305 kvm_vm_free(vm); 306 } 307 308 static void test_zero_memory_regions(void) 309 { 310 struct kvm_vcpu *vcpu; 311 struct kvm_run *run; 312 struct kvm_vm *vm; 313 314 pr_info("Testing KVM_RUN with zero added memory regions\n"); 315 316 vm = vm_create_barebones(); 317 vcpu = __vm_vcpu_add(vm, 0); 318 319 vm_ioctl(vm, KVM_SET_NR_MMU_PAGES, (void *)64ul); 320 vcpu_run(vcpu); 321 322 run = vcpu->run; 323 TEST_ASSERT(run->exit_reason == KVM_EXIT_INTERNAL_ERROR, 324 "Unexpected exit_reason = %u\n", run->exit_reason); 325 326 kvm_vm_free(vm); 327 } 328 #endif /* __x86_64__ */ 329 330 /* 331 * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any 332 * tentative to add further slots should fail. 333 */ 334 static void test_add_max_memory_regions(void) 335 { 336 int ret; 337 struct kvm_vm *vm; 338 uint32_t max_mem_slots; 339 uint32_t slot; 340 void *mem, *mem_aligned, *mem_extra; 341 size_t alignment; 342 343 #ifdef __s390x__ 344 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */ 345 alignment = 0x100000; 346 #else 347 alignment = 1; 348 #endif 349 350 max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS); 351 TEST_ASSERT(max_mem_slots > 0, 352 "KVM_CAP_NR_MEMSLOTS should be greater than 0"); 353 pr_info("Allowed number of memory slots: %i\n", max_mem_slots); 354 355 vm = vm_create_barebones(); 356 357 /* Check it can be added memory slots up to the maximum allowed */ 358 pr_info("Adding slots 0..%i, each memory region with %dK size\n", 359 (max_mem_slots - 1), MEM_REGION_SIZE >> 10); 360 361 mem = mmap(NULL, (size_t)max_mem_slots * MEM_REGION_SIZE + alignment, 362 PROT_READ | PROT_WRITE, 363 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); 364 TEST_ASSERT(mem != MAP_FAILED, "Failed to mmap() host"); 365 mem_aligned = (void *)(((size_t) mem + alignment - 1) & ~(alignment - 1)); 366 367 for (slot = 0; slot < max_mem_slots; slot++) 368 vm_set_user_memory_region(vm, slot, 0, 369 ((uint64_t)slot * MEM_REGION_SIZE), 370 MEM_REGION_SIZE, 371 mem_aligned + (uint64_t)slot * MEM_REGION_SIZE); 372 373 /* Check it cannot be added memory slots beyond the limit */ 374 mem_extra = mmap(NULL, MEM_REGION_SIZE, PROT_READ | PROT_WRITE, 375 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 376 TEST_ASSERT(mem_extra != MAP_FAILED, "Failed to mmap() host"); 377 378 ret = __vm_set_user_memory_region(vm, max_mem_slots, 0, 379 (uint64_t)max_mem_slots * MEM_REGION_SIZE, 380 MEM_REGION_SIZE, mem_extra); 381 TEST_ASSERT(ret == -1 && errno == EINVAL, 382 "Adding one more memory slot should fail with EINVAL"); 383 384 munmap(mem, (size_t)max_mem_slots * MEM_REGION_SIZE + alignment); 385 munmap(mem_extra, MEM_REGION_SIZE); 386 kvm_vm_free(vm); 387 } 388 389 int main(int argc, char *argv[]) 390 { 391 #ifdef __x86_64__ 392 int i, loops; 393 #endif 394 395 #ifdef __x86_64__ 396 /* 397 * FIXME: the zero-memslot test fails on aarch64 and s390x because 398 * KVM_RUN fails with ENOEXEC or EFAULT. 399 */ 400 test_zero_memory_regions(); 401 #endif 402 403 test_add_max_memory_regions(); 404 405 #ifdef __x86_64__ 406 if (argc > 1) 407 loops = atoi_positive("Number of iterations", argv[1]); 408 else 409 loops = 10; 410 411 pr_info("Testing MOVE of in-use region, %d loops\n", loops); 412 for (i = 0; i < loops; i++) 413 test_move_memory_region(); 414 415 pr_info("Testing DELETE of in-use region, %d loops\n", loops); 416 for (i = 0; i < loops; i++) 417 test_delete_memory_region(); 418 #endif 419 420 return 0; 421 } 422