1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KVM dirty page logging test 4 * 5 * Copyright (C) 2018, Red Hat, Inc. 6 */ 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <linux/bitmap.h> 10 #include <linux/bitops.h> 11 12 #include "test_util.h" 13 #include "kvm_util.h" 14 #include "processor.h" 15 #include "svm_util.h" 16 #include "vmx.h" 17 18 /* The memory slot index to track dirty pages */ 19 #define TEST_MEM_SLOT_INDEX 1 20 21 /* 22 * Allocate four pages total. Two pages are used to verify that the KVM marks 23 * the accessed page/GFN as marked dirty, but not the "other" page. Times two 24 * so that each "normal" page can be accessed from L2 via an aliased L2 GVA+GPA 25 * (when TDP is enabled), to verify KVM marks _L1's_ page/GFN as dirty (to 26 * detect failures, L2 => L1 GPAs can't be identity mapped in the TDP page 27 * tables, as marking L2's GPA dirty would get a false pass if L1 == L2). 28 */ 29 #define TEST_MEM_PAGES 4 30 31 #define TEST_MEM_BASE 0xc0000000 32 #define TEST_MEM_ALIAS_BASE 0xc0002000 33 34 #define TEST_GUEST_ADDR(base, idx) ((base) + (idx) * PAGE_SIZE) 35 36 #define TEST_GVA(idx) TEST_GUEST_ADDR(TEST_MEM_BASE, idx) 37 #define TEST_GPA(idx) TEST_GUEST_ADDR(TEST_MEM_BASE, idx) 38 39 #define TEST_ALIAS_GPA(idx) TEST_GUEST_ADDR(TEST_MEM_ALIAS_BASE, idx) 40 41 #define TEST_HVA(vm, idx) addr_gpa2hva(vm, TEST_GPA(idx)) 42 43 #define L2_GUEST_STACK_SIZE 64 44 45 /* Use the page offset bits to communicate the access+fault type. */ 46 #define TEST_SYNC_READ_FAULT BIT(0) 47 #define TEST_SYNC_WRITE_FAULT BIT(1) 48 #define TEST_SYNC_NO_FAULT BIT(2) 49 50 static void l2_guest_code(vm_vaddr_t base) 51 { 52 vm_vaddr_t page0 = TEST_GUEST_ADDR(base, 0); 53 vm_vaddr_t page1 = TEST_GUEST_ADDR(base, 1); 54 55 READ_ONCE(*(u64 *)page0); 56 GUEST_SYNC(page0 | TEST_SYNC_READ_FAULT); 57 WRITE_ONCE(*(u64 *)page0, 1); 58 GUEST_SYNC(page0 | TEST_SYNC_WRITE_FAULT); 59 READ_ONCE(*(u64 *)page0); 60 GUEST_SYNC(page0 | TEST_SYNC_NO_FAULT); 61 62 WRITE_ONCE(*(u64 *)page1, 1); 63 GUEST_SYNC(page1 | TEST_SYNC_WRITE_FAULT); 64 WRITE_ONCE(*(u64 *)page1, 1); 65 GUEST_SYNC(page1 | TEST_SYNC_WRITE_FAULT); 66 READ_ONCE(*(u64 *)page1); 67 GUEST_SYNC(page1 | TEST_SYNC_NO_FAULT); 68 69 /* Exit to L1 and never come back. */ 70 vmcall(); 71 } 72 73 static void l2_guest_code_tdp_enabled(void) 74 { 75 /* 76 * Use the aliased virtual addresses when running with TDP to verify 77 * that KVM correctly handles the case where a page is dirtied via a 78 * different GPA than would be used by L1. 79 */ 80 l2_guest_code(TEST_MEM_ALIAS_BASE); 81 } 82 83 static void l2_guest_code_tdp_disabled(void) 84 { 85 /* 86 * Use the "normal" virtual addresses when running without TDP enabled, 87 * in which case L2 will use the same page tables as L1, and thus needs 88 * to use the same virtual addresses that are mapped into L1. 89 */ 90 l2_guest_code(TEST_MEM_BASE); 91 } 92 93 void l1_vmx_code(struct vmx_pages *vmx) 94 { 95 unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; 96 void *l2_rip; 97 98 GUEST_ASSERT(vmx->vmcs_gpa); 99 GUEST_ASSERT(prepare_for_vmx_operation(vmx)); 100 GUEST_ASSERT(load_vmcs(vmx)); 101 102 if (vmx->eptp_gpa) 103 l2_rip = l2_guest_code_tdp_enabled; 104 else 105 l2_rip = l2_guest_code_tdp_disabled; 106 107 prepare_vmcs(vmx, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]); 108 109 GUEST_SYNC(TEST_SYNC_NO_FAULT); 110 GUEST_ASSERT(!vmlaunch()); 111 GUEST_SYNC(TEST_SYNC_NO_FAULT); 112 GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); 113 GUEST_DONE(); 114 } 115 116 static void l1_svm_code(struct svm_test_data *svm) 117 { 118 unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; 119 void *l2_rip; 120 121 if (svm->ncr3_gpa) 122 l2_rip = l2_guest_code_tdp_enabled; 123 else 124 l2_rip = l2_guest_code_tdp_disabled; 125 126 generic_svm_setup(svm, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]); 127 128 GUEST_SYNC(TEST_SYNC_NO_FAULT); 129 run_guest(svm->vmcb, svm->vmcb_gpa); 130 GUEST_SYNC(TEST_SYNC_NO_FAULT); 131 GUEST_ASSERT_EQ(svm->vmcb->control.exit_code, SVM_EXIT_VMMCALL); 132 GUEST_DONE(); 133 } 134 135 static void l1_guest_code(void *data) 136 { 137 if (this_cpu_has(X86_FEATURE_VMX)) 138 l1_vmx_code(data); 139 else 140 l1_svm_code(data); 141 } 142 143 static void test_handle_ucall_sync(struct kvm_vm *vm, u64 arg, 144 unsigned long *bmap) 145 { 146 vm_vaddr_t gva = arg & ~(PAGE_SIZE - 1); 147 int page_nr, i; 148 149 /* 150 * Extract the page number of underlying physical page, which is also 151 * the _L1_ page number. The dirty bitmap _must_ be updated based on 152 * the L1 GPA, not L2 GPA, i.e. whether or not L2 used an aliased GPA 153 * (i.e. if TDP enabled for L2) is irrelevant with respect to the dirty 154 * bitmap and which underlying physical page is accessed. 155 * 156 * Note, gva will be '0' if there was no access, i.e. if the purpose of 157 * the sync is to verify all pages are clean. 158 */ 159 if (!gva) 160 page_nr = 0; 161 else if (gva >= TEST_MEM_ALIAS_BASE) 162 page_nr = (gva - TEST_MEM_ALIAS_BASE) >> PAGE_SHIFT; 163 else 164 page_nr = (gva - TEST_MEM_BASE) >> PAGE_SHIFT; 165 TEST_ASSERT(page_nr == 0 || page_nr == 1, 166 "Test bug, unexpected frame number '%u' for arg = %lx", page_nr, arg); 167 TEST_ASSERT(gva || (arg & TEST_SYNC_NO_FAULT), 168 "Test bug, gva must be valid if a fault is expected"); 169 170 kvm_vm_get_dirty_log(vm, TEST_MEM_SLOT_INDEX, bmap); 171 172 /* 173 * Check all pages to verify the correct physical page was modified (or 174 * not), and that all pages are clean/dirty as expected. 175 * 176 * If a fault of any kind is expected, the target page should be dirty 177 * as the Dirty bit is set in the gPTE. KVM should create a writable 178 * SPTE even on a read fault, *and* KVM must mark the GFN as dirty 179 * when doing so. 180 */ 181 for (i = 0; i < TEST_MEM_PAGES; i++) { 182 if (i == page_nr && (arg & TEST_SYNC_WRITE_FAULT)) 183 TEST_ASSERT(*(u64 *)TEST_HVA(vm, i) == 1, 184 "Page %u incorrectly not written by guest", i); 185 else 186 TEST_ASSERT(*(u64 *)TEST_HVA(vm, i) == 0xaaaaaaaaaaaaaaaaULL, 187 "Page %u incorrectly written by guest", i); 188 189 if (i == page_nr && !(arg & TEST_SYNC_NO_FAULT)) 190 TEST_ASSERT(test_bit(i, bmap), 191 "Page %u incorrectly reported clean on %s fault", 192 i, arg & TEST_SYNC_READ_FAULT ? "read" : "write"); 193 else 194 TEST_ASSERT(!test_bit(i, bmap), 195 "Page %u incorrectly reported dirty", i); 196 } 197 } 198 199 static void test_dirty_log(bool nested_tdp) 200 { 201 vm_vaddr_t nested_gva = 0; 202 unsigned long *bmap; 203 struct kvm_vcpu *vcpu; 204 struct kvm_vm *vm; 205 struct ucall uc; 206 bool done = false; 207 208 pr_info("Nested TDP: %s\n", nested_tdp ? "enabled" : "disabled"); 209 210 /* Create VM */ 211 vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); 212 if (nested_tdp) 213 vm_enable_tdp(vm); 214 215 if (kvm_cpu_has(X86_FEATURE_VMX)) 216 vcpu_alloc_vmx(vm, &nested_gva); 217 else 218 vcpu_alloc_svm(vm, &nested_gva); 219 220 vcpu_args_set(vcpu, 1, nested_gva); 221 222 /* Add an extra memory slot for testing dirty logging */ 223 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 224 TEST_MEM_BASE, 225 TEST_MEM_SLOT_INDEX, 226 TEST_MEM_PAGES, 227 KVM_MEM_LOG_DIRTY_PAGES); 228 229 /* 230 * Add an identity map for GVA range [0xc0000000, 0xc0004000). This 231 * affects both L1 and L2. However... 232 */ 233 virt_map(vm, TEST_MEM_BASE, TEST_MEM_BASE, TEST_MEM_PAGES); 234 235 /* 236 * ... pages in the L2 GPA address range [0xc0002000, 0xc0004000) will 237 * map to [0xc0000000, 0xc0002000) when TDP is enabled (for L2). 238 * 239 * When TDP is disabled, the L2 guest code will still access the same L1 240 * GPAs as the TDP enabled case. 241 * 242 * Set the Dirty bit in the PTEs used by L2 so that KVM will create 243 * writable SPTEs when handling read faults (if the Dirty bit isn't 244 * set, KVM must intercept the next write to emulate the Dirty bit 245 * update). 246 */ 247 if (nested_tdp) { 248 tdp_identity_map_default_memslots(vm); 249 tdp_map(vm, TEST_ALIAS_GPA(0), TEST_GPA(0), PAGE_SIZE); 250 tdp_map(vm, TEST_ALIAS_GPA(1), TEST_GPA(1), PAGE_SIZE); 251 252 *tdp_get_pte(vm, TEST_ALIAS_GPA(0)) |= PTE_DIRTY_MASK(&vm->stage2_mmu); 253 *tdp_get_pte(vm, TEST_ALIAS_GPA(1)) |= PTE_DIRTY_MASK(&vm->stage2_mmu); 254 } else { 255 *vm_get_pte(vm, TEST_GVA(0)) |= PTE_DIRTY_MASK(&vm->mmu); 256 *vm_get_pte(vm, TEST_GVA(1)) |= PTE_DIRTY_MASK(&vm->mmu); 257 } 258 259 bmap = bitmap_zalloc(TEST_MEM_PAGES); 260 261 while (!done) { 262 memset(TEST_HVA(vm, 0), 0xaa, TEST_MEM_PAGES * PAGE_SIZE); 263 264 vcpu_run(vcpu); 265 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); 266 267 switch (get_ucall(vcpu, &uc)) { 268 case UCALL_ABORT: 269 REPORT_GUEST_ASSERT(uc); 270 /* NOT REACHED */ 271 case UCALL_SYNC: 272 test_handle_ucall_sync(vm, uc.args[1], bmap); 273 break; 274 case UCALL_DONE: 275 done = true; 276 break; 277 default: 278 TEST_FAIL("Unknown ucall %lu", uc.cmd); 279 } 280 } 281 } 282 283 int main(int argc, char *argv[]) 284 { 285 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || kvm_cpu_has(X86_FEATURE_SVM)); 286 287 test_dirty_log(/*nested_tdp=*/false); 288 289 if (kvm_cpu_has_tdp()) 290 test_dirty_log(/*nested_tdp=*/true); 291 292 return 0; 293 } 294