1 // SPDX-License-Identifier: GPL-2.0 2 #include <signal.h> 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <sys/ioctl.h> 7 8 #include <linux/sizes.h> 9 10 #include <kvm_util.h> 11 #include <processor.h> 12 13 #include "ucall_common.h" 14 15 struct kvm_coalesced_io { 16 struct kvm_coalesced_mmio_ring *ring; 17 uint32_t ring_size; 18 uint64_t mmio_gpa; 19 uint64_t *mmio; 20 21 /* 22 * x86-only, but define pio_port for all architectures to minimize the 23 * amount of #ifdeffery and complexity, without having to sacrifice 24 * verbose error messages. 25 */ 26 uint8_t pio_port; 27 }; 28 29 static struct kvm_coalesced_io kvm_builtin_io_ring; 30 31 #ifdef __x86_64__ 32 static const int has_pio = 1; 33 #else 34 static const int has_pio = 0; 35 #endif 36 37 static void guest_code(struct kvm_coalesced_io *io) 38 { 39 int i, j; 40 41 for (;;) { 42 for (j = 0; j < 1 + has_pio; j++) { 43 /* 44 * KVM always leaves one free entry, i.e. exits to 45 * userspace before the last entry is filled. 46 */ 47 for (i = 0; i < io->ring_size - 1; i++) { 48 #ifdef __x86_64__ 49 if (i & 1) 50 outl(io->pio_port, io->pio_port + i); 51 else 52 #endif 53 WRITE_ONCE(*io->mmio, io->mmio_gpa + i); 54 } 55 #ifdef __x86_64__ 56 if (j & 1) 57 outl(io->pio_port, io->pio_port + i); 58 else 59 #endif 60 WRITE_ONCE(*io->mmio, io->mmio_gpa + i); 61 } 62 GUEST_SYNC(0); 63 64 WRITE_ONCE(*io->mmio, io->mmio_gpa + i); 65 #ifdef __x86_64__ 66 outl(io->pio_port, io->pio_port + i); 67 #endif 68 } 69 } 70 71 static void vcpu_run_and_verify_io_exit(struct kvm_vcpu *vcpu, 72 struct kvm_coalesced_io *io, 73 uint32_t ring_start, 74 uint32_t expected_exit) 75 { 76 const bool want_pio = expected_exit == KVM_EXIT_IO; 77 struct kvm_coalesced_mmio_ring *ring = io->ring; 78 struct kvm_run *run = vcpu->run; 79 uint32_t pio_value; 80 81 WRITE_ONCE(ring->first, ring_start); 82 WRITE_ONCE(ring->last, ring_start); 83 84 vcpu_run(vcpu); 85 86 /* 87 * Annoyingly, reading PIO data is safe only for PIO exits, otherwise 88 * data_offset is garbage, e.g. an MMIO gpa. 89 */ 90 if (run->exit_reason == KVM_EXIT_IO) 91 pio_value = *(uint32_t *)((void *)run + run->io.data_offset); 92 else 93 pio_value = 0; 94 95 TEST_ASSERT((!want_pio && (run->exit_reason == KVM_EXIT_MMIO && run->mmio.is_write && 96 run->mmio.phys_addr == io->mmio_gpa && run->mmio.len == 8 && 97 *(uint64_t *)run->mmio.data == io->mmio_gpa + io->ring_size - 1)) || 98 (want_pio && (run->exit_reason == KVM_EXIT_IO && run->io.port == io->pio_port && 99 run->io.direction == KVM_EXIT_IO_OUT && run->io.count == 1 && 100 pio_value == io->pio_port + io->ring_size - 1)), 101 "For start = %u, expected exit on %u-byte %s write 0x%llx = %lx, got exit_reason = %u (%s)\n " 102 "(MMIO addr = 0x%llx, write = %u, len = %u, data = %lx)\n " 103 "(PIO port = 0x%x, write = %u, len = %u, count = %u, data = %x", 104 ring_start, want_pio ? 4 : 8, want_pio ? "PIO" : "MMIO", 105 want_pio ? (unsigned long long)io->pio_port : io->mmio_gpa, 106 (want_pio ? io->pio_port : io->mmio_gpa) + io->ring_size - 1, run->exit_reason, 107 run->exit_reason == KVM_EXIT_MMIO ? "MMIO" : run->exit_reason == KVM_EXIT_IO ? "PIO" : "other", 108 run->mmio.phys_addr, run->mmio.is_write, run->mmio.len, *(uint64_t *)run->mmio.data, 109 run->io.port, run->io.direction, run->io.size, run->io.count, pio_value); 110 } 111 112 static void vcpu_run_and_verify_coalesced_io(struct kvm_vcpu *vcpu, 113 struct kvm_coalesced_io *io, 114 uint32_t ring_start, 115 uint32_t expected_exit) 116 { 117 struct kvm_coalesced_mmio_ring *ring = io->ring; 118 int i; 119 120 vcpu_run_and_verify_io_exit(vcpu, io, ring_start, expected_exit); 121 122 TEST_ASSERT((ring->last + 1) % io->ring_size == ring->first, 123 "Expected ring to be full (minus 1), first = %u, last = %u, max = %u, start = %u", 124 ring->first, ring->last, io->ring_size, ring_start); 125 126 for (i = 0; i < io->ring_size - 1; i++) { 127 uint32_t idx = (ring->first + i) % io->ring_size; 128 struct kvm_coalesced_mmio *entry = &ring->coalesced_mmio[idx]; 129 130 #ifdef __x86_64__ 131 if (i & 1) 132 TEST_ASSERT(entry->phys_addr == io->pio_port && 133 entry->len == 4 && entry->pio && 134 *(uint32_t *)entry->data == io->pio_port + i, 135 "Wanted 4-byte port I/O 0x%x = 0x%x in entry %u, got %u-byte %s 0x%llx = 0x%x", 136 io->pio_port, io->pio_port + i, i, 137 entry->len, entry->pio ? "PIO" : "MMIO", 138 entry->phys_addr, *(uint32_t *)entry->data); 139 else 140 #endif 141 TEST_ASSERT(entry->phys_addr == io->mmio_gpa && 142 entry->len == 8 && !entry->pio, 143 "Wanted 8-byte MMIO to 0x%lx = %lx in entry %u, got %u-byte %s 0x%llx = 0x%lx", 144 io->mmio_gpa, io->mmio_gpa + i, i, 145 entry->len, entry->pio ? "PIO" : "MMIO", 146 entry->phys_addr, *(uint64_t *)entry->data); 147 } 148 } 149 150 static void test_coalesced_io(struct kvm_vcpu *vcpu, 151 struct kvm_coalesced_io *io, uint32_t ring_start) 152 { 153 struct kvm_coalesced_mmio_ring *ring = io->ring; 154 155 kvm_vm_register_coalesced_io(vcpu->vm, io->mmio_gpa, 8, false /* pio */); 156 #ifdef __x86_64__ 157 kvm_vm_register_coalesced_io(vcpu->vm, io->pio_port, 8, true /* pio */); 158 #endif 159 160 vcpu_run_and_verify_coalesced_io(vcpu, io, ring_start, KVM_EXIT_MMIO); 161 #ifdef __x86_64__ 162 vcpu_run_and_verify_coalesced_io(vcpu, io, ring_start, KVM_EXIT_IO); 163 #endif 164 165 /* 166 * Verify ucall, which may use non-coalesced MMIO or PIO, generates an 167 * immediate exit. 168 */ 169 WRITE_ONCE(ring->first, ring_start); 170 WRITE_ONCE(ring->last, ring_start); 171 vcpu_run(vcpu); 172 TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC); 173 TEST_ASSERT_EQ(ring->first, ring_start); 174 TEST_ASSERT_EQ(ring->last, ring_start); 175 176 /* Verify that non-coalesced MMIO/PIO generates an exit to userspace. */ 177 kvm_vm_unregister_coalesced_io(vcpu->vm, io->mmio_gpa, 8, false /* pio */); 178 vcpu_run_and_verify_io_exit(vcpu, io, ring_start, KVM_EXIT_MMIO); 179 180 #ifdef __x86_64__ 181 kvm_vm_unregister_coalesced_io(vcpu->vm, io->pio_port, 8, true /* pio */); 182 vcpu_run_and_verify_io_exit(vcpu, io, ring_start, KVM_EXIT_IO); 183 #endif 184 } 185 186 int main(int argc, char *argv[]) 187 { 188 struct kvm_vcpu *vcpu; 189 struct kvm_vm *vm; 190 int i; 191 192 TEST_REQUIRE(kvm_has_cap(KVM_CAP_COALESCED_MMIO)); 193 194 #ifdef __x86_64__ 195 TEST_REQUIRE(kvm_has_cap(KVM_CAP_COALESCED_PIO)); 196 #endif 197 198 vm = vm_create_with_one_vcpu(&vcpu, guest_code); 199 200 kvm_builtin_io_ring = (struct kvm_coalesced_io) { 201 /* 202 * The I/O ring is a kernel-allocated page whose address is 203 * relative to each vCPU's run page, with the page offset 204 * provided by KVM in the return of KVM_CAP_COALESCED_MMIO. 205 */ 206 .ring = (void *)vcpu->run + 207 (kvm_check_cap(KVM_CAP_COALESCED_MMIO) * getpagesize()), 208 209 /* 210 * The size of the I/O ring is fixed, but KVM defines the sized 211 * based on the kernel's PAGE_SIZE. Thus, userspace must query 212 * the host's page size at runtime to compute the ring size. 213 */ 214 .ring_size = (getpagesize() - sizeof(struct kvm_coalesced_mmio_ring)) / 215 sizeof(struct kvm_coalesced_mmio), 216 217 /* 218 * Arbitrary address+port (MMIO mustn't overlap memslots), with 219 * the MMIO GPA identity mapped in the guest. 220 */ 221 .mmio_gpa = 4ull * SZ_1G, 222 .mmio = (uint64_t *)(4ull * SZ_1G), 223 .pio_port = 0x80, 224 }; 225 226 virt_map(vm, (uint64_t)kvm_builtin_io_ring.mmio, kvm_builtin_io_ring.mmio_gpa, 1); 227 228 sync_global_to_guest(vm, kvm_builtin_io_ring); 229 vcpu_args_set(vcpu, 1, &kvm_builtin_io_ring); 230 231 for (i = 0; i < kvm_builtin_io_ring.ring_size; i++) 232 test_coalesced_io(vcpu, &kvm_builtin_io_ring, i); 233 234 kvm_vm_free(vm); 235 return 0; 236 } 237