// SPDX-License-Identifier: GPL-2.0 #include #include #include #include #include #include #include #include #include "ucall_common.h" struct kvm_coalesced_io { struct kvm_coalesced_mmio_ring *ring; uint32_t ring_size; uint64_t mmio_gpa; uint64_t *mmio; /* * x86-only, but define pio_port for all architectures to minimize the * amount of #ifdeffery and complexity, without having to sacrifice * verbose error messages. */ uint8_t pio_port; }; static struct kvm_coalesced_io kvm_builtin_io_ring; #ifdef __x86_64__ static const int has_pio = 1; #else static const int has_pio = 0; #endif static void guest_code(struct kvm_coalesced_io *io) { int i, j; for (;;) { for (j = 0; j < 1 + has_pio; j++) { /* * KVM always leaves one free entry, i.e. exits to * userspace before the last entry is filled. */ for (i = 0; i < io->ring_size - 1; i++) { #ifdef __x86_64__ if (i & 1) outl(io->pio_port, io->pio_port + i); else #endif WRITE_ONCE(*io->mmio, io->mmio_gpa + i); } #ifdef __x86_64__ if (j & 1) outl(io->pio_port, io->pio_port + i); else #endif WRITE_ONCE(*io->mmio, io->mmio_gpa + i); } GUEST_SYNC(0); WRITE_ONCE(*io->mmio, io->mmio_gpa + i); #ifdef __x86_64__ outl(io->pio_port, io->pio_port + i); #endif } } static void vcpu_run_and_verify_io_exit(struct kvm_vcpu *vcpu, struct kvm_coalesced_io *io, uint32_t ring_start, uint32_t expected_exit) { const bool want_pio = expected_exit == KVM_EXIT_IO; struct kvm_coalesced_mmio_ring *ring = io->ring; struct kvm_run *run = vcpu->run; uint32_t pio_value; WRITE_ONCE(ring->first, ring_start); WRITE_ONCE(ring->last, ring_start); vcpu_run(vcpu); /* * Annoyingly, reading PIO data is safe only for PIO exits, otherwise * data_offset is garbage, e.g. an MMIO gpa. */ if (run->exit_reason == KVM_EXIT_IO) pio_value = *(uint32_t *)((void *)run + run->io.data_offset); else pio_value = 0; TEST_ASSERT((!want_pio && (run->exit_reason == KVM_EXIT_MMIO && run->mmio.is_write && run->mmio.phys_addr == io->mmio_gpa && run->mmio.len == 8 && *(uint64_t *)run->mmio.data == io->mmio_gpa + io->ring_size - 1)) || (want_pio && (run->exit_reason == KVM_EXIT_IO && run->io.port == io->pio_port && run->io.direction == KVM_EXIT_IO_OUT && run->io.count == 1 && pio_value == io->pio_port + io->ring_size - 1)), "For start = %u, expected exit on %u-byte %s write 0x%llx = %lx, got exit_reason = %u (%s)\n " "(MMIO addr = 0x%llx, write = %u, len = %u, data = %lx)\n " "(PIO port = 0x%x, write = %u, len = %u, count = %u, data = %x", ring_start, want_pio ? 4 : 8, want_pio ? "PIO" : "MMIO", want_pio ? (unsigned long long)io->pio_port : io->mmio_gpa, (want_pio ? io->pio_port : io->mmio_gpa) + io->ring_size - 1, run->exit_reason, run->exit_reason == KVM_EXIT_MMIO ? "MMIO" : run->exit_reason == KVM_EXIT_IO ? "PIO" : "other", run->mmio.phys_addr, run->mmio.is_write, run->mmio.len, *(uint64_t *)run->mmio.data, run->io.port, run->io.direction, run->io.size, run->io.count, pio_value); } static void vcpu_run_and_verify_coalesced_io(struct kvm_vcpu *vcpu, struct kvm_coalesced_io *io, uint32_t ring_start, uint32_t expected_exit) { struct kvm_coalesced_mmio_ring *ring = io->ring; int i; vcpu_run_and_verify_io_exit(vcpu, io, ring_start, expected_exit); TEST_ASSERT((ring->last + 1) % io->ring_size == ring->first, "Expected ring to be full (minus 1), first = %u, last = %u, max = %u, start = %u", ring->first, ring->last, io->ring_size, ring_start); for (i = 0; i < io->ring_size - 1; i++) { uint32_t idx = (ring->first + i) % io->ring_size; struct kvm_coalesced_mmio *entry = &ring->coalesced_mmio[idx]; #ifdef __x86_64__ if (i & 1) TEST_ASSERT(entry->phys_addr == io->pio_port && entry->len == 4 && entry->pio && *(uint32_t *)entry->data == io->pio_port + i, "Wanted 4-byte port I/O 0x%x = 0x%x in entry %u, got %u-byte %s 0x%llx = 0x%x", io->pio_port, io->pio_port + i, i, entry->len, entry->pio ? "PIO" : "MMIO", entry->phys_addr, *(uint32_t *)entry->data); else #endif TEST_ASSERT(entry->phys_addr == io->mmio_gpa && entry->len == 8 && !entry->pio, "Wanted 8-byte MMIO to 0x%lx = %lx in entry %u, got %u-byte %s 0x%llx = 0x%lx", io->mmio_gpa, io->mmio_gpa + i, i, entry->len, entry->pio ? "PIO" : "MMIO", entry->phys_addr, *(uint64_t *)entry->data); } } static void test_coalesced_io(struct kvm_vcpu *vcpu, struct kvm_coalesced_io *io, uint32_t ring_start) { struct kvm_coalesced_mmio_ring *ring = io->ring; kvm_vm_register_coalesced_io(vcpu->vm, io->mmio_gpa, 8, false /* pio */); #ifdef __x86_64__ kvm_vm_register_coalesced_io(vcpu->vm, io->pio_port, 8, true /* pio */); #endif vcpu_run_and_verify_coalesced_io(vcpu, io, ring_start, KVM_EXIT_MMIO); #ifdef __x86_64__ vcpu_run_and_verify_coalesced_io(vcpu, io, ring_start, KVM_EXIT_IO); #endif /* * Verify ucall, which may use non-coalesced MMIO or PIO, generates an * immediate exit. */ WRITE_ONCE(ring->first, ring_start); WRITE_ONCE(ring->last, ring_start); vcpu_run(vcpu); TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC); TEST_ASSERT_EQ(ring->first, ring_start); TEST_ASSERT_EQ(ring->last, ring_start); /* Verify that non-coalesced MMIO/PIO generates an exit to userspace. */ kvm_vm_unregister_coalesced_io(vcpu->vm, io->mmio_gpa, 8, false /* pio */); vcpu_run_and_verify_io_exit(vcpu, io, ring_start, KVM_EXIT_MMIO); #ifdef __x86_64__ kvm_vm_unregister_coalesced_io(vcpu->vm, io->pio_port, 8, true /* pio */); vcpu_run_and_verify_io_exit(vcpu, io, ring_start, KVM_EXIT_IO); #endif } int main(int argc, char *argv[]) { struct kvm_vcpu *vcpu; struct kvm_vm *vm; int i; TEST_REQUIRE(kvm_has_cap(KVM_CAP_COALESCED_MMIO)); #ifdef __x86_64__ TEST_REQUIRE(kvm_has_cap(KVM_CAP_COALESCED_PIO)); #endif vm = vm_create_with_one_vcpu(&vcpu, guest_code); kvm_builtin_io_ring = (struct kvm_coalesced_io) { /* * The I/O ring is a kernel-allocated page whose address is * relative to each vCPU's run page, with the page offset * provided by KVM in the return of KVM_CAP_COALESCED_MMIO. */ .ring = (void *)vcpu->run + (kvm_check_cap(KVM_CAP_COALESCED_MMIO) * getpagesize()), /* * The size of the I/O ring is fixed, but KVM defines the sized * based on the kernel's PAGE_SIZE. Thus, userspace must query * the host's page size at runtime to compute the ring size. */ .ring_size = (getpagesize() - sizeof(struct kvm_coalesced_mmio_ring)) / sizeof(struct kvm_coalesced_mmio), /* * Arbitrary address+port (MMIO mustn't overlap memslots), with * the MMIO GPA identity mapped in the guest. */ .mmio_gpa = 4ull * SZ_1G, .mmio = (uint64_t *)(4ull * SZ_1G), .pio_port = 0x80, }; virt_map(vm, (uint64_t)kvm_builtin_io_ring.mmio, kvm_builtin_io_ring.mmio_gpa, 1); sync_global_to_guest(vm, kvm_builtin_io_ring); vcpu_args_set(vcpu, 1, &kvm_builtin_io_ring); for (i = 0; i < kvm_builtin_io_ring.ring_size; i++) test_coalesced_io(vcpu, &kvm_builtin_io_ring, i); kvm_vm_free(vm); return 0; }