1 // SPDX-License-Identifier: GPL-2.0 2 #include "kvm_util.h" 3 #include "test_util.h" 4 #include "apic.h" 5 #include "processor.h" 6 #include "proc_util.h" 7 8 #include <libvfio.h> 9 #include <linux/sizes.h> 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <unistd.h> 13 #include <pthread.h> 14 #include <sys/eventfd.h> 15 #include <sys/sysinfo.h> 16 17 static u64 timeout_ns = 2ULL * 1000 * 1000 * 1000; 18 static bool guest_ready_for_irqs[KVM_MAX_VCPUS]; 19 static bool guest_received_irq[KVM_MAX_VCPUS]; 20 static bool guest_received_nmi[KVM_MAX_VCPUS]; 21 static bool x2apic = true; 22 static bool irq_affinity; 23 static bool done; 24 25 #define GUEST_RECEIVED_IRQ(__vcpu) \ 26 SYNC_FROM_GUEST_AND_READ((__vcpu)->vm, guest_received_irq[(__vcpu)->id]) 27 #define GUEST_RECEIVED_NMI(__vcpu) \ 28 SYNC_FROM_GUEST_AND_READ((__vcpu)->vm, guest_received_nmi[(__vcpu)->id]) 29 30 #define GUEST_RECEIVED_INTERRUPT(__vcpu, __nmi) \ 31 ((__nmi) ? GUEST_RECEIVED_NMI(__vcpu) : GUEST_RECEIVED_IRQ(__vcpu)) 32 33 static u32 guest_get_vcpu_id(void) 34 { 35 if (x2apic) 36 return x2apic_read_reg(APIC_ID); 37 else 38 return xapic_read_reg(APIC_ID) >> 24; 39 } 40 41 static void guest_irq_handler(struct ex_regs *regs) 42 { 43 WRITE_ONCE(guest_received_irq[guest_get_vcpu_id()], true); 44 45 if (x2apic) 46 x2apic_write_reg(APIC_EOI, 0); 47 else 48 xapic_write_reg(APIC_EOI, 0); 49 } 50 51 static void guest_nmi_handler(struct ex_regs *regs) 52 { 53 WRITE_ONCE(guest_received_nmi[guest_get_vcpu_id()], true); 54 } 55 56 static void guest_code(void) 57 { 58 if (x2apic) 59 x2apic_enable(); 60 else 61 xapic_enable(); 62 63 sti_nop(); 64 65 WRITE_ONCE(guest_ready_for_irqs[guest_get_vcpu_id()], true); 66 67 while (!READ_ONCE(done)) 68 cpu_relax(); 69 70 GUEST_DONE(); 71 } 72 73 static void *vcpu_thread_main(void *arg) 74 { 75 struct kvm_vcpu *vcpu = arg; 76 struct ucall uc; 77 78 vcpu_run(vcpu); 79 TEST_ASSERT_EQ(UCALL_DONE, get_ucall(vcpu, &uc)); 80 81 return NULL; 82 } 83 84 static int vfio_setup_msi(struct vfio_pci_device *device) 85 { 86 const int flags = MAP_SHARED | MAP_ANONYMOUS; 87 const int prot = PROT_READ | PROT_WRITE; 88 struct iova_allocator *allocator; 89 struct dma_region *region; 90 91 /* Sanity check that the device+driver can actually send MSIs. */ 92 TEST_REQUIRE(device->driver.ops); 93 TEST_REQUIRE(device->driver.ops->send_msi); 94 95 /* 96 * Set up a DMA-able region for the driver to use. Very few devices 97 * provide a way to arbitrarily send interrupts (MSIs), e.g. by writing 98 * an MMIO register. Instead, most devices send MSIs when an action is 99 * completed, and practically all actions involve DMA of some form. 100 */ 101 allocator = iova_allocator_init(device->iommu); 102 103 region = &device->driver.region; 104 region->size = SZ_2M; 105 region->iova = iova_allocator_alloc(allocator, region->size); 106 region->vaddr = kvm_mmap(region->size, prot, flags, -1); 107 TEST_ASSERT(region->vaddr != MAP_FAILED, "mmap() failed\n"); 108 iommu_map(device->iommu, region); 109 110 iova_allocator_cleanup(allocator); 111 112 vfio_pci_driver_init(device); 113 114 return device->driver.msi; 115 } 116 117 static void trigger_interrupt(struct vfio_pci_device *device, int eventfd) 118 { 119 if (device) 120 vfio_pci_driver_send_msi(device); 121 else 122 eventfd_write(eventfd, 1); 123 } 124 125 126 static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, 127 u8 vector, bool use_nmi) 128 { 129 struct { 130 struct kvm_irq_routing header; 131 struct kvm_irq_routing_entry entry; 132 } routing = { 133 .header.nr = 1, 134 .entry = { 135 .gsi = gsi, 136 .type = KVM_IRQ_ROUTING_MSI, 137 .u.msi.address_lo = 0xFEE00000 | (vcpu->id & GENMASK(7, 0)) << 12, 138 .u.msi.address_hi = vcpu->id & GENMASK(31, 8), 139 .u.msi.data = use_nmi ? NMI_VECTOR | (4 << 8) : vector, 140 }, 141 }; 142 143 vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing.header); 144 } 145 146 static void kvm_set_empty_gsi_routing(struct kvm_vm *vm) 147 { 148 struct kvm_irq_routing routing = {}; 149 150 vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing); 151 } 152 153 static const char *probe_iommu_type(void) 154 { 155 int io_fd; 156 157 io_fd = open("/dev/iommu", O_RDONLY); 158 if (io_fd >= 0) { 159 close(io_fd); 160 return MODE_IOMMUFD; 161 } 162 163 io_fd = __open_path_or_exit("/dev/vfio/vfio", O_RDONLY, 164 "Is VFIO (or IOMMUFD) loaded and enabled?"); 165 close(io_fd); 166 return MODE_VFIO_TYPE1_IOMMU; 167 } 168 169 static void help(const char *name) 170 { 171 printf("Usage: %s [-a] [-d <segment:bus:device.function>] [-e] [-h] [-i nr_irqs] [-m] [-n] [-t iommu_type] [-v nr_vcpus] [-x]\n", name); 172 printf("\n"); 173 printf("Tests KVM interrupt routing and delivery via irqfd.\n"); 174 printf("-a Affine the device's host IRQ to a random physical CPU\n"); 175 printf("-d Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n"); 176 printf("-e Set empty GSI routing in-between some interrupts\n"); 177 printf("-i The number of IRQs to generate during the test\n"); 178 printf("-m Pin target vCPU to random physical CPU before triggering interrupt\n"); 179 printf("-n Deliver 50 percent of IRQs as non-maskable interrupts\n"); 180 printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); 181 printf("-v Number of vCPUS to run\n"); 182 printf("-x Use xAPIC mode instead of x2APIC mode in the guest\n"); 183 printf("\n"); 184 exit(KSFT_FAIL); 185 } 186 187 int main(int argc, char **argv) 188 { 189 /* 190 * Pick a random vector and a random GSI to use for device IRQ. 191 * 192 * Pick an IRQ vector in range [32, UINT8_MAX]. Min value is 32 because 193 * Linux/x86 reserves vectors 0-31 for exceptions and architecture 194 * defined NMIs and interrupts. 195 * 196 * Pick a GSI in range [24, KVM_MAX_IRQ_ROUTES - 1]. The min value is 24 197 * because KVM reserves GSIs 0-15 for legacy ISA IRQs and 16-23 only go 198 * to the IOAPIC. The max is KVM_MAX_IRQ_ROUTES - 1, because 199 * KVM_MAX_IRQ_ROUTES is exclusive. 200 */ 201 u32 gsi = kvm_random_u64_in_range(&kvm_rng, 24, KVM_MAX_IRQ_ROUTES - 1); 202 u8 vector = kvm_random_u64_in_range(&kvm_rng, 32, UINT8_MAX); 203 204 pthread_t vcpu_threads[KVM_MAX_VCPUS]; 205 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; 206 struct vfio_pci_device *device = NULL; 207 int nr_irqs = 1000, nr_vcpus = 1; 208 bool set_empty_routing = false; 209 const char *device_bdf = NULL; 210 const char *iommu_type = NULL; 211 int i, j, c, msix, eventfd; 212 bool migrate_vcpus = false; 213 cpu_set_t available_cpus; 214 bool use_nmi = false; 215 struct iommu *iommu; 216 struct kvm_vm *vm; 217 int irq, irq_cpu; 218 219 while ((c = getopt(argc, argv, "ad:ehi:mnt:v:x")) != -1) { 220 switch (c) { 221 case 'a': 222 irq_affinity = true; 223 break; 224 case 'd': 225 device_bdf = optarg; 226 break; 227 case 'e': 228 set_empty_routing = true; 229 break; 230 case 'i': 231 nr_irqs = atoi_positive("Number of IRQs", optarg); 232 break; 233 case 'm': 234 migrate_vcpus = true; 235 break; 236 case 'n': 237 use_nmi = true; 238 break; 239 case 't': 240 iommu_type = optarg; 241 break; 242 case 'v': 243 nr_vcpus = atoi_positive("Number of vCPUS", optarg); 244 TEST_ASSERT(nr_vcpus <= KVM_MAX_VCPUS, 245 "KVM selftests support at most %u vCPUs", KVM_MAX_VCPUS); 246 break; 247 case 'x': 248 x2apic = false; 249 break; 250 case 'h': 251 default: 252 help(argv[0]); 253 } 254 } 255 256 TEST_REQUIRE(kvm_arch_has_default_irqchip()); 257 258 vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); 259 vm_enable_cap(vm, KVM_CAP_X2APIC_API, KVM_X2APIC_API_USE_32BIT_IDS | 260 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK); 261 262 vm_install_exception_handler(vm, vector, guest_irq_handler); 263 vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); 264 265 if (!x2apic) { 266 TEST_ASSERT(nr_vcpus < 256, "xAPIC can only target IDs [0-254] (255 vCPUs)"); 267 virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); 268 } 269 270 if (device_bdf) { 271 if (!iommu_type) 272 iommu_type = probe_iommu_type(); 273 iommu = iommu_init(iommu_type); 274 device = vfio_pci_device_init(device_bdf, iommu); 275 msix = vfio_setup_msi(device); 276 irq = vfio_msix_to_host_irq(device_bdf, msix); 277 eventfd = device->msi_eventfds[msix]; 278 printf("Using device %s MSI-X[%d] (IRQ-%u)\n", device_bdf, msix, 279 irq); 280 } else { 281 TEST_ASSERT(!irq_affinity, 282 "Setting IRQ affinity (-a) requires a backing device (-d)"); 283 284 eventfd = kvm_new_eventfd(); 285 irq = -1; 286 } 287 288 pr_info("Injecting interrupts for GSI %d (guest vector 0x%x) %d times\n", 289 gsi, vector, nr_irqs); 290 291 kvm_assign_irqfd(vm, gsi, eventfd); 292 293 sync_global_to_guest(vm, x2apic); 294 295 if (migrate_vcpus) 296 kvm_sched_getaffinity(0, sizeof(available_cpus), &available_cpus); 297 298 for (i = 0; i < nr_vcpus; i++) 299 kvm_pthread_create(&vcpu_threads[i], NULL, vcpu_thread_main, vcpus[i]); 300 301 for (i = 0; i < nr_vcpus; i++) { 302 struct kvm_vcpu *vcpu = vcpus[i]; 303 304 while (!SYNC_FROM_GUEST_AND_READ(vm, guest_ready_for_irqs[vcpu->id])) 305 continue; 306 } 307 308 irq_cpu = -1; 309 310 for (i = 0; i < nr_irqs; i++) { 311 const bool do_set_empty_routing = set_empty_routing && (i & BIT(3)); 312 const bool do_use_nmi = use_nmi && (i & BIT(2)); 313 struct kvm_vcpu *vcpu = vcpus[i % nr_vcpus]; 314 struct timespec start; 315 316 if (do_set_empty_routing) 317 kvm_set_empty_gsi_routing(vm); 318 319 kvm_route_msi(vm, gsi, vcpu, vector, do_use_nmi); 320 321 if (irq_affinity) { 322 irq_cpu = kvm_random_u64(&kvm_rng) % get_nprocs(); 323 proc_irq_set_smp_affinity(irq, irq_cpu); 324 } 325 326 if (migrate_vcpus) 327 pin_task_to_random_cpu(vcpu_threads[i % nr_vcpus], &available_cpus); 328 329 for (j = 0; j < nr_vcpus; j++) { 330 TEST_ASSERT(!GUEST_RECEIVED_IRQ(vcpus[j]), 331 "IRQ flag for vCPU %d not clear prior to test", 332 vcpus[j]->id); 333 TEST_ASSERT(!GUEST_RECEIVED_NMI(vcpus[j]), 334 "NMI flag for vCPU %d not clear prior to test", 335 vcpus[j]->id); 336 } 337 338 trigger_interrupt(device, eventfd); 339 340 clock_gettime(CLOCK_MONOTONIC, &start); 341 while (!GUEST_RECEIVED_INTERRUPT(vcpu, do_use_nmi) && 342 timespec_to_ns(timespec_elapsed(start)) <= timeout_ns) 343 cpu_relax(); 344 345 TEST_ASSERT(GUEST_RECEIVED_INTERRUPT(vcpu, do_use_nmi), 346 "vCPU %d timed out waiting for %s (vector 0x%x) from GSI %d (via CPU %d)\n", 347 vcpu->id, do_use_nmi ? "NMI" : "IRQ", 348 do_use_nmi ? NMI_VECTOR : vector, gsi, irq_cpu); 349 350 if (do_use_nmi) 351 WRITE_AND_SYNC_TO_GUEST(vm, guest_received_nmi[vcpu->id], false); 352 else 353 WRITE_AND_SYNC_TO_GUEST(vm, guest_received_irq[vcpu->id], false); 354 } 355 356 WRITE_AND_SYNC_TO_GUEST(vm, done, true); 357 358 for (i = 0; i < nr_vcpus; i++) 359 kvm_pthread_join(vcpu_threads[i], NULL); 360 361 return 0; 362 } 363