1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * vgic_lpi_stress - Stress test for KVM's ITS emulation 4 * 5 * Copyright (c) 2024 Google LLC 6 */ 7 8 #include <linux/sizes.h> 9 #include <pthread.h> 10 #include <stdatomic.h> 11 #include <sys/sysinfo.h> 12 13 #include "kvm_util.h" 14 #include "gic.h" 15 #include "gic_v3.h" 16 #include "gic_v3_its.h" 17 #include "processor.h" 18 #include "ucall.h" 19 #include "vgic.h" 20 21 #define TEST_MEMSLOT_INDEX 1 22 23 #define GIC_LPI_OFFSET 8192 24 25 static size_t nr_iterations = 1000; 26 27 static struct kvm_vm *vm; 28 static struct kvm_vcpu **vcpus; 29 static int its_fd; 30 31 static struct test_data { 32 bool request_vcpus_stop; 33 u32 nr_cpus; 34 u32 nr_devices; 35 u32 nr_event_ids; 36 37 gpa_t device_table; 38 gpa_t collection_table; 39 gpa_t cmdq_base; 40 void *cmdq_base_va; 41 gpa_t itt_tables; 42 43 gpa_t lpi_prop_table; 44 gpa_t lpi_pend_tables; 45 } test_data = { 46 .nr_cpus = 1, 47 .nr_devices = 1, 48 .nr_event_ids = 16, 49 }; 50 51 static void guest_irq_handler(struct ex_regs *regs) 52 { 53 u32 intid = gic_get_and_ack_irq(); 54 55 if (intid == IAR_SPURIOUS) 56 return; 57 58 GUEST_ASSERT(intid >= GIC_LPI_OFFSET); 59 gic_set_eoi(intid); 60 } 61 62 static void guest_setup_its_mappings(void) 63 { 64 u32 coll_id, device_id, event_id, intid = GIC_LPI_OFFSET; 65 u32 nr_events = test_data.nr_event_ids; 66 u32 nr_devices = test_data.nr_devices; 67 u32 nr_cpus = test_data.nr_cpus; 68 69 for (coll_id = 0; coll_id < nr_cpus; coll_id++) 70 its_send_mapc_cmd(test_data.cmdq_base_va, coll_id, coll_id, true); 71 72 /* Round-robin the LPIs to all of the vCPUs in the VM */ 73 coll_id = 0; 74 for (device_id = 0; device_id < nr_devices; device_id++) { 75 gpa_t itt_base = test_data.itt_tables + (device_id * SZ_64K); 76 77 its_send_mapd_cmd(test_data.cmdq_base_va, device_id, 78 itt_base, SZ_64K, true); 79 80 for (event_id = 0; event_id < nr_events; event_id++) { 81 its_send_mapti_cmd(test_data.cmdq_base_va, device_id, 82 event_id, coll_id, intid++); 83 84 coll_id = (coll_id + 1) % test_data.nr_cpus; 85 } 86 } 87 } 88 89 static void guest_invalidate_all_rdists(void) 90 { 91 int i; 92 93 for (i = 0; i < test_data.nr_cpus; i++) 94 its_send_invall_cmd(test_data.cmdq_base_va, i); 95 } 96 97 static void guest_setup_gic(void) 98 { 99 static atomic_int nr_cpus_ready = 0; 100 u32 cpuid = guest_get_vcpuid(); 101 102 gic_init(GIC_V3, test_data.nr_cpus); 103 gic_rdist_enable_lpis(test_data.lpi_prop_table, SZ_64K, 104 test_data.lpi_pend_tables + (cpuid * SZ_64K)); 105 106 atomic_fetch_add(&nr_cpus_ready, 1); 107 108 if (cpuid > 0) 109 return; 110 111 while (atomic_load(&nr_cpus_ready) < test_data.nr_cpus) 112 cpu_relax(); 113 114 its_init(test_data.collection_table, SZ_64K, 115 test_data.device_table, SZ_64K, 116 test_data.cmdq_base, SZ_64K); 117 118 guest_setup_its_mappings(); 119 guest_invalidate_all_rdists(); 120 121 /* SYNC to ensure ITS setup is complete */ 122 for (cpuid = 0; cpuid < test_data.nr_cpus; cpuid++) 123 its_send_sync_cmd(test_data.cmdq_base_va, cpuid); 124 } 125 126 static void guest_code(size_t nr_lpis) 127 { 128 guest_setup_gic(); 129 local_irq_enable(); 130 131 GUEST_SYNC(0); 132 133 /* 134 * Don't use WFI here to avoid blocking the vCPU thread indefinitely and 135 * never getting the stop signal. 136 */ 137 while (!READ_ONCE(test_data.request_vcpus_stop)) 138 cpu_relax(); 139 140 GUEST_DONE(); 141 } 142 143 static void setup_memslot(void) 144 { 145 gpa_t gpa_base; 146 size_t pages; 147 size_t sz; 148 149 /* 150 * For the ITS: 151 * - A single level device table 152 * - A single level collection table 153 * - The command queue 154 * - An ITT for each device 155 */ 156 sz = (3 + test_data.nr_devices) * SZ_64K; 157 158 /* 159 * For the redistributors: 160 * - A shared LPI configuration table 161 * - An LPI pending table for each vCPU 162 */ 163 sz += (1 + test_data.nr_cpus) * SZ_64K; 164 165 pages = sz / vm->page_size; 166 gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; 167 vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, 168 gpa_base, TEST_MEMSLOT_INDEX, pages); 169 } 170 171 #define LPI_PROP_DEFAULT_PRIO 0xa0 172 173 static void configure_lpis(void) 174 { 175 size_t nr_lpis = test_data.nr_devices * test_data.nr_event_ids; 176 u8 *tbl = addr_gpa2hva(vm, test_data.lpi_prop_table); 177 size_t i; 178 179 for (i = 0; i < nr_lpis; i++) { 180 tbl[i] = LPI_PROP_DEFAULT_PRIO | 181 LPI_PROP_GROUP1 | 182 LPI_PROP_ENABLED; 183 } 184 } 185 186 static void setup_test_data(void) 187 { 188 size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K); 189 u32 nr_devices = test_data.nr_devices; 190 u32 nr_cpus = test_data.nr_cpus; 191 gpa_t cmdq_base; 192 193 test_data.device_table = vm_phy_pages_alloc(vm, pages_per_64k, 194 MEM_REGION_TEST_EXTRA); 195 196 test_data.collection_table = vm_phy_pages_alloc(vm, pages_per_64k, 197 MEM_REGION_TEST_EXTRA); 198 199 cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, MEM_REGION_TEST_EXTRA); 200 virt_map(vm, cmdq_base, cmdq_base, pages_per_64k); 201 test_data.cmdq_base = cmdq_base; 202 test_data.cmdq_base_va = (void *)cmdq_base; 203 204 test_data.itt_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_devices, 205 MEM_REGION_TEST_EXTRA); 206 207 test_data.lpi_prop_table = vm_phy_pages_alloc(vm, pages_per_64k, 208 MEM_REGION_TEST_EXTRA); 209 configure_lpis(); 210 211 test_data.lpi_pend_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_cpus, 212 MEM_REGION_TEST_EXTRA); 213 214 sync_global_to_guest(vm, test_data); 215 } 216 217 static void setup_gic(void) 218 { 219 its_fd = vgic_its_setup(vm); 220 } 221 222 static void signal_lpi(u32 device_id, u32 event_id) 223 { 224 gpa_t db_addr = GITS_BASE_GPA + GITS_TRANSLATER; 225 226 struct kvm_msi msi = { 227 .address_lo = db_addr, 228 .address_hi = db_addr >> 32, 229 .data = event_id, 230 .devid = device_id, 231 .flags = KVM_MSI_VALID_DEVID, 232 }; 233 234 /* 235 * KVM_SIGNAL_MSI returns 1 if the MSI wasn't 'blocked' by the VM, 236 * which for arm64 implies having a valid translation in the ITS. 237 */ 238 TEST_ASSERT(__vm_ioctl(vm, KVM_SIGNAL_MSI, &msi) == 1, 239 "KVM_SIGNAL_MSI ioctl failed"); 240 } 241 242 static pthread_barrier_t test_setup_barrier; 243 244 static void *lpi_worker_thread(void *data) 245 { 246 u32 device_id = (size_t)data; 247 u32 event_id; 248 size_t i; 249 250 pthread_barrier_wait(&test_setup_barrier); 251 252 for (i = 0; i < nr_iterations; i++) 253 for (event_id = 0; event_id < test_data.nr_event_ids; event_id++) 254 signal_lpi(device_id, event_id); 255 256 return NULL; 257 } 258 259 static void *vcpu_worker_thread(void *data) 260 { 261 struct kvm_vcpu *vcpu = data; 262 struct ucall uc; 263 264 while (true) { 265 vcpu_run(vcpu); 266 267 switch (get_ucall(vcpu, &uc)) { 268 case UCALL_SYNC: 269 pthread_barrier_wait(&test_setup_barrier); 270 continue; 271 case UCALL_DONE: 272 return NULL; 273 case UCALL_ABORT: 274 REPORT_GUEST_ASSERT(uc); 275 break; 276 default: 277 TEST_FAIL("Unknown ucall: %lu", uc.cmd); 278 } 279 } 280 281 return NULL; 282 } 283 284 static void report_stats(struct timespec delta) 285 { 286 double nr_lpis; 287 double time; 288 289 nr_lpis = test_data.nr_devices * test_data.nr_event_ids * nr_iterations; 290 291 time = delta.tv_sec; 292 time += ((double)delta.tv_nsec) / NSEC_PER_SEC; 293 294 pr_info("Rate: %.2f LPIs/sec\n", nr_lpis / time); 295 } 296 297 static void run_test(void) 298 { 299 u32 nr_devices = test_data.nr_devices; 300 u32 nr_vcpus = test_data.nr_cpus; 301 pthread_t *lpi_threads = malloc(nr_devices * sizeof(pthread_t)); 302 pthread_t *vcpu_threads = malloc(nr_vcpus * sizeof(pthread_t)); 303 struct timespec start, delta; 304 size_t i; 305 306 TEST_ASSERT(lpi_threads && vcpu_threads, "Failed to allocate pthread arrays"); 307 308 pthread_barrier_init(&test_setup_barrier, NULL, nr_vcpus + nr_devices + 1); 309 310 for (i = 0; i < nr_vcpus; i++) 311 kvm_pthread_create(&vcpu_threads[i], NULL, vcpu_worker_thread, vcpus[i]); 312 313 for (i = 0; i < nr_devices; i++) 314 kvm_pthread_create(&lpi_threads[i], NULL, lpi_worker_thread, (void *)i); 315 316 pthread_barrier_wait(&test_setup_barrier); 317 318 clock_gettime(CLOCK_MONOTONIC, &start); 319 320 for (i = 0; i < nr_devices; i++) 321 kvm_pthread_join(lpi_threads[i], NULL); 322 323 delta = timespec_elapsed(start); 324 write_guest_global(vm, test_data.request_vcpus_stop, true); 325 326 for (i = 0; i < nr_vcpus; i++) 327 kvm_pthread_join(vcpu_threads[i], NULL); 328 329 report_stats(delta); 330 } 331 332 static void setup_vm(void) 333 { 334 int i; 335 336 vcpus = malloc(test_data.nr_cpus * sizeof(struct kvm_vcpu *)); 337 TEST_ASSERT(vcpus, "Failed to allocate vCPU array"); 338 339 vm = vm_create_with_vcpus(test_data.nr_cpus, guest_code, vcpus); 340 341 vm_init_descriptor_tables(vm); 342 for (i = 0; i < test_data.nr_cpus; i++) 343 vcpu_init_descriptor_tables(vcpus[i]); 344 345 vm_install_exception_handler(vm, VECTOR_IRQ_CURRENT, guest_irq_handler); 346 347 setup_memslot(); 348 349 setup_gic(); 350 351 setup_test_data(); 352 } 353 354 static void destroy_vm(void) 355 { 356 close(its_fd); 357 kvm_vm_free(vm); 358 free(vcpus); 359 } 360 361 static void pr_usage(const char *name) 362 { 363 pr_info("%s [-v NR_VCPUS] [-d NR_DEVICES] [-e NR_EVENTS] [-i ITERS] -h\n", name); 364 pr_info(" -v:\tnumber of vCPUs (default: %u)\n", test_data.nr_cpus); 365 pr_info(" -d:\tnumber of devices (default: %u)\n", test_data.nr_devices); 366 pr_info(" -e:\tnumber of event IDs per device (default: %u)\n", test_data.nr_event_ids); 367 pr_info(" -i:\tnumber of iterations (default: %lu)\n", nr_iterations); 368 } 369 370 int main(int argc, char **argv) 371 { 372 u32 nr_threads; 373 int c; 374 375 TEST_REQUIRE(kvm_supports_vgic_v3()); 376 377 while ((c = getopt(argc, argv, "hv:d:e:i:")) != -1) { 378 switch (c) { 379 case 'v': 380 test_data.nr_cpus = atoi(optarg); 381 break; 382 case 'd': 383 test_data.nr_devices = atoi(optarg); 384 break; 385 case 'e': 386 test_data.nr_event_ids = atoi(optarg); 387 break; 388 case 'i': 389 nr_iterations = strtoul(optarg, NULL, 0); 390 break; 391 case 'h': 392 default: 393 pr_usage(argv[0]); 394 return 1; 395 } 396 } 397 398 nr_threads = test_data.nr_cpus + test_data.nr_devices; 399 if (nr_threads > get_nprocs()) 400 pr_info("WARNING: running %u threads on %d CPUs; performance is degraded.\n", 401 nr_threads, get_nprocs()); 402 403 setup_vm(); 404 405 run_test(); 406 407 destroy_vm(); 408 409 return 0; 410 } 411