1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kvm eventfd support - use eventfd objects to signal various KVM events 4 * 5 * Copyright 2009 Novell. All Rights Reserved. 6 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 7 * 8 * Author: 9 * Gregory Haskins <ghaskins@novell.com> 10 */ 11 12 #include <linux/kvm_host.h> 13 #include <linux/kvm.h> 14 #include <linux/kvm_irqfd.h> 15 #include <linux/workqueue.h> 16 #include <linux/syscalls.h> 17 #include <linux/wait.h> 18 #include <linux/poll.h> 19 #include <linux/file.h> 20 #include <linux/list.h> 21 #include <linux/eventfd.h> 22 #include <linux/kernel.h> 23 #include <linux/srcu.h> 24 #include <linux/slab.h> 25 #include <linux/seqlock.h> 26 #include <linux/irqbypass.h> 27 #include <trace/events/kvm.h> 28 29 #include <kvm/iodev.h> 30 31 #ifdef CONFIG_HAVE_KVM_IRQFD 32 33 static struct workqueue_struct *irqfd_cleanup_wq; 34 35 bool __attribute__((weak)) 36 kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args) 37 { 38 return true; 39 } 40 41 static void 42 irqfd_inject(struct work_struct *work) 43 { 44 struct kvm_kernel_irqfd *irqfd = 45 container_of(work, struct kvm_kernel_irqfd, inject); 46 struct kvm *kvm = irqfd->kvm; 47 48 if (!irqfd->resampler) { 49 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1, 50 false); 51 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0, 52 false); 53 } else 54 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 55 irqfd->gsi, 1, false); 56 } 57 58 /* 59 * Since resampler irqfds share an IRQ source ID, we de-assert once 60 * then notify all of the resampler irqfds using this GSI. We can't 61 * do multiple de-asserts or we risk racing with incoming re-asserts. 62 */ 63 static void 64 irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian) 65 { 66 struct kvm_kernel_irqfd_resampler *resampler; 67 struct kvm *kvm; 68 struct kvm_kernel_irqfd *irqfd; 69 int idx; 70 71 resampler = container_of(kian, 72 struct kvm_kernel_irqfd_resampler, notifier); 73 kvm = resampler->kvm; 74 75 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 76 resampler->notifier.gsi, 0, false); 77 78 idx = srcu_read_lock(&kvm->irq_srcu); 79 80 list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link) 81 eventfd_signal(irqfd->resamplefd, 1); 82 83 srcu_read_unlock(&kvm->irq_srcu, idx); 84 } 85 86 static void 87 irqfd_resampler_shutdown(struct kvm_kernel_irqfd *irqfd) 88 { 89 struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler; 90 struct kvm *kvm = resampler->kvm; 91 92 mutex_lock(&kvm->irqfds.resampler_lock); 93 94 list_del_rcu(&irqfd->resampler_link); 95 synchronize_srcu(&kvm->irq_srcu); 96 97 if (list_empty(&resampler->list)) { 98 list_del(&resampler->link); 99 kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier); 100 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 101 resampler->notifier.gsi, 0, false); 102 kfree(resampler); 103 } 104 105 mutex_unlock(&kvm->irqfds.resampler_lock); 106 } 107 108 /* 109 * Race-free decouple logic (ordering is critical) 110 */ 111 static void 112 irqfd_shutdown(struct work_struct *work) 113 { 114 struct kvm_kernel_irqfd *irqfd = 115 container_of(work, struct kvm_kernel_irqfd, shutdown); 116 struct kvm *kvm = irqfd->kvm; 117 u64 cnt; 118 119 /* Make sure irqfd has been initialized in assign path. */ 120 synchronize_srcu(&kvm->irq_srcu); 121 122 /* 123 * Synchronize with the wait-queue and unhook ourselves to prevent 124 * further events. 125 */ 126 eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); 127 128 /* 129 * We know no new events will be scheduled at this point, so block 130 * until all previously outstanding events have completed 131 */ 132 flush_work(&irqfd->inject); 133 134 if (irqfd->resampler) { 135 irqfd_resampler_shutdown(irqfd); 136 eventfd_ctx_put(irqfd->resamplefd); 137 } 138 139 /* 140 * It is now safe to release the object's resources 141 */ 142 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 143 irq_bypass_unregister_consumer(&irqfd->consumer); 144 #endif 145 eventfd_ctx_put(irqfd->eventfd); 146 kfree(irqfd); 147 } 148 149 150 /* assumes kvm->irqfds.lock is held */ 151 static bool 152 irqfd_is_active(struct kvm_kernel_irqfd *irqfd) 153 { 154 return list_empty(&irqfd->list) ? false : true; 155 } 156 157 /* 158 * Mark the irqfd as inactive and schedule it for removal 159 * 160 * assumes kvm->irqfds.lock is held 161 */ 162 static void 163 irqfd_deactivate(struct kvm_kernel_irqfd *irqfd) 164 { 165 BUG_ON(!irqfd_is_active(irqfd)); 166 167 list_del_init(&irqfd->list); 168 169 queue_work(irqfd_cleanup_wq, &irqfd->shutdown); 170 } 171 172 int __attribute__((weak)) kvm_arch_set_irq_inatomic( 173 struct kvm_kernel_irq_routing_entry *irq, 174 struct kvm *kvm, int irq_source_id, 175 int level, 176 bool line_status) 177 { 178 return -EWOULDBLOCK; 179 } 180 181 /* 182 * Called with wqh->lock held and interrupts disabled 183 */ 184 static int 185 irqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 186 { 187 struct kvm_kernel_irqfd *irqfd = 188 container_of(wait, struct kvm_kernel_irqfd, wait); 189 __poll_t flags = key_to_poll(key); 190 struct kvm_kernel_irq_routing_entry irq; 191 struct kvm *kvm = irqfd->kvm; 192 unsigned seq; 193 int idx; 194 int ret = 0; 195 196 if (flags & EPOLLIN) { 197 u64 cnt; 198 eventfd_ctx_do_read(irqfd->eventfd, &cnt); 199 200 idx = srcu_read_lock(&kvm->irq_srcu); 201 do { 202 seq = read_seqcount_begin(&irqfd->irq_entry_sc); 203 irq = irqfd->irq_entry; 204 } while (read_seqcount_retry(&irqfd->irq_entry_sc, seq)); 205 /* An event has been signaled, inject an interrupt */ 206 if (kvm_arch_set_irq_inatomic(&irq, kvm, 207 KVM_USERSPACE_IRQ_SOURCE_ID, 1, 208 false) == -EWOULDBLOCK) 209 schedule_work(&irqfd->inject); 210 srcu_read_unlock(&kvm->irq_srcu, idx); 211 ret = 1; 212 } 213 214 if (flags & EPOLLHUP) { 215 /* The eventfd is closing, detach from KVM */ 216 unsigned long iflags; 217 218 spin_lock_irqsave(&kvm->irqfds.lock, iflags); 219 220 /* 221 * We must check if someone deactivated the irqfd before 222 * we could acquire the irqfds.lock since the item is 223 * deactivated from the KVM side before it is unhooked from 224 * the wait-queue. If it is already deactivated, we can 225 * simply return knowing the other side will cleanup for us. 226 * We cannot race against the irqfd going away since the 227 * other side is required to acquire wqh->lock, which we hold 228 */ 229 if (irqfd_is_active(irqfd)) 230 irqfd_deactivate(irqfd); 231 232 spin_unlock_irqrestore(&kvm->irqfds.lock, iflags); 233 } 234 235 return ret; 236 } 237 238 static void 239 irqfd_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, 240 poll_table *pt) 241 { 242 struct kvm_kernel_irqfd *irqfd = 243 container_of(pt, struct kvm_kernel_irqfd, pt); 244 add_wait_queue_priority(wqh, &irqfd->wait); 245 } 246 247 /* Must be called under irqfds.lock */ 248 static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd) 249 { 250 struct kvm_kernel_irq_routing_entry *e; 251 struct kvm_kernel_irq_routing_entry entries[KVM_NR_IRQCHIPS]; 252 int n_entries; 253 254 n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi); 255 256 write_seqcount_begin(&irqfd->irq_entry_sc); 257 258 e = entries; 259 if (n_entries == 1) 260 irqfd->irq_entry = *e; 261 else 262 irqfd->irq_entry.type = 0; 263 264 write_seqcount_end(&irqfd->irq_entry_sc); 265 } 266 267 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 268 void __attribute__((weak)) kvm_arch_irq_bypass_stop( 269 struct irq_bypass_consumer *cons) 270 { 271 } 272 273 void __attribute__((weak)) kvm_arch_irq_bypass_start( 274 struct irq_bypass_consumer *cons) 275 { 276 } 277 278 int __attribute__((weak)) kvm_arch_update_irqfd_routing( 279 struct kvm *kvm, unsigned int host_irq, 280 uint32_t guest_irq, bool set) 281 { 282 return 0; 283 } 284 285 bool __attribute__((weak)) kvm_arch_irqfd_route_changed( 286 struct kvm_kernel_irq_routing_entry *old, 287 struct kvm_kernel_irq_routing_entry *new) 288 { 289 return true; 290 } 291 #endif 292 293 static int 294 kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args) 295 { 296 struct kvm_kernel_irqfd *irqfd, *tmp; 297 struct fd f; 298 struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL; 299 int ret; 300 __poll_t events; 301 int idx; 302 303 if (!kvm_arch_intc_initialized(kvm)) 304 return -EAGAIN; 305 306 if (!kvm_arch_irqfd_allowed(kvm, args)) 307 return -EINVAL; 308 309 irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL_ACCOUNT); 310 if (!irqfd) 311 return -ENOMEM; 312 313 irqfd->kvm = kvm; 314 irqfd->gsi = args->gsi; 315 INIT_LIST_HEAD(&irqfd->list); 316 INIT_WORK(&irqfd->inject, irqfd_inject); 317 INIT_WORK(&irqfd->shutdown, irqfd_shutdown); 318 seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock); 319 320 f = fdget(args->fd); 321 if (!f.file) { 322 ret = -EBADF; 323 goto out; 324 } 325 326 eventfd = eventfd_ctx_fileget(f.file); 327 if (IS_ERR(eventfd)) { 328 ret = PTR_ERR(eventfd); 329 goto fail; 330 } 331 332 irqfd->eventfd = eventfd; 333 334 if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) { 335 struct kvm_kernel_irqfd_resampler *resampler; 336 337 resamplefd = eventfd_ctx_fdget(args->resamplefd); 338 if (IS_ERR(resamplefd)) { 339 ret = PTR_ERR(resamplefd); 340 goto fail; 341 } 342 343 irqfd->resamplefd = resamplefd; 344 INIT_LIST_HEAD(&irqfd->resampler_link); 345 346 mutex_lock(&kvm->irqfds.resampler_lock); 347 348 list_for_each_entry(resampler, 349 &kvm->irqfds.resampler_list, link) { 350 if (resampler->notifier.gsi == irqfd->gsi) { 351 irqfd->resampler = resampler; 352 break; 353 } 354 } 355 356 if (!irqfd->resampler) { 357 resampler = kzalloc(sizeof(*resampler), 358 GFP_KERNEL_ACCOUNT); 359 if (!resampler) { 360 ret = -ENOMEM; 361 mutex_unlock(&kvm->irqfds.resampler_lock); 362 goto fail; 363 } 364 365 resampler->kvm = kvm; 366 INIT_LIST_HEAD(&resampler->list); 367 resampler->notifier.gsi = irqfd->gsi; 368 resampler->notifier.irq_acked = irqfd_resampler_ack; 369 INIT_LIST_HEAD(&resampler->link); 370 371 list_add(&resampler->link, &kvm->irqfds.resampler_list); 372 kvm_register_irq_ack_notifier(kvm, 373 &resampler->notifier); 374 irqfd->resampler = resampler; 375 } 376 377 list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list); 378 synchronize_srcu(&kvm->irq_srcu); 379 380 mutex_unlock(&kvm->irqfds.resampler_lock); 381 } 382 383 /* 384 * Install our own custom wake-up handling so we are notified via 385 * a callback whenever someone signals the underlying eventfd 386 */ 387 init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup); 388 init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc); 389 390 spin_lock_irq(&kvm->irqfds.lock); 391 392 ret = 0; 393 list_for_each_entry(tmp, &kvm->irqfds.items, list) { 394 if (irqfd->eventfd != tmp->eventfd) 395 continue; 396 /* This fd is used for another irq already. */ 397 ret = -EBUSY; 398 spin_unlock_irq(&kvm->irqfds.lock); 399 goto fail; 400 } 401 402 idx = srcu_read_lock(&kvm->irq_srcu); 403 irqfd_update(kvm, irqfd); 404 405 list_add_tail(&irqfd->list, &kvm->irqfds.items); 406 407 spin_unlock_irq(&kvm->irqfds.lock); 408 409 /* 410 * Check if there was an event already pending on the eventfd 411 * before we registered, and trigger it as if we didn't miss it. 412 */ 413 events = vfs_poll(f.file, &irqfd->pt); 414 415 if (events & EPOLLIN) 416 schedule_work(&irqfd->inject); 417 418 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 419 if (kvm_arch_has_irq_bypass()) { 420 irqfd->consumer.token = (void *)irqfd->eventfd; 421 irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer; 422 irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer; 423 irqfd->consumer.stop = kvm_arch_irq_bypass_stop; 424 irqfd->consumer.start = kvm_arch_irq_bypass_start; 425 ret = irq_bypass_register_consumer(&irqfd->consumer); 426 if (ret) 427 pr_info("irq bypass consumer (token %p) registration fails: %d\n", 428 irqfd->consumer.token, ret); 429 } 430 #endif 431 432 srcu_read_unlock(&kvm->irq_srcu, idx); 433 434 /* 435 * do not drop the file until the irqfd is fully initialized, otherwise 436 * we might race against the EPOLLHUP 437 */ 438 fdput(f); 439 return 0; 440 441 fail: 442 if (irqfd->resampler) 443 irqfd_resampler_shutdown(irqfd); 444 445 if (resamplefd && !IS_ERR(resamplefd)) 446 eventfd_ctx_put(resamplefd); 447 448 if (eventfd && !IS_ERR(eventfd)) 449 eventfd_ctx_put(eventfd); 450 451 fdput(f); 452 453 out: 454 kfree(irqfd); 455 return ret; 456 } 457 458 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin) 459 { 460 struct kvm_irq_ack_notifier *kian; 461 int gsi, idx; 462 463 idx = srcu_read_lock(&kvm->irq_srcu); 464 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin); 465 if (gsi != -1) 466 hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list, 467 link, srcu_read_lock_held(&kvm->irq_srcu)) 468 if (kian->gsi == gsi) { 469 srcu_read_unlock(&kvm->irq_srcu, idx); 470 return true; 471 } 472 473 srcu_read_unlock(&kvm->irq_srcu, idx); 474 475 return false; 476 } 477 EXPORT_SYMBOL_GPL(kvm_irq_has_notifier); 478 479 void kvm_notify_acked_gsi(struct kvm *kvm, int gsi) 480 { 481 struct kvm_irq_ack_notifier *kian; 482 483 hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list, 484 link, srcu_read_lock_held(&kvm->irq_srcu)) 485 if (kian->gsi == gsi) 486 kian->irq_acked(kian); 487 } 488 489 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin) 490 { 491 int gsi, idx; 492 493 trace_kvm_ack_irq(irqchip, pin); 494 495 idx = srcu_read_lock(&kvm->irq_srcu); 496 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin); 497 if (gsi != -1) 498 kvm_notify_acked_gsi(kvm, gsi); 499 srcu_read_unlock(&kvm->irq_srcu, idx); 500 } 501 502 void kvm_register_irq_ack_notifier(struct kvm *kvm, 503 struct kvm_irq_ack_notifier *kian) 504 { 505 mutex_lock(&kvm->irq_lock); 506 hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list); 507 mutex_unlock(&kvm->irq_lock); 508 kvm_arch_post_irq_ack_notifier_list_update(kvm); 509 } 510 511 void kvm_unregister_irq_ack_notifier(struct kvm *kvm, 512 struct kvm_irq_ack_notifier *kian) 513 { 514 mutex_lock(&kvm->irq_lock); 515 hlist_del_init_rcu(&kian->link); 516 mutex_unlock(&kvm->irq_lock); 517 synchronize_srcu(&kvm->irq_srcu); 518 kvm_arch_post_irq_ack_notifier_list_update(kvm); 519 } 520 #endif 521 522 void 523 kvm_eventfd_init(struct kvm *kvm) 524 { 525 #ifdef CONFIG_HAVE_KVM_IRQFD 526 spin_lock_init(&kvm->irqfds.lock); 527 INIT_LIST_HEAD(&kvm->irqfds.items); 528 INIT_LIST_HEAD(&kvm->irqfds.resampler_list); 529 mutex_init(&kvm->irqfds.resampler_lock); 530 #endif 531 INIT_LIST_HEAD(&kvm->ioeventfds); 532 } 533 534 #ifdef CONFIG_HAVE_KVM_IRQFD 535 /* 536 * shutdown any irqfd's that match fd+gsi 537 */ 538 static int 539 kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args) 540 { 541 struct kvm_kernel_irqfd *irqfd, *tmp; 542 struct eventfd_ctx *eventfd; 543 544 eventfd = eventfd_ctx_fdget(args->fd); 545 if (IS_ERR(eventfd)) 546 return PTR_ERR(eventfd); 547 548 spin_lock_irq(&kvm->irqfds.lock); 549 550 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) { 551 if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) { 552 /* 553 * This clearing of irq_entry.type is needed for when 554 * another thread calls kvm_irq_routing_update before 555 * we flush workqueue below (we synchronize with 556 * kvm_irq_routing_update using irqfds.lock). 557 */ 558 write_seqcount_begin(&irqfd->irq_entry_sc); 559 irqfd->irq_entry.type = 0; 560 write_seqcount_end(&irqfd->irq_entry_sc); 561 irqfd_deactivate(irqfd); 562 } 563 } 564 565 spin_unlock_irq(&kvm->irqfds.lock); 566 eventfd_ctx_put(eventfd); 567 568 /* 569 * Block until we know all outstanding shutdown jobs have completed 570 * so that we guarantee there will not be any more interrupts on this 571 * gsi once this deassign function returns. 572 */ 573 flush_workqueue(irqfd_cleanup_wq); 574 575 return 0; 576 } 577 578 int 579 kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 580 { 581 if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE)) 582 return -EINVAL; 583 584 if (args->flags & KVM_IRQFD_FLAG_DEASSIGN) 585 return kvm_irqfd_deassign(kvm, args); 586 587 return kvm_irqfd_assign(kvm, args); 588 } 589 590 /* 591 * This function is called as the kvm VM fd is being released. Shutdown all 592 * irqfds that still remain open 593 */ 594 void 595 kvm_irqfd_release(struct kvm *kvm) 596 { 597 struct kvm_kernel_irqfd *irqfd, *tmp; 598 599 spin_lock_irq(&kvm->irqfds.lock); 600 601 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) 602 irqfd_deactivate(irqfd); 603 604 spin_unlock_irq(&kvm->irqfds.lock); 605 606 /* 607 * Block until we know all outstanding shutdown jobs have completed 608 * since we do not take a kvm* reference. 609 */ 610 flush_workqueue(irqfd_cleanup_wq); 611 612 } 613 614 /* 615 * Take note of a change in irq routing. 616 * Caller must invoke synchronize_srcu(&kvm->irq_srcu) afterwards. 617 */ 618 void kvm_irq_routing_update(struct kvm *kvm) 619 { 620 struct kvm_kernel_irqfd *irqfd; 621 622 spin_lock_irq(&kvm->irqfds.lock); 623 624 list_for_each_entry(irqfd, &kvm->irqfds.items, list) { 625 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 626 /* Under irqfds.lock, so can read irq_entry safely */ 627 struct kvm_kernel_irq_routing_entry old = irqfd->irq_entry; 628 #endif 629 630 irqfd_update(kvm, irqfd); 631 632 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 633 if (irqfd->producer && 634 kvm_arch_irqfd_route_changed(&old, &irqfd->irq_entry)) { 635 int ret = kvm_arch_update_irqfd_routing( 636 irqfd->kvm, irqfd->producer->irq, 637 irqfd->gsi, 1); 638 WARN_ON(ret); 639 } 640 #endif 641 } 642 643 spin_unlock_irq(&kvm->irqfds.lock); 644 } 645 646 /* 647 * create a host-wide workqueue for issuing deferred shutdown requests 648 * aggregated from all vm* instances. We need our own isolated 649 * queue to ease flushing work items when a VM exits. 650 */ 651 int kvm_irqfd_init(void) 652 { 653 irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", 0, 0); 654 if (!irqfd_cleanup_wq) 655 return -ENOMEM; 656 657 return 0; 658 } 659 660 void kvm_irqfd_exit(void) 661 { 662 destroy_workqueue(irqfd_cleanup_wq); 663 } 664 #endif 665 666 /* 667 * -------------------------------------------------------------------- 668 * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal. 669 * 670 * userspace can register a PIO/MMIO address with an eventfd for receiving 671 * notification when the memory has been touched. 672 * -------------------------------------------------------------------- 673 */ 674 675 struct _ioeventfd { 676 struct list_head list; 677 u64 addr; 678 int length; 679 struct eventfd_ctx *eventfd; 680 u64 datamatch; 681 struct kvm_io_device dev; 682 u8 bus_idx; 683 bool wildcard; 684 }; 685 686 static inline struct _ioeventfd * 687 to_ioeventfd(struct kvm_io_device *dev) 688 { 689 return container_of(dev, struct _ioeventfd, dev); 690 } 691 692 static void 693 ioeventfd_release(struct _ioeventfd *p) 694 { 695 eventfd_ctx_put(p->eventfd); 696 list_del(&p->list); 697 kfree(p); 698 } 699 700 static bool 701 ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) 702 { 703 u64 _val; 704 705 if (addr != p->addr) 706 /* address must be precise for a hit */ 707 return false; 708 709 if (!p->length) 710 /* length = 0 means only look at the address, so always a hit */ 711 return true; 712 713 if (len != p->length) 714 /* address-range must be precise for a hit */ 715 return false; 716 717 if (p->wildcard) 718 /* all else equal, wildcard is always a hit */ 719 return true; 720 721 /* otherwise, we have to actually compare the data */ 722 723 BUG_ON(!IS_ALIGNED((unsigned long)val, len)); 724 725 switch (len) { 726 case 1: 727 _val = *(u8 *)val; 728 break; 729 case 2: 730 _val = *(u16 *)val; 731 break; 732 case 4: 733 _val = *(u32 *)val; 734 break; 735 case 8: 736 _val = *(u64 *)val; 737 break; 738 default: 739 return false; 740 } 741 742 return _val == p->datamatch; 743 } 744 745 /* MMIO/PIO writes trigger an event if the addr/val match */ 746 static int 747 ioeventfd_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, gpa_t addr, 748 int len, const void *val) 749 { 750 struct _ioeventfd *p = to_ioeventfd(this); 751 752 if (!ioeventfd_in_range(p, addr, len, val)) 753 return -EOPNOTSUPP; 754 755 eventfd_signal(p->eventfd, 1); 756 return 0; 757 } 758 759 /* 760 * This function is called as KVM is completely shutting down. We do not 761 * need to worry about locking just nuke anything we have as quickly as possible 762 */ 763 static void 764 ioeventfd_destructor(struct kvm_io_device *this) 765 { 766 struct _ioeventfd *p = to_ioeventfd(this); 767 768 ioeventfd_release(p); 769 } 770 771 static const struct kvm_io_device_ops ioeventfd_ops = { 772 .write = ioeventfd_write, 773 .destructor = ioeventfd_destructor, 774 }; 775 776 /* assumes kvm->slots_lock held */ 777 static bool 778 ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p) 779 { 780 struct _ioeventfd *_p; 781 782 list_for_each_entry(_p, &kvm->ioeventfds, list) 783 if (_p->bus_idx == p->bus_idx && 784 _p->addr == p->addr && 785 (!_p->length || !p->length || 786 (_p->length == p->length && 787 (_p->wildcard || p->wildcard || 788 _p->datamatch == p->datamatch)))) 789 return true; 790 791 return false; 792 } 793 794 static enum kvm_bus ioeventfd_bus_from_flags(__u32 flags) 795 { 796 if (flags & KVM_IOEVENTFD_FLAG_PIO) 797 return KVM_PIO_BUS; 798 if (flags & KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY) 799 return KVM_VIRTIO_CCW_NOTIFY_BUS; 800 return KVM_MMIO_BUS; 801 } 802 803 static int kvm_assign_ioeventfd_idx(struct kvm *kvm, 804 enum kvm_bus bus_idx, 805 struct kvm_ioeventfd *args) 806 { 807 808 struct eventfd_ctx *eventfd; 809 struct _ioeventfd *p; 810 int ret; 811 812 eventfd = eventfd_ctx_fdget(args->fd); 813 if (IS_ERR(eventfd)) 814 return PTR_ERR(eventfd); 815 816 p = kzalloc(sizeof(*p), GFP_KERNEL_ACCOUNT); 817 if (!p) { 818 ret = -ENOMEM; 819 goto fail; 820 } 821 822 INIT_LIST_HEAD(&p->list); 823 p->addr = args->addr; 824 p->bus_idx = bus_idx; 825 p->length = args->len; 826 p->eventfd = eventfd; 827 828 /* The datamatch feature is optional, otherwise this is a wildcard */ 829 if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH) 830 p->datamatch = args->datamatch; 831 else 832 p->wildcard = true; 833 834 mutex_lock(&kvm->slots_lock); 835 836 /* Verify that there isn't a match already */ 837 if (ioeventfd_check_collision(kvm, p)) { 838 ret = -EEXIST; 839 goto unlock_fail; 840 } 841 842 kvm_iodevice_init(&p->dev, &ioeventfd_ops); 843 844 ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length, 845 &p->dev); 846 if (ret < 0) 847 goto unlock_fail; 848 849 kvm_get_bus(kvm, bus_idx)->ioeventfd_count++; 850 list_add_tail(&p->list, &kvm->ioeventfds); 851 852 mutex_unlock(&kvm->slots_lock); 853 854 return 0; 855 856 unlock_fail: 857 mutex_unlock(&kvm->slots_lock); 858 859 fail: 860 kfree(p); 861 eventfd_ctx_put(eventfd); 862 863 return ret; 864 } 865 866 static int 867 kvm_deassign_ioeventfd_idx(struct kvm *kvm, enum kvm_bus bus_idx, 868 struct kvm_ioeventfd *args) 869 { 870 struct _ioeventfd *p, *tmp; 871 struct eventfd_ctx *eventfd; 872 struct kvm_io_bus *bus; 873 int ret = -ENOENT; 874 bool wildcard; 875 876 eventfd = eventfd_ctx_fdget(args->fd); 877 if (IS_ERR(eventfd)) 878 return PTR_ERR(eventfd); 879 880 wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH); 881 882 mutex_lock(&kvm->slots_lock); 883 884 list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) { 885 886 if (p->bus_idx != bus_idx || 887 p->eventfd != eventfd || 888 p->addr != args->addr || 889 p->length != args->len || 890 p->wildcard != wildcard) 891 continue; 892 893 if (!p->wildcard && p->datamatch != args->datamatch) 894 continue; 895 896 kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev); 897 bus = kvm_get_bus(kvm, bus_idx); 898 if (bus) 899 bus->ioeventfd_count--; 900 ioeventfd_release(p); 901 ret = 0; 902 break; 903 } 904 905 mutex_unlock(&kvm->slots_lock); 906 907 eventfd_ctx_put(eventfd); 908 909 return ret; 910 } 911 912 static int kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 913 { 914 enum kvm_bus bus_idx = ioeventfd_bus_from_flags(args->flags); 915 int ret = kvm_deassign_ioeventfd_idx(kvm, bus_idx, args); 916 917 if (!args->len && bus_idx == KVM_MMIO_BUS) 918 kvm_deassign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args); 919 920 return ret; 921 } 922 923 static int 924 kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 925 { 926 enum kvm_bus bus_idx; 927 int ret; 928 929 bus_idx = ioeventfd_bus_from_flags(args->flags); 930 /* must be natural-word sized, or 0 to ignore length */ 931 switch (args->len) { 932 case 0: 933 case 1: 934 case 2: 935 case 4: 936 case 8: 937 break; 938 default: 939 return -EINVAL; 940 } 941 942 /* check for range overflow */ 943 if (args->addr + args->len < args->addr) 944 return -EINVAL; 945 946 /* check for extra flags that we don't understand */ 947 if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK) 948 return -EINVAL; 949 950 /* ioeventfd with no length can't be combined with DATAMATCH */ 951 if (!args->len && (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH)) 952 return -EINVAL; 953 954 ret = kvm_assign_ioeventfd_idx(kvm, bus_idx, args); 955 if (ret) 956 goto fail; 957 958 /* When length is ignored, MMIO is also put on a separate bus, for 959 * faster lookups. 960 */ 961 if (!args->len && bus_idx == KVM_MMIO_BUS) { 962 ret = kvm_assign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args); 963 if (ret < 0) 964 goto fast_fail; 965 } 966 967 return 0; 968 969 fast_fail: 970 kvm_deassign_ioeventfd_idx(kvm, bus_idx, args); 971 fail: 972 return ret; 973 } 974 975 int 976 kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 977 { 978 if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN) 979 return kvm_deassign_ioeventfd(kvm, args); 980 981 return kvm_assign_ioeventfd(kvm, args); 982 } 983