1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO PCI interrupt handling 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/device.h> 14 #include <linux/interrupt.h> 15 #include <linux/eventfd.h> 16 #include <linux/msi.h> 17 #include <linux/pci.h> 18 #include <linux/file.h> 19 #include <linux/vfio.h> 20 #include <linux/wait.h> 21 #include <linux/slab.h> 22 23 #include "vfio_pci_priv.h" 24 25 struct vfio_pci_irq_ctx { 26 struct vfio_pci_core_device *vdev; 27 struct eventfd_ctx *trigger; 28 struct virqfd *unmask; 29 struct virqfd *mask; 30 char *name; 31 bool masked; 32 struct irq_bypass_producer producer; 33 }; 34 35 static bool irq_is(struct vfio_pci_core_device *vdev, int type) 36 { 37 return vdev->irq_type == type; 38 } 39 40 static bool is_intx(struct vfio_pci_core_device *vdev) 41 { 42 return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX; 43 } 44 45 static bool is_irq_none(struct vfio_pci_core_device *vdev) 46 { 47 return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX || 48 vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX || 49 vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX); 50 } 51 52 static 53 struct vfio_pci_irq_ctx *vfio_irq_ctx_get(struct vfio_pci_core_device *vdev, 54 unsigned long index) 55 { 56 return xa_load(&vdev->ctx, index); 57 } 58 59 static void vfio_irq_ctx_free(struct vfio_pci_core_device *vdev, 60 struct vfio_pci_irq_ctx *ctx, unsigned long index) 61 { 62 xa_erase(&vdev->ctx, index); 63 kfree(ctx); 64 } 65 66 static struct vfio_pci_irq_ctx * 67 vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index) 68 { 69 struct vfio_pci_irq_ctx *ctx; 70 int ret; 71 72 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); 73 if (!ctx) 74 return NULL; 75 76 ret = xa_insert(&vdev->ctx, index, ctx, GFP_KERNEL_ACCOUNT); 77 if (ret) { 78 kfree(ctx); 79 return NULL; 80 } 81 82 return ctx; 83 } 84 85 /* 86 * INTx 87 */ 88 static void vfio_send_intx_eventfd(void *opaque, void *unused) 89 { 90 struct vfio_pci_core_device *vdev = opaque; 91 92 if (likely(is_intx(vdev) && !vdev->virq_disabled)) { 93 struct vfio_pci_irq_ctx *ctx; 94 struct eventfd_ctx *trigger; 95 96 ctx = vfio_irq_ctx_get(vdev, 0); 97 if (WARN_ON_ONCE(!ctx)) 98 return; 99 100 trigger = READ_ONCE(ctx->trigger); 101 if (likely(trigger)) 102 eventfd_signal(trigger); 103 } 104 } 105 106 /* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */ 107 static bool __vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 108 { 109 struct pci_dev *pdev = vdev->pdev; 110 struct vfio_pci_irq_ctx *ctx; 111 unsigned long flags; 112 bool masked_changed = false; 113 114 lockdep_assert_held(&vdev->igate); 115 116 spin_lock_irqsave(&vdev->irqlock, flags); 117 118 /* 119 * Masking can come from interrupt, ioctl, or config space 120 * via INTx disable. The latter means this can get called 121 * even when not using intx delivery. In this case, just 122 * try to have the physical bit follow the virtual bit. 123 */ 124 if (unlikely(!is_intx(vdev))) { 125 if (vdev->pci_2_3) 126 pci_intx(pdev, 0); 127 goto out_unlock; 128 } 129 130 ctx = vfio_irq_ctx_get(vdev, 0); 131 if (WARN_ON_ONCE(!ctx)) 132 goto out_unlock; 133 134 if (!ctx->masked) { 135 /* 136 * Can't use check_and_mask here because we always want to 137 * mask, not just when something is pending. 138 */ 139 if (vdev->pci_2_3) 140 pci_intx(pdev, 0); 141 else 142 disable_irq_nosync(pdev->irq); 143 144 ctx->masked = true; 145 masked_changed = true; 146 } 147 148 out_unlock: 149 spin_unlock_irqrestore(&vdev->irqlock, flags); 150 return masked_changed; 151 } 152 153 bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 154 { 155 bool mask_changed; 156 157 mutex_lock(&vdev->igate); 158 mask_changed = __vfio_pci_intx_mask(vdev); 159 mutex_unlock(&vdev->igate); 160 161 return mask_changed; 162 } 163 164 /* 165 * If this is triggered by an eventfd, we can't call eventfd_signal 166 * or else we'll deadlock on the eventfd wait queue. Return >0 when 167 * a signal is necessary, which can then be handled via a work queue 168 * or directly depending on the caller. 169 */ 170 static int vfio_pci_intx_unmask_handler(void *opaque, void *unused) 171 { 172 struct vfio_pci_core_device *vdev = opaque; 173 struct pci_dev *pdev = vdev->pdev; 174 struct vfio_pci_irq_ctx *ctx; 175 unsigned long flags; 176 int ret = 0; 177 178 spin_lock_irqsave(&vdev->irqlock, flags); 179 180 /* 181 * Unmasking comes from ioctl or config, so again, have the 182 * physical bit follow the virtual even when not using INTx. 183 */ 184 if (unlikely(!is_intx(vdev))) { 185 if (vdev->pci_2_3) 186 pci_intx(pdev, 1); 187 goto out_unlock; 188 } 189 190 ctx = vfio_irq_ctx_get(vdev, 0); 191 if (WARN_ON_ONCE(!ctx)) 192 goto out_unlock; 193 194 if (ctx->masked && !vdev->virq_disabled) { 195 /* 196 * A pending interrupt here would immediately trigger, 197 * but we can avoid that overhead by just re-sending 198 * the interrupt to the user. 199 */ 200 if (vdev->pci_2_3) { 201 if (!pci_check_and_unmask_intx(pdev)) 202 ret = 1; 203 } else 204 enable_irq(pdev->irq); 205 206 ctx->masked = (ret > 0); 207 } 208 209 out_unlock: 210 spin_unlock_irqrestore(&vdev->irqlock, flags); 211 212 return ret; 213 } 214 215 static void __vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 216 { 217 lockdep_assert_held(&vdev->igate); 218 219 if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0) 220 vfio_send_intx_eventfd(vdev, NULL); 221 } 222 223 void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 224 { 225 mutex_lock(&vdev->igate); 226 __vfio_pci_intx_unmask(vdev); 227 mutex_unlock(&vdev->igate); 228 } 229 230 static irqreturn_t vfio_intx_handler(int irq, void *dev_id) 231 { 232 struct vfio_pci_irq_ctx *ctx = dev_id; 233 struct vfio_pci_core_device *vdev = ctx->vdev; 234 unsigned long flags; 235 int ret = IRQ_NONE; 236 237 spin_lock_irqsave(&vdev->irqlock, flags); 238 239 if (!vdev->pci_2_3) { 240 disable_irq_nosync(vdev->pdev->irq); 241 ctx->masked = true; 242 ret = IRQ_HANDLED; 243 } else if (!ctx->masked && /* may be shared */ 244 pci_check_and_mask_intx(vdev->pdev)) { 245 ctx->masked = true; 246 ret = IRQ_HANDLED; 247 } 248 249 spin_unlock_irqrestore(&vdev->irqlock, flags); 250 251 if (ret == IRQ_HANDLED) 252 vfio_send_intx_eventfd(vdev, NULL); 253 254 return ret; 255 } 256 257 static int vfio_intx_enable(struct vfio_pci_core_device *vdev, 258 struct eventfd_ctx *trigger) 259 { 260 struct pci_dev *pdev = vdev->pdev; 261 struct vfio_pci_irq_ctx *ctx; 262 unsigned long irqflags; 263 char *name; 264 int ret; 265 266 if (!is_irq_none(vdev)) 267 return -EINVAL; 268 269 if (!pdev->irq) 270 return -ENODEV; 271 272 name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-intx(%s)", pci_name(pdev)); 273 if (!name) 274 return -ENOMEM; 275 276 ctx = vfio_irq_ctx_alloc(vdev, 0); 277 if (!ctx) 278 return -ENOMEM; 279 280 ctx->name = name; 281 ctx->trigger = trigger; 282 ctx->vdev = vdev; 283 284 /* 285 * Fill the initial masked state based on virq_disabled. After 286 * enable, changing the DisINTx bit in vconfig directly changes INTx 287 * masking. igate prevents races during setup, once running masked 288 * is protected via irqlock. 289 * 290 * Devices supporting DisINTx also reflect the current mask state in 291 * the physical DisINTx bit, which is not affected during IRQ setup. 292 * 293 * Devices without DisINTx support require an exclusive interrupt. 294 * IRQ masking is performed at the IRQ chip. Again, igate protects 295 * against races during setup and IRQ handlers and irqfds are not 296 * yet active, therefore masked is stable and can be used to 297 * conditionally auto-enable the IRQ. 298 * 299 * irq_type must be stable while the IRQ handler is registered, 300 * therefore it must be set before request_irq(). 301 */ 302 ctx->masked = vdev->virq_disabled; 303 if (vdev->pci_2_3) { 304 pci_intx(pdev, !ctx->masked); 305 irqflags = IRQF_SHARED; 306 } else { 307 irqflags = ctx->masked ? IRQF_NO_AUTOEN : 0; 308 } 309 310 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; 311 312 ret = request_irq(pdev->irq, vfio_intx_handler, 313 irqflags, ctx->name, ctx); 314 if (ret) { 315 vdev->irq_type = VFIO_PCI_NUM_IRQS; 316 kfree(name); 317 vfio_irq_ctx_free(vdev, ctx, 0); 318 return ret; 319 } 320 321 return 0; 322 } 323 324 static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, 325 struct eventfd_ctx *trigger) 326 { 327 struct pci_dev *pdev = vdev->pdev; 328 struct vfio_pci_irq_ctx *ctx; 329 struct eventfd_ctx *old; 330 331 ctx = vfio_irq_ctx_get(vdev, 0); 332 if (WARN_ON_ONCE(!ctx)) 333 return -EINVAL; 334 335 old = ctx->trigger; 336 337 WRITE_ONCE(ctx->trigger, trigger); 338 339 /* Releasing an old ctx requires synchronizing in-flight users */ 340 if (old) { 341 synchronize_irq(pdev->irq); 342 vfio_virqfd_flush_thread(&ctx->unmask); 343 eventfd_ctx_put(old); 344 } 345 346 return 0; 347 } 348 349 static void vfio_intx_disable(struct vfio_pci_core_device *vdev) 350 { 351 struct pci_dev *pdev = vdev->pdev; 352 struct vfio_pci_irq_ctx *ctx; 353 354 ctx = vfio_irq_ctx_get(vdev, 0); 355 WARN_ON_ONCE(!ctx); 356 if (ctx) { 357 vfio_virqfd_disable(&ctx->unmask); 358 vfio_virqfd_disable(&ctx->mask); 359 free_irq(pdev->irq, ctx); 360 if (ctx->trigger) 361 eventfd_ctx_put(ctx->trigger); 362 kfree(ctx->name); 363 vfio_irq_ctx_free(vdev, ctx, 0); 364 } 365 vdev->irq_type = VFIO_PCI_NUM_IRQS; 366 } 367 368 /* 369 * MSI/MSI-X 370 */ 371 static irqreturn_t vfio_msihandler(int irq, void *arg) 372 { 373 struct eventfd_ctx *trigger = arg; 374 375 eventfd_signal(trigger); 376 return IRQ_HANDLED; 377 } 378 379 static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix) 380 { 381 struct pci_dev *pdev = vdev->pdev; 382 unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; 383 int ret; 384 u16 cmd; 385 386 if (!is_irq_none(vdev)) 387 return -EINVAL; 388 389 /* return the number of supported vectors if we can't get all: */ 390 cmd = vfio_pci_memory_lock_and_enable(vdev); 391 ret = pci_alloc_irq_vectors(pdev, 1, nvec, flag); 392 if (ret < nvec) { 393 if (ret > 0) 394 pci_free_irq_vectors(pdev); 395 vfio_pci_memory_unlock_and_restore(vdev, cmd); 396 return ret; 397 } 398 vfio_pci_memory_unlock_and_restore(vdev, cmd); 399 400 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : 401 VFIO_PCI_MSI_IRQ_INDEX; 402 403 if (!msix) { 404 /* 405 * Compute the virtual hardware field for max msi vectors - 406 * it is the log base 2 of the number of vectors. 407 */ 408 vdev->msi_qmax = fls(nvec * 2 - 1) - 1; 409 } 410 411 return 0; 412 } 413 414 /* 415 * vfio_msi_alloc_irq() returns the Linux IRQ number of an MSI or MSI-X device 416 * interrupt vector. If a Linux IRQ number is not available then a new 417 * interrupt is allocated if dynamic MSI-X is supported. 418 * 419 * Where is vfio_msi_free_irq()? Allocated interrupts are maintained, 420 * essentially forming a cache that subsequent allocations can draw from. 421 * Interrupts are freed using pci_free_irq_vectors() when MSI/MSI-X is 422 * disabled. 423 */ 424 static int vfio_msi_alloc_irq(struct vfio_pci_core_device *vdev, 425 unsigned int vector, bool msix) 426 { 427 struct pci_dev *pdev = vdev->pdev; 428 struct msi_map map; 429 int irq; 430 u16 cmd; 431 432 irq = pci_irq_vector(pdev, vector); 433 if (WARN_ON_ONCE(irq == 0)) 434 return -EINVAL; 435 if (irq > 0 || !msix || !vdev->has_dyn_msix) 436 return irq; 437 438 cmd = vfio_pci_memory_lock_and_enable(vdev); 439 map = pci_msix_alloc_irq_at(pdev, vector, NULL); 440 vfio_pci_memory_unlock_and_restore(vdev, cmd); 441 442 return map.index < 0 ? map.index : map.virq; 443 } 444 445 static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev, 446 unsigned int vector, int fd, bool msix) 447 { 448 struct pci_dev *pdev = vdev->pdev; 449 struct vfio_pci_irq_ctx *ctx; 450 struct eventfd_ctx *trigger; 451 int irq = -EINVAL, ret; 452 u16 cmd; 453 454 ctx = vfio_irq_ctx_get(vdev, vector); 455 456 if (ctx) { 457 irq_bypass_unregister_producer(&ctx->producer); 458 irq = pci_irq_vector(pdev, vector); 459 cmd = vfio_pci_memory_lock_and_enable(vdev); 460 free_irq(irq, ctx->trigger); 461 vfio_pci_memory_unlock_and_restore(vdev, cmd); 462 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 463 kfree(ctx->name); 464 eventfd_ctx_put(ctx->trigger); 465 vfio_irq_ctx_free(vdev, ctx, vector); 466 } 467 468 if (fd < 0) 469 return 0; 470 471 if (irq == -EINVAL) { 472 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 473 irq = vfio_msi_alloc_irq(vdev, vector, msix); 474 if (irq < 0) 475 return irq; 476 } 477 478 ctx = vfio_irq_ctx_alloc(vdev, vector); 479 if (!ctx) 480 return -ENOMEM; 481 482 ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-msi%s[%d](%s)", 483 msix ? "x" : "", vector, pci_name(pdev)); 484 if (!ctx->name) { 485 ret = -ENOMEM; 486 goto out_free_ctx; 487 } 488 489 trigger = eventfd_ctx_fdget(fd); 490 if (IS_ERR(trigger)) { 491 ret = PTR_ERR(trigger); 492 goto out_free_name; 493 } 494 495 /* 496 * If the vector was previously allocated, refresh the on-device 497 * message data before enabling in case it had been cleared or 498 * corrupted (e.g. due to backdoor resets) since writing. 499 */ 500 cmd = vfio_pci_memory_lock_and_enable(vdev); 501 if (msix) { 502 struct msi_msg msg; 503 504 get_cached_msi_msg(irq, &msg); 505 pci_write_msi_msg(irq, &msg); 506 } 507 508 ret = request_irq(irq, vfio_msihandler, 0, ctx->name, trigger); 509 vfio_pci_memory_unlock_and_restore(vdev, cmd); 510 if (ret) 511 goto out_put_eventfd_ctx; 512 513 ctx->producer.token = trigger; 514 ctx->producer.irq = irq; 515 ret = irq_bypass_register_producer(&ctx->producer); 516 if (unlikely(ret)) { 517 dev_info(&pdev->dev, 518 "irq bypass producer (token %p) registration fails: %d\n", 519 ctx->producer.token, ret); 520 521 ctx->producer.token = NULL; 522 } 523 ctx->trigger = trigger; 524 525 return 0; 526 527 out_put_eventfd_ctx: 528 eventfd_ctx_put(trigger); 529 out_free_name: 530 kfree(ctx->name); 531 out_free_ctx: 532 vfio_irq_ctx_free(vdev, ctx, vector); 533 return ret; 534 } 535 536 static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start, 537 unsigned count, int32_t *fds, bool msix) 538 { 539 unsigned int i, j; 540 int ret = 0; 541 542 for (i = 0, j = start; i < count && !ret; i++, j++) { 543 int fd = fds ? fds[i] : -1; 544 ret = vfio_msi_set_vector_signal(vdev, j, fd, msix); 545 } 546 547 if (ret) { 548 for (i = start; i < j; i++) 549 vfio_msi_set_vector_signal(vdev, i, -1, msix); 550 } 551 552 return ret; 553 } 554 555 static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix) 556 { 557 struct pci_dev *pdev = vdev->pdev; 558 struct vfio_pci_irq_ctx *ctx; 559 unsigned long i; 560 u16 cmd; 561 562 xa_for_each(&vdev->ctx, i, ctx) { 563 vfio_virqfd_disable(&ctx->unmask); 564 vfio_virqfd_disable(&ctx->mask); 565 vfio_msi_set_vector_signal(vdev, i, -1, msix); 566 } 567 568 cmd = vfio_pci_memory_lock_and_enable(vdev); 569 pci_free_irq_vectors(pdev); 570 vfio_pci_memory_unlock_and_restore(vdev, cmd); 571 572 /* 573 * Both disable paths above use pci_intx_for_msi() to clear DisINTx 574 * via their shutdown paths. Restore for NoINTx devices. 575 */ 576 if (vdev->nointx) 577 pci_intx(pdev, 0); 578 579 vdev->irq_type = VFIO_PCI_NUM_IRQS; 580 } 581 582 /* 583 * IOCTL support 584 */ 585 static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev, 586 unsigned index, unsigned start, 587 unsigned count, uint32_t flags, void *data) 588 { 589 if (!is_intx(vdev) || start != 0 || count != 1) 590 return -EINVAL; 591 592 if (flags & VFIO_IRQ_SET_DATA_NONE) { 593 __vfio_pci_intx_unmask(vdev); 594 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 595 uint8_t unmask = *(uint8_t *)data; 596 if (unmask) 597 __vfio_pci_intx_unmask(vdev); 598 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 599 struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, 0); 600 int32_t fd = *(int32_t *)data; 601 602 if (WARN_ON_ONCE(!ctx)) 603 return -EINVAL; 604 if (fd >= 0) 605 return vfio_virqfd_enable((void *) vdev, 606 vfio_pci_intx_unmask_handler, 607 vfio_send_intx_eventfd, NULL, 608 &ctx->unmask, fd); 609 610 vfio_virqfd_disable(&ctx->unmask); 611 } 612 613 return 0; 614 } 615 616 static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev, 617 unsigned index, unsigned start, 618 unsigned count, uint32_t flags, void *data) 619 { 620 if (!is_intx(vdev) || start != 0 || count != 1) 621 return -EINVAL; 622 623 if (flags & VFIO_IRQ_SET_DATA_NONE) { 624 __vfio_pci_intx_mask(vdev); 625 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 626 uint8_t mask = *(uint8_t *)data; 627 if (mask) 628 __vfio_pci_intx_mask(vdev); 629 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 630 return -ENOTTY; /* XXX implement me */ 631 } 632 633 return 0; 634 } 635 636 static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev, 637 unsigned index, unsigned start, 638 unsigned count, uint32_t flags, void *data) 639 { 640 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 641 vfio_intx_disable(vdev); 642 return 0; 643 } 644 645 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) 646 return -EINVAL; 647 648 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 649 struct eventfd_ctx *trigger = NULL; 650 int32_t fd = *(int32_t *)data; 651 int ret; 652 653 if (fd >= 0) { 654 trigger = eventfd_ctx_fdget(fd); 655 if (IS_ERR(trigger)) 656 return PTR_ERR(trigger); 657 } 658 659 if (is_intx(vdev)) 660 ret = vfio_intx_set_signal(vdev, trigger); 661 else 662 ret = vfio_intx_enable(vdev, trigger); 663 664 if (ret && trigger) 665 eventfd_ctx_put(trigger); 666 667 return ret; 668 } 669 670 if (!is_intx(vdev)) 671 return -EINVAL; 672 673 if (flags & VFIO_IRQ_SET_DATA_NONE) { 674 vfio_send_intx_eventfd(vdev, NULL); 675 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 676 uint8_t trigger = *(uint8_t *)data; 677 if (trigger) 678 vfio_send_intx_eventfd(vdev, NULL); 679 } 680 return 0; 681 } 682 683 static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, 684 unsigned index, unsigned start, 685 unsigned count, uint32_t flags, void *data) 686 { 687 struct vfio_pci_irq_ctx *ctx; 688 unsigned int i; 689 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; 690 691 if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 692 vfio_msi_disable(vdev, msix); 693 return 0; 694 } 695 696 if (!(irq_is(vdev, index) || is_irq_none(vdev))) 697 return -EINVAL; 698 699 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 700 int32_t *fds = data; 701 int ret; 702 703 if (vdev->irq_type == index) 704 return vfio_msi_set_block(vdev, start, count, 705 fds, msix); 706 707 ret = vfio_msi_enable(vdev, start + count, msix); 708 if (ret) 709 return ret; 710 711 ret = vfio_msi_set_block(vdev, start, count, fds, msix); 712 if (ret) 713 vfio_msi_disable(vdev, msix); 714 715 return ret; 716 } 717 718 if (!irq_is(vdev, index)) 719 return -EINVAL; 720 721 for (i = start; i < start + count; i++) { 722 ctx = vfio_irq_ctx_get(vdev, i); 723 if (!ctx) 724 continue; 725 if (flags & VFIO_IRQ_SET_DATA_NONE) { 726 eventfd_signal(ctx->trigger); 727 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 728 uint8_t *bools = data; 729 if (bools[i - start]) 730 eventfd_signal(ctx->trigger); 731 } 732 } 733 return 0; 734 } 735 736 static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, 737 unsigned int count, uint32_t flags, 738 void *data) 739 { 740 /* DATA_NONE/DATA_BOOL enables loopback testing */ 741 if (flags & VFIO_IRQ_SET_DATA_NONE) { 742 if (*ctx) { 743 if (count) { 744 eventfd_signal(*ctx); 745 } else { 746 eventfd_ctx_put(*ctx); 747 *ctx = NULL; 748 } 749 return 0; 750 } 751 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 752 uint8_t trigger; 753 754 if (!count) 755 return -EINVAL; 756 757 trigger = *(uint8_t *)data; 758 if (trigger && *ctx) 759 eventfd_signal(*ctx); 760 761 return 0; 762 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 763 int32_t fd; 764 765 if (!count) 766 return -EINVAL; 767 768 fd = *(int32_t *)data; 769 if (fd == -1) { 770 if (*ctx) 771 eventfd_ctx_put(*ctx); 772 *ctx = NULL; 773 } else if (fd >= 0) { 774 struct eventfd_ctx *efdctx; 775 776 efdctx = eventfd_ctx_fdget(fd); 777 if (IS_ERR(efdctx)) 778 return PTR_ERR(efdctx); 779 780 if (*ctx) 781 eventfd_ctx_put(*ctx); 782 783 *ctx = efdctx; 784 } 785 return 0; 786 } 787 788 return -EINVAL; 789 } 790 791 static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev, 792 unsigned index, unsigned start, 793 unsigned count, uint32_t flags, void *data) 794 { 795 if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1) 796 return -EINVAL; 797 798 return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger, 799 count, flags, data); 800 } 801 802 static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, 803 unsigned index, unsigned start, 804 unsigned count, uint32_t flags, void *data) 805 { 806 if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1) 807 return -EINVAL; 808 809 return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger, 810 count, flags, data); 811 } 812 813 int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, 814 unsigned index, unsigned start, unsigned count, 815 void *data) 816 { 817 int (*func)(struct vfio_pci_core_device *vdev, unsigned index, 818 unsigned start, unsigned count, uint32_t flags, 819 void *data) = NULL; 820 821 switch (index) { 822 case VFIO_PCI_INTX_IRQ_INDEX: 823 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 824 case VFIO_IRQ_SET_ACTION_MASK: 825 func = vfio_pci_set_intx_mask; 826 break; 827 case VFIO_IRQ_SET_ACTION_UNMASK: 828 func = vfio_pci_set_intx_unmask; 829 break; 830 case VFIO_IRQ_SET_ACTION_TRIGGER: 831 func = vfio_pci_set_intx_trigger; 832 break; 833 } 834 break; 835 case VFIO_PCI_MSI_IRQ_INDEX: 836 case VFIO_PCI_MSIX_IRQ_INDEX: 837 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 838 case VFIO_IRQ_SET_ACTION_MASK: 839 case VFIO_IRQ_SET_ACTION_UNMASK: 840 /* XXX Need masking support exported */ 841 break; 842 case VFIO_IRQ_SET_ACTION_TRIGGER: 843 func = vfio_pci_set_msi_trigger; 844 break; 845 } 846 break; 847 case VFIO_PCI_ERR_IRQ_INDEX: 848 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 849 case VFIO_IRQ_SET_ACTION_TRIGGER: 850 if (pci_is_pcie(vdev->pdev)) 851 func = vfio_pci_set_err_trigger; 852 break; 853 } 854 break; 855 case VFIO_PCI_REQ_IRQ_INDEX: 856 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 857 case VFIO_IRQ_SET_ACTION_TRIGGER: 858 func = vfio_pci_set_req_trigger; 859 break; 860 } 861 break; 862 } 863 864 if (!func) 865 return -ENOTTY; 866 867 return func(vdev, index, start, count, flags, data); 868 } 869