1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO PCI interrupt handling 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/device.h> 14 #include <linux/interrupt.h> 15 #include <linux/eventfd.h> 16 #include <linux/msi.h> 17 #include <linux/pci.h> 18 #include <linux/file.h> 19 #include <linux/vfio.h> 20 #include <linux/wait.h> 21 #include <linux/slab.h> 22 23 #include "vfio_pci_priv.h" 24 25 struct vfio_pci_irq_ctx { 26 struct vfio_pci_core_device *vdev; 27 struct eventfd_ctx *trigger; 28 struct virqfd *unmask; 29 struct virqfd *mask; 30 char *name; 31 bool masked; 32 struct irq_bypass_producer producer; 33 }; 34 35 static bool irq_is(struct vfio_pci_core_device *vdev, int type) 36 { 37 return vdev->irq_type == type; 38 } 39 40 static bool is_intx(struct vfio_pci_core_device *vdev) 41 { 42 return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX; 43 } 44 45 static bool is_irq_none(struct vfio_pci_core_device *vdev) 46 { 47 return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX || 48 vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX || 49 vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX); 50 } 51 52 static 53 struct vfio_pci_irq_ctx *vfio_irq_ctx_get(struct vfio_pci_core_device *vdev, 54 unsigned long index) 55 { 56 return xa_load(&vdev->ctx, index); 57 } 58 59 static void vfio_irq_ctx_free(struct vfio_pci_core_device *vdev, 60 struct vfio_pci_irq_ctx *ctx, unsigned long index) 61 { 62 xa_erase(&vdev->ctx, index); 63 kfree(ctx); 64 } 65 66 static struct vfio_pci_irq_ctx * 67 vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index) 68 { 69 struct vfio_pci_irq_ctx *ctx; 70 int ret; 71 72 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); 73 if (!ctx) 74 return NULL; 75 76 ret = xa_insert(&vdev->ctx, index, ctx, GFP_KERNEL_ACCOUNT); 77 if (ret) { 78 kfree(ctx); 79 return NULL; 80 } 81 82 return ctx; 83 } 84 85 /* 86 * INTx 87 */ 88 static void vfio_send_intx_eventfd(void *opaque, void *data) 89 { 90 struct vfio_pci_core_device *vdev = opaque; 91 92 if (likely(is_intx(vdev) && !vdev->virq_disabled)) { 93 struct vfio_pci_irq_ctx *ctx = data; 94 struct eventfd_ctx *trigger = READ_ONCE(ctx->trigger); 95 96 if (likely(trigger)) 97 eventfd_signal(trigger); 98 } 99 } 100 101 /* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */ 102 static bool __vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 103 { 104 struct pci_dev *pdev = vdev->pdev; 105 struct vfio_pci_irq_ctx *ctx; 106 unsigned long flags; 107 bool masked_changed = false; 108 109 lockdep_assert_held(&vdev->igate); 110 111 spin_lock_irqsave(&vdev->irqlock, flags); 112 113 /* 114 * Masking can come from interrupt, ioctl, or config space 115 * via INTx disable. The latter means this can get called 116 * even when not using intx delivery. In this case, just 117 * try to have the physical bit follow the virtual bit. 118 */ 119 if (unlikely(!is_intx(vdev))) { 120 if (vdev->pci_2_3) 121 pci_intx(pdev, 0); 122 goto out_unlock; 123 } 124 125 ctx = vfio_irq_ctx_get(vdev, 0); 126 if (WARN_ON_ONCE(!ctx)) 127 goto out_unlock; 128 129 if (!ctx->masked) { 130 /* 131 * Can't use check_and_mask here because we always want to 132 * mask, not just when something is pending. 133 */ 134 if (vdev->pci_2_3) 135 pci_intx(pdev, 0); 136 else 137 disable_irq_nosync(pdev->irq); 138 139 ctx->masked = true; 140 masked_changed = true; 141 } 142 143 out_unlock: 144 spin_unlock_irqrestore(&vdev->irqlock, flags); 145 return masked_changed; 146 } 147 148 bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 149 { 150 bool mask_changed; 151 152 mutex_lock(&vdev->igate); 153 mask_changed = __vfio_pci_intx_mask(vdev); 154 mutex_unlock(&vdev->igate); 155 156 return mask_changed; 157 } 158 159 /* 160 * If this is triggered by an eventfd, we can't call eventfd_signal 161 * or else we'll deadlock on the eventfd wait queue. Return >0 when 162 * a signal is necessary, which can then be handled via a work queue 163 * or directly depending on the caller. 164 */ 165 static int vfio_pci_intx_unmask_handler(void *opaque, void *data) 166 { 167 struct vfio_pci_core_device *vdev = opaque; 168 struct pci_dev *pdev = vdev->pdev; 169 struct vfio_pci_irq_ctx *ctx = data; 170 unsigned long flags; 171 int ret = 0; 172 173 spin_lock_irqsave(&vdev->irqlock, flags); 174 175 /* 176 * Unmasking comes from ioctl or config, so again, have the 177 * physical bit follow the virtual even when not using INTx. 178 */ 179 if (unlikely(!is_intx(vdev))) { 180 if (vdev->pci_2_3) 181 pci_intx(pdev, 1); 182 goto out_unlock; 183 } 184 185 if (ctx->masked && !vdev->virq_disabled) { 186 /* 187 * A pending interrupt here would immediately trigger, 188 * but we can avoid that overhead by just re-sending 189 * the interrupt to the user. 190 */ 191 if (vdev->pci_2_3) { 192 if (!pci_check_and_unmask_intx(pdev)) 193 ret = 1; 194 } else 195 enable_irq(pdev->irq); 196 197 ctx->masked = (ret > 0); 198 } 199 200 out_unlock: 201 spin_unlock_irqrestore(&vdev->irqlock, flags); 202 203 return ret; 204 } 205 206 static void __vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 207 { 208 struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, 0); 209 210 lockdep_assert_held(&vdev->igate); 211 212 if (vfio_pci_intx_unmask_handler(vdev, ctx) > 0) 213 vfio_send_intx_eventfd(vdev, ctx); 214 } 215 216 void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 217 { 218 mutex_lock(&vdev->igate); 219 __vfio_pci_intx_unmask(vdev); 220 mutex_unlock(&vdev->igate); 221 } 222 223 static irqreturn_t vfio_intx_handler(int irq, void *dev_id) 224 { 225 struct vfio_pci_irq_ctx *ctx = dev_id; 226 struct vfio_pci_core_device *vdev = ctx->vdev; 227 unsigned long flags; 228 int ret = IRQ_NONE; 229 230 spin_lock_irqsave(&vdev->irqlock, flags); 231 232 if (!vdev->pci_2_3) { 233 disable_irq_nosync(vdev->pdev->irq); 234 ctx->masked = true; 235 ret = IRQ_HANDLED; 236 } else if (!ctx->masked && /* may be shared */ 237 pci_check_and_mask_intx(vdev->pdev)) { 238 ctx->masked = true; 239 ret = IRQ_HANDLED; 240 } 241 242 spin_unlock_irqrestore(&vdev->irqlock, flags); 243 244 if (ret == IRQ_HANDLED) 245 vfio_send_intx_eventfd(vdev, ctx); 246 247 return ret; 248 } 249 250 static int vfio_intx_enable(struct vfio_pci_core_device *vdev, 251 struct eventfd_ctx *trigger) 252 { 253 struct pci_dev *pdev = vdev->pdev; 254 struct vfio_pci_irq_ctx *ctx; 255 unsigned long irqflags; 256 char *name; 257 int ret; 258 259 if (!is_irq_none(vdev)) 260 return -EINVAL; 261 262 if (!pdev->irq) 263 return -ENODEV; 264 265 name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-intx(%s)", pci_name(pdev)); 266 if (!name) 267 return -ENOMEM; 268 269 ctx = vfio_irq_ctx_alloc(vdev, 0); 270 if (!ctx) { 271 kfree(name); 272 return -ENOMEM; 273 } 274 275 ctx->name = name; 276 ctx->trigger = trigger; 277 ctx->vdev = vdev; 278 279 /* 280 * Fill the initial masked state based on virq_disabled. After 281 * enable, changing the DisINTx bit in vconfig directly changes INTx 282 * masking. igate prevents races during setup, once running masked 283 * is protected via irqlock. 284 * 285 * Devices supporting DisINTx also reflect the current mask state in 286 * the physical DisINTx bit, which is not affected during IRQ setup. 287 * 288 * Devices without DisINTx support require an exclusive interrupt. 289 * IRQ masking is performed at the IRQ chip. Again, igate protects 290 * against races during setup and IRQ handlers and irqfds are not 291 * yet active, therefore masked is stable and can be used to 292 * conditionally auto-enable the IRQ. 293 * 294 * irq_type must be stable while the IRQ handler is registered, 295 * therefore it must be set before request_irq(). 296 */ 297 ctx->masked = vdev->virq_disabled; 298 if (vdev->pci_2_3) { 299 pci_intx(pdev, !ctx->masked); 300 irqflags = IRQF_SHARED; 301 } else { 302 irqflags = ctx->masked ? IRQF_NO_AUTOEN : 0; 303 } 304 305 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; 306 307 ret = request_irq(pdev->irq, vfio_intx_handler, 308 irqflags, ctx->name, ctx); 309 if (ret) { 310 vdev->irq_type = VFIO_PCI_NUM_IRQS; 311 kfree(name); 312 vfio_irq_ctx_free(vdev, ctx, 0); 313 return ret; 314 } 315 316 return 0; 317 } 318 319 static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, 320 struct eventfd_ctx *trigger) 321 { 322 struct pci_dev *pdev = vdev->pdev; 323 struct vfio_pci_irq_ctx *ctx; 324 struct eventfd_ctx *old; 325 326 ctx = vfio_irq_ctx_get(vdev, 0); 327 if (WARN_ON_ONCE(!ctx)) 328 return -EINVAL; 329 330 old = ctx->trigger; 331 332 WRITE_ONCE(ctx->trigger, trigger); 333 334 /* Releasing an old ctx requires synchronizing in-flight users */ 335 if (old) { 336 synchronize_irq(pdev->irq); 337 vfio_virqfd_flush_thread(&ctx->unmask); 338 eventfd_ctx_put(old); 339 } 340 341 return 0; 342 } 343 344 static void vfio_intx_disable(struct vfio_pci_core_device *vdev) 345 { 346 struct pci_dev *pdev = vdev->pdev; 347 struct vfio_pci_irq_ctx *ctx; 348 349 ctx = vfio_irq_ctx_get(vdev, 0); 350 WARN_ON_ONCE(!ctx); 351 if (ctx) { 352 vfio_virqfd_disable(&ctx->unmask); 353 vfio_virqfd_disable(&ctx->mask); 354 free_irq(pdev->irq, ctx); 355 if (ctx->trigger) 356 eventfd_ctx_put(ctx->trigger); 357 kfree(ctx->name); 358 vfio_irq_ctx_free(vdev, ctx, 0); 359 } 360 vdev->irq_type = VFIO_PCI_NUM_IRQS; 361 } 362 363 /* 364 * MSI/MSI-X 365 */ 366 static irqreturn_t vfio_msihandler(int irq, void *arg) 367 { 368 struct eventfd_ctx *trigger = arg; 369 370 eventfd_signal(trigger); 371 return IRQ_HANDLED; 372 } 373 374 static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix) 375 { 376 struct pci_dev *pdev = vdev->pdev; 377 unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; 378 int ret; 379 u16 cmd; 380 381 if (!is_irq_none(vdev)) 382 return -EINVAL; 383 384 /* return the number of supported vectors if we can't get all: */ 385 cmd = vfio_pci_memory_lock_and_enable(vdev); 386 ret = pci_alloc_irq_vectors(pdev, 1, nvec, flag); 387 if (ret < nvec) { 388 if (ret > 0) 389 pci_free_irq_vectors(pdev); 390 vfio_pci_memory_unlock_and_restore(vdev, cmd); 391 return ret; 392 } 393 vfio_pci_memory_unlock_and_restore(vdev, cmd); 394 395 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : 396 VFIO_PCI_MSI_IRQ_INDEX; 397 398 if (!msix) { 399 /* 400 * Compute the virtual hardware field for max msi vectors - 401 * it is the log base 2 of the number of vectors. 402 */ 403 vdev->msi_qmax = fls(nvec * 2 - 1) - 1; 404 } 405 406 return 0; 407 } 408 409 /* 410 * vfio_msi_alloc_irq() returns the Linux IRQ number of an MSI or MSI-X device 411 * interrupt vector. If a Linux IRQ number is not available then a new 412 * interrupt is allocated if dynamic MSI-X is supported. 413 * 414 * Where is vfio_msi_free_irq()? Allocated interrupts are maintained, 415 * essentially forming a cache that subsequent allocations can draw from. 416 * Interrupts are freed using pci_free_irq_vectors() when MSI/MSI-X is 417 * disabled. 418 */ 419 static int vfio_msi_alloc_irq(struct vfio_pci_core_device *vdev, 420 unsigned int vector, bool msix) 421 { 422 struct pci_dev *pdev = vdev->pdev; 423 struct msi_map map; 424 int irq; 425 u16 cmd; 426 427 irq = pci_irq_vector(pdev, vector); 428 if (WARN_ON_ONCE(irq == 0)) 429 return -EINVAL; 430 if (irq > 0 || !msix || !vdev->has_dyn_msix) 431 return irq; 432 433 cmd = vfio_pci_memory_lock_and_enable(vdev); 434 map = pci_msix_alloc_irq_at(pdev, vector, NULL); 435 vfio_pci_memory_unlock_and_restore(vdev, cmd); 436 437 return map.index < 0 ? map.index : map.virq; 438 } 439 440 static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev, 441 unsigned int vector, int fd, bool msix) 442 { 443 struct pci_dev *pdev = vdev->pdev; 444 struct vfio_pci_irq_ctx *ctx; 445 struct eventfd_ctx *trigger; 446 int irq = -EINVAL, ret; 447 u16 cmd; 448 449 ctx = vfio_irq_ctx_get(vdev, vector); 450 451 if (ctx) { 452 irq_bypass_unregister_producer(&ctx->producer); 453 irq = pci_irq_vector(pdev, vector); 454 cmd = vfio_pci_memory_lock_and_enable(vdev); 455 free_irq(irq, ctx->trigger); 456 vfio_pci_memory_unlock_and_restore(vdev, cmd); 457 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 458 kfree(ctx->name); 459 eventfd_ctx_put(ctx->trigger); 460 vfio_irq_ctx_free(vdev, ctx, vector); 461 } 462 463 if (fd < 0) 464 return 0; 465 466 if (irq == -EINVAL) { 467 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 468 irq = vfio_msi_alloc_irq(vdev, vector, msix); 469 if (irq < 0) 470 return irq; 471 } 472 473 ctx = vfio_irq_ctx_alloc(vdev, vector); 474 if (!ctx) 475 return -ENOMEM; 476 477 ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-msi%s[%d](%s)", 478 msix ? "x" : "", vector, pci_name(pdev)); 479 if (!ctx->name) { 480 ret = -ENOMEM; 481 goto out_free_ctx; 482 } 483 484 trigger = eventfd_ctx_fdget(fd); 485 if (IS_ERR(trigger)) { 486 ret = PTR_ERR(trigger); 487 goto out_free_name; 488 } 489 490 /* 491 * If the vector was previously allocated, refresh the on-device 492 * message data before enabling in case it had been cleared or 493 * corrupted (e.g. due to backdoor resets) since writing. 494 */ 495 cmd = vfio_pci_memory_lock_and_enable(vdev); 496 if (msix) { 497 struct msi_msg msg; 498 499 get_cached_msi_msg(irq, &msg); 500 pci_write_msi_msg(irq, &msg); 501 } 502 503 ret = request_irq(irq, vfio_msihandler, 0, ctx->name, trigger); 504 vfio_pci_memory_unlock_and_restore(vdev, cmd); 505 if (ret) 506 goto out_put_eventfd_ctx; 507 508 ctx->producer.token = trigger; 509 ctx->producer.irq = irq; 510 ret = irq_bypass_register_producer(&ctx->producer); 511 if (unlikely(ret)) { 512 dev_info(&pdev->dev, 513 "irq bypass producer (token %p) registration fails: %d\n", 514 ctx->producer.token, ret); 515 516 ctx->producer.token = NULL; 517 } 518 ctx->trigger = trigger; 519 520 return 0; 521 522 out_put_eventfd_ctx: 523 eventfd_ctx_put(trigger); 524 out_free_name: 525 kfree(ctx->name); 526 out_free_ctx: 527 vfio_irq_ctx_free(vdev, ctx, vector); 528 return ret; 529 } 530 531 static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start, 532 unsigned count, int32_t *fds, bool msix) 533 { 534 unsigned int i, j; 535 int ret = 0; 536 537 for (i = 0, j = start; i < count && !ret; i++, j++) { 538 int fd = fds ? fds[i] : -1; 539 ret = vfio_msi_set_vector_signal(vdev, j, fd, msix); 540 } 541 542 if (ret) { 543 for (i = start; i < j; i++) 544 vfio_msi_set_vector_signal(vdev, i, -1, msix); 545 } 546 547 return ret; 548 } 549 550 static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix) 551 { 552 struct pci_dev *pdev = vdev->pdev; 553 struct vfio_pci_irq_ctx *ctx; 554 unsigned long i; 555 u16 cmd; 556 557 xa_for_each(&vdev->ctx, i, ctx) { 558 vfio_virqfd_disable(&ctx->unmask); 559 vfio_virqfd_disable(&ctx->mask); 560 vfio_msi_set_vector_signal(vdev, i, -1, msix); 561 } 562 563 cmd = vfio_pci_memory_lock_and_enable(vdev); 564 pci_free_irq_vectors(pdev); 565 vfio_pci_memory_unlock_and_restore(vdev, cmd); 566 567 /* 568 * Both disable paths above use pci_intx_for_msi() to clear DisINTx 569 * via their shutdown paths. Restore for NoINTx devices. 570 */ 571 if (vdev->nointx) 572 pci_intx(pdev, 0); 573 574 vdev->irq_type = VFIO_PCI_NUM_IRQS; 575 } 576 577 /* 578 * IOCTL support 579 */ 580 static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev, 581 unsigned index, unsigned start, 582 unsigned count, uint32_t flags, void *data) 583 { 584 if (!is_intx(vdev) || start != 0 || count != 1) 585 return -EINVAL; 586 587 if (flags & VFIO_IRQ_SET_DATA_NONE) { 588 __vfio_pci_intx_unmask(vdev); 589 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 590 uint8_t unmask = *(uint8_t *)data; 591 if (unmask) 592 __vfio_pci_intx_unmask(vdev); 593 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 594 struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, 0); 595 int32_t fd = *(int32_t *)data; 596 597 if (WARN_ON_ONCE(!ctx)) 598 return -EINVAL; 599 if (fd >= 0) 600 return vfio_virqfd_enable((void *) vdev, 601 vfio_pci_intx_unmask_handler, 602 vfio_send_intx_eventfd, ctx, 603 &ctx->unmask, fd); 604 605 vfio_virqfd_disable(&ctx->unmask); 606 } 607 608 return 0; 609 } 610 611 static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev, 612 unsigned index, unsigned start, 613 unsigned count, uint32_t flags, void *data) 614 { 615 if (!is_intx(vdev) || start != 0 || count != 1) 616 return -EINVAL; 617 618 if (flags & VFIO_IRQ_SET_DATA_NONE) { 619 __vfio_pci_intx_mask(vdev); 620 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 621 uint8_t mask = *(uint8_t *)data; 622 if (mask) 623 __vfio_pci_intx_mask(vdev); 624 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 625 return -ENOTTY; /* XXX implement me */ 626 } 627 628 return 0; 629 } 630 631 static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev, 632 unsigned index, unsigned start, 633 unsigned count, uint32_t flags, void *data) 634 { 635 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 636 vfio_intx_disable(vdev); 637 return 0; 638 } 639 640 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) 641 return -EINVAL; 642 643 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 644 struct eventfd_ctx *trigger = NULL; 645 int32_t fd = *(int32_t *)data; 646 int ret; 647 648 if (fd >= 0) { 649 trigger = eventfd_ctx_fdget(fd); 650 if (IS_ERR(trigger)) 651 return PTR_ERR(trigger); 652 } 653 654 if (is_intx(vdev)) 655 ret = vfio_intx_set_signal(vdev, trigger); 656 else 657 ret = vfio_intx_enable(vdev, trigger); 658 659 if (ret && trigger) 660 eventfd_ctx_put(trigger); 661 662 return ret; 663 } 664 665 if (!is_intx(vdev)) 666 return -EINVAL; 667 668 if (flags & VFIO_IRQ_SET_DATA_NONE) { 669 vfio_send_intx_eventfd(vdev, vfio_irq_ctx_get(vdev, 0)); 670 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 671 uint8_t trigger = *(uint8_t *)data; 672 if (trigger) 673 vfio_send_intx_eventfd(vdev, vfio_irq_ctx_get(vdev, 0)); 674 } 675 return 0; 676 } 677 678 static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, 679 unsigned index, unsigned start, 680 unsigned count, uint32_t flags, void *data) 681 { 682 struct vfio_pci_irq_ctx *ctx; 683 unsigned int i; 684 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; 685 686 if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 687 vfio_msi_disable(vdev, msix); 688 return 0; 689 } 690 691 if (!(irq_is(vdev, index) || is_irq_none(vdev))) 692 return -EINVAL; 693 694 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 695 int32_t *fds = data; 696 int ret; 697 698 if (vdev->irq_type == index) 699 return vfio_msi_set_block(vdev, start, count, 700 fds, msix); 701 702 ret = vfio_msi_enable(vdev, start + count, msix); 703 if (ret) 704 return ret; 705 706 ret = vfio_msi_set_block(vdev, start, count, fds, msix); 707 if (ret) 708 vfio_msi_disable(vdev, msix); 709 710 return ret; 711 } 712 713 if (!irq_is(vdev, index)) 714 return -EINVAL; 715 716 for (i = start; i < start + count; i++) { 717 ctx = vfio_irq_ctx_get(vdev, i); 718 if (!ctx) 719 continue; 720 if (flags & VFIO_IRQ_SET_DATA_NONE) { 721 eventfd_signal(ctx->trigger); 722 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 723 uint8_t *bools = data; 724 if (bools[i - start]) 725 eventfd_signal(ctx->trigger); 726 } 727 } 728 return 0; 729 } 730 731 static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, 732 unsigned int count, uint32_t flags, 733 void *data) 734 { 735 /* DATA_NONE/DATA_BOOL enables loopback testing */ 736 if (flags & VFIO_IRQ_SET_DATA_NONE) { 737 if (*ctx) { 738 if (count) { 739 eventfd_signal(*ctx); 740 } else { 741 eventfd_ctx_put(*ctx); 742 *ctx = NULL; 743 } 744 return 0; 745 } 746 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 747 uint8_t trigger; 748 749 if (!count) 750 return -EINVAL; 751 752 trigger = *(uint8_t *)data; 753 if (trigger && *ctx) 754 eventfd_signal(*ctx); 755 756 return 0; 757 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 758 int32_t fd; 759 760 if (!count) 761 return -EINVAL; 762 763 fd = *(int32_t *)data; 764 if (fd == -1) { 765 if (*ctx) 766 eventfd_ctx_put(*ctx); 767 *ctx = NULL; 768 } else if (fd >= 0) { 769 struct eventfd_ctx *efdctx; 770 771 efdctx = eventfd_ctx_fdget(fd); 772 if (IS_ERR(efdctx)) 773 return PTR_ERR(efdctx); 774 775 if (*ctx) 776 eventfd_ctx_put(*ctx); 777 778 *ctx = efdctx; 779 } 780 return 0; 781 } 782 783 return -EINVAL; 784 } 785 786 static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev, 787 unsigned index, unsigned start, 788 unsigned count, uint32_t flags, void *data) 789 { 790 if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1) 791 return -EINVAL; 792 793 return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger, 794 count, flags, data); 795 } 796 797 static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, 798 unsigned index, unsigned start, 799 unsigned count, uint32_t flags, void *data) 800 { 801 if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1) 802 return -EINVAL; 803 804 return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger, 805 count, flags, data); 806 } 807 808 int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, 809 unsigned index, unsigned start, unsigned count, 810 void *data) 811 { 812 int (*func)(struct vfio_pci_core_device *vdev, unsigned index, 813 unsigned start, unsigned count, uint32_t flags, 814 void *data) = NULL; 815 816 switch (index) { 817 case VFIO_PCI_INTX_IRQ_INDEX: 818 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 819 case VFIO_IRQ_SET_ACTION_MASK: 820 func = vfio_pci_set_intx_mask; 821 break; 822 case VFIO_IRQ_SET_ACTION_UNMASK: 823 func = vfio_pci_set_intx_unmask; 824 break; 825 case VFIO_IRQ_SET_ACTION_TRIGGER: 826 func = vfio_pci_set_intx_trigger; 827 break; 828 } 829 break; 830 case VFIO_PCI_MSI_IRQ_INDEX: 831 case VFIO_PCI_MSIX_IRQ_INDEX: 832 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 833 case VFIO_IRQ_SET_ACTION_MASK: 834 case VFIO_IRQ_SET_ACTION_UNMASK: 835 /* XXX Need masking support exported */ 836 break; 837 case VFIO_IRQ_SET_ACTION_TRIGGER: 838 func = vfio_pci_set_msi_trigger; 839 break; 840 } 841 break; 842 case VFIO_PCI_ERR_IRQ_INDEX: 843 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 844 case VFIO_IRQ_SET_ACTION_TRIGGER: 845 if (pci_is_pcie(vdev->pdev)) 846 func = vfio_pci_set_err_trigger; 847 break; 848 } 849 break; 850 case VFIO_PCI_REQ_IRQ_INDEX: 851 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 852 case VFIO_IRQ_SET_ACTION_TRIGGER: 853 func = vfio_pci_set_req_trigger; 854 break; 855 } 856 break; 857 } 858 859 if (!func) 860 return -ENOTTY; 861 862 return func(vdev, index, start, count, flags, data); 863 } 864