1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020 Intel Corporation 4 * Author: Johannes Berg <johannes@sipsolutions.net> 5 */ 6 #include <linux/module.h> 7 #include <linux/pci.h> 8 #include <linux/virtio.h> 9 #include <linux/virtio_config.h> 10 #include <linux/logic_iomem.h> 11 #include <linux/of_platform.h> 12 #include <linux/irqdomain.h> 13 #include <linux/virtio_pcidev.h> 14 #include <linux/virtio-uml.h> 15 #include <linux/delay.h> 16 #include <linux/msi.h> 17 #include <linux/unaligned.h> 18 #include <irq_kern.h> 19 20 #define MAX_DEVICES 8 21 #define MAX_MSI_VECTORS 32 22 #define CFG_SPACE_SIZE 4096 23 24 /* for MSI-X we have a 32-bit payload */ 25 #define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32)) 26 #define NUM_IRQ_MSGS 10 27 28 struct um_pci_message_buffer { 29 struct virtio_pcidev_msg hdr; 30 u8 data[8]; 31 }; 32 33 struct um_pci_device { 34 struct virtio_device *vdev; 35 36 /* for now just standard BARs */ 37 u8 resptr[PCI_STD_NUM_BARS]; 38 39 struct virtqueue *cmd_vq, *irq_vq; 40 41 #define UM_PCI_WRITE_BUFS 20 42 struct um_pci_message_buffer bufs[UM_PCI_WRITE_BUFS + 1]; 43 void *extra_ptrs[UM_PCI_WRITE_BUFS + 1]; 44 DECLARE_BITMAP(used_bufs, UM_PCI_WRITE_BUFS); 45 46 #define UM_PCI_STAT_WAITING 0 47 unsigned long status; 48 49 int irq; 50 51 bool platform; 52 }; 53 54 struct um_pci_device_reg { 55 struct um_pci_device *dev; 56 void __iomem *iomem; 57 }; 58 59 static struct pci_host_bridge *bridge; 60 static DEFINE_MUTEX(um_pci_mtx); 61 static struct um_pci_device *um_pci_platform_device; 62 static struct um_pci_device_reg um_pci_devices[MAX_DEVICES]; 63 static struct fwnode_handle *um_pci_fwnode; 64 static struct irq_domain *um_pci_inner_domain; 65 static struct irq_domain *um_pci_msi_domain; 66 static unsigned long um_pci_msi_used[BITS_TO_LONGS(MAX_MSI_VECTORS)]; 67 68 static unsigned int um_pci_max_delay_us = 40000; 69 module_param_named(max_delay_us, um_pci_max_delay_us, uint, 0644); 70 71 static int um_pci_get_buf(struct um_pci_device *dev, bool *posted) 72 { 73 int i; 74 75 for (i = 0; i < UM_PCI_WRITE_BUFS; i++) { 76 if (!test_and_set_bit(i, dev->used_bufs)) 77 return i; 78 } 79 80 *posted = false; 81 return UM_PCI_WRITE_BUFS; 82 } 83 84 static void um_pci_free_buf(struct um_pci_device *dev, void *buf) 85 { 86 int i; 87 88 if (buf == &dev->bufs[UM_PCI_WRITE_BUFS]) { 89 kfree(dev->extra_ptrs[UM_PCI_WRITE_BUFS]); 90 dev->extra_ptrs[UM_PCI_WRITE_BUFS] = NULL; 91 return; 92 } 93 94 for (i = 0; i < UM_PCI_WRITE_BUFS; i++) { 95 if (buf == &dev->bufs[i]) { 96 kfree(dev->extra_ptrs[i]); 97 dev->extra_ptrs[i] = NULL; 98 WARN_ON(!test_and_clear_bit(i, dev->used_bufs)); 99 return; 100 } 101 } 102 103 WARN_ON(1); 104 } 105 106 static int um_pci_send_cmd(struct um_pci_device *dev, 107 struct virtio_pcidev_msg *cmd, 108 unsigned int cmd_size, 109 const void *extra, unsigned int extra_size, 110 void *out, unsigned int out_size) 111 { 112 struct scatterlist out_sg, extra_sg, in_sg; 113 struct scatterlist *sgs_list[] = { 114 [0] = &out_sg, 115 [1] = extra ? &extra_sg : &in_sg, 116 [2] = extra ? &in_sg : NULL, 117 }; 118 struct um_pci_message_buffer *buf; 119 int delay_count = 0; 120 bool bounce_out; 121 int ret, len; 122 int buf_idx; 123 bool posted; 124 125 if (WARN_ON(cmd_size < sizeof(*cmd) || cmd_size > sizeof(*buf))) 126 return -EINVAL; 127 128 switch (cmd->op) { 129 case VIRTIO_PCIDEV_OP_CFG_WRITE: 130 case VIRTIO_PCIDEV_OP_MMIO_WRITE: 131 case VIRTIO_PCIDEV_OP_MMIO_MEMSET: 132 /* in PCI, writes are posted, so don't wait */ 133 posted = !out; 134 WARN_ON(!posted); 135 break; 136 default: 137 posted = false; 138 break; 139 } 140 141 bounce_out = !posted && cmd_size <= sizeof(*cmd) && 142 out && out_size <= sizeof(buf->data); 143 144 buf_idx = um_pci_get_buf(dev, &posted); 145 buf = &dev->bufs[buf_idx]; 146 memcpy(buf, cmd, cmd_size); 147 148 if (posted && extra && extra_size > sizeof(buf) - cmd_size) { 149 dev->extra_ptrs[buf_idx] = kmemdup(extra, extra_size, 150 GFP_ATOMIC); 151 152 if (!dev->extra_ptrs[buf_idx]) { 153 um_pci_free_buf(dev, buf); 154 return -ENOMEM; 155 } 156 extra = dev->extra_ptrs[buf_idx]; 157 } else if (extra && extra_size <= sizeof(buf) - cmd_size) { 158 memcpy((u8 *)buf + cmd_size, extra, extra_size); 159 cmd_size += extra_size; 160 extra_size = 0; 161 extra = NULL; 162 cmd = (void *)buf; 163 } else { 164 cmd = (void *)buf; 165 } 166 167 sg_init_one(&out_sg, cmd, cmd_size); 168 if (extra) 169 sg_init_one(&extra_sg, extra, extra_size); 170 /* allow stack for small buffers */ 171 if (bounce_out) 172 sg_init_one(&in_sg, buf->data, out_size); 173 else if (out) 174 sg_init_one(&in_sg, out, out_size); 175 176 /* add to internal virtio queue */ 177 ret = virtqueue_add_sgs(dev->cmd_vq, sgs_list, 178 extra ? 2 : 1, 179 out ? 1 : 0, 180 cmd, GFP_ATOMIC); 181 if (ret) { 182 um_pci_free_buf(dev, buf); 183 return ret; 184 } 185 186 if (posted) { 187 virtqueue_kick(dev->cmd_vq); 188 return 0; 189 } 190 191 /* kick and poll for getting a response on the queue */ 192 set_bit(UM_PCI_STAT_WAITING, &dev->status); 193 virtqueue_kick(dev->cmd_vq); 194 ret = 0; 195 196 while (1) { 197 void *completed = virtqueue_get_buf(dev->cmd_vq, &len); 198 199 if (completed == buf) 200 break; 201 202 if (completed) 203 um_pci_free_buf(dev, completed); 204 205 if (WARN_ONCE(virtqueue_is_broken(dev->cmd_vq) || 206 ++delay_count > um_pci_max_delay_us, 207 "um virt-pci delay: %d", delay_count)) { 208 ret = -EIO; 209 break; 210 } 211 udelay(1); 212 } 213 clear_bit(UM_PCI_STAT_WAITING, &dev->status); 214 215 if (bounce_out) 216 memcpy(out, buf->data, out_size); 217 218 um_pci_free_buf(dev, buf); 219 220 return ret; 221 } 222 223 static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset, 224 int size) 225 { 226 struct um_pci_device_reg *reg = priv; 227 struct um_pci_device *dev = reg->dev; 228 struct virtio_pcidev_msg hdr = { 229 .op = VIRTIO_PCIDEV_OP_CFG_READ, 230 .size = size, 231 .addr = offset, 232 }; 233 /* max 8, we might not use it all */ 234 u8 data[8]; 235 236 if (!dev) 237 return ULONG_MAX; 238 239 memset(data, 0xff, sizeof(data)); 240 241 switch (size) { 242 case 1: 243 case 2: 244 case 4: 245 #ifdef CONFIG_64BIT 246 case 8: 247 #endif 248 break; 249 default: 250 WARN(1, "invalid config space read size %d\n", size); 251 return ULONG_MAX; 252 } 253 254 if (um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, data, size)) 255 return ULONG_MAX; 256 257 switch (size) { 258 case 1: 259 return data[0]; 260 case 2: 261 return le16_to_cpup((void *)data); 262 case 4: 263 return le32_to_cpup((void *)data); 264 #ifdef CONFIG_64BIT 265 case 8: 266 return le64_to_cpup((void *)data); 267 #endif 268 default: 269 return ULONG_MAX; 270 } 271 } 272 273 static void um_pci_cfgspace_write(void *priv, unsigned int offset, int size, 274 unsigned long val) 275 { 276 struct um_pci_device_reg *reg = priv; 277 struct um_pci_device *dev = reg->dev; 278 struct { 279 struct virtio_pcidev_msg hdr; 280 /* maximum size - we may only use parts of it */ 281 u8 data[8]; 282 } msg = { 283 .hdr = { 284 .op = VIRTIO_PCIDEV_OP_CFG_WRITE, 285 .size = size, 286 .addr = offset, 287 }, 288 }; 289 290 if (!dev) 291 return; 292 293 switch (size) { 294 case 1: 295 msg.data[0] = (u8)val; 296 break; 297 case 2: 298 put_unaligned_le16(val, (void *)msg.data); 299 break; 300 case 4: 301 put_unaligned_le32(val, (void *)msg.data); 302 break; 303 #ifdef CONFIG_64BIT 304 case 8: 305 put_unaligned_le64(val, (void *)msg.data); 306 break; 307 #endif 308 default: 309 WARN(1, "invalid config space write size %d\n", size); 310 return; 311 } 312 313 WARN_ON(um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0)); 314 } 315 316 static const struct logic_iomem_ops um_pci_device_cfgspace_ops = { 317 .read = um_pci_cfgspace_read, 318 .write = um_pci_cfgspace_write, 319 }; 320 321 static void um_pci_bar_copy_from(void *priv, void *buffer, 322 unsigned int offset, int size) 323 { 324 u8 *resptr = priv; 325 struct um_pci_device *dev = container_of(resptr - *resptr, 326 struct um_pci_device, 327 resptr[0]); 328 struct virtio_pcidev_msg hdr = { 329 .op = VIRTIO_PCIDEV_OP_MMIO_READ, 330 .bar = *resptr, 331 .size = size, 332 .addr = offset, 333 }; 334 335 memset(buffer, 0xff, size); 336 337 um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, buffer, size); 338 } 339 340 static unsigned long um_pci_bar_read(void *priv, unsigned int offset, 341 int size) 342 { 343 /* 8 is maximum size - we may only use parts of it */ 344 u8 data[8]; 345 346 switch (size) { 347 case 1: 348 case 2: 349 case 4: 350 #ifdef CONFIG_64BIT 351 case 8: 352 #endif 353 break; 354 default: 355 WARN(1, "invalid config space read size %d\n", size); 356 return ULONG_MAX; 357 } 358 359 um_pci_bar_copy_from(priv, data, offset, size); 360 361 switch (size) { 362 case 1: 363 return data[0]; 364 case 2: 365 return le16_to_cpup((void *)data); 366 case 4: 367 return le32_to_cpup((void *)data); 368 #ifdef CONFIG_64BIT 369 case 8: 370 return le64_to_cpup((void *)data); 371 #endif 372 default: 373 return ULONG_MAX; 374 } 375 } 376 377 static void um_pci_bar_copy_to(void *priv, unsigned int offset, 378 const void *buffer, int size) 379 { 380 u8 *resptr = priv; 381 struct um_pci_device *dev = container_of(resptr - *resptr, 382 struct um_pci_device, 383 resptr[0]); 384 struct virtio_pcidev_msg hdr = { 385 .op = VIRTIO_PCIDEV_OP_MMIO_WRITE, 386 .bar = *resptr, 387 .size = size, 388 .addr = offset, 389 }; 390 391 um_pci_send_cmd(dev, &hdr, sizeof(hdr), buffer, size, NULL, 0); 392 } 393 394 static void um_pci_bar_write(void *priv, unsigned int offset, int size, 395 unsigned long val) 396 { 397 /* maximum size - we may only use parts of it */ 398 u8 data[8]; 399 400 switch (size) { 401 case 1: 402 data[0] = (u8)val; 403 break; 404 case 2: 405 put_unaligned_le16(val, (void *)data); 406 break; 407 case 4: 408 put_unaligned_le32(val, (void *)data); 409 break; 410 #ifdef CONFIG_64BIT 411 case 8: 412 put_unaligned_le64(val, (void *)data); 413 break; 414 #endif 415 default: 416 WARN(1, "invalid config space write size %d\n", size); 417 return; 418 } 419 420 um_pci_bar_copy_to(priv, offset, data, size); 421 } 422 423 static void um_pci_bar_set(void *priv, unsigned int offset, u8 value, int size) 424 { 425 u8 *resptr = priv; 426 struct um_pci_device *dev = container_of(resptr - *resptr, 427 struct um_pci_device, 428 resptr[0]); 429 struct { 430 struct virtio_pcidev_msg hdr; 431 u8 data; 432 } msg = { 433 .hdr = { 434 .op = VIRTIO_PCIDEV_OP_CFG_WRITE, 435 .bar = *resptr, 436 .size = size, 437 .addr = offset, 438 }, 439 .data = value, 440 }; 441 442 um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0); 443 } 444 445 static const struct logic_iomem_ops um_pci_device_bar_ops = { 446 .read = um_pci_bar_read, 447 .write = um_pci_bar_write, 448 .set = um_pci_bar_set, 449 .copy_from = um_pci_bar_copy_from, 450 .copy_to = um_pci_bar_copy_to, 451 }; 452 453 static void __iomem *um_pci_map_bus(struct pci_bus *bus, unsigned int devfn, 454 int where) 455 { 456 struct um_pci_device_reg *dev; 457 unsigned int busn = bus->number; 458 459 if (busn > 0) 460 return NULL; 461 462 /* not allowing functions for now ... */ 463 if (devfn % 8) 464 return NULL; 465 466 if (devfn / 8 >= ARRAY_SIZE(um_pci_devices)) 467 return NULL; 468 469 dev = &um_pci_devices[devfn / 8]; 470 if (!dev) 471 return NULL; 472 473 return (void __iomem *)((unsigned long)dev->iomem + where); 474 } 475 476 static struct pci_ops um_pci_ops = { 477 .map_bus = um_pci_map_bus, 478 .read = pci_generic_config_read, 479 .write = pci_generic_config_write, 480 }; 481 482 static void um_pci_rescan(void) 483 { 484 pci_lock_rescan_remove(); 485 pci_rescan_bus(bridge->bus); 486 pci_unlock_rescan_remove(); 487 } 488 489 static void um_pci_irq_vq_addbuf(struct virtqueue *vq, void *buf, bool kick) 490 { 491 struct scatterlist sg[1]; 492 493 sg_init_one(sg, buf, MAX_IRQ_MSG_SIZE); 494 if (virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC)) 495 kfree(buf); 496 else if (kick) 497 virtqueue_kick(vq); 498 } 499 500 static void um_pci_handle_irq_message(struct virtqueue *vq, 501 struct virtio_pcidev_msg *msg) 502 { 503 struct virtio_device *vdev = vq->vdev; 504 struct um_pci_device *dev = vdev->priv; 505 506 if (!dev->irq) 507 return; 508 509 /* we should properly chain interrupts, but on ARCH=um we don't care */ 510 511 switch (msg->op) { 512 case VIRTIO_PCIDEV_OP_INT: 513 generic_handle_irq(dev->irq); 514 break; 515 case VIRTIO_PCIDEV_OP_MSI: 516 /* our MSI message is just the interrupt number */ 517 if (msg->size == sizeof(u32)) 518 generic_handle_irq(le32_to_cpup((void *)msg->data)); 519 else 520 generic_handle_irq(le16_to_cpup((void *)msg->data)); 521 break; 522 case VIRTIO_PCIDEV_OP_PME: 523 /* nothing to do - we already woke up due to the message */ 524 break; 525 default: 526 dev_err(&vdev->dev, "unexpected virt-pci message %d\n", msg->op); 527 break; 528 } 529 } 530 531 static void um_pci_cmd_vq_cb(struct virtqueue *vq) 532 { 533 struct virtio_device *vdev = vq->vdev; 534 struct um_pci_device *dev = vdev->priv; 535 void *cmd; 536 int len; 537 538 if (test_bit(UM_PCI_STAT_WAITING, &dev->status)) 539 return; 540 541 while ((cmd = virtqueue_get_buf(vq, &len))) 542 um_pci_free_buf(dev, cmd); 543 } 544 545 static void um_pci_irq_vq_cb(struct virtqueue *vq) 546 { 547 struct virtio_pcidev_msg *msg; 548 int len; 549 550 while ((msg = virtqueue_get_buf(vq, &len))) { 551 if (len >= sizeof(*msg)) 552 um_pci_handle_irq_message(vq, msg); 553 554 /* recycle the message buffer */ 555 um_pci_irq_vq_addbuf(vq, msg, true); 556 } 557 } 558 559 #ifdef CONFIG_OF 560 /* Copied from arch/x86/kernel/devicetree.c */ 561 struct device_node *pcibios_get_phb_of_node(struct pci_bus *bus) 562 { 563 struct device_node *np; 564 565 for_each_node_by_type(np, "pci") { 566 const void *prop; 567 unsigned int bus_min; 568 569 prop = of_get_property(np, "bus-range", NULL); 570 if (!prop) 571 continue; 572 bus_min = be32_to_cpup(prop); 573 if (bus->number == bus_min) 574 return np; 575 } 576 return NULL; 577 } 578 #endif 579 580 static int um_pci_init_vqs(struct um_pci_device *dev) 581 { 582 struct virtqueue_info vqs_info[] = { 583 { "cmd", um_pci_cmd_vq_cb }, 584 { "irq", um_pci_irq_vq_cb }, 585 }; 586 struct virtqueue *vqs[2]; 587 int err, i; 588 589 err = virtio_find_vqs(dev->vdev, 2, vqs, vqs_info, NULL); 590 if (err) 591 return err; 592 593 dev->cmd_vq = vqs[0]; 594 dev->irq_vq = vqs[1]; 595 596 virtio_device_ready(dev->vdev); 597 598 for (i = 0; i < NUM_IRQ_MSGS; i++) { 599 void *msg = kzalloc(MAX_IRQ_MSG_SIZE, GFP_KERNEL); 600 601 if (msg) 602 um_pci_irq_vq_addbuf(dev->irq_vq, msg, false); 603 } 604 605 virtqueue_kick(dev->irq_vq); 606 607 return 0; 608 } 609 610 static void __um_pci_virtio_platform_remove(struct virtio_device *vdev, 611 struct um_pci_device *dev) 612 { 613 virtio_reset_device(vdev); 614 vdev->config->del_vqs(vdev); 615 616 mutex_lock(&um_pci_mtx); 617 um_pci_platform_device = NULL; 618 mutex_unlock(&um_pci_mtx); 619 620 kfree(dev); 621 } 622 623 static int um_pci_virtio_platform_probe(struct virtio_device *vdev, 624 struct um_pci_device *dev) 625 { 626 int ret; 627 628 dev->platform = true; 629 630 mutex_lock(&um_pci_mtx); 631 632 if (um_pci_platform_device) { 633 mutex_unlock(&um_pci_mtx); 634 ret = -EBUSY; 635 goto out_free; 636 } 637 638 ret = um_pci_init_vqs(dev); 639 if (ret) { 640 mutex_unlock(&um_pci_mtx); 641 goto out_free; 642 } 643 644 um_pci_platform_device = dev; 645 646 mutex_unlock(&um_pci_mtx); 647 648 ret = of_platform_default_populate(vdev->dev.of_node, NULL, &vdev->dev); 649 if (ret) 650 __um_pci_virtio_platform_remove(vdev, dev); 651 652 return ret; 653 654 out_free: 655 kfree(dev); 656 return ret; 657 } 658 659 static int um_pci_virtio_probe(struct virtio_device *vdev) 660 { 661 struct um_pci_device *dev; 662 int i, free = -1; 663 int err = -ENOSPC; 664 665 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 666 if (!dev) 667 return -ENOMEM; 668 669 dev->vdev = vdev; 670 vdev->priv = dev; 671 672 if (of_device_is_compatible(vdev->dev.of_node, "simple-bus")) 673 return um_pci_virtio_platform_probe(vdev, dev); 674 675 mutex_lock(&um_pci_mtx); 676 for (i = 0; i < MAX_DEVICES; i++) { 677 if (um_pci_devices[i].dev) 678 continue; 679 free = i; 680 break; 681 } 682 683 if (free < 0) 684 goto error; 685 686 err = um_pci_init_vqs(dev); 687 if (err) 688 goto error; 689 690 dev->irq = irq_alloc_desc(numa_node_id()); 691 if (dev->irq < 0) { 692 err = dev->irq; 693 goto err_reset; 694 } 695 um_pci_devices[free].dev = dev; 696 vdev->priv = dev; 697 698 mutex_unlock(&um_pci_mtx); 699 700 device_set_wakeup_enable(&vdev->dev, true); 701 702 /* 703 * In order to do suspend-resume properly, don't allow VQs 704 * to be suspended. 705 */ 706 virtio_uml_set_no_vq_suspend(vdev, true); 707 708 um_pci_rescan(); 709 return 0; 710 err_reset: 711 virtio_reset_device(vdev); 712 vdev->config->del_vqs(vdev); 713 error: 714 mutex_unlock(&um_pci_mtx); 715 kfree(dev); 716 return err; 717 } 718 719 static void um_pci_virtio_remove(struct virtio_device *vdev) 720 { 721 struct um_pci_device *dev = vdev->priv; 722 int i; 723 724 if (dev->platform) { 725 of_platform_depopulate(&vdev->dev); 726 __um_pci_virtio_platform_remove(vdev, dev); 727 return; 728 } 729 730 device_set_wakeup_enable(&vdev->dev, false); 731 732 mutex_lock(&um_pci_mtx); 733 for (i = 0; i < MAX_DEVICES; i++) { 734 if (um_pci_devices[i].dev != dev) 735 continue; 736 737 um_pci_devices[i].dev = NULL; 738 irq_free_desc(dev->irq); 739 740 break; 741 } 742 mutex_unlock(&um_pci_mtx); 743 744 if (i < MAX_DEVICES) { 745 struct pci_dev *pci_dev; 746 747 pci_dev = pci_get_slot(bridge->bus, i); 748 if (pci_dev) 749 pci_stop_and_remove_bus_device_locked(pci_dev); 750 } 751 752 /* Stop all virtqueues */ 753 virtio_reset_device(vdev); 754 dev->cmd_vq = NULL; 755 dev->irq_vq = NULL; 756 vdev->config->del_vqs(vdev); 757 758 kfree(dev); 759 } 760 761 static struct virtio_device_id id_table[] = { 762 { CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID, VIRTIO_DEV_ANY_ID }, 763 { 0 }, 764 }; 765 MODULE_DEVICE_TABLE(virtio, id_table); 766 767 static struct virtio_driver um_pci_virtio_driver = { 768 .driver.name = "virtio-pci", 769 .id_table = id_table, 770 .probe = um_pci_virtio_probe, 771 .remove = um_pci_virtio_remove, 772 }; 773 774 static struct resource virt_cfgspace_resource = { 775 .name = "PCI config space", 776 .start = 0xf0000000 - MAX_DEVICES * CFG_SPACE_SIZE, 777 .end = 0xf0000000 - 1, 778 .flags = IORESOURCE_MEM, 779 }; 780 781 static long um_pci_map_cfgspace(unsigned long offset, size_t size, 782 const struct logic_iomem_ops **ops, 783 void **priv) 784 { 785 if (WARN_ON(size > CFG_SPACE_SIZE || offset % CFG_SPACE_SIZE)) 786 return -EINVAL; 787 788 if (offset / CFG_SPACE_SIZE < MAX_DEVICES) { 789 *ops = &um_pci_device_cfgspace_ops; 790 *priv = &um_pci_devices[offset / CFG_SPACE_SIZE]; 791 return 0; 792 } 793 794 WARN(1, "cannot map offset 0x%lx/0x%zx\n", offset, size); 795 return -ENOENT; 796 } 797 798 static const struct logic_iomem_region_ops um_pci_cfgspace_ops = { 799 .map = um_pci_map_cfgspace, 800 }; 801 802 static struct resource virt_iomem_resource = { 803 .name = "PCI iomem", 804 .start = 0xf0000000, 805 .end = 0xffffffff, 806 .flags = IORESOURCE_MEM, 807 }; 808 809 struct um_pci_map_iomem_data { 810 unsigned long offset; 811 size_t size; 812 const struct logic_iomem_ops **ops; 813 void **priv; 814 long ret; 815 }; 816 817 static int um_pci_map_iomem_walk(struct pci_dev *pdev, void *_data) 818 { 819 struct um_pci_map_iomem_data *data = _data; 820 struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8]; 821 struct um_pci_device *dev; 822 int i; 823 824 if (!reg->dev) 825 return 0; 826 827 for (i = 0; i < ARRAY_SIZE(dev->resptr); i++) { 828 struct resource *r = &pdev->resource[i]; 829 830 if ((r->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM) 831 continue; 832 833 /* 834 * must be the whole or part of the resource, 835 * not allowed to only overlap 836 */ 837 if (data->offset < r->start || data->offset > r->end) 838 continue; 839 if (data->offset + data->size - 1 > r->end) 840 continue; 841 842 dev = reg->dev; 843 *data->ops = &um_pci_device_bar_ops; 844 dev->resptr[i] = i; 845 *data->priv = &dev->resptr[i]; 846 data->ret = data->offset - r->start; 847 848 /* no need to continue */ 849 return 1; 850 } 851 852 return 0; 853 } 854 855 static long um_pci_map_iomem(unsigned long offset, size_t size, 856 const struct logic_iomem_ops **ops, 857 void **priv) 858 { 859 struct um_pci_map_iomem_data data = { 860 /* we want the full address here */ 861 .offset = offset + virt_iomem_resource.start, 862 .size = size, 863 .ops = ops, 864 .priv = priv, 865 .ret = -ENOENT, 866 }; 867 868 pci_walk_bus(bridge->bus, um_pci_map_iomem_walk, &data); 869 return data.ret; 870 } 871 872 static const struct logic_iomem_region_ops um_pci_iomem_ops = { 873 .map = um_pci_map_iomem, 874 }; 875 876 static void um_pci_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) 877 { 878 /* 879 * This is a very low address and not actually valid 'physical' memory 880 * in UML, so we can simply map MSI(-X) vectors to there, it cannot be 881 * legitimately written to by the device in any other way. 882 * We use the (virtual) IRQ number here as the message to simplify the 883 * code that receives the message, where for now we simply trust the 884 * device to send the correct message. 885 */ 886 msg->address_hi = 0; 887 msg->address_lo = 0xa0000; 888 msg->data = data->irq; 889 } 890 891 static struct irq_chip um_pci_msi_bottom_irq_chip = { 892 .name = "UM virtio MSI", 893 .irq_compose_msi_msg = um_pci_compose_msi_msg, 894 }; 895 896 static int um_pci_inner_domain_alloc(struct irq_domain *domain, 897 unsigned int virq, unsigned int nr_irqs, 898 void *args) 899 { 900 unsigned long bit; 901 902 WARN_ON(nr_irqs != 1); 903 904 mutex_lock(&um_pci_mtx); 905 bit = find_first_zero_bit(um_pci_msi_used, MAX_MSI_VECTORS); 906 if (bit >= MAX_MSI_VECTORS) { 907 mutex_unlock(&um_pci_mtx); 908 return -ENOSPC; 909 } 910 911 set_bit(bit, um_pci_msi_used); 912 mutex_unlock(&um_pci_mtx); 913 914 irq_domain_set_info(domain, virq, bit, &um_pci_msi_bottom_irq_chip, 915 domain->host_data, handle_simple_irq, 916 NULL, NULL); 917 918 return 0; 919 } 920 921 static void um_pci_inner_domain_free(struct irq_domain *domain, 922 unsigned int virq, unsigned int nr_irqs) 923 { 924 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 925 926 mutex_lock(&um_pci_mtx); 927 928 if (!test_bit(d->hwirq, um_pci_msi_used)) 929 pr_err("trying to free unused MSI#%lu\n", d->hwirq); 930 else 931 __clear_bit(d->hwirq, um_pci_msi_used); 932 933 mutex_unlock(&um_pci_mtx); 934 } 935 936 static const struct irq_domain_ops um_pci_inner_domain_ops = { 937 .alloc = um_pci_inner_domain_alloc, 938 .free = um_pci_inner_domain_free, 939 }; 940 941 static struct irq_chip um_pci_msi_irq_chip = { 942 .name = "UM virtio PCIe MSI", 943 .irq_mask = pci_msi_mask_irq, 944 .irq_unmask = pci_msi_unmask_irq, 945 }; 946 947 static struct msi_domain_info um_pci_msi_domain_info = { 948 .flags = MSI_FLAG_USE_DEF_DOM_OPS | 949 MSI_FLAG_USE_DEF_CHIP_OPS | 950 MSI_FLAG_PCI_MSIX, 951 .chip = &um_pci_msi_irq_chip, 952 }; 953 954 static struct resource busn_resource = { 955 .name = "PCI busn", 956 .start = 0, 957 .end = 0, 958 .flags = IORESOURCE_BUS, 959 }; 960 961 static int um_pci_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin) 962 { 963 struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8]; 964 965 if (WARN_ON(!reg->dev)) 966 return -EINVAL; 967 968 /* Yes, we map all pins to the same IRQ ... doesn't matter for now. */ 969 return reg->dev->irq; 970 } 971 972 void *pci_root_bus_fwnode(struct pci_bus *bus) 973 { 974 return um_pci_fwnode; 975 } 976 977 static long um_pci_map_platform(unsigned long offset, size_t size, 978 const struct logic_iomem_ops **ops, 979 void **priv) 980 { 981 if (!um_pci_platform_device) 982 return -ENOENT; 983 984 *ops = &um_pci_device_bar_ops; 985 *priv = &um_pci_platform_device->resptr[0]; 986 987 return offset; 988 } 989 990 static const struct logic_iomem_region_ops um_pci_platform_ops = { 991 .map = um_pci_map_platform, 992 }; 993 994 static struct resource virt_platform_resource = { 995 .name = "platform", 996 .start = 0x10000000, 997 .end = 0x1fffffff, 998 .flags = IORESOURCE_MEM, 999 }; 1000 1001 static int __init um_pci_init(void) 1002 { 1003 struct irq_domain_info inner_domain_info = { 1004 .size = MAX_MSI_VECTORS, 1005 .hwirq_max = MAX_MSI_VECTORS, 1006 .ops = &um_pci_inner_domain_ops, 1007 }; 1008 int err, i; 1009 1010 WARN_ON(logic_iomem_add_region(&virt_cfgspace_resource, 1011 &um_pci_cfgspace_ops)); 1012 WARN_ON(logic_iomem_add_region(&virt_iomem_resource, 1013 &um_pci_iomem_ops)); 1014 WARN_ON(logic_iomem_add_region(&virt_platform_resource, 1015 &um_pci_platform_ops)); 1016 1017 if (WARN(CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID < 0, 1018 "No virtio device ID configured for PCI - no PCI support\n")) 1019 return 0; 1020 1021 bridge = pci_alloc_host_bridge(0); 1022 if (!bridge) { 1023 err = -ENOMEM; 1024 goto free; 1025 } 1026 1027 um_pci_fwnode = irq_domain_alloc_named_fwnode("um-pci"); 1028 if (!um_pci_fwnode) { 1029 err = -ENOMEM; 1030 goto free; 1031 } 1032 1033 inner_domain_info.fwnode = um_pci_fwnode; 1034 um_pci_inner_domain = irq_domain_instantiate(&inner_domain_info); 1035 if (IS_ERR(um_pci_inner_domain)) { 1036 err = PTR_ERR(um_pci_inner_domain); 1037 goto free; 1038 } 1039 1040 um_pci_msi_domain = pci_msi_create_irq_domain(um_pci_fwnode, 1041 &um_pci_msi_domain_info, 1042 um_pci_inner_domain); 1043 if (!um_pci_msi_domain) { 1044 err = -ENOMEM; 1045 goto free; 1046 } 1047 1048 pci_add_resource(&bridge->windows, &virt_iomem_resource); 1049 pci_add_resource(&bridge->windows, &busn_resource); 1050 bridge->ops = &um_pci_ops; 1051 bridge->map_irq = um_pci_map_irq; 1052 1053 for (i = 0; i < MAX_DEVICES; i++) { 1054 resource_size_t start; 1055 1056 start = virt_cfgspace_resource.start + i * CFG_SPACE_SIZE; 1057 um_pci_devices[i].iomem = ioremap(start, CFG_SPACE_SIZE); 1058 if (WARN(!um_pci_devices[i].iomem, "failed to map %d\n", i)) { 1059 err = -ENOMEM; 1060 goto free; 1061 } 1062 } 1063 1064 err = pci_host_probe(bridge); 1065 if (err) 1066 goto free; 1067 1068 err = register_virtio_driver(&um_pci_virtio_driver); 1069 if (err) 1070 goto free; 1071 return 0; 1072 free: 1073 if (!IS_ERR_OR_NULL(um_pci_inner_domain)) 1074 irq_domain_remove(um_pci_inner_domain); 1075 if (um_pci_fwnode) 1076 irq_domain_free_fwnode(um_pci_fwnode); 1077 if (bridge) { 1078 pci_free_resource_list(&bridge->windows); 1079 pci_free_host_bridge(bridge); 1080 } 1081 return err; 1082 } 1083 module_init(um_pci_init); 1084 1085 static void __exit um_pci_exit(void) 1086 { 1087 unregister_virtio_driver(&um_pci_virtio_driver); 1088 irq_domain_remove(um_pci_msi_domain); 1089 irq_domain_remove(um_pci_inner_domain); 1090 pci_free_resource_list(&bridge->windows); 1091 pci_free_host_bridge(bridge); 1092 } 1093 module_exit(um_pci_exit); 1094