1d1b054daSYu Zhao /* 2d1b054daSYu Zhao * drivers/pci/iov.c 3d1b054daSYu Zhao * 4d1b054daSYu Zhao * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 5d1b054daSYu Zhao * 6d1b054daSYu Zhao * PCI Express I/O Virtualization (IOV) support. 7d1b054daSYu Zhao * Single Root IOV 1.0 8d1b054daSYu Zhao */ 9d1b054daSYu Zhao 10d1b054daSYu Zhao #include <linux/pci.h> 11d1b054daSYu Zhao #include <linux/mutex.h> 12d1b054daSYu Zhao #include <linux/string.h> 13d1b054daSYu Zhao #include <linux/delay.h> 14d1b054daSYu Zhao #include "pci.h" 15d1b054daSYu Zhao 16dd7cc44dSYu Zhao #define VIRTFN_ID_LEN 16 17d1b054daSYu Zhao 18a28724b0SYu Zhao static inline u8 virtfn_bus(struct pci_dev *dev, int id) 19a28724b0SYu Zhao { 20a28724b0SYu Zhao return dev->bus->number + ((dev->devfn + dev->sriov->offset + 21a28724b0SYu Zhao dev->sriov->stride * id) >> 8); 22a28724b0SYu Zhao } 23a28724b0SYu Zhao 24a28724b0SYu Zhao static inline u8 virtfn_devfn(struct pci_dev *dev, int id) 25a28724b0SYu Zhao { 26a28724b0SYu Zhao return (dev->devfn + dev->sriov->offset + 27a28724b0SYu Zhao dev->sriov->stride * id) & 0xff; 28a28724b0SYu Zhao } 29a28724b0SYu Zhao 30dd7cc44dSYu Zhao static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 31dd7cc44dSYu Zhao { 32dd7cc44dSYu Zhao int rc; 33dd7cc44dSYu Zhao struct pci_bus *child; 34dd7cc44dSYu Zhao 35dd7cc44dSYu Zhao if (bus->number == busnr) 36dd7cc44dSYu Zhao return bus; 37dd7cc44dSYu Zhao 38dd7cc44dSYu Zhao child = pci_find_bus(pci_domain_nr(bus), busnr); 39dd7cc44dSYu Zhao if (child) 40dd7cc44dSYu Zhao return child; 41dd7cc44dSYu Zhao 42dd7cc44dSYu Zhao child = pci_add_new_bus(bus, NULL, busnr); 43dd7cc44dSYu Zhao if (!child) 44dd7cc44dSYu Zhao return NULL; 45dd7cc44dSYu Zhao 46dd7cc44dSYu Zhao child->subordinate = busnr; 47dd7cc44dSYu Zhao child->dev.parent = bus->bridge; 48dd7cc44dSYu Zhao rc = pci_bus_add_child(child); 49dd7cc44dSYu Zhao if (rc) { 50dd7cc44dSYu Zhao pci_remove_bus(child); 51dd7cc44dSYu Zhao return NULL; 52dd7cc44dSYu Zhao } 53dd7cc44dSYu Zhao 54dd7cc44dSYu Zhao return child; 55dd7cc44dSYu Zhao } 56dd7cc44dSYu Zhao 57dd7cc44dSYu Zhao static void virtfn_remove_bus(struct pci_bus *bus, int busnr) 58dd7cc44dSYu Zhao { 59dd7cc44dSYu Zhao struct pci_bus *child; 60dd7cc44dSYu Zhao 61dd7cc44dSYu Zhao if (bus->number == busnr) 62dd7cc44dSYu Zhao return; 63dd7cc44dSYu Zhao 64dd7cc44dSYu Zhao child = pci_find_bus(pci_domain_nr(bus), busnr); 65dd7cc44dSYu Zhao BUG_ON(!child); 66dd7cc44dSYu Zhao 67dd7cc44dSYu Zhao if (list_empty(&child->devices)) 68dd7cc44dSYu Zhao pci_remove_bus(child); 69dd7cc44dSYu Zhao } 70dd7cc44dSYu Zhao 71dd7cc44dSYu Zhao static int virtfn_add(struct pci_dev *dev, int id, int reset) 72dd7cc44dSYu Zhao { 73dd7cc44dSYu Zhao int i; 74dd7cc44dSYu Zhao int rc; 75dd7cc44dSYu Zhao u64 size; 76dd7cc44dSYu Zhao char buf[VIRTFN_ID_LEN]; 77dd7cc44dSYu Zhao struct pci_dev *virtfn; 78dd7cc44dSYu Zhao struct resource *res; 79dd7cc44dSYu Zhao struct pci_sriov *iov = dev->sriov; 80dd7cc44dSYu Zhao 81dd7cc44dSYu Zhao virtfn = alloc_pci_dev(); 82dd7cc44dSYu Zhao if (!virtfn) 83dd7cc44dSYu Zhao return -ENOMEM; 84dd7cc44dSYu Zhao 85dd7cc44dSYu Zhao mutex_lock(&iov->dev->sriov->lock); 86dd7cc44dSYu Zhao virtfn->bus = virtfn_add_bus(dev->bus, virtfn_bus(dev, id)); 87dd7cc44dSYu Zhao if (!virtfn->bus) { 88dd7cc44dSYu Zhao kfree(virtfn); 89dd7cc44dSYu Zhao mutex_unlock(&iov->dev->sriov->lock); 90dd7cc44dSYu Zhao return -ENOMEM; 91dd7cc44dSYu Zhao } 92dd7cc44dSYu Zhao virtfn->devfn = virtfn_devfn(dev, id); 93dd7cc44dSYu Zhao virtfn->vendor = dev->vendor; 94dd7cc44dSYu Zhao pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device); 95dd7cc44dSYu Zhao pci_setup_device(virtfn); 96dd7cc44dSYu Zhao virtfn->dev.parent = dev->dev.parent; 97dd7cc44dSYu Zhao 98dd7cc44dSYu Zhao for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 99dd7cc44dSYu Zhao res = dev->resource + PCI_IOV_RESOURCES + i; 100dd7cc44dSYu Zhao if (!res->parent) 101dd7cc44dSYu Zhao continue; 102dd7cc44dSYu Zhao virtfn->resource[i].name = pci_name(virtfn); 103dd7cc44dSYu Zhao virtfn->resource[i].flags = res->flags; 104dd7cc44dSYu Zhao size = resource_size(res); 105dd7cc44dSYu Zhao do_div(size, iov->total); 106dd7cc44dSYu Zhao virtfn->resource[i].start = res->start + size * id; 107dd7cc44dSYu Zhao virtfn->resource[i].end = virtfn->resource[i].start + size - 1; 108dd7cc44dSYu Zhao rc = request_resource(res, &virtfn->resource[i]); 109dd7cc44dSYu Zhao BUG_ON(rc); 110dd7cc44dSYu Zhao } 111dd7cc44dSYu Zhao 112dd7cc44dSYu Zhao if (reset) 113dd7cc44dSYu Zhao pci_execute_reset_function(virtfn); 114dd7cc44dSYu Zhao 115dd7cc44dSYu Zhao pci_device_add(virtfn, virtfn->bus); 116dd7cc44dSYu Zhao mutex_unlock(&iov->dev->sriov->lock); 117dd7cc44dSYu Zhao 118dd7cc44dSYu Zhao virtfn->physfn = pci_dev_get(dev); 119dd7cc44dSYu Zhao virtfn->is_virtfn = 1; 120dd7cc44dSYu Zhao 121dd7cc44dSYu Zhao rc = pci_bus_add_device(virtfn); 122dd7cc44dSYu Zhao if (rc) 123dd7cc44dSYu Zhao goto failed1; 124dd7cc44dSYu Zhao sprintf(buf, "virtfn%u", id); 125dd7cc44dSYu Zhao rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 126dd7cc44dSYu Zhao if (rc) 127dd7cc44dSYu Zhao goto failed1; 128dd7cc44dSYu Zhao rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 129dd7cc44dSYu Zhao if (rc) 130dd7cc44dSYu Zhao goto failed2; 131dd7cc44dSYu Zhao 132dd7cc44dSYu Zhao kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 133dd7cc44dSYu Zhao 134dd7cc44dSYu Zhao return 0; 135dd7cc44dSYu Zhao 136dd7cc44dSYu Zhao failed2: 137dd7cc44dSYu Zhao sysfs_remove_link(&dev->dev.kobj, buf); 138dd7cc44dSYu Zhao failed1: 139dd7cc44dSYu Zhao pci_dev_put(dev); 140dd7cc44dSYu Zhao mutex_lock(&iov->dev->sriov->lock); 141dd7cc44dSYu Zhao pci_remove_bus_device(virtfn); 142dd7cc44dSYu Zhao virtfn_remove_bus(dev->bus, virtfn_bus(dev, id)); 143dd7cc44dSYu Zhao mutex_unlock(&iov->dev->sriov->lock); 144dd7cc44dSYu Zhao 145dd7cc44dSYu Zhao return rc; 146dd7cc44dSYu Zhao } 147dd7cc44dSYu Zhao 148dd7cc44dSYu Zhao static void virtfn_remove(struct pci_dev *dev, int id, int reset) 149dd7cc44dSYu Zhao { 150dd7cc44dSYu Zhao char buf[VIRTFN_ID_LEN]; 151dd7cc44dSYu Zhao struct pci_bus *bus; 152dd7cc44dSYu Zhao struct pci_dev *virtfn; 153dd7cc44dSYu Zhao struct pci_sriov *iov = dev->sriov; 154dd7cc44dSYu Zhao 155dd7cc44dSYu Zhao bus = pci_find_bus(pci_domain_nr(dev->bus), virtfn_bus(dev, id)); 156dd7cc44dSYu Zhao if (!bus) 157dd7cc44dSYu Zhao return; 158dd7cc44dSYu Zhao 159dd7cc44dSYu Zhao virtfn = pci_get_slot(bus, virtfn_devfn(dev, id)); 160dd7cc44dSYu Zhao if (!virtfn) 161dd7cc44dSYu Zhao return; 162dd7cc44dSYu Zhao 163dd7cc44dSYu Zhao pci_dev_put(virtfn); 164dd7cc44dSYu Zhao 165dd7cc44dSYu Zhao if (reset) { 166dd7cc44dSYu Zhao device_release_driver(&virtfn->dev); 167dd7cc44dSYu Zhao pci_execute_reset_function(virtfn); 168dd7cc44dSYu Zhao } 169dd7cc44dSYu Zhao 170dd7cc44dSYu Zhao sprintf(buf, "virtfn%u", id); 171dd7cc44dSYu Zhao sysfs_remove_link(&dev->dev.kobj, buf); 172dd7cc44dSYu Zhao sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 173dd7cc44dSYu Zhao 174dd7cc44dSYu Zhao mutex_lock(&iov->dev->sriov->lock); 175dd7cc44dSYu Zhao pci_remove_bus_device(virtfn); 176dd7cc44dSYu Zhao virtfn_remove_bus(dev->bus, virtfn_bus(dev, id)); 177dd7cc44dSYu Zhao mutex_unlock(&iov->dev->sriov->lock); 178dd7cc44dSYu Zhao 179dd7cc44dSYu Zhao pci_dev_put(dev); 180dd7cc44dSYu Zhao } 181dd7cc44dSYu Zhao 18274bb1bccSYu Zhao static int sriov_migration(struct pci_dev *dev) 18374bb1bccSYu Zhao { 18474bb1bccSYu Zhao u16 status; 18574bb1bccSYu Zhao struct pci_sriov *iov = dev->sriov; 18674bb1bccSYu Zhao 18774bb1bccSYu Zhao if (!iov->nr_virtfn) 18874bb1bccSYu Zhao return 0; 18974bb1bccSYu Zhao 19074bb1bccSYu Zhao if (!(iov->cap & PCI_SRIOV_CAP_VFM)) 19174bb1bccSYu Zhao return 0; 19274bb1bccSYu Zhao 19374bb1bccSYu Zhao pci_read_config_word(dev, iov->pos + PCI_SRIOV_STATUS, &status); 19474bb1bccSYu Zhao if (!(status & PCI_SRIOV_STATUS_VFM)) 19574bb1bccSYu Zhao return 0; 19674bb1bccSYu Zhao 19774bb1bccSYu Zhao schedule_work(&iov->mtask); 19874bb1bccSYu Zhao 19974bb1bccSYu Zhao return 1; 20074bb1bccSYu Zhao } 20174bb1bccSYu Zhao 20274bb1bccSYu Zhao static void sriov_migration_task(struct work_struct *work) 20374bb1bccSYu Zhao { 20474bb1bccSYu Zhao int i; 20574bb1bccSYu Zhao u8 state; 20674bb1bccSYu Zhao u16 status; 20774bb1bccSYu Zhao struct pci_sriov *iov = container_of(work, struct pci_sriov, mtask); 20874bb1bccSYu Zhao 20974bb1bccSYu Zhao for (i = iov->initial; i < iov->nr_virtfn; i++) { 21074bb1bccSYu Zhao state = readb(iov->mstate + i); 21174bb1bccSYu Zhao if (state == PCI_SRIOV_VFM_MI) { 21274bb1bccSYu Zhao writeb(PCI_SRIOV_VFM_AV, iov->mstate + i); 21374bb1bccSYu Zhao state = readb(iov->mstate + i); 21474bb1bccSYu Zhao if (state == PCI_SRIOV_VFM_AV) 21574bb1bccSYu Zhao virtfn_add(iov->self, i, 1); 21674bb1bccSYu Zhao } else if (state == PCI_SRIOV_VFM_MO) { 21774bb1bccSYu Zhao virtfn_remove(iov->self, i, 1); 21874bb1bccSYu Zhao writeb(PCI_SRIOV_VFM_UA, iov->mstate + i); 21974bb1bccSYu Zhao state = readb(iov->mstate + i); 22074bb1bccSYu Zhao if (state == PCI_SRIOV_VFM_AV) 22174bb1bccSYu Zhao virtfn_add(iov->self, i, 0); 22274bb1bccSYu Zhao } 22374bb1bccSYu Zhao } 22474bb1bccSYu Zhao 22574bb1bccSYu Zhao pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status); 22674bb1bccSYu Zhao status &= ~PCI_SRIOV_STATUS_VFM; 22774bb1bccSYu Zhao pci_write_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, status); 22874bb1bccSYu Zhao } 22974bb1bccSYu Zhao 23074bb1bccSYu Zhao static int sriov_enable_migration(struct pci_dev *dev, int nr_virtfn) 23174bb1bccSYu Zhao { 23274bb1bccSYu Zhao int bir; 23374bb1bccSYu Zhao u32 table; 23474bb1bccSYu Zhao resource_size_t pa; 23574bb1bccSYu Zhao struct pci_sriov *iov = dev->sriov; 23674bb1bccSYu Zhao 23774bb1bccSYu Zhao if (nr_virtfn <= iov->initial) 23874bb1bccSYu Zhao return 0; 23974bb1bccSYu Zhao 24074bb1bccSYu Zhao pci_read_config_dword(dev, iov->pos + PCI_SRIOV_VFM, &table); 24174bb1bccSYu Zhao bir = PCI_SRIOV_VFM_BIR(table); 24274bb1bccSYu Zhao if (bir > PCI_STD_RESOURCE_END) 24374bb1bccSYu Zhao return -EIO; 24474bb1bccSYu Zhao 24574bb1bccSYu Zhao table = PCI_SRIOV_VFM_OFFSET(table); 24674bb1bccSYu Zhao if (table + nr_virtfn > pci_resource_len(dev, bir)) 24774bb1bccSYu Zhao return -EIO; 24874bb1bccSYu Zhao 24974bb1bccSYu Zhao pa = pci_resource_start(dev, bir) + table; 25074bb1bccSYu Zhao iov->mstate = ioremap(pa, nr_virtfn); 25174bb1bccSYu Zhao if (!iov->mstate) 25274bb1bccSYu Zhao return -ENOMEM; 25374bb1bccSYu Zhao 25474bb1bccSYu Zhao INIT_WORK(&iov->mtask, sriov_migration_task); 25574bb1bccSYu Zhao 25674bb1bccSYu Zhao iov->ctrl |= PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR; 25774bb1bccSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 25874bb1bccSYu Zhao 25974bb1bccSYu Zhao return 0; 26074bb1bccSYu Zhao } 26174bb1bccSYu Zhao 26274bb1bccSYu Zhao static void sriov_disable_migration(struct pci_dev *dev) 26374bb1bccSYu Zhao { 26474bb1bccSYu Zhao struct pci_sriov *iov = dev->sriov; 26574bb1bccSYu Zhao 26674bb1bccSYu Zhao iov->ctrl &= ~(PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR); 26774bb1bccSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 26874bb1bccSYu Zhao 26974bb1bccSYu Zhao cancel_work_sync(&iov->mtask); 27074bb1bccSYu Zhao iounmap(iov->mstate); 27174bb1bccSYu Zhao } 27274bb1bccSYu Zhao 273dd7cc44dSYu Zhao static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 274dd7cc44dSYu Zhao { 275dd7cc44dSYu Zhao int rc; 276dd7cc44dSYu Zhao int i, j; 277dd7cc44dSYu Zhao int nres; 278dd7cc44dSYu Zhao u16 offset, stride, initial; 279dd7cc44dSYu Zhao struct resource *res; 280dd7cc44dSYu Zhao struct pci_dev *pdev; 281dd7cc44dSYu Zhao struct pci_sriov *iov = dev->sriov; 282dd7cc44dSYu Zhao 283dd7cc44dSYu Zhao if (!nr_virtfn) 284dd7cc44dSYu Zhao return 0; 285dd7cc44dSYu Zhao 286dd7cc44dSYu Zhao if (iov->nr_virtfn) 287dd7cc44dSYu Zhao return -EINVAL; 288dd7cc44dSYu Zhao 289dd7cc44dSYu Zhao pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 290dd7cc44dSYu Zhao if (initial > iov->total || 291dd7cc44dSYu Zhao (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total))) 292dd7cc44dSYu Zhao return -EIO; 293dd7cc44dSYu Zhao 294dd7cc44dSYu Zhao if (nr_virtfn < 0 || nr_virtfn > iov->total || 295dd7cc44dSYu Zhao (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 296dd7cc44dSYu Zhao return -EINVAL; 297dd7cc44dSYu Zhao 298dd7cc44dSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 299dd7cc44dSYu Zhao pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &offset); 300dd7cc44dSYu Zhao pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &stride); 301dd7cc44dSYu Zhao if (!offset || (nr_virtfn > 1 && !stride)) 302dd7cc44dSYu Zhao return -EIO; 303dd7cc44dSYu Zhao 304dd7cc44dSYu Zhao nres = 0; 305dd7cc44dSYu Zhao for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 306dd7cc44dSYu Zhao res = dev->resource + PCI_IOV_RESOURCES + i; 307dd7cc44dSYu Zhao if (res->parent) 308dd7cc44dSYu Zhao nres++; 309dd7cc44dSYu Zhao } 310dd7cc44dSYu Zhao if (nres != iov->nres) { 311dd7cc44dSYu Zhao dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n"); 312dd7cc44dSYu Zhao return -ENOMEM; 313dd7cc44dSYu Zhao } 314dd7cc44dSYu Zhao 315dd7cc44dSYu Zhao iov->offset = offset; 316dd7cc44dSYu Zhao iov->stride = stride; 317dd7cc44dSYu Zhao 318dd7cc44dSYu Zhao if (virtfn_bus(dev, nr_virtfn - 1) > dev->bus->subordinate) { 319dd7cc44dSYu Zhao dev_err(&dev->dev, "SR-IOV: bus number out of range\n"); 320dd7cc44dSYu Zhao return -ENOMEM; 321dd7cc44dSYu Zhao } 322dd7cc44dSYu Zhao 323dd7cc44dSYu Zhao if (iov->link != dev->devfn) { 324dd7cc44dSYu Zhao pdev = pci_get_slot(dev->bus, iov->link); 325dd7cc44dSYu Zhao if (!pdev) 326dd7cc44dSYu Zhao return -ENODEV; 327dd7cc44dSYu Zhao 328dd7cc44dSYu Zhao pci_dev_put(pdev); 329dd7cc44dSYu Zhao 330dd7cc44dSYu Zhao if (!pdev->is_physfn) 331dd7cc44dSYu Zhao return -ENODEV; 332dd7cc44dSYu Zhao 333dd7cc44dSYu Zhao rc = sysfs_create_link(&dev->dev.kobj, 334dd7cc44dSYu Zhao &pdev->dev.kobj, "dep_link"); 335dd7cc44dSYu Zhao if (rc) 336dd7cc44dSYu Zhao return rc; 337dd7cc44dSYu Zhao } 338dd7cc44dSYu Zhao 339dd7cc44dSYu Zhao iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 340dd7cc44dSYu Zhao pci_block_user_cfg_access(dev); 341dd7cc44dSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 342dd7cc44dSYu Zhao msleep(100); 343dd7cc44dSYu Zhao pci_unblock_user_cfg_access(dev); 344dd7cc44dSYu Zhao 345dd7cc44dSYu Zhao iov->initial = initial; 346dd7cc44dSYu Zhao if (nr_virtfn < initial) 347dd7cc44dSYu Zhao initial = nr_virtfn; 348dd7cc44dSYu Zhao 349dd7cc44dSYu Zhao for (i = 0; i < initial; i++) { 350dd7cc44dSYu Zhao rc = virtfn_add(dev, i, 0); 351dd7cc44dSYu Zhao if (rc) 352dd7cc44dSYu Zhao goto failed; 353dd7cc44dSYu Zhao } 354dd7cc44dSYu Zhao 35574bb1bccSYu Zhao if (iov->cap & PCI_SRIOV_CAP_VFM) { 35674bb1bccSYu Zhao rc = sriov_enable_migration(dev, nr_virtfn); 35774bb1bccSYu Zhao if (rc) 35874bb1bccSYu Zhao goto failed; 35974bb1bccSYu Zhao } 36074bb1bccSYu Zhao 361dd7cc44dSYu Zhao kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 362dd7cc44dSYu Zhao iov->nr_virtfn = nr_virtfn; 363dd7cc44dSYu Zhao 364dd7cc44dSYu Zhao return 0; 365dd7cc44dSYu Zhao 366dd7cc44dSYu Zhao failed: 367dd7cc44dSYu Zhao for (j = 0; j < i; j++) 368dd7cc44dSYu Zhao virtfn_remove(dev, j, 0); 369dd7cc44dSYu Zhao 370dd7cc44dSYu Zhao iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 371dd7cc44dSYu Zhao pci_block_user_cfg_access(dev); 372dd7cc44dSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 373dd7cc44dSYu Zhao ssleep(1); 374dd7cc44dSYu Zhao pci_unblock_user_cfg_access(dev); 375dd7cc44dSYu Zhao 376dd7cc44dSYu Zhao if (iov->link != dev->devfn) 377dd7cc44dSYu Zhao sysfs_remove_link(&dev->dev.kobj, "dep_link"); 378dd7cc44dSYu Zhao 379dd7cc44dSYu Zhao return rc; 380dd7cc44dSYu Zhao } 381dd7cc44dSYu Zhao 382dd7cc44dSYu Zhao static void sriov_disable(struct pci_dev *dev) 383dd7cc44dSYu Zhao { 384dd7cc44dSYu Zhao int i; 385dd7cc44dSYu Zhao struct pci_sriov *iov = dev->sriov; 386dd7cc44dSYu Zhao 387dd7cc44dSYu Zhao if (!iov->nr_virtfn) 388dd7cc44dSYu Zhao return; 389dd7cc44dSYu Zhao 39074bb1bccSYu Zhao if (iov->cap & PCI_SRIOV_CAP_VFM) 39174bb1bccSYu Zhao sriov_disable_migration(dev); 39274bb1bccSYu Zhao 393dd7cc44dSYu Zhao for (i = 0; i < iov->nr_virtfn; i++) 394dd7cc44dSYu Zhao virtfn_remove(dev, i, 0); 395dd7cc44dSYu Zhao 396dd7cc44dSYu Zhao iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 397dd7cc44dSYu Zhao pci_block_user_cfg_access(dev); 398dd7cc44dSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 399dd7cc44dSYu Zhao ssleep(1); 400dd7cc44dSYu Zhao pci_unblock_user_cfg_access(dev); 401dd7cc44dSYu Zhao 402dd7cc44dSYu Zhao if (iov->link != dev->devfn) 403dd7cc44dSYu Zhao sysfs_remove_link(&dev->dev.kobj, "dep_link"); 404dd7cc44dSYu Zhao 405dd7cc44dSYu Zhao iov->nr_virtfn = 0; 406dd7cc44dSYu Zhao } 407dd7cc44dSYu Zhao 408d1b054daSYu Zhao static int sriov_init(struct pci_dev *dev, int pos) 409d1b054daSYu Zhao { 410d1b054daSYu Zhao int i; 411d1b054daSYu Zhao int rc; 412d1b054daSYu Zhao int nres; 413d1b054daSYu Zhao u32 pgsz; 414d1b054daSYu Zhao u16 ctrl, total, offset, stride; 415d1b054daSYu Zhao struct pci_sriov *iov; 416d1b054daSYu Zhao struct resource *res; 417d1b054daSYu Zhao struct pci_dev *pdev; 418d1b054daSYu Zhao 419d1b054daSYu Zhao if (dev->pcie_type != PCI_EXP_TYPE_RC_END && 420d1b054daSYu Zhao dev->pcie_type != PCI_EXP_TYPE_ENDPOINT) 421d1b054daSYu Zhao return -ENODEV; 422d1b054daSYu Zhao 423d1b054daSYu Zhao pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 424d1b054daSYu Zhao if (ctrl & PCI_SRIOV_CTRL_VFE) { 425d1b054daSYu Zhao pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 426d1b054daSYu Zhao ssleep(1); 427d1b054daSYu Zhao } 428d1b054daSYu Zhao 429d1b054daSYu Zhao pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 430d1b054daSYu Zhao if (!total) 431d1b054daSYu Zhao return 0; 432d1b054daSYu Zhao 433d1b054daSYu Zhao ctrl = 0; 434d1b054daSYu Zhao list_for_each_entry(pdev, &dev->bus->devices, bus_list) 435d1b054daSYu Zhao if (pdev->is_physfn) 436d1b054daSYu Zhao goto found; 437d1b054daSYu Zhao 438d1b054daSYu Zhao pdev = NULL; 439d1b054daSYu Zhao if (pci_ari_enabled(dev->bus)) 440d1b054daSYu Zhao ctrl |= PCI_SRIOV_CTRL_ARI; 441d1b054daSYu Zhao 442d1b054daSYu Zhao found: 443d1b054daSYu Zhao pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 444d1b054daSYu Zhao pci_write_config_word(dev, pos + PCI_SRIOV_NUM_VF, total); 445d1b054daSYu Zhao pci_read_config_word(dev, pos + PCI_SRIOV_VF_OFFSET, &offset); 446d1b054daSYu Zhao pci_read_config_word(dev, pos + PCI_SRIOV_VF_STRIDE, &stride); 447d1b054daSYu Zhao if (!offset || (total > 1 && !stride)) 448d1b054daSYu Zhao return -EIO; 449d1b054daSYu Zhao 450d1b054daSYu Zhao pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 451d1b054daSYu Zhao i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 452d1b054daSYu Zhao pgsz &= ~((1 << i) - 1); 453d1b054daSYu Zhao if (!pgsz) 454d1b054daSYu Zhao return -EIO; 455d1b054daSYu Zhao 456d1b054daSYu Zhao pgsz &= ~(pgsz - 1); 457d1b054daSYu Zhao pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 458d1b054daSYu Zhao 459d1b054daSYu Zhao nres = 0; 460d1b054daSYu Zhao for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 461d1b054daSYu Zhao res = dev->resource + PCI_IOV_RESOURCES + i; 462d1b054daSYu Zhao i += __pci_read_base(dev, pci_bar_unknown, res, 463d1b054daSYu Zhao pos + PCI_SRIOV_BAR + i * 4); 464d1b054daSYu Zhao if (!res->flags) 465d1b054daSYu Zhao continue; 466d1b054daSYu Zhao if (resource_size(res) & (PAGE_SIZE - 1)) { 467d1b054daSYu Zhao rc = -EIO; 468d1b054daSYu Zhao goto failed; 469d1b054daSYu Zhao } 470d1b054daSYu Zhao res->end = res->start + resource_size(res) * total - 1; 471d1b054daSYu Zhao nres++; 472d1b054daSYu Zhao } 473d1b054daSYu Zhao 474d1b054daSYu Zhao iov = kzalloc(sizeof(*iov), GFP_KERNEL); 475d1b054daSYu Zhao if (!iov) { 476d1b054daSYu Zhao rc = -ENOMEM; 477d1b054daSYu Zhao goto failed; 478d1b054daSYu Zhao } 479d1b054daSYu Zhao 480d1b054daSYu Zhao iov->pos = pos; 481d1b054daSYu Zhao iov->nres = nres; 482d1b054daSYu Zhao iov->ctrl = ctrl; 483d1b054daSYu Zhao iov->total = total; 484d1b054daSYu Zhao iov->offset = offset; 485d1b054daSYu Zhao iov->stride = stride; 486d1b054daSYu Zhao iov->pgsz = pgsz; 487d1b054daSYu Zhao iov->self = dev; 488d1b054daSYu Zhao pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 489d1b054daSYu Zhao pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 490*4d135dbeSYu Zhao if (dev->pcie_type == PCI_EXP_TYPE_RC_END) 491*4d135dbeSYu Zhao iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 492d1b054daSYu Zhao 493d1b054daSYu Zhao if (pdev) 494d1b054daSYu Zhao iov->dev = pci_dev_get(pdev); 495d1b054daSYu Zhao else { 496d1b054daSYu Zhao iov->dev = dev; 497d1b054daSYu Zhao mutex_init(&iov->lock); 498d1b054daSYu Zhao } 499d1b054daSYu Zhao 500d1b054daSYu Zhao dev->sriov = iov; 501d1b054daSYu Zhao dev->is_physfn = 1; 502d1b054daSYu Zhao 503d1b054daSYu Zhao return 0; 504d1b054daSYu Zhao 505d1b054daSYu Zhao failed: 506d1b054daSYu Zhao for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 507d1b054daSYu Zhao res = dev->resource + PCI_IOV_RESOURCES + i; 508d1b054daSYu Zhao res->flags = 0; 509d1b054daSYu Zhao } 510d1b054daSYu Zhao 511d1b054daSYu Zhao return rc; 512d1b054daSYu Zhao } 513d1b054daSYu Zhao 514d1b054daSYu Zhao static void sriov_release(struct pci_dev *dev) 515d1b054daSYu Zhao { 516dd7cc44dSYu Zhao BUG_ON(dev->sriov->nr_virtfn); 517dd7cc44dSYu Zhao 518d1b054daSYu Zhao if (dev == dev->sriov->dev) 519d1b054daSYu Zhao mutex_destroy(&dev->sriov->lock); 520d1b054daSYu Zhao else 521d1b054daSYu Zhao pci_dev_put(dev->sriov->dev); 522d1b054daSYu Zhao 523d1b054daSYu Zhao kfree(dev->sriov); 524d1b054daSYu Zhao dev->sriov = NULL; 525d1b054daSYu Zhao } 526d1b054daSYu Zhao 5278c5cdb6aSYu Zhao static void sriov_restore_state(struct pci_dev *dev) 5288c5cdb6aSYu Zhao { 5298c5cdb6aSYu Zhao int i; 5308c5cdb6aSYu Zhao u16 ctrl; 5318c5cdb6aSYu Zhao struct pci_sriov *iov = dev->sriov; 5328c5cdb6aSYu Zhao 5338c5cdb6aSYu Zhao pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 5348c5cdb6aSYu Zhao if (ctrl & PCI_SRIOV_CTRL_VFE) 5358c5cdb6aSYu Zhao return; 5368c5cdb6aSYu Zhao 5378c5cdb6aSYu Zhao for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) 5388c5cdb6aSYu Zhao pci_update_resource(dev, i); 5398c5cdb6aSYu Zhao 5408c5cdb6aSYu Zhao pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 541dd7cc44dSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, iov->nr_virtfn); 5428c5cdb6aSYu Zhao pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 5438c5cdb6aSYu Zhao if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 5448c5cdb6aSYu Zhao msleep(100); 5458c5cdb6aSYu Zhao } 5468c5cdb6aSYu Zhao 547d1b054daSYu Zhao /** 548d1b054daSYu Zhao * pci_iov_init - initialize the IOV capability 549d1b054daSYu Zhao * @dev: the PCI device 550d1b054daSYu Zhao * 551d1b054daSYu Zhao * Returns 0 on success, or negative on failure. 552d1b054daSYu Zhao */ 553d1b054daSYu Zhao int pci_iov_init(struct pci_dev *dev) 554d1b054daSYu Zhao { 555d1b054daSYu Zhao int pos; 556d1b054daSYu Zhao 557d1b054daSYu Zhao if (!dev->is_pcie) 558d1b054daSYu Zhao return -ENODEV; 559d1b054daSYu Zhao 560d1b054daSYu Zhao pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 561d1b054daSYu Zhao if (pos) 562d1b054daSYu Zhao return sriov_init(dev, pos); 563d1b054daSYu Zhao 564d1b054daSYu Zhao return -ENODEV; 565d1b054daSYu Zhao } 566d1b054daSYu Zhao 567d1b054daSYu Zhao /** 568d1b054daSYu Zhao * pci_iov_release - release resources used by the IOV capability 569d1b054daSYu Zhao * @dev: the PCI device 570d1b054daSYu Zhao */ 571d1b054daSYu Zhao void pci_iov_release(struct pci_dev *dev) 572d1b054daSYu Zhao { 573d1b054daSYu Zhao if (dev->is_physfn) 574d1b054daSYu Zhao sriov_release(dev); 575d1b054daSYu Zhao } 576d1b054daSYu Zhao 577d1b054daSYu Zhao /** 578d1b054daSYu Zhao * pci_iov_resource_bar - get position of the SR-IOV BAR 579d1b054daSYu Zhao * @dev: the PCI device 580d1b054daSYu Zhao * @resno: the resource number 581d1b054daSYu Zhao * @type: the BAR type to be filled in 582d1b054daSYu Zhao * 583d1b054daSYu Zhao * Returns position of the BAR encapsulated in the SR-IOV capability. 584d1b054daSYu Zhao */ 585d1b054daSYu Zhao int pci_iov_resource_bar(struct pci_dev *dev, int resno, 586d1b054daSYu Zhao enum pci_bar_type *type) 587d1b054daSYu Zhao { 588d1b054daSYu Zhao if (resno < PCI_IOV_RESOURCES || resno > PCI_IOV_RESOURCE_END) 589d1b054daSYu Zhao return 0; 590d1b054daSYu Zhao 591d1b054daSYu Zhao BUG_ON(!dev->is_physfn); 592d1b054daSYu Zhao 593d1b054daSYu Zhao *type = pci_bar_unknown; 594d1b054daSYu Zhao 595d1b054daSYu Zhao return dev->sriov->pos + PCI_SRIOV_BAR + 596d1b054daSYu Zhao 4 * (resno - PCI_IOV_RESOURCES); 597d1b054daSYu Zhao } 5988c5cdb6aSYu Zhao 5998c5cdb6aSYu Zhao /** 6008c5cdb6aSYu Zhao * pci_restore_iov_state - restore the state of the IOV capability 6018c5cdb6aSYu Zhao * @dev: the PCI device 6028c5cdb6aSYu Zhao */ 6038c5cdb6aSYu Zhao void pci_restore_iov_state(struct pci_dev *dev) 6048c5cdb6aSYu Zhao { 6058c5cdb6aSYu Zhao if (dev->is_physfn) 6068c5cdb6aSYu Zhao sriov_restore_state(dev); 6078c5cdb6aSYu Zhao } 608a28724b0SYu Zhao 609a28724b0SYu Zhao /** 610a28724b0SYu Zhao * pci_iov_bus_range - find bus range used by Virtual Function 611a28724b0SYu Zhao * @bus: the PCI bus 612a28724b0SYu Zhao * 613a28724b0SYu Zhao * Returns max number of buses (exclude current one) used by Virtual 614a28724b0SYu Zhao * Functions. 615a28724b0SYu Zhao */ 616a28724b0SYu Zhao int pci_iov_bus_range(struct pci_bus *bus) 617a28724b0SYu Zhao { 618a28724b0SYu Zhao int max = 0; 619a28724b0SYu Zhao u8 busnr; 620a28724b0SYu Zhao struct pci_dev *dev; 621a28724b0SYu Zhao 622a28724b0SYu Zhao list_for_each_entry(dev, &bus->devices, bus_list) { 623a28724b0SYu Zhao if (!dev->is_physfn) 624a28724b0SYu Zhao continue; 625a28724b0SYu Zhao busnr = virtfn_bus(dev, dev->sriov->total - 1); 626a28724b0SYu Zhao if (busnr > max) 627a28724b0SYu Zhao max = busnr; 628a28724b0SYu Zhao } 629a28724b0SYu Zhao 630a28724b0SYu Zhao return max ? max - bus->number : 0; 631a28724b0SYu Zhao } 632dd7cc44dSYu Zhao 633dd7cc44dSYu Zhao /** 634dd7cc44dSYu Zhao * pci_enable_sriov - enable the SR-IOV capability 635dd7cc44dSYu Zhao * @dev: the PCI device 63652a8873bSRandy Dunlap * @nr_virtfn: number of virtual functions to enable 637dd7cc44dSYu Zhao * 638dd7cc44dSYu Zhao * Returns 0 on success, or negative on failure. 639dd7cc44dSYu Zhao */ 640dd7cc44dSYu Zhao int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 641dd7cc44dSYu Zhao { 642dd7cc44dSYu Zhao might_sleep(); 643dd7cc44dSYu Zhao 644dd7cc44dSYu Zhao if (!dev->is_physfn) 645dd7cc44dSYu Zhao return -ENODEV; 646dd7cc44dSYu Zhao 647dd7cc44dSYu Zhao return sriov_enable(dev, nr_virtfn); 648dd7cc44dSYu Zhao } 649dd7cc44dSYu Zhao EXPORT_SYMBOL_GPL(pci_enable_sriov); 650dd7cc44dSYu Zhao 651dd7cc44dSYu Zhao /** 652dd7cc44dSYu Zhao * pci_disable_sriov - disable the SR-IOV capability 653dd7cc44dSYu Zhao * @dev: the PCI device 654dd7cc44dSYu Zhao */ 655dd7cc44dSYu Zhao void pci_disable_sriov(struct pci_dev *dev) 656dd7cc44dSYu Zhao { 657dd7cc44dSYu Zhao might_sleep(); 658dd7cc44dSYu Zhao 659dd7cc44dSYu Zhao if (!dev->is_physfn) 660dd7cc44dSYu Zhao return; 661dd7cc44dSYu Zhao 662dd7cc44dSYu Zhao sriov_disable(dev); 663dd7cc44dSYu Zhao } 664dd7cc44dSYu Zhao EXPORT_SYMBOL_GPL(pci_disable_sriov); 66574bb1bccSYu Zhao 66674bb1bccSYu Zhao /** 66774bb1bccSYu Zhao * pci_sriov_migration - notify SR-IOV core of Virtual Function Migration 66874bb1bccSYu Zhao * @dev: the PCI device 66974bb1bccSYu Zhao * 67074bb1bccSYu Zhao * Returns IRQ_HANDLED if the IRQ is handled, or IRQ_NONE if not. 67174bb1bccSYu Zhao * 67274bb1bccSYu Zhao * Physical Function driver is responsible to register IRQ handler using 67374bb1bccSYu Zhao * VF Migration Interrupt Message Number, and call this function when the 67474bb1bccSYu Zhao * interrupt is generated by the hardware. 67574bb1bccSYu Zhao */ 67674bb1bccSYu Zhao irqreturn_t pci_sriov_migration(struct pci_dev *dev) 67774bb1bccSYu Zhao { 67874bb1bccSYu Zhao if (!dev->is_physfn) 67974bb1bccSYu Zhao return IRQ_NONE; 68074bb1bccSYu Zhao 68174bb1bccSYu Zhao return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE; 68274bb1bccSYu Zhao } 68374bb1bccSYu Zhao EXPORT_SYMBOL_GPL(pci_sriov_migration); 684