1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Single Root IOV 1.0 5 * Address Translation Service 1.0 6 * 7 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 8 */ 9 10 #include <linux/pci.h> 11 #include <linux/slab.h> 12 #include <linux/export.h> 13 #include <linux/string.h> 14 #include <linux/delay.h> 15 #include "pci.h" 16 17 #define VIRTFN_ID_LEN 16 18 19 int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id) 20 { 21 if (!dev->is_physfn) 22 return -EINVAL; 23 return dev->bus->number + ((dev->devfn + dev->sriov->offset + 24 dev->sriov->stride * vf_id) >> 8); 25 } 26 27 int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id) 28 { 29 if (!dev->is_physfn) 30 return -EINVAL; 31 return (dev->devfn + dev->sriov->offset + 32 dev->sriov->stride * vf_id) & 0xff; 33 } 34 EXPORT_SYMBOL_GPL(pci_iov_virtfn_devfn); 35 36 /* 37 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may 38 * change when NumVFs changes. 39 * 40 * Update iov->offset and iov->stride when NumVFs is written. 41 */ 42 static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn) 43 { 44 struct pci_sriov *iov = dev->sriov; 45 46 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 47 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset); 48 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride); 49 } 50 51 /* 52 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride 53 * determine how many additional bus numbers will be consumed by VFs. 54 * 55 * Iterate over all valid NumVFs, validate offset and stride, and calculate 56 * the maximum number of bus numbers that could ever be required. 57 */ 58 static int compute_max_vf_buses(struct pci_dev *dev) 59 { 60 struct pci_sriov *iov = dev->sriov; 61 int nr_virtfn, busnr, rc = 0; 62 63 for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) { 64 pci_iov_set_numvfs(dev, nr_virtfn); 65 if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) { 66 rc = -EIO; 67 goto out; 68 } 69 70 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 71 if (busnr > iov->max_VF_buses) 72 iov->max_VF_buses = busnr; 73 } 74 75 out: 76 pci_iov_set_numvfs(dev, 0); 77 return rc; 78 } 79 80 static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 81 { 82 struct pci_bus *child; 83 84 if (bus->number == busnr) 85 return bus; 86 87 child = pci_find_bus(pci_domain_nr(bus), busnr); 88 if (child) 89 return child; 90 91 child = pci_add_new_bus(bus, NULL, busnr); 92 if (!child) 93 return NULL; 94 95 pci_bus_insert_busn_res(child, busnr, busnr); 96 97 return child; 98 } 99 100 static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus) 101 { 102 if (physbus != virtbus && list_empty(&virtbus->devices)) 103 pci_remove_bus(virtbus); 104 } 105 106 resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) 107 { 108 if (!dev->is_physfn) 109 return 0; 110 111 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES]; 112 } 113 114 static void pci_read_vf_config_common(struct pci_dev *virtfn) 115 { 116 struct pci_dev *physfn = virtfn->physfn; 117 118 /* 119 * Some config registers are the same across all associated VFs. 120 * Read them once from VF0 so we can skip reading them from the 121 * other VFs. 122 * 123 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to 124 * have the same Revision ID and Subsystem ID, but we assume they 125 * do. 126 */ 127 pci_read_config_dword(virtfn, PCI_CLASS_REVISION, 128 &physfn->sriov->class); 129 pci_read_config_byte(virtfn, PCI_HEADER_TYPE, 130 &physfn->sriov->hdr_type); 131 pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID, 132 &physfn->sriov->subsystem_vendor); 133 pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID, 134 &physfn->sriov->subsystem_device); 135 } 136 137 int pci_iov_sysfs_link(struct pci_dev *dev, 138 struct pci_dev *virtfn, int id) 139 { 140 char buf[VIRTFN_ID_LEN]; 141 int rc; 142 143 sprintf(buf, "virtfn%u", id); 144 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 145 if (rc) 146 goto failed; 147 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 148 if (rc) 149 goto failed1; 150 151 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 152 153 return 0; 154 155 failed1: 156 sysfs_remove_link(&dev->dev.kobj, buf); 157 failed: 158 return rc; 159 } 160 161 #ifdef CONFIG_PCI_MSI 162 static ssize_t sriov_vf_total_msix_show(struct device *dev, 163 struct device_attribute *attr, 164 char *buf) 165 { 166 struct pci_dev *pdev = to_pci_dev(dev); 167 u32 vf_total_msix = 0; 168 169 device_lock(dev); 170 if (!pdev->driver || !pdev->driver->sriov_get_vf_total_msix) 171 goto unlock; 172 173 vf_total_msix = pdev->driver->sriov_get_vf_total_msix(pdev); 174 unlock: 175 device_unlock(dev); 176 return sysfs_emit(buf, "%u\n", vf_total_msix); 177 } 178 static DEVICE_ATTR_RO(sriov_vf_total_msix); 179 180 static ssize_t sriov_vf_msix_count_store(struct device *dev, 181 struct device_attribute *attr, 182 const char *buf, size_t count) 183 { 184 struct pci_dev *vf_dev = to_pci_dev(dev); 185 struct pci_dev *pdev = pci_physfn(vf_dev); 186 int val, ret; 187 188 ret = kstrtoint(buf, 0, &val); 189 if (ret) 190 return ret; 191 192 if (val < 0) 193 return -EINVAL; 194 195 device_lock(&pdev->dev); 196 if (!pdev->driver || !pdev->driver->sriov_set_msix_vec_count) { 197 ret = -EOPNOTSUPP; 198 goto err_pdev; 199 } 200 201 device_lock(&vf_dev->dev); 202 if (vf_dev->driver) { 203 /* 204 * A driver is already attached to this VF and has configured 205 * itself based on the current MSI-X vector count. Changing 206 * the vector size could mess up the driver, so block it. 207 */ 208 ret = -EBUSY; 209 goto err_dev; 210 } 211 212 ret = pdev->driver->sriov_set_msix_vec_count(vf_dev, val); 213 214 err_dev: 215 device_unlock(&vf_dev->dev); 216 err_pdev: 217 device_unlock(&pdev->dev); 218 return ret ? : count; 219 } 220 static DEVICE_ATTR_WO(sriov_vf_msix_count); 221 #endif 222 223 static struct attribute *sriov_vf_dev_attrs[] = { 224 #ifdef CONFIG_PCI_MSI 225 &dev_attr_sriov_vf_msix_count.attr, 226 #endif 227 NULL, 228 }; 229 230 static umode_t sriov_vf_attrs_are_visible(struct kobject *kobj, 231 struct attribute *a, int n) 232 { 233 struct device *dev = kobj_to_dev(kobj); 234 struct pci_dev *pdev = to_pci_dev(dev); 235 236 if (!pdev->is_virtfn) 237 return 0; 238 239 return a->mode; 240 } 241 242 const struct attribute_group sriov_vf_dev_attr_group = { 243 .attrs = sriov_vf_dev_attrs, 244 .is_visible = sriov_vf_attrs_are_visible, 245 }; 246 247 int pci_iov_add_virtfn(struct pci_dev *dev, int id) 248 { 249 int i; 250 int rc = -ENOMEM; 251 u64 size; 252 struct pci_dev *virtfn; 253 struct resource *res; 254 struct pci_sriov *iov = dev->sriov; 255 struct pci_bus *bus; 256 257 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id)); 258 if (!bus) 259 goto failed; 260 261 virtfn = pci_alloc_dev(bus); 262 if (!virtfn) 263 goto failed0; 264 265 virtfn->devfn = pci_iov_virtfn_devfn(dev, id); 266 virtfn->vendor = dev->vendor; 267 virtfn->device = iov->vf_device; 268 virtfn->is_virtfn = 1; 269 virtfn->physfn = pci_dev_get(dev); 270 virtfn->no_command_memory = 1; 271 272 if (id == 0) 273 pci_read_vf_config_common(virtfn); 274 275 rc = pci_setup_device(virtfn); 276 if (rc) 277 goto failed1; 278 279 virtfn->dev.parent = dev->dev.parent; 280 virtfn->multifunction = 0; 281 282 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 283 res = &dev->resource[i + PCI_IOV_RESOURCES]; 284 if (!res->parent) 285 continue; 286 virtfn->resource[i].name = pci_name(virtfn); 287 virtfn->resource[i].flags = res->flags; 288 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); 289 virtfn->resource[i].start = res->start + size * id; 290 virtfn->resource[i].end = virtfn->resource[i].start + size - 1; 291 rc = request_resource(res, &virtfn->resource[i]); 292 BUG_ON(rc); 293 } 294 295 pci_device_add(virtfn, virtfn->bus); 296 rc = pci_iov_sysfs_link(dev, virtfn, id); 297 if (rc) 298 goto failed1; 299 300 pci_bus_add_device(virtfn); 301 302 return 0; 303 304 failed1: 305 pci_stop_and_remove_bus_device(virtfn); 306 pci_dev_put(dev); 307 failed0: 308 virtfn_remove_bus(dev->bus, bus); 309 failed: 310 311 return rc; 312 } 313 314 void pci_iov_remove_virtfn(struct pci_dev *dev, int id) 315 { 316 char buf[VIRTFN_ID_LEN]; 317 struct pci_dev *virtfn; 318 319 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus), 320 pci_iov_virtfn_bus(dev, id), 321 pci_iov_virtfn_devfn(dev, id)); 322 if (!virtfn) 323 return; 324 325 sprintf(buf, "virtfn%u", id); 326 sysfs_remove_link(&dev->dev.kobj, buf); 327 /* 328 * pci_stop_dev() could have been called for this virtfn already, 329 * so the directory for the virtfn may have been removed before. 330 * Double check to avoid spurious sysfs warnings. 331 */ 332 if (virtfn->dev.kobj.sd) 333 sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 334 335 pci_stop_and_remove_bus_device(virtfn); 336 virtfn_remove_bus(dev->bus, virtfn->bus); 337 338 /* balance pci_get_domain_bus_and_slot() */ 339 pci_dev_put(virtfn); 340 pci_dev_put(dev); 341 } 342 343 static ssize_t sriov_totalvfs_show(struct device *dev, 344 struct device_attribute *attr, 345 char *buf) 346 { 347 struct pci_dev *pdev = to_pci_dev(dev); 348 349 return sysfs_emit(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); 350 } 351 352 static ssize_t sriov_numvfs_show(struct device *dev, 353 struct device_attribute *attr, 354 char *buf) 355 { 356 struct pci_dev *pdev = to_pci_dev(dev); 357 u16 num_vfs; 358 359 /* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */ 360 device_lock(&pdev->dev); 361 num_vfs = pdev->sriov->num_VFs; 362 device_unlock(&pdev->dev); 363 364 return sysfs_emit(buf, "%u\n", num_vfs); 365 } 366 367 /* 368 * num_vfs > 0; number of VFs to enable 369 * num_vfs = 0; disable all VFs 370 * 371 * Note: SRIOV spec does not allow partial VF 372 * disable, so it's all or none. 373 */ 374 static ssize_t sriov_numvfs_store(struct device *dev, 375 struct device_attribute *attr, 376 const char *buf, size_t count) 377 { 378 struct pci_dev *pdev = to_pci_dev(dev); 379 int ret; 380 u16 num_vfs; 381 382 ret = kstrtou16(buf, 0, &num_vfs); 383 if (ret < 0) 384 return ret; 385 386 if (num_vfs > pci_sriov_get_totalvfs(pdev)) 387 return -ERANGE; 388 389 device_lock(&pdev->dev); 390 391 if (num_vfs == pdev->sriov->num_VFs) 392 goto exit; 393 394 /* is PF driver loaded */ 395 if (!pdev->driver) { 396 pci_info(pdev, "no driver bound to device; cannot configure SR-IOV\n"); 397 ret = -ENOENT; 398 goto exit; 399 } 400 401 /* is PF driver loaded w/callback */ 402 if (!pdev->driver->sriov_configure) { 403 pci_info(pdev, "driver does not support SR-IOV configuration via sysfs\n"); 404 ret = -ENOENT; 405 goto exit; 406 } 407 408 if (num_vfs == 0) { 409 /* disable VFs */ 410 ret = pdev->driver->sriov_configure(pdev, 0); 411 goto exit; 412 } 413 414 /* enable VFs */ 415 if (pdev->sriov->num_VFs) { 416 pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n", 417 pdev->sriov->num_VFs, num_vfs); 418 ret = -EBUSY; 419 goto exit; 420 } 421 422 ret = pdev->driver->sriov_configure(pdev, num_vfs); 423 if (ret < 0) 424 goto exit; 425 426 if (ret != num_vfs) 427 pci_warn(pdev, "%d VFs requested; only %d enabled\n", 428 num_vfs, ret); 429 430 exit: 431 device_unlock(&pdev->dev); 432 433 if (ret < 0) 434 return ret; 435 436 return count; 437 } 438 439 static ssize_t sriov_offset_show(struct device *dev, 440 struct device_attribute *attr, 441 char *buf) 442 { 443 struct pci_dev *pdev = to_pci_dev(dev); 444 445 return sysfs_emit(buf, "%u\n", pdev->sriov->offset); 446 } 447 448 static ssize_t sriov_stride_show(struct device *dev, 449 struct device_attribute *attr, 450 char *buf) 451 { 452 struct pci_dev *pdev = to_pci_dev(dev); 453 454 return sysfs_emit(buf, "%u\n", pdev->sriov->stride); 455 } 456 457 static ssize_t sriov_vf_device_show(struct device *dev, 458 struct device_attribute *attr, 459 char *buf) 460 { 461 struct pci_dev *pdev = to_pci_dev(dev); 462 463 return sysfs_emit(buf, "%x\n", pdev->sriov->vf_device); 464 } 465 466 static ssize_t sriov_drivers_autoprobe_show(struct device *dev, 467 struct device_attribute *attr, 468 char *buf) 469 { 470 struct pci_dev *pdev = to_pci_dev(dev); 471 472 return sysfs_emit(buf, "%u\n", pdev->sriov->drivers_autoprobe); 473 } 474 475 static ssize_t sriov_drivers_autoprobe_store(struct device *dev, 476 struct device_attribute *attr, 477 const char *buf, size_t count) 478 { 479 struct pci_dev *pdev = to_pci_dev(dev); 480 bool drivers_autoprobe; 481 482 if (kstrtobool(buf, &drivers_autoprobe) < 0) 483 return -EINVAL; 484 485 pdev->sriov->drivers_autoprobe = drivers_autoprobe; 486 487 return count; 488 } 489 490 static DEVICE_ATTR_RO(sriov_totalvfs); 491 static DEVICE_ATTR_RW(sriov_numvfs); 492 static DEVICE_ATTR_RO(sriov_offset); 493 static DEVICE_ATTR_RO(sriov_stride); 494 static DEVICE_ATTR_RO(sriov_vf_device); 495 static DEVICE_ATTR_RW(sriov_drivers_autoprobe); 496 497 static struct attribute *sriov_pf_dev_attrs[] = { 498 &dev_attr_sriov_totalvfs.attr, 499 &dev_attr_sriov_numvfs.attr, 500 &dev_attr_sriov_offset.attr, 501 &dev_attr_sriov_stride.attr, 502 &dev_attr_sriov_vf_device.attr, 503 &dev_attr_sriov_drivers_autoprobe.attr, 504 #ifdef CONFIG_PCI_MSI 505 &dev_attr_sriov_vf_total_msix.attr, 506 #endif 507 NULL, 508 }; 509 510 static umode_t sriov_pf_attrs_are_visible(struct kobject *kobj, 511 struct attribute *a, int n) 512 { 513 struct device *dev = kobj_to_dev(kobj); 514 515 if (!dev_is_pf(dev)) 516 return 0; 517 518 return a->mode; 519 } 520 521 const struct attribute_group sriov_pf_dev_attr_group = { 522 .attrs = sriov_pf_dev_attrs, 523 .is_visible = sriov_pf_attrs_are_visible, 524 }; 525 526 int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) 527 { 528 return 0; 529 } 530 531 int __weak pcibios_sriov_disable(struct pci_dev *pdev) 532 { 533 return 0; 534 } 535 536 static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) 537 { 538 unsigned int i; 539 int rc; 540 541 if (dev->no_vf_scan) 542 return 0; 543 544 for (i = 0; i < num_vfs; i++) { 545 rc = pci_iov_add_virtfn(dev, i); 546 if (rc) 547 goto failed; 548 } 549 return 0; 550 failed: 551 while (i--) 552 pci_iov_remove_virtfn(dev, i); 553 554 return rc; 555 } 556 557 static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 558 { 559 int rc; 560 int i; 561 int nres; 562 u16 initial; 563 struct resource *res; 564 struct pci_dev *pdev; 565 struct pci_sriov *iov = dev->sriov; 566 int bars = 0; 567 int bus; 568 569 if (!nr_virtfn) 570 return 0; 571 572 if (iov->num_VFs) 573 return -EINVAL; 574 575 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 576 if (initial > iov->total_VFs || 577 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs))) 578 return -EIO; 579 580 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs || 581 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 582 return -EINVAL; 583 584 nres = 0; 585 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 586 bars |= (1 << (i + PCI_IOV_RESOURCES)); 587 res = &dev->resource[i + PCI_IOV_RESOURCES]; 588 if (res->parent) 589 nres++; 590 } 591 if (nres != iov->nres) { 592 pci_err(dev, "not enough MMIO resources for SR-IOV\n"); 593 return -ENOMEM; 594 } 595 596 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 597 if (bus > dev->bus->busn_res.end) { 598 pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n", 599 nr_virtfn, bus, &dev->bus->busn_res); 600 return -ENOMEM; 601 } 602 603 if (pci_enable_resources(dev, bars)) { 604 pci_err(dev, "SR-IOV: IOV BARS not allocated\n"); 605 return -ENOMEM; 606 } 607 608 if (iov->link != dev->devfn) { 609 pdev = pci_get_slot(dev->bus, iov->link); 610 if (!pdev) 611 return -ENODEV; 612 613 if (!pdev->is_physfn) { 614 pci_dev_put(pdev); 615 return -ENOSYS; 616 } 617 618 rc = sysfs_create_link(&dev->dev.kobj, 619 &pdev->dev.kobj, "dep_link"); 620 pci_dev_put(pdev); 621 if (rc) 622 return rc; 623 } 624 625 iov->initial_VFs = initial; 626 if (nr_virtfn < initial) 627 initial = nr_virtfn; 628 629 rc = pcibios_sriov_enable(dev, initial); 630 if (rc) { 631 pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc); 632 goto err_pcibios; 633 } 634 635 pci_iov_set_numvfs(dev, nr_virtfn); 636 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 637 pci_cfg_access_lock(dev); 638 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 639 msleep(100); 640 pci_cfg_access_unlock(dev); 641 642 rc = sriov_add_vfs(dev, initial); 643 if (rc) 644 goto err_pcibios; 645 646 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 647 iov->num_VFs = nr_virtfn; 648 649 return 0; 650 651 err_pcibios: 652 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 653 pci_cfg_access_lock(dev); 654 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 655 ssleep(1); 656 pci_cfg_access_unlock(dev); 657 658 pcibios_sriov_disable(dev); 659 660 if (iov->link != dev->devfn) 661 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 662 663 pci_iov_set_numvfs(dev, 0); 664 return rc; 665 } 666 667 static void sriov_del_vfs(struct pci_dev *dev) 668 { 669 struct pci_sriov *iov = dev->sriov; 670 int i; 671 672 for (i = 0; i < iov->num_VFs; i++) 673 pci_iov_remove_virtfn(dev, i); 674 } 675 676 static void sriov_disable(struct pci_dev *dev) 677 { 678 struct pci_sriov *iov = dev->sriov; 679 680 if (!iov->num_VFs) 681 return; 682 683 sriov_del_vfs(dev); 684 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 685 pci_cfg_access_lock(dev); 686 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 687 ssleep(1); 688 pci_cfg_access_unlock(dev); 689 690 pcibios_sriov_disable(dev); 691 692 if (iov->link != dev->devfn) 693 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 694 695 iov->num_VFs = 0; 696 pci_iov_set_numvfs(dev, 0); 697 } 698 699 static int sriov_init(struct pci_dev *dev, int pos) 700 { 701 int i, bar64; 702 int rc; 703 int nres; 704 u32 pgsz; 705 u16 ctrl, total; 706 struct pci_sriov *iov; 707 struct resource *res; 708 struct pci_dev *pdev; 709 710 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 711 if (ctrl & PCI_SRIOV_CTRL_VFE) { 712 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 713 ssleep(1); 714 } 715 716 ctrl = 0; 717 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 718 if (pdev->is_physfn) 719 goto found; 720 721 pdev = NULL; 722 if (pci_ari_enabled(dev->bus)) 723 ctrl |= PCI_SRIOV_CTRL_ARI; 724 725 found: 726 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 727 728 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 729 if (!total) 730 return 0; 731 732 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 733 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 734 pgsz &= ~((1 << i) - 1); 735 if (!pgsz) 736 return -EIO; 737 738 pgsz &= ~(pgsz - 1); 739 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 740 741 iov = kzalloc(sizeof(*iov), GFP_KERNEL); 742 if (!iov) 743 return -ENOMEM; 744 745 nres = 0; 746 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 747 res = &dev->resource[i + PCI_IOV_RESOURCES]; 748 /* 749 * If it is already FIXED, don't change it, something 750 * (perhaps EA or header fixups) wants it this way. 751 */ 752 if (res->flags & IORESOURCE_PCI_FIXED) 753 bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0; 754 else 755 bar64 = __pci_read_base(dev, pci_bar_unknown, res, 756 pos + PCI_SRIOV_BAR + i * 4); 757 if (!res->flags) 758 continue; 759 if (resource_size(res) & (PAGE_SIZE - 1)) { 760 rc = -EIO; 761 goto failed; 762 } 763 iov->barsz[i] = resource_size(res); 764 res->end = res->start + resource_size(res) * total - 1; 765 pci_info(dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n", 766 i, res, i, total); 767 i += bar64; 768 nres++; 769 } 770 771 iov->pos = pos; 772 iov->nres = nres; 773 iov->ctrl = ctrl; 774 iov->total_VFs = total; 775 iov->driver_max_VFs = total; 776 pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device); 777 iov->pgsz = pgsz; 778 iov->self = dev; 779 iov->drivers_autoprobe = true; 780 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 781 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 782 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) 783 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 784 785 if (pdev) 786 iov->dev = pci_dev_get(pdev); 787 else 788 iov->dev = dev; 789 790 dev->sriov = iov; 791 dev->is_physfn = 1; 792 rc = compute_max_vf_buses(dev); 793 if (rc) 794 goto fail_max_buses; 795 796 return 0; 797 798 fail_max_buses: 799 dev->sriov = NULL; 800 dev->is_physfn = 0; 801 failed: 802 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 803 res = &dev->resource[i + PCI_IOV_RESOURCES]; 804 res->flags = 0; 805 } 806 807 kfree(iov); 808 return rc; 809 } 810 811 static void sriov_release(struct pci_dev *dev) 812 { 813 BUG_ON(dev->sriov->num_VFs); 814 815 if (dev != dev->sriov->dev) 816 pci_dev_put(dev->sriov->dev); 817 818 kfree(dev->sriov); 819 dev->sriov = NULL; 820 } 821 822 static void sriov_restore_state(struct pci_dev *dev) 823 { 824 int i; 825 u16 ctrl; 826 struct pci_sriov *iov = dev->sriov; 827 828 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 829 if (ctrl & PCI_SRIOV_CTRL_VFE) 830 return; 831 832 /* 833 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because 834 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI. 835 */ 836 ctrl &= ~PCI_SRIOV_CTRL_ARI; 837 ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI; 838 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl); 839 840 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) 841 pci_update_resource(dev, i + PCI_IOV_RESOURCES); 842 843 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 844 pci_iov_set_numvfs(dev, iov->num_VFs); 845 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 846 if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 847 msleep(100); 848 } 849 850 /** 851 * pci_iov_init - initialize the IOV capability 852 * @dev: the PCI device 853 * 854 * Returns 0 on success, or negative on failure. 855 */ 856 int pci_iov_init(struct pci_dev *dev) 857 { 858 int pos; 859 860 if (!pci_is_pcie(dev)) 861 return -ENODEV; 862 863 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 864 if (pos) 865 return sriov_init(dev, pos); 866 867 return -ENODEV; 868 } 869 870 /** 871 * pci_iov_release - release resources used by the IOV capability 872 * @dev: the PCI device 873 */ 874 void pci_iov_release(struct pci_dev *dev) 875 { 876 if (dev->is_physfn) 877 sriov_release(dev); 878 } 879 880 /** 881 * pci_iov_remove - clean up SR-IOV state after PF driver is detached 882 * @dev: the PCI device 883 */ 884 void pci_iov_remove(struct pci_dev *dev) 885 { 886 struct pci_sriov *iov = dev->sriov; 887 888 if (!dev->is_physfn) 889 return; 890 891 iov->driver_max_VFs = iov->total_VFs; 892 if (iov->num_VFs) 893 pci_warn(dev, "driver left SR-IOV enabled after remove\n"); 894 } 895 896 /** 897 * pci_iov_update_resource - update a VF BAR 898 * @dev: the PCI device 899 * @resno: the resource number 900 * 901 * Update a VF BAR in the SR-IOV capability of a PF. 902 */ 903 void pci_iov_update_resource(struct pci_dev *dev, int resno) 904 { 905 struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL; 906 struct resource *res = dev->resource + resno; 907 int vf_bar = resno - PCI_IOV_RESOURCES; 908 struct pci_bus_region region; 909 u16 cmd; 910 u32 new; 911 int reg; 912 913 /* 914 * The generic pci_restore_bars() path calls this for all devices, 915 * including VFs and non-SR-IOV devices. If this is not a PF, we 916 * have nothing to do. 917 */ 918 if (!iov) 919 return; 920 921 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd); 922 if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) { 923 dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n", 924 vf_bar, res); 925 return; 926 } 927 928 /* 929 * Ignore unimplemented BARs, unused resource slots for 64-bit 930 * BARs, and non-movable resources, e.g., those described via 931 * Enhanced Allocation. 932 */ 933 if (!res->flags) 934 return; 935 936 if (res->flags & IORESOURCE_UNSET) 937 return; 938 939 if (res->flags & IORESOURCE_PCI_FIXED) 940 return; 941 942 pcibios_resource_to_bus(dev->bus, ®ion, res); 943 new = region.start; 944 new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; 945 946 reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar; 947 pci_write_config_dword(dev, reg, new); 948 if (res->flags & IORESOURCE_MEM_64) { 949 new = region.start >> 16 >> 16; 950 pci_write_config_dword(dev, reg + 4, new); 951 } 952 } 953 954 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev, 955 int resno) 956 { 957 return pci_iov_resource_size(dev, resno); 958 } 959 960 /** 961 * pci_sriov_resource_alignment - get resource alignment for VF BAR 962 * @dev: the PCI device 963 * @resno: the resource number 964 * 965 * Returns the alignment of the VF BAR found in the SR-IOV capability. 966 * This is not the same as the resource size which is defined as 967 * the VF BAR size multiplied by the number of VFs. The alignment 968 * is just the VF BAR size. 969 */ 970 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno) 971 { 972 return pcibios_iov_resource_alignment(dev, resno); 973 } 974 975 /** 976 * pci_restore_iov_state - restore the state of the IOV capability 977 * @dev: the PCI device 978 */ 979 void pci_restore_iov_state(struct pci_dev *dev) 980 { 981 if (dev->is_physfn) 982 sriov_restore_state(dev); 983 } 984 985 /** 986 * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs 987 * @dev: the PCI device 988 * @auto_probe: set VF drivers auto probe flag 989 */ 990 void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe) 991 { 992 if (dev->is_physfn) 993 dev->sriov->drivers_autoprobe = auto_probe; 994 } 995 996 /** 997 * pci_iov_bus_range - find bus range used by Virtual Function 998 * @bus: the PCI bus 999 * 1000 * Returns max number of buses (exclude current one) used by Virtual 1001 * Functions. 1002 */ 1003 int pci_iov_bus_range(struct pci_bus *bus) 1004 { 1005 int max = 0; 1006 struct pci_dev *dev; 1007 1008 list_for_each_entry(dev, &bus->devices, bus_list) { 1009 if (!dev->is_physfn) 1010 continue; 1011 if (dev->sriov->max_VF_buses > max) 1012 max = dev->sriov->max_VF_buses; 1013 } 1014 1015 return max ? max - bus->number : 0; 1016 } 1017 1018 /** 1019 * pci_enable_sriov - enable the SR-IOV capability 1020 * @dev: the PCI device 1021 * @nr_virtfn: number of virtual functions to enable 1022 * 1023 * Returns 0 on success, or negative on failure. 1024 */ 1025 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 1026 { 1027 might_sleep(); 1028 1029 if (!dev->is_physfn) 1030 return -ENOSYS; 1031 1032 return sriov_enable(dev, nr_virtfn); 1033 } 1034 EXPORT_SYMBOL_GPL(pci_enable_sriov); 1035 1036 /** 1037 * pci_disable_sriov - disable the SR-IOV capability 1038 * @dev: the PCI device 1039 */ 1040 void pci_disable_sriov(struct pci_dev *dev) 1041 { 1042 might_sleep(); 1043 1044 if (!dev->is_physfn) 1045 return; 1046 1047 sriov_disable(dev); 1048 } 1049 EXPORT_SYMBOL_GPL(pci_disable_sriov); 1050 1051 /** 1052 * pci_num_vf - return number of VFs associated with a PF device_release_driver 1053 * @dev: the PCI device 1054 * 1055 * Returns number of VFs, or 0 if SR-IOV is not enabled. 1056 */ 1057 int pci_num_vf(struct pci_dev *dev) 1058 { 1059 if (!dev->is_physfn) 1060 return 0; 1061 1062 return dev->sriov->num_VFs; 1063 } 1064 EXPORT_SYMBOL_GPL(pci_num_vf); 1065 1066 /** 1067 * pci_vfs_assigned - returns number of VFs are assigned to a guest 1068 * @dev: the PCI device 1069 * 1070 * Returns number of VFs belonging to this device that are assigned to a guest. 1071 * If device is not a physical function returns 0. 1072 */ 1073 int pci_vfs_assigned(struct pci_dev *dev) 1074 { 1075 struct pci_dev *vfdev; 1076 unsigned int vfs_assigned = 0; 1077 unsigned short dev_id; 1078 1079 /* only search if we are a PF */ 1080 if (!dev->is_physfn) 1081 return 0; 1082 1083 /* 1084 * determine the device ID for the VFs, the vendor ID will be the 1085 * same as the PF so there is no need to check for that one 1086 */ 1087 dev_id = dev->sriov->vf_device; 1088 1089 /* loop through all the VFs to see if we own any that are assigned */ 1090 vfdev = pci_get_device(dev->vendor, dev_id, NULL); 1091 while (vfdev) { 1092 /* 1093 * It is considered assigned if it is a virtual function with 1094 * our dev as the physical function and the assigned bit is set 1095 */ 1096 if (vfdev->is_virtfn && (vfdev->physfn == dev) && 1097 pci_is_dev_assigned(vfdev)) 1098 vfs_assigned++; 1099 1100 vfdev = pci_get_device(dev->vendor, dev_id, vfdev); 1101 } 1102 1103 return vfs_assigned; 1104 } 1105 EXPORT_SYMBOL_GPL(pci_vfs_assigned); 1106 1107 /** 1108 * pci_sriov_set_totalvfs -- reduce the TotalVFs available 1109 * @dev: the PCI PF device 1110 * @numvfs: number that should be used for TotalVFs supported 1111 * 1112 * Should be called from PF driver's probe routine with 1113 * device's mutex held. 1114 * 1115 * Returns 0 if PF is an SRIOV-capable device and 1116 * value of numvfs valid. If not a PF return -ENOSYS; 1117 * if numvfs is invalid return -EINVAL; 1118 * if VFs already enabled, return -EBUSY. 1119 */ 1120 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) 1121 { 1122 if (!dev->is_physfn) 1123 return -ENOSYS; 1124 1125 if (numvfs > dev->sriov->total_VFs) 1126 return -EINVAL; 1127 1128 /* Shouldn't change if VFs already enabled */ 1129 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE) 1130 return -EBUSY; 1131 1132 dev->sriov->driver_max_VFs = numvfs; 1133 return 0; 1134 } 1135 EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs); 1136 1137 /** 1138 * pci_sriov_get_totalvfs -- get total VFs supported on this device 1139 * @dev: the PCI PF device 1140 * 1141 * For a PCIe device with SRIOV support, return the PCIe 1142 * SRIOV capability value of TotalVFs or the value of driver_max_VFs 1143 * if the driver reduced it. Otherwise 0. 1144 */ 1145 int pci_sriov_get_totalvfs(struct pci_dev *dev) 1146 { 1147 if (!dev->is_physfn) 1148 return 0; 1149 1150 return dev->sriov->driver_max_VFs; 1151 } 1152 EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs); 1153 1154 /** 1155 * pci_sriov_configure_simple - helper to configure SR-IOV 1156 * @dev: the PCI device 1157 * @nr_virtfn: number of virtual functions to enable, 0 to disable 1158 * 1159 * Enable or disable SR-IOV for devices that don't require any PF setup 1160 * before enabling SR-IOV. Return value is negative on error, or number of 1161 * VFs allocated on success. 1162 */ 1163 int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn) 1164 { 1165 int rc; 1166 1167 might_sleep(); 1168 1169 if (!dev->is_physfn) 1170 return -ENODEV; 1171 1172 if (pci_vfs_assigned(dev)) { 1173 pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n"); 1174 return -EPERM; 1175 } 1176 1177 if (nr_virtfn == 0) { 1178 sriov_disable(dev); 1179 return 0; 1180 } 1181 1182 rc = sriov_enable(dev, nr_virtfn); 1183 if (rc < 0) 1184 return rc; 1185 1186 return nr_virtfn; 1187 } 1188 EXPORT_SYMBOL_GPL(pci_sriov_configure_simple); 1189