1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Bus Services, see include/linux/pci.h for further explanation. 4 * 5 * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter, 6 * David Mosberger-Tang 7 * 8 * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz> 9 */ 10 11 #include <linux/acpi.h> 12 #include <linux/kernel.h> 13 #include <linux/delay.h> 14 #include <linux/dmi.h> 15 #include <linux/init.h> 16 #include <linux/of.h> 17 #include <linux/of_pci.h> 18 #include <linux/pci.h> 19 #include <linux/pm.h> 20 #include <linux/slab.h> 21 #include <linux/module.h> 22 #include <linux/spinlock.h> 23 #include <linux/string.h> 24 #include <linux/log2.h> 25 #include <linux/logic_pio.h> 26 #include <linux/pm_wakeup.h> 27 #include <linux/interrupt.h> 28 #include <linux/device.h> 29 #include <linux/pm_runtime.h> 30 #include <linux/pci_hotplug.h> 31 #include <linux/vmalloc.h> 32 #include <linux/pci-ats.h> 33 #include <asm/setup.h> 34 #include <asm/dma.h> 35 #include <linux/aer.h> 36 #include "pci.h" 37 38 const char *pci_power_names[] = { 39 "error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown", 40 }; 41 EXPORT_SYMBOL_GPL(pci_power_names); 42 43 int isa_dma_bridge_buggy; 44 EXPORT_SYMBOL(isa_dma_bridge_buggy); 45 46 int pci_pci_problems; 47 EXPORT_SYMBOL(pci_pci_problems); 48 49 unsigned int pci_pm_d3_delay; 50 51 static void pci_pme_list_scan(struct work_struct *work); 52 53 static LIST_HEAD(pci_pme_list); 54 static DEFINE_MUTEX(pci_pme_list_mutex); 55 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan); 56 57 struct pci_pme_device { 58 struct list_head list; 59 struct pci_dev *dev; 60 }; 61 62 #define PME_TIMEOUT 1000 /* How long between PME checks */ 63 64 static void pci_dev_d3_sleep(struct pci_dev *dev) 65 { 66 unsigned int delay = dev->d3_delay; 67 68 if (delay < pci_pm_d3_delay) 69 delay = pci_pm_d3_delay; 70 71 if (delay) 72 msleep(delay); 73 } 74 75 #ifdef CONFIG_PCI_DOMAINS 76 int pci_domains_supported = 1; 77 #endif 78 79 #define DEFAULT_CARDBUS_IO_SIZE (256) 80 #define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024) 81 /* pci=cbmemsize=nnM,cbiosize=nn can override this */ 82 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE; 83 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE; 84 85 #define DEFAULT_HOTPLUG_IO_SIZE (256) 86 #define DEFAULT_HOTPLUG_MEM_SIZE (2*1024*1024) 87 /* pci=hpmemsize=nnM,hpiosize=nn can override this */ 88 unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE; 89 unsigned long pci_hotplug_mem_size = DEFAULT_HOTPLUG_MEM_SIZE; 90 91 #define DEFAULT_HOTPLUG_BUS_SIZE 1 92 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 93 94 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT; 95 96 /* 97 * The default CLS is used if arch didn't set CLS explicitly and not 98 * all pci devices agree on the same value. Arch can override either 99 * the dfl or actual value as it sees fit. Don't forget this is 100 * measured in 32-bit words, not bytes. 101 */ 102 u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2; 103 u8 pci_cache_line_size; 104 105 /* 106 * If we set up a device for bus mastering, we need to check the latency 107 * timer as certain BIOSes forget to set it properly. 108 */ 109 unsigned int pcibios_max_latency = 255; 110 111 /* If set, the PCIe ARI capability will not be used. */ 112 static bool pcie_ari_disabled; 113 114 /* If set, the PCIe ATS capability will not be used. */ 115 static bool pcie_ats_disabled; 116 117 /* If set, the PCI config space of each device is printed during boot. */ 118 bool pci_early_dump; 119 120 bool pci_ats_disabled(void) 121 { 122 return pcie_ats_disabled; 123 } 124 125 /* Disable bridge_d3 for all PCIe ports */ 126 static bool pci_bridge_d3_disable; 127 /* Force bridge_d3 for all PCIe ports */ 128 static bool pci_bridge_d3_force; 129 130 static int __init pcie_port_pm_setup(char *str) 131 { 132 if (!strcmp(str, "off")) 133 pci_bridge_d3_disable = true; 134 else if (!strcmp(str, "force")) 135 pci_bridge_d3_force = true; 136 return 1; 137 } 138 __setup("pcie_port_pm=", pcie_port_pm_setup); 139 140 /* Time to wait after a reset for device to become responsive */ 141 #define PCIE_RESET_READY_POLL_MS 60000 142 143 /** 144 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children 145 * @bus: pointer to PCI bus structure to search 146 * 147 * Given a PCI bus, returns the highest PCI bus number present in the set 148 * including the given PCI bus and its list of child PCI buses. 149 */ 150 unsigned char pci_bus_max_busnr(struct pci_bus *bus) 151 { 152 struct pci_bus *tmp; 153 unsigned char max, n; 154 155 max = bus->busn_res.end; 156 list_for_each_entry(tmp, &bus->children, node) { 157 n = pci_bus_max_busnr(tmp); 158 if (n > max) 159 max = n; 160 } 161 return max; 162 } 163 EXPORT_SYMBOL_GPL(pci_bus_max_busnr); 164 165 #ifdef CONFIG_HAS_IOMEM 166 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) 167 { 168 struct resource *res = &pdev->resource[bar]; 169 170 /* 171 * Make sure the BAR is actually a memory resource, not an IO resource 172 */ 173 if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) { 174 pci_warn(pdev, "can't ioremap BAR %d: %pR\n", bar, res); 175 return NULL; 176 } 177 return ioremap_nocache(res->start, resource_size(res)); 178 } 179 EXPORT_SYMBOL_GPL(pci_ioremap_bar); 180 181 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar) 182 { 183 /* 184 * Make sure the BAR is actually a memory resource, not an IO resource 185 */ 186 if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { 187 WARN_ON(1); 188 return NULL; 189 } 190 return ioremap_wc(pci_resource_start(pdev, bar), 191 pci_resource_len(pdev, bar)); 192 } 193 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar); 194 #endif 195 196 /** 197 * pci_dev_str_match_path - test if a path string matches a device 198 * @dev: the PCI device to test 199 * @p: string to match the device against 200 * @endptr: pointer to the string after the match 201 * 202 * Test if a string (typically from a kernel parameter) formatted as a 203 * path of device/function addresses matches a PCI device. The string must 204 * be of the form: 205 * 206 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 207 * 208 * A path for a device can be obtained using 'lspci -t'. Using a path 209 * is more robust against bus renumbering than using only a single bus, 210 * device and function address. 211 * 212 * Returns 1 if the string matches the device, 0 if it does not and 213 * a negative error code if it fails to parse the string. 214 */ 215 static int pci_dev_str_match_path(struct pci_dev *dev, const char *path, 216 const char **endptr) 217 { 218 int ret; 219 int seg, bus, slot, func; 220 char *wpath, *p; 221 char end; 222 223 *endptr = strchrnul(path, ';'); 224 225 wpath = kmemdup_nul(path, *endptr - path, GFP_KERNEL); 226 if (!wpath) 227 return -ENOMEM; 228 229 while (1) { 230 p = strrchr(wpath, '/'); 231 if (!p) 232 break; 233 ret = sscanf(p, "/%x.%x%c", &slot, &func, &end); 234 if (ret != 2) { 235 ret = -EINVAL; 236 goto free_and_exit; 237 } 238 239 if (dev->devfn != PCI_DEVFN(slot, func)) { 240 ret = 0; 241 goto free_and_exit; 242 } 243 244 /* 245 * Note: we don't need to get a reference to the upstream 246 * bridge because we hold a reference to the top level 247 * device which should hold a reference to the bridge, 248 * and so on. 249 */ 250 dev = pci_upstream_bridge(dev); 251 if (!dev) { 252 ret = 0; 253 goto free_and_exit; 254 } 255 256 *p = 0; 257 } 258 259 ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot, 260 &func, &end); 261 if (ret != 4) { 262 seg = 0; 263 ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end); 264 if (ret != 3) { 265 ret = -EINVAL; 266 goto free_and_exit; 267 } 268 } 269 270 ret = (seg == pci_domain_nr(dev->bus) && 271 bus == dev->bus->number && 272 dev->devfn == PCI_DEVFN(slot, func)); 273 274 free_and_exit: 275 kfree(wpath); 276 return ret; 277 } 278 279 /** 280 * pci_dev_str_match - test if a string matches a device 281 * @dev: the PCI device to test 282 * @p: string to match the device against 283 * @endptr: pointer to the string after the match 284 * 285 * Test if a string (typically from a kernel parameter) matches a specified 286 * PCI device. The string may be of one of the following formats: 287 * 288 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 289 * pci:<vendor>:<device>[:<subvendor>:<subdevice>] 290 * 291 * The first format specifies a PCI bus/device/function address which 292 * may change if new hardware is inserted, if motherboard firmware changes, 293 * or due to changes caused in kernel parameters. If the domain is 294 * left unspecified, it is taken to be 0. In order to be robust against 295 * bus renumbering issues, a path of PCI device/function numbers may be used 296 * to address the specific device. The path for a device can be determined 297 * through the use of 'lspci -t'. 298 * 299 * The second format matches devices using IDs in the configuration 300 * space which may match multiple devices in the system. A value of 0 301 * for any field will match all devices. (Note: this differs from 302 * in-kernel code that uses PCI_ANY_ID which is ~0; this is for 303 * legacy reasons and convenience so users don't have to specify 304 * FFFFFFFFs on the command line.) 305 * 306 * Returns 1 if the string matches the device, 0 if it does not and 307 * a negative error code if the string cannot be parsed. 308 */ 309 static int pci_dev_str_match(struct pci_dev *dev, const char *p, 310 const char **endptr) 311 { 312 int ret; 313 int count; 314 unsigned short vendor, device, subsystem_vendor, subsystem_device; 315 316 if (strncmp(p, "pci:", 4) == 0) { 317 /* PCI vendor/device (subvendor/subdevice) IDs are specified */ 318 p += 4; 319 ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device, 320 &subsystem_vendor, &subsystem_device, &count); 321 if (ret != 4) { 322 ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count); 323 if (ret != 2) 324 return -EINVAL; 325 326 subsystem_vendor = 0; 327 subsystem_device = 0; 328 } 329 330 p += count; 331 332 if ((!vendor || vendor == dev->vendor) && 333 (!device || device == dev->device) && 334 (!subsystem_vendor || 335 subsystem_vendor == dev->subsystem_vendor) && 336 (!subsystem_device || 337 subsystem_device == dev->subsystem_device)) 338 goto found; 339 } else { 340 /* 341 * PCI Bus, Device, Function IDs are specified 342 * (optionally, may include a path of devfns following it) 343 */ 344 ret = pci_dev_str_match_path(dev, p, &p); 345 if (ret < 0) 346 return ret; 347 else if (ret) 348 goto found; 349 } 350 351 *endptr = p; 352 return 0; 353 354 found: 355 *endptr = p; 356 return 1; 357 } 358 359 static int __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn, 360 u8 pos, int cap, int *ttl) 361 { 362 u8 id; 363 u16 ent; 364 365 pci_bus_read_config_byte(bus, devfn, pos, &pos); 366 367 while ((*ttl)--) { 368 if (pos < 0x40) 369 break; 370 pos &= ~3; 371 pci_bus_read_config_word(bus, devfn, pos, &ent); 372 373 id = ent & 0xff; 374 if (id == 0xff) 375 break; 376 if (id == cap) 377 return pos; 378 pos = (ent >> 8); 379 } 380 return 0; 381 } 382 383 static int __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, 384 u8 pos, int cap) 385 { 386 int ttl = PCI_FIND_CAP_TTL; 387 388 return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl); 389 } 390 391 int pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) 392 { 393 return __pci_find_next_cap(dev->bus, dev->devfn, 394 pos + PCI_CAP_LIST_NEXT, cap); 395 } 396 EXPORT_SYMBOL_GPL(pci_find_next_capability); 397 398 static int __pci_bus_find_cap_start(struct pci_bus *bus, 399 unsigned int devfn, u8 hdr_type) 400 { 401 u16 status; 402 403 pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status); 404 if (!(status & PCI_STATUS_CAP_LIST)) 405 return 0; 406 407 switch (hdr_type) { 408 case PCI_HEADER_TYPE_NORMAL: 409 case PCI_HEADER_TYPE_BRIDGE: 410 return PCI_CAPABILITY_LIST; 411 case PCI_HEADER_TYPE_CARDBUS: 412 return PCI_CB_CAPABILITY_LIST; 413 } 414 415 return 0; 416 } 417 418 /** 419 * pci_find_capability - query for devices' capabilities 420 * @dev: PCI device to query 421 * @cap: capability code 422 * 423 * Tell if a device supports a given PCI capability. 424 * Returns the address of the requested capability structure within the 425 * device's PCI configuration space or 0 in case the device does not 426 * support it. Possible values for @cap: 427 * 428 * %PCI_CAP_ID_PM Power Management 429 * %PCI_CAP_ID_AGP Accelerated Graphics Port 430 * %PCI_CAP_ID_VPD Vital Product Data 431 * %PCI_CAP_ID_SLOTID Slot Identification 432 * %PCI_CAP_ID_MSI Message Signalled Interrupts 433 * %PCI_CAP_ID_CHSWP CompactPCI HotSwap 434 * %PCI_CAP_ID_PCIX PCI-X 435 * %PCI_CAP_ID_EXP PCI Express 436 */ 437 int pci_find_capability(struct pci_dev *dev, int cap) 438 { 439 int pos; 440 441 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 442 if (pos) 443 pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap); 444 445 return pos; 446 } 447 EXPORT_SYMBOL(pci_find_capability); 448 449 /** 450 * pci_bus_find_capability - query for devices' capabilities 451 * @bus: the PCI bus to query 452 * @devfn: PCI device to query 453 * @cap: capability code 454 * 455 * Like pci_find_capability() but works for pci devices that do not have a 456 * pci_dev structure set up yet. 457 * 458 * Returns the address of the requested capability structure within the 459 * device's PCI configuration space or 0 in case the device does not 460 * support it. 461 */ 462 int pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap) 463 { 464 int pos; 465 u8 hdr_type; 466 467 pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type); 468 469 pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f); 470 if (pos) 471 pos = __pci_find_next_cap(bus, devfn, pos, cap); 472 473 return pos; 474 } 475 EXPORT_SYMBOL(pci_bus_find_capability); 476 477 /** 478 * pci_find_next_ext_capability - Find an extended capability 479 * @dev: PCI device to query 480 * @start: address at which to start looking (0 to start at beginning of list) 481 * @cap: capability code 482 * 483 * Returns the address of the next matching extended capability structure 484 * within the device's PCI configuration space or 0 if the device does 485 * not support it. Some capabilities can occur several times, e.g., the 486 * vendor-specific capability, and this provides a way to find them all. 487 */ 488 int pci_find_next_ext_capability(struct pci_dev *dev, int start, int cap) 489 { 490 u32 header; 491 int ttl; 492 int pos = PCI_CFG_SPACE_SIZE; 493 494 /* minimum 8 bytes per capability */ 495 ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; 496 497 if (dev->cfg_size <= PCI_CFG_SPACE_SIZE) 498 return 0; 499 500 if (start) 501 pos = start; 502 503 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 504 return 0; 505 506 /* 507 * If we have no capabilities, this is indicated by cap ID, 508 * cap version and next pointer all being 0. 509 */ 510 if (header == 0) 511 return 0; 512 513 while (ttl-- > 0) { 514 if (PCI_EXT_CAP_ID(header) == cap && pos != start) 515 return pos; 516 517 pos = PCI_EXT_CAP_NEXT(header); 518 if (pos < PCI_CFG_SPACE_SIZE) 519 break; 520 521 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 522 break; 523 } 524 525 return 0; 526 } 527 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); 528 529 /** 530 * pci_find_ext_capability - Find an extended capability 531 * @dev: PCI device to query 532 * @cap: capability code 533 * 534 * Returns the address of the requested extended capability structure 535 * within the device's PCI configuration space or 0 if the device does 536 * not support it. Possible values for @cap: 537 * 538 * %PCI_EXT_CAP_ID_ERR Advanced Error Reporting 539 * %PCI_EXT_CAP_ID_VC Virtual Channel 540 * %PCI_EXT_CAP_ID_DSN Device Serial Number 541 * %PCI_EXT_CAP_ID_PWR Power Budgeting 542 */ 543 int pci_find_ext_capability(struct pci_dev *dev, int cap) 544 { 545 return pci_find_next_ext_capability(dev, 0, cap); 546 } 547 EXPORT_SYMBOL_GPL(pci_find_ext_capability); 548 549 static int __pci_find_next_ht_cap(struct pci_dev *dev, int pos, int ht_cap) 550 { 551 int rc, ttl = PCI_FIND_CAP_TTL; 552 u8 cap, mask; 553 554 if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST) 555 mask = HT_3BIT_CAP_MASK; 556 else 557 mask = HT_5BIT_CAP_MASK; 558 559 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos, 560 PCI_CAP_ID_HT, &ttl); 561 while (pos) { 562 rc = pci_read_config_byte(dev, pos + 3, &cap); 563 if (rc != PCIBIOS_SUCCESSFUL) 564 return 0; 565 566 if ((cap & mask) == ht_cap) 567 return pos; 568 569 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, 570 pos + PCI_CAP_LIST_NEXT, 571 PCI_CAP_ID_HT, &ttl); 572 } 573 574 return 0; 575 } 576 /** 577 * pci_find_next_ht_capability - query a device's Hypertransport capabilities 578 * @dev: PCI device to query 579 * @pos: Position from which to continue searching 580 * @ht_cap: Hypertransport capability code 581 * 582 * To be used in conjunction with pci_find_ht_capability() to search for 583 * all capabilities matching @ht_cap. @pos should always be a value returned 584 * from pci_find_ht_capability(). 585 * 586 * NB. To be 100% safe against broken PCI devices, the caller should take 587 * steps to avoid an infinite loop. 588 */ 589 int pci_find_next_ht_capability(struct pci_dev *dev, int pos, int ht_cap) 590 { 591 return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap); 592 } 593 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability); 594 595 /** 596 * pci_find_ht_capability - query a device's Hypertransport capabilities 597 * @dev: PCI device to query 598 * @ht_cap: Hypertransport capability code 599 * 600 * Tell if a device supports a given Hypertransport capability. 601 * Returns an address within the device's PCI configuration space 602 * or 0 in case the device does not support the request capability. 603 * The address points to the PCI capability, of type PCI_CAP_ID_HT, 604 * which has a Hypertransport capability matching @ht_cap. 605 */ 606 int pci_find_ht_capability(struct pci_dev *dev, int ht_cap) 607 { 608 int pos; 609 610 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 611 if (pos) 612 pos = __pci_find_next_ht_cap(dev, pos, ht_cap); 613 614 return pos; 615 } 616 EXPORT_SYMBOL_GPL(pci_find_ht_capability); 617 618 /** 619 * pci_find_parent_resource - return resource region of parent bus of given region 620 * @dev: PCI device structure contains resources to be searched 621 * @res: child resource record for which parent is sought 622 * 623 * For given resource region of given device, return the resource 624 * region of parent bus the given region is contained in. 625 */ 626 struct resource *pci_find_parent_resource(const struct pci_dev *dev, 627 struct resource *res) 628 { 629 const struct pci_bus *bus = dev->bus; 630 struct resource *r; 631 int i; 632 633 pci_bus_for_each_resource(bus, r, i) { 634 if (!r) 635 continue; 636 if (resource_contains(r, res)) { 637 638 /* 639 * If the window is prefetchable but the BAR is 640 * not, the allocator made a mistake. 641 */ 642 if (r->flags & IORESOURCE_PREFETCH && 643 !(res->flags & IORESOURCE_PREFETCH)) 644 return NULL; 645 646 /* 647 * If we're below a transparent bridge, there may 648 * be both a positively-decoded aperture and a 649 * subtractively-decoded region that contain the BAR. 650 * We want the positively-decoded one, so this depends 651 * on pci_bus_for_each_resource() giving us those 652 * first. 653 */ 654 return r; 655 } 656 } 657 return NULL; 658 } 659 EXPORT_SYMBOL(pci_find_parent_resource); 660 661 /** 662 * pci_find_resource - Return matching PCI device resource 663 * @dev: PCI device to query 664 * @res: Resource to look for 665 * 666 * Goes over standard PCI resources (BARs) and checks if the given resource 667 * is partially or fully contained in any of them. In that case the 668 * matching resource is returned, %NULL otherwise. 669 */ 670 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res) 671 { 672 int i; 673 674 for (i = 0; i < PCI_ROM_RESOURCE; i++) { 675 struct resource *r = &dev->resource[i]; 676 677 if (r->start && resource_contains(r, res)) 678 return r; 679 } 680 681 return NULL; 682 } 683 EXPORT_SYMBOL(pci_find_resource); 684 685 /** 686 * pci_find_pcie_root_port - return PCIe Root Port 687 * @dev: PCI device to query 688 * 689 * Traverse up the parent chain and return the PCIe Root Port PCI Device 690 * for a given PCI Device. 691 */ 692 struct pci_dev *pci_find_pcie_root_port(struct pci_dev *dev) 693 { 694 struct pci_dev *bridge, *highest_pcie_bridge = dev; 695 696 bridge = pci_upstream_bridge(dev); 697 while (bridge && pci_is_pcie(bridge)) { 698 highest_pcie_bridge = bridge; 699 bridge = pci_upstream_bridge(bridge); 700 } 701 702 if (pci_pcie_type(highest_pcie_bridge) != PCI_EXP_TYPE_ROOT_PORT) 703 return NULL; 704 705 return highest_pcie_bridge; 706 } 707 EXPORT_SYMBOL(pci_find_pcie_root_port); 708 709 /** 710 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos 711 * @dev: the PCI device to operate on 712 * @pos: config space offset of status word 713 * @mask: mask of bit(s) to care about in status word 714 * 715 * Return 1 when mask bit(s) in status word clear, 0 otherwise. 716 */ 717 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask) 718 { 719 int i; 720 721 /* Wait for Transaction Pending bit clean */ 722 for (i = 0; i < 4; i++) { 723 u16 status; 724 if (i) 725 msleep((1 << (i - 1)) * 100); 726 727 pci_read_config_word(dev, pos, &status); 728 if (!(status & mask)) 729 return 1; 730 } 731 732 return 0; 733 } 734 735 /** 736 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up) 737 * @dev: PCI device to have its BARs restored 738 * 739 * Restore the BAR values for a given device, so as to make it 740 * accessible by its driver. 741 */ 742 static void pci_restore_bars(struct pci_dev *dev) 743 { 744 int i; 745 746 for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) 747 pci_update_resource(dev, i); 748 } 749 750 static const struct pci_platform_pm_ops *pci_platform_pm; 751 752 int pci_set_platform_pm(const struct pci_platform_pm_ops *ops) 753 { 754 if (!ops->is_manageable || !ops->set_state || !ops->get_state || 755 !ops->choose_state || !ops->set_wakeup || !ops->need_resume) 756 return -EINVAL; 757 pci_platform_pm = ops; 758 return 0; 759 } 760 761 static inline bool platform_pci_power_manageable(struct pci_dev *dev) 762 { 763 return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false; 764 } 765 766 static inline int platform_pci_set_power_state(struct pci_dev *dev, 767 pci_power_t t) 768 { 769 return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS; 770 } 771 772 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev) 773 { 774 return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN; 775 } 776 777 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev) 778 { 779 return pci_platform_pm ? 780 pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR; 781 } 782 783 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable) 784 { 785 return pci_platform_pm ? 786 pci_platform_pm->set_wakeup(dev, enable) : -ENODEV; 787 } 788 789 static inline bool platform_pci_need_resume(struct pci_dev *dev) 790 { 791 return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false; 792 } 793 794 /** 795 * pci_raw_set_power_state - Use PCI PM registers to set the power state of 796 * given PCI device 797 * @dev: PCI device to handle. 798 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. 799 * 800 * RETURN VALUE: 801 * -EINVAL if the requested state is invalid. 802 * -EIO if device does not support PCI PM or its PM capabilities register has a 803 * wrong version, or device doesn't support the requested state. 804 * 0 if device already is in the requested state. 805 * 0 if device's power state has been successfully changed. 806 */ 807 static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state) 808 { 809 u16 pmcsr; 810 bool need_restore = false; 811 812 /* Check if we're already there */ 813 if (dev->current_state == state) 814 return 0; 815 816 if (!dev->pm_cap) 817 return -EIO; 818 819 if (state < PCI_D0 || state > PCI_D3hot) 820 return -EINVAL; 821 822 /* Validate current state: 823 * Can enter D0 from any state, but if we can only go deeper 824 * to sleep if we're already in a low power state 825 */ 826 if (state != PCI_D0 && dev->current_state <= PCI_D3cold 827 && dev->current_state > state) { 828 pci_err(dev, "invalid power transition (from state %d to %d)\n", 829 dev->current_state, state); 830 return -EINVAL; 831 } 832 833 /* check if this device supports the desired state */ 834 if ((state == PCI_D1 && !dev->d1_support) 835 || (state == PCI_D2 && !dev->d2_support)) 836 return -EIO; 837 838 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 839 840 /* If we're (effectively) in D3, force entire word to 0. 841 * This doesn't affect PME_Status, disables PME_En, and 842 * sets PowerState to 0. 843 */ 844 switch (dev->current_state) { 845 case PCI_D0: 846 case PCI_D1: 847 case PCI_D2: 848 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 849 pmcsr |= state; 850 break; 851 case PCI_D3hot: 852 case PCI_D3cold: 853 case PCI_UNKNOWN: /* Boot-up */ 854 if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot 855 && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET)) 856 need_restore = true; 857 /* Fall-through: force to D0 */ 858 default: 859 pmcsr = 0; 860 break; 861 } 862 863 /* enter specified state */ 864 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 865 866 /* Mandatory power management transition delays */ 867 /* see PCI PM 1.1 5.6.1 table 18 */ 868 if (state == PCI_D3hot || dev->current_state == PCI_D3hot) 869 pci_dev_d3_sleep(dev); 870 else if (state == PCI_D2 || dev->current_state == PCI_D2) 871 udelay(PCI_PM_D2_DELAY); 872 873 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 874 dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); 875 if (dev->current_state != state && printk_ratelimit()) 876 pci_info(dev, "Refused to change power state, currently in D%d\n", 877 dev->current_state); 878 879 /* 880 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT 881 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning 882 * from D3hot to D0 _may_ perform an internal reset, thereby 883 * going to "D0 Uninitialized" rather than "D0 Initialized". 884 * For example, at least some versions of the 3c905B and the 885 * 3c556B exhibit this behaviour. 886 * 887 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave 888 * devices in a D3hot state at boot. Consequently, we need to 889 * restore at least the BARs so that the device will be 890 * accessible to its driver. 891 */ 892 if (need_restore) 893 pci_restore_bars(dev); 894 895 if (dev->bus->self) 896 pcie_aspm_pm_state_change(dev->bus->self); 897 898 return 0; 899 } 900 901 /** 902 * pci_update_current_state - Read power state of given device and cache it 903 * @dev: PCI device to handle. 904 * @state: State to cache in case the device doesn't have the PM capability 905 * 906 * The power state is read from the PMCSR register, which however is 907 * inaccessible in D3cold. The platform firmware is therefore queried first 908 * to detect accessibility of the register. In case the platform firmware 909 * reports an incorrect state or the device isn't power manageable by the 910 * platform at all, we try to detect D3cold by testing accessibility of the 911 * vendor ID in config space. 912 */ 913 void pci_update_current_state(struct pci_dev *dev, pci_power_t state) 914 { 915 if (platform_pci_get_power_state(dev) == PCI_D3cold || 916 !pci_device_is_present(dev)) { 917 dev->current_state = PCI_D3cold; 918 } else if (dev->pm_cap) { 919 u16 pmcsr; 920 921 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 922 dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); 923 } else { 924 dev->current_state = state; 925 } 926 } 927 928 /** 929 * pci_power_up - Put the given device into D0 forcibly 930 * @dev: PCI device to power up 931 */ 932 void pci_power_up(struct pci_dev *dev) 933 { 934 if (platform_pci_power_manageable(dev)) 935 platform_pci_set_power_state(dev, PCI_D0); 936 937 pci_raw_set_power_state(dev, PCI_D0); 938 pci_update_current_state(dev, PCI_D0); 939 } 940 941 /** 942 * pci_platform_power_transition - Use platform to change device power state 943 * @dev: PCI device to handle. 944 * @state: State to put the device into. 945 */ 946 static int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state) 947 { 948 int error; 949 950 if (platform_pci_power_manageable(dev)) { 951 error = platform_pci_set_power_state(dev, state); 952 if (!error) 953 pci_update_current_state(dev, state); 954 } else 955 error = -ENODEV; 956 957 if (error && !dev->pm_cap) /* Fall back to PCI_D0 */ 958 dev->current_state = PCI_D0; 959 960 return error; 961 } 962 963 /** 964 * pci_wakeup - Wake up a PCI device 965 * @pci_dev: Device to handle. 966 * @ign: ignored parameter 967 */ 968 static int pci_wakeup(struct pci_dev *pci_dev, void *ign) 969 { 970 pci_wakeup_event(pci_dev); 971 pm_request_resume(&pci_dev->dev); 972 return 0; 973 } 974 975 /** 976 * pci_wakeup_bus - Walk given bus and wake up devices on it 977 * @bus: Top bus of the subtree to walk. 978 */ 979 void pci_wakeup_bus(struct pci_bus *bus) 980 { 981 if (bus) 982 pci_walk_bus(bus, pci_wakeup, NULL); 983 } 984 985 /** 986 * __pci_start_power_transition - Start power transition of a PCI device 987 * @dev: PCI device to handle. 988 * @state: State to put the device into. 989 */ 990 static void __pci_start_power_transition(struct pci_dev *dev, pci_power_t state) 991 { 992 if (state == PCI_D0) { 993 pci_platform_power_transition(dev, PCI_D0); 994 /* 995 * Mandatory power management transition delays, see 996 * PCI Express Base Specification Revision 2.0 Section 997 * 6.6.1: Conventional Reset. Do not delay for 998 * devices powered on/off by corresponding bridge, 999 * because have already delayed for the bridge. 1000 */ 1001 if (dev->runtime_d3cold) { 1002 if (dev->d3cold_delay) 1003 msleep(dev->d3cold_delay); 1004 /* 1005 * When powering on a bridge from D3cold, the 1006 * whole hierarchy may be powered on into 1007 * D0uninitialized state, resume them to give 1008 * them a chance to suspend again 1009 */ 1010 pci_wakeup_bus(dev->subordinate); 1011 } 1012 } 1013 } 1014 1015 /** 1016 * __pci_dev_set_current_state - Set current state of a PCI device 1017 * @dev: Device to handle 1018 * @data: pointer to state to be set 1019 */ 1020 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data) 1021 { 1022 pci_power_t state = *(pci_power_t *)data; 1023 1024 dev->current_state = state; 1025 return 0; 1026 } 1027 1028 /** 1029 * pci_bus_set_current_state - Walk given bus and set current state of devices 1030 * @bus: Top bus of the subtree to walk. 1031 * @state: state to be set 1032 */ 1033 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state) 1034 { 1035 if (bus) 1036 pci_walk_bus(bus, __pci_dev_set_current_state, &state); 1037 } 1038 1039 /** 1040 * __pci_complete_power_transition - Complete power transition of a PCI device 1041 * @dev: PCI device to handle. 1042 * @state: State to put the device into. 1043 * 1044 * This function should not be called directly by device drivers. 1045 */ 1046 int __pci_complete_power_transition(struct pci_dev *dev, pci_power_t state) 1047 { 1048 int ret; 1049 1050 if (state <= PCI_D0) 1051 return -EINVAL; 1052 ret = pci_platform_power_transition(dev, state); 1053 /* Power off the bridge may power off the whole hierarchy */ 1054 if (!ret && state == PCI_D3cold) 1055 pci_bus_set_current_state(dev->subordinate, PCI_D3cold); 1056 return ret; 1057 } 1058 EXPORT_SYMBOL_GPL(__pci_complete_power_transition); 1059 1060 /** 1061 * pci_set_power_state - Set the power state of a PCI device 1062 * @dev: PCI device to handle. 1063 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. 1064 * 1065 * Transition a device to a new power state, using the platform firmware and/or 1066 * the device's PCI PM registers. 1067 * 1068 * RETURN VALUE: 1069 * -EINVAL if the requested state is invalid. 1070 * -EIO if device does not support PCI PM or its PM capabilities register has a 1071 * wrong version, or device doesn't support the requested state. 1072 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported. 1073 * 0 if device already is in the requested state. 1074 * 0 if the transition is to D3 but D3 is not supported. 1075 * 0 if device's power state has been successfully changed. 1076 */ 1077 int pci_set_power_state(struct pci_dev *dev, pci_power_t state) 1078 { 1079 int error; 1080 1081 /* bound the state we're entering */ 1082 if (state > PCI_D3cold) 1083 state = PCI_D3cold; 1084 else if (state < PCI_D0) 1085 state = PCI_D0; 1086 else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev)) 1087 /* 1088 * If the device or the parent bridge do not support PCI PM, 1089 * ignore the request if we're doing anything other than putting 1090 * it into D0 (which would only happen on boot). 1091 */ 1092 return 0; 1093 1094 /* Check if we're already there */ 1095 if (dev->current_state == state) 1096 return 0; 1097 1098 __pci_start_power_transition(dev, state); 1099 1100 /* This device is quirked not to be put into D3, so 1101 don't put it in D3 */ 1102 if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3)) 1103 return 0; 1104 1105 /* 1106 * To put device in D3cold, we put device into D3hot in native 1107 * way, then put device into D3cold with platform ops 1108 */ 1109 error = pci_raw_set_power_state(dev, state > PCI_D3hot ? 1110 PCI_D3hot : state); 1111 1112 if (!__pci_complete_power_transition(dev, state)) 1113 error = 0; 1114 1115 return error; 1116 } 1117 EXPORT_SYMBOL(pci_set_power_state); 1118 1119 /** 1120 * pci_choose_state - Choose the power state of a PCI device 1121 * @dev: PCI device to be suspended 1122 * @state: target sleep state for the whole system. This is the value 1123 * that is passed to suspend() function. 1124 * 1125 * Returns PCI power state suitable for given device and given system 1126 * message. 1127 */ 1128 1129 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) 1130 { 1131 pci_power_t ret; 1132 1133 if (!dev->pm_cap) 1134 return PCI_D0; 1135 1136 ret = platform_pci_choose_state(dev); 1137 if (ret != PCI_POWER_ERROR) 1138 return ret; 1139 1140 switch (state.event) { 1141 case PM_EVENT_ON: 1142 return PCI_D0; 1143 case PM_EVENT_FREEZE: 1144 case PM_EVENT_PRETHAW: 1145 /* REVISIT both freeze and pre-thaw "should" use D0 */ 1146 case PM_EVENT_SUSPEND: 1147 case PM_EVENT_HIBERNATE: 1148 return PCI_D3hot; 1149 default: 1150 pci_info(dev, "unrecognized suspend event %d\n", 1151 state.event); 1152 BUG(); 1153 } 1154 return PCI_D0; 1155 } 1156 EXPORT_SYMBOL(pci_choose_state); 1157 1158 #define PCI_EXP_SAVE_REGS 7 1159 1160 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev, 1161 u16 cap, bool extended) 1162 { 1163 struct pci_cap_saved_state *tmp; 1164 1165 hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) { 1166 if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap) 1167 return tmp; 1168 } 1169 return NULL; 1170 } 1171 1172 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap) 1173 { 1174 return _pci_find_saved_cap(dev, cap, false); 1175 } 1176 1177 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap) 1178 { 1179 return _pci_find_saved_cap(dev, cap, true); 1180 } 1181 1182 static int pci_save_pcie_state(struct pci_dev *dev) 1183 { 1184 int i = 0; 1185 struct pci_cap_saved_state *save_state; 1186 u16 *cap; 1187 1188 if (!pci_is_pcie(dev)) 1189 return 0; 1190 1191 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1192 if (!save_state) { 1193 pci_err(dev, "buffer not found in %s\n", __func__); 1194 return -ENOMEM; 1195 } 1196 1197 cap = (u16 *)&save_state->cap.data[0]; 1198 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]); 1199 pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]); 1200 pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]); 1201 pcie_capability_read_word(dev, PCI_EXP_RTCTL, &cap[i++]); 1202 pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]); 1203 pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]); 1204 pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]); 1205 1206 return 0; 1207 } 1208 1209 static void pci_restore_pcie_state(struct pci_dev *dev) 1210 { 1211 int i = 0; 1212 struct pci_cap_saved_state *save_state; 1213 u16 *cap; 1214 1215 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1216 if (!save_state) 1217 return; 1218 1219 cap = (u16 *)&save_state->cap.data[0]; 1220 pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]); 1221 pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]); 1222 pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]); 1223 pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]); 1224 pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]); 1225 pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]); 1226 pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]); 1227 } 1228 1229 1230 static int pci_save_pcix_state(struct pci_dev *dev) 1231 { 1232 int pos; 1233 struct pci_cap_saved_state *save_state; 1234 1235 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1236 if (!pos) 1237 return 0; 1238 1239 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1240 if (!save_state) { 1241 pci_err(dev, "buffer not found in %s\n", __func__); 1242 return -ENOMEM; 1243 } 1244 1245 pci_read_config_word(dev, pos + PCI_X_CMD, 1246 (u16 *)save_state->cap.data); 1247 1248 return 0; 1249 } 1250 1251 static void pci_restore_pcix_state(struct pci_dev *dev) 1252 { 1253 int i = 0, pos; 1254 struct pci_cap_saved_state *save_state; 1255 u16 *cap; 1256 1257 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1258 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1259 if (!save_state || !pos) 1260 return; 1261 cap = (u16 *)&save_state->cap.data[0]; 1262 1263 pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]); 1264 } 1265 1266 1267 /** 1268 * pci_save_state - save the PCI configuration space of a device before suspending 1269 * @dev: - PCI device that we're dealing with 1270 */ 1271 int pci_save_state(struct pci_dev *dev) 1272 { 1273 int i; 1274 /* XXX: 100% dword access ok here? */ 1275 for (i = 0; i < 16; i++) 1276 pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]); 1277 dev->state_saved = true; 1278 1279 i = pci_save_pcie_state(dev); 1280 if (i != 0) 1281 return i; 1282 1283 i = pci_save_pcix_state(dev); 1284 if (i != 0) 1285 return i; 1286 1287 return pci_save_vc_state(dev); 1288 } 1289 EXPORT_SYMBOL(pci_save_state); 1290 1291 static void pci_restore_config_dword(struct pci_dev *pdev, int offset, 1292 u32 saved_val, int retry, bool force) 1293 { 1294 u32 val; 1295 1296 pci_read_config_dword(pdev, offset, &val); 1297 if (!force && val == saved_val) 1298 return; 1299 1300 for (;;) { 1301 pci_dbg(pdev, "restoring config space at offset %#x (was %#x, writing %#x)\n", 1302 offset, val, saved_val); 1303 pci_write_config_dword(pdev, offset, saved_val); 1304 if (retry-- <= 0) 1305 return; 1306 1307 pci_read_config_dword(pdev, offset, &val); 1308 if (val == saved_val) 1309 return; 1310 1311 mdelay(1); 1312 } 1313 } 1314 1315 static void pci_restore_config_space_range(struct pci_dev *pdev, 1316 int start, int end, int retry, 1317 bool force) 1318 { 1319 int index; 1320 1321 for (index = end; index >= start; index--) 1322 pci_restore_config_dword(pdev, 4 * index, 1323 pdev->saved_config_space[index], 1324 retry, force); 1325 } 1326 1327 static void pci_restore_config_space(struct pci_dev *pdev) 1328 { 1329 if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) { 1330 pci_restore_config_space_range(pdev, 10, 15, 0, false); 1331 /* Restore BARs before the command register. */ 1332 pci_restore_config_space_range(pdev, 4, 9, 10, false); 1333 pci_restore_config_space_range(pdev, 0, 3, 0, false); 1334 } else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 1335 pci_restore_config_space_range(pdev, 12, 15, 0, false); 1336 1337 /* 1338 * Force rewriting of prefetch registers to avoid S3 resume 1339 * issues on Intel PCI bridges that occur when these 1340 * registers are not explicitly written. 1341 */ 1342 pci_restore_config_space_range(pdev, 9, 11, 0, true); 1343 pci_restore_config_space_range(pdev, 0, 8, 0, false); 1344 } else { 1345 pci_restore_config_space_range(pdev, 0, 15, 0, false); 1346 } 1347 } 1348 1349 static void pci_restore_rebar_state(struct pci_dev *pdev) 1350 { 1351 unsigned int pos, nbars, i; 1352 u32 ctrl; 1353 1354 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 1355 if (!pos) 1356 return; 1357 1358 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1359 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 1360 PCI_REBAR_CTRL_NBAR_SHIFT; 1361 1362 for (i = 0; i < nbars; i++, pos += 8) { 1363 struct resource *res; 1364 int bar_idx, size; 1365 1366 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1367 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 1368 res = pdev->resource + bar_idx; 1369 size = order_base_2((resource_size(res) >> 20) | 1) - 1; 1370 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 1371 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 1372 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 1373 } 1374 } 1375 1376 /** 1377 * pci_restore_state - Restore the saved state of a PCI device 1378 * @dev: - PCI device that we're dealing with 1379 */ 1380 void pci_restore_state(struct pci_dev *dev) 1381 { 1382 if (!dev->state_saved) 1383 return; 1384 1385 /* PCI Express register must be restored first */ 1386 pci_restore_pcie_state(dev); 1387 pci_restore_pasid_state(dev); 1388 pci_restore_pri_state(dev); 1389 pci_restore_ats_state(dev); 1390 pci_restore_vc_state(dev); 1391 pci_restore_rebar_state(dev); 1392 1393 pci_cleanup_aer_error_status_regs(dev); 1394 1395 pci_restore_config_space(dev); 1396 1397 pci_restore_pcix_state(dev); 1398 pci_restore_msi_state(dev); 1399 1400 /* Restore ACS and IOV configuration state */ 1401 pci_enable_acs(dev); 1402 pci_restore_iov_state(dev); 1403 1404 dev->state_saved = false; 1405 } 1406 EXPORT_SYMBOL(pci_restore_state); 1407 1408 struct pci_saved_state { 1409 u32 config_space[16]; 1410 struct pci_cap_saved_data cap[0]; 1411 }; 1412 1413 /** 1414 * pci_store_saved_state - Allocate and return an opaque struct containing 1415 * the device saved state. 1416 * @dev: PCI device that we're dealing with 1417 * 1418 * Return NULL if no state or error. 1419 */ 1420 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev) 1421 { 1422 struct pci_saved_state *state; 1423 struct pci_cap_saved_state *tmp; 1424 struct pci_cap_saved_data *cap; 1425 size_t size; 1426 1427 if (!dev->state_saved) 1428 return NULL; 1429 1430 size = sizeof(*state) + sizeof(struct pci_cap_saved_data); 1431 1432 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) 1433 size += sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1434 1435 state = kzalloc(size, GFP_KERNEL); 1436 if (!state) 1437 return NULL; 1438 1439 memcpy(state->config_space, dev->saved_config_space, 1440 sizeof(state->config_space)); 1441 1442 cap = state->cap; 1443 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) { 1444 size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1445 memcpy(cap, &tmp->cap, len); 1446 cap = (struct pci_cap_saved_data *)((u8 *)cap + len); 1447 } 1448 /* Empty cap_save terminates list */ 1449 1450 return state; 1451 } 1452 EXPORT_SYMBOL_GPL(pci_store_saved_state); 1453 1454 /** 1455 * pci_load_saved_state - Reload the provided save state into struct pci_dev. 1456 * @dev: PCI device that we're dealing with 1457 * @state: Saved state returned from pci_store_saved_state() 1458 */ 1459 int pci_load_saved_state(struct pci_dev *dev, 1460 struct pci_saved_state *state) 1461 { 1462 struct pci_cap_saved_data *cap; 1463 1464 dev->state_saved = false; 1465 1466 if (!state) 1467 return 0; 1468 1469 memcpy(dev->saved_config_space, state->config_space, 1470 sizeof(state->config_space)); 1471 1472 cap = state->cap; 1473 while (cap->size) { 1474 struct pci_cap_saved_state *tmp; 1475 1476 tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended); 1477 if (!tmp || tmp->cap.size != cap->size) 1478 return -EINVAL; 1479 1480 memcpy(tmp->cap.data, cap->data, tmp->cap.size); 1481 cap = (struct pci_cap_saved_data *)((u8 *)cap + 1482 sizeof(struct pci_cap_saved_data) + cap->size); 1483 } 1484 1485 dev->state_saved = true; 1486 return 0; 1487 } 1488 EXPORT_SYMBOL_GPL(pci_load_saved_state); 1489 1490 /** 1491 * pci_load_and_free_saved_state - Reload the save state pointed to by state, 1492 * and free the memory allocated for it. 1493 * @dev: PCI device that we're dealing with 1494 * @state: Pointer to saved state returned from pci_store_saved_state() 1495 */ 1496 int pci_load_and_free_saved_state(struct pci_dev *dev, 1497 struct pci_saved_state **state) 1498 { 1499 int ret = pci_load_saved_state(dev, *state); 1500 kfree(*state); 1501 *state = NULL; 1502 return ret; 1503 } 1504 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state); 1505 1506 int __weak pcibios_enable_device(struct pci_dev *dev, int bars) 1507 { 1508 return pci_enable_resources(dev, bars); 1509 } 1510 1511 static int do_pci_enable_device(struct pci_dev *dev, int bars) 1512 { 1513 int err; 1514 struct pci_dev *bridge; 1515 u16 cmd; 1516 u8 pin; 1517 1518 err = pci_set_power_state(dev, PCI_D0); 1519 if (err < 0 && err != -EIO) 1520 return err; 1521 1522 bridge = pci_upstream_bridge(dev); 1523 if (bridge) 1524 pcie_aspm_powersave_config_link(bridge); 1525 1526 err = pcibios_enable_device(dev, bars); 1527 if (err < 0) 1528 return err; 1529 pci_fixup_device(pci_fixup_enable, dev); 1530 1531 if (dev->msi_enabled || dev->msix_enabled) 1532 return 0; 1533 1534 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin); 1535 if (pin) { 1536 pci_read_config_word(dev, PCI_COMMAND, &cmd); 1537 if (cmd & PCI_COMMAND_INTX_DISABLE) 1538 pci_write_config_word(dev, PCI_COMMAND, 1539 cmd & ~PCI_COMMAND_INTX_DISABLE); 1540 } 1541 1542 return 0; 1543 } 1544 1545 /** 1546 * pci_reenable_device - Resume abandoned device 1547 * @dev: PCI device to be resumed 1548 * 1549 * Note this function is a backend of pci_default_resume and is not supposed 1550 * to be called by normal code, write proper resume handler and use it instead. 1551 */ 1552 int pci_reenable_device(struct pci_dev *dev) 1553 { 1554 if (pci_is_enabled(dev)) 1555 return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1); 1556 return 0; 1557 } 1558 EXPORT_SYMBOL(pci_reenable_device); 1559 1560 static void pci_enable_bridge(struct pci_dev *dev) 1561 { 1562 struct pci_dev *bridge; 1563 int retval; 1564 1565 bridge = pci_upstream_bridge(dev); 1566 if (bridge) 1567 pci_enable_bridge(bridge); 1568 1569 if (pci_is_enabled(dev)) { 1570 if (!dev->is_busmaster) 1571 pci_set_master(dev); 1572 return; 1573 } 1574 1575 retval = pci_enable_device(dev); 1576 if (retval) 1577 pci_err(dev, "Error enabling bridge (%d), continuing\n", 1578 retval); 1579 pci_set_master(dev); 1580 } 1581 1582 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) 1583 { 1584 struct pci_dev *bridge; 1585 int err; 1586 int i, bars = 0; 1587 1588 /* 1589 * Power state could be unknown at this point, either due to a fresh 1590 * boot or a device removal call. So get the current power state 1591 * so that things like MSI message writing will behave as expected 1592 * (e.g. if the device really is in D0 at enable time). 1593 */ 1594 if (dev->pm_cap) { 1595 u16 pmcsr; 1596 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1597 dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); 1598 } 1599 1600 if (atomic_inc_return(&dev->enable_cnt) > 1) 1601 return 0; /* already enabled */ 1602 1603 bridge = pci_upstream_bridge(dev); 1604 if (bridge) 1605 pci_enable_bridge(bridge); 1606 1607 /* only skip sriov related */ 1608 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 1609 if (dev->resource[i].flags & flags) 1610 bars |= (1 << i); 1611 for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++) 1612 if (dev->resource[i].flags & flags) 1613 bars |= (1 << i); 1614 1615 err = do_pci_enable_device(dev, bars); 1616 if (err < 0) 1617 atomic_dec(&dev->enable_cnt); 1618 return err; 1619 } 1620 1621 /** 1622 * pci_enable_device_io - Initialize a device for use with IO space 1623 * @dev: PCI device to be initialized 1624 * 1625 * Initialize device before it's used by a driver. Ask low-level code 1626 * to enable I/O resources. Wake up the device if it was suspended. 1627 * Beware, this function can fail. 1628 */ 1629 int pci_enable_device_io(struct pci_dev *dev) 1630 { 1631 return pci_enable_device_flags(dev, IORESOURCE_IO); 1632 } 1633 EXPORT_SYMBOL(pci_enable_device_io); 1634 1635 /** 1636 * pci_enable_device_mem - Initialize a device for use with Memory space 1637 * @dev: PCI device to be initialized 1638 * 1639 * Initialize device before it's used by a driver. Ask low-level code 1640 * to enable Memory resources. Wake up the device if it was suspended. 1641 * Beware, this function can fail. 1642 */ 1643 int pci_enable_device_mem(struct pci_dev *dev) 1644 { 1645 return pci_enable_device_flags(dev, IORESOURCE_MEM); 1646 } 1647 EXPORT_SYMBOL(pci_enable_device_mem); 1648 1649 /** 1650 * pci_enable_device - Initialize device before it's used by a driver. 1651 * @dev: PCI device to be initialized 1652 * 1653 * Initialize device before it's used by a driver. Ask low-level code 1654 * to enable I/O and memory. Wake up the device if it was suspended. 1655 * Beware, this function can fail. 1656 * 1657 * Note we don't actually enable the device many times if we call 1658 * this function repeatedly (we just increment the count). 1659 */ 1660 int pci_enable_device(struct pci_dev *dev) 1661 { 1662 return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO); 1663 } 1664 EXPORT_SYMBOL(pci_enable_device); 1665 1666 /* 1667 * Managed PCI resources. This manages device on/off, intx/msi/msix 1668 * on/off and BAR regions. pci_dev itself records msi/msix status, so 1669 * there's no need to track it separately. pci_devres is initialized 1670 * when a device is enabled using managed PCI device enable interface. 1671 */ 1672 struct pci_devres { 1673 unsigned int enabled:1; 1674 unsigned int pinned:1; 1675 unsigned int orig_intx:1; 1676 unsigned int restore_intx:1; 1677 unsigned int mwi:1; 1678 u32 region_mask; 1679 }; 1680 1681 static void pcim_release(struct device *gendev, void *res) 1682 { 1683 struct pci_dev *dev = to_pci_dev(gendev); 1684 struct pci_devres *this = res; 1685 int i; 1686 1687 if (dev->msi_enabled) 1688 pci_disable_msi(dev); 1689 if (dev->msix_enabled) 1690 pci_disable_msix(dev); 1691 1692 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) 1693 if (this->region_mask & (1 << i)) 1694 pci_release_region(dev, i); 1695 1696 if (this->mwi) 1697 pci_clear_mwi(dev); 1698 1699 if (this->restore_intx) 1700 pci_intx(dev, this->orig_intx); 1701 1702 if (this->enabled && !this->pinned) 1703 pci_disable_device(dev); 1704 } 1705 1706 static struct pci_devres *get_pci_dr(struct pci_dev *pdev) 1707 { 1708 struct pci_devres *dr, *new_dr; 1709 1710 dr = devres_find(&pdev->dev, pcim_release, NULL, NULL); 1711 if (dr) 1712 return dr; 1713 1714 new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL); 1715 if (!new_dr) 1716 return NULL; 1717 return devres_get(&pdev->dev, new_dr, NULL, NULL); 1718 } 1719 1720 static struct pci_devres *find_pci_dr(struct pci_dev *pdev) 1721 { 1722 if (pci_is_managed(pdev)) 1723 return devres_find(&pdev->dev, pcim_release, NULL, NULL); 1724 return NULL; 1725 } 1726 1727 /** 1728 * pcim_enable_device - Managed pci_enable_device() 1729 * @pdev: PCI device to be initialized 1730 * 1731 * Managed pci_enable_device(). 1732 */ 1733 int pcim_enable_device(struct pci_dev *pdev) 1734 { 1735 struct pci_devres *dr; 1736 int rc; 1737 1738 dr = get_pci_dr(pdev); 1739 if (unlikely(!dr)) 1740 return -ENOMEM; 1741 if (dr->enabled) 1742 return 0; 1743 1744 rc = pci_enable_device(pdev); 1745 if (!rc) { 1746 pdev->is_managed = 1; 1747 dr->enabled = 1; 1748 } 1749 return rc; 1750 } 1751 EXPORT_SYMBOL(pcim_enable_device); 1752 1753 /** 1754 * pcim_pin_device - Pin managed PCI device 1755 * @pdev: PCI device to pin 1756 * 1757 * Pin managed PCI device @pdev. Pinned device won't be disabled on 1758 * driver detach. @pdev must have been enabled with 1759 * pcim_enable_device(). 1760 */ 1761 void pcim_pin_device(struct pci_dev *pdev) 1762 { 1763 struct pci_devres *dr; 1764 1765 dr = find_pci_dr(pdev); 1766 WARN_ON(!dr || !dr->enabled); 1767 if (dr) 1768 dr->pinned = 1; 1769 } 1770 EXPORT_SYMBOL(pcim_pin_device); 1771 1772 /* 1773 * pcibios_add_device - provide arch specific hooks when adding device dev 1774 * @dev: the PCI device being added 1775 * 1776 * Permits the platform to provide architecture specific functionality when 1777 * devices are added. This is the default implementation. Architecture 1778 * implementations can override this. 1779 */ 1780 int __weak pcibios_add_device(struct pci_dev *dev) 1781 { 1782 return 0; 1783 } 1784 1785 /** 1786 * pcibios_release_device - provide arch specific hooks when releasing device dev 1787 * @dev: the PCI device being released 1788 * 1789 * Permits the platform to provide architecture specific functionality when 1790 * devices are released. This is the default implementation. Architecture 1791 * implementations can override this. 1792 */ 1793 void __weak pcibios_release_device(struct pci_dev *dev) {} 1794 1795 /** 1796 * pcibios_disable_device - disable arch specific PCI resources for device dev 1797 * @dev: the PCI device to disable 1798 * 1799 * Disables architecture specific PCI resources for the device. This 1800 * is the default implementation. Architecture implementations can 1801 * override this. 1802 */ 1803 void __weak pcibios_disable_device(struct pci_dev *dev) {} 1804 1805 /** 1806 * pcibios_penalize_isa_irq - penalize an ISA IRQ 1807 * @irq: ISA IRQ to penalize 1808 * @active: IRQ active or not 1809 * 1810 * Permits the platform to provide architecture-specific functionality when 1811 * penalizing ISA IRQs. This is the default implementation. Architecture 1812 * implementations can override this. 1813 */ 1814 void __weak pcibios_penalize_isa_irq(int irq, int active) {} 1815 1816 static void do_pci_disable_device(struct pci_dev *dev) 1817 { 1818 u16 pci_command; 1819 1820 pci_read_config_word(dev, PCI_COMMAND, &pci_command); 1821 if (pci_command & PCI_COMMAND_MASTER) { 1822 pci_command &= ~PCI_COMMAND_MASTER; 1823 pci_write_config_word(dev, PCI_COMMAND, pci_command); 1824 } 1825 1826 pcibios_disable_device(dev); 1827 } 1828 1829 /** 1830 * pci_disable_enabled_device - Disable device without updating enable_cnt 1831 * @dev: PCI device to disable 1832 * 1833 * NOTE: This function is a backend of PCI power management routines and is 1834 * not supposed to be called drivers. 1835 */ 1836 void pci_disable_enabled_device(struct pci_dev *dev) 1837 { 1838 if (pci_is_enabled(dev)) 1839 do_pci_disable_device(dev); 1840 } 1841 1842 /** 1843 * pci_disable_device - Disable PCI device after use 1844 * @dev: PCI device to be disabled 1845 * 1846 * Signal to the system that the PCI device is not in use by the system 1847 * anymore. This only involves disabling PCI bus-mastering, if active. 1848 * 1849 * Note we don't actually disable the device until all callers of 1850 * pci_enable_device() have called pci_disable_device(). 1851 */ 1852 void pci_disable_device(struct pci_dev *dev) 1853 { 1854 struct pci_devres *dr; 1855 1856 dr = find_pci_dr(dev); 1857 if (dr) 1858 dr->enabled = 0; 1859 1860 dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0, 1861 "disabling already-disabled device"); 1862 1863 if (atomic_dec_return(&dev->enable_cnt) != 0) 1864 return; 1865 1866 do_pci_disable_device(dev); 1867 1868 dev->is_busmaster = 0; 1869 } 1870 EXPORT_SYMBOL(pci_disable_device); 1871 1872 /** 1873 * pcibios_set_pcie_reset_state - set reset state for device dev 1874 * @dev: the PCIe device reset 1875 * @state: Reset state to enter into 1876 * 1877 * 1878 * Sets the PCIe reset state for the device. This is the default 1879 * implementation. Architecture implementations can override this. 1880 */ 1881 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev, 1882 enum pcie_reset_state state) 1883 { 1884 return -EINVAL; 1885 } 1886 1887 /** 1888 * pci_set_pcie_reset_state - set reset state for device dev 1889 * @dev: the PCIe device reset 1890 * @state: Reset state to enter into 1891 * 1892 * 1893 * Sets the PCI reset state for the device. 1894 */ 1895 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 1896 { 1897 return pcibios_set_pcie_reset_state(dev, state); 1898 } 1899 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state); 1900 1901 /** 1902 * pcie_clear_root_pme_status - Clear root port PME interrupt status. 1903 * @dev: PCIe root port or event collector. 1904 */ 1905 void pcie_clear_root_pme_status(struct pci_dev *dev) 1906 { 1907 pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME); 1908 } 1909 1910 /** 1911 * pci_check_pme_status - Check if given device has generated PME. 1912 * @dev: Device to check. 1913 * 1914 * Check the PME status of the device and if set, clear it and clear PME enable 1915 * (if set). Return 'true' if PME status and PME enable were both set or 1916 * 'false' otherwise. 1917 */ 1918 bool pci_check_pme_status(struct pci_dev *dev) 1919 { 1920 int pmcsr_pos; 1921 u16 pmcsr; 1922 bool ret = false; 1923 1924 if (!dev->pm_cap) 1925 return false; 1926 1927 pmcsr_pos = dev->pm_cap + PCI_PM_CTRL; 1928 pci_read_config_word(dev, pmcsr_pos, &pmcsr); 1929 if (!(pmcsr & PCI_PM_CTRL_PME_STATUS)) 1930 return false; 1931 1932 /* Clear PME status. */ 1933 pmcsr |= PCI_PM_CTRL_PME_STATUS; 1934 if (pmcsr & PCI_PM_CTRL_PME_ENABLE) { 1935 /* Disable PME to avoid interrupt flood. */ 1936 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 1937 ret = true; 1938 } 1939 1940 pci_write_config_word(dev, pmcsr_pos, pmcsr); 1941 1942 return ret; 1943 } 1944 1945 /** 1946 * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set. 1947 * @dev: Device to handle. 1948 * @pme_poll_reset: Whether or not to reset the device's pme_poll flag. 1949 * 1950 * Check if @dev has generated PME and queue a resume request for it in that 1951 * case. 1952 */ 1953 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset) 1954 { 1955 if (pme_poll_reset && dev->pme_poll) 1956 dev->pme_poll = false; 1957 1958 if (pci_check_pme_status(dev)) { 1959 pci_wakeup_event(dev); 1960 pm_request_resume(&dev->dev); 1961 } 1962 return 0; 1963 } 1964 1965 /** 1966 * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary. 1967 * @bus: Top bus of the subtree to walk. 1968 */ 1969 void pci_pme_wakeup_bus(struct pci_bus *bus) 1970 { 1971 if (bus) 1972 pci_walk_bus(bus, pci_pme_wakeup, (void *)true); 1973 } 1974 1975 1976 /** 1977 * pci_pme_capable - check the capability of PCI device to generate PME# 1978 * @dev: PCI device to handle. 1979 * @state: PCI state from which device will issue PME#. 1980 */ 1981 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state) 1982 { 1983 if (!dev->pm_cap) 1984 return false; 1985 1986 return !!(dev->pme_support & (1 << state)); 1987 } 1988 EXPORT_SYMBOL(pci_pme_capable); 1989 1990 static void pci_pme_list_scan(struct work_struct *work) 1991 { 1992 struct pci_pme_device *pme_dev, *n; 1993 1994 mutex_lock(&pci_pme_list_mutex); 1995 list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 1996 if (pme_dev->dev->pme_poll) { 1997 struct pci_dev *bridge; 1998 1999 bridge = pme_dev->dev->bus->self; 2000 /* 2001 * If bridge is in low power state, the 2002 * configuration space of subordinate devices 2003 * may be not accessible 2004 */ 2005 if (bridge && bridge->current_state != PCI_D0) 2006 continue; 2007 pci_pme_wakeup(pme_dev->dev, NULL); 2008 } else { 2009 list_del(&pme_dev->list); 2010 kfree(pme_dev); 2011 } 2012 } 2013 if (!list_empty(&pci_pme_list)) 2014 queue_delayed_work(system_freezable_wq, &pci_pme_work, 2015 msecs_to_jiffies(PME_TIMEOUT)); 2016 mutex_unlock(&pci_pme_list_mutex); 2017 } 2018 2019 static void __pci_pme_active(struct pci_dev *dev, bool enable) 2020 { 2021 u16 pmcsr; 2022 2023 if (!dev->pme_support) 2024 return; 2025 2026 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2027 /* Clear PME_Status by writing 1 to it and enable PME# */ 2028 pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE; 2029 if (!enable) 2030 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2031 2032 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2033 } 2034 2035 /** 2036 * pci_pme_restore - Restore PME configuration after config space restore. 2037 * @dev: PCI device to update. 2038 */ 2039 void pci_pme_restore(struct pci_dev *dev) 2040 { 2041 u16 pmcsr; 2042 2043 if (!dev->pme_support) 2044 return; 2045 2046 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2047 if (dev->wakeup_prepared) { 2048 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 2049 pmcsr &= ~PCI_PM_CTRL_PME_STATUS; 2050 } else { 2051 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2052 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2053 } 2054 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2055 } 2056 2057 /** 2058 * pci_pme_active - enable or disable PCI device's PME# function 2059 * @dev: PCI device to handle. 2060 * @enable: 'true' to enable PME# generation; 'false' to disable it. 2061 * 2062 * The caller must verify that the device is capable of generating PME# before 2063 * calling this function with @enable equal to 'true'. 2064 */ 2065 void pci_pme_active(struct pci_dev *dev, bool enable) 2066 { 2067 __pci_pme_active(dev, enable); 2068 2069 /* 2070 * PCI (as opposed to PCIe) PME requires that the device have 2071 * its PME# line hooked up correctly. Not all hardware vendors 2072 * do this, so the PME never gets delivered and the device 2073 * remains asleep. The easiest way around this is to 2074 * periodically walk the list of suspended devices and check 2075 * whether any have their PME flag set. The assumption is that 2076 * we'll wake up often enough anyway that this won't be a huge 2077 * hit, and the power savings from the devices will still be a 2078 * win. 2079 * 2080 * Although PCIe uses in-band PME message instead of PME# line 2081 * to report PME, PME does not work for some PCIe devices in 2082 * reality. For example, there are devices that set their PME 2083 * status bits, but don't really bother to send a PME message; 2084 * there are PCI Express Root Ports that don't bother to 2085 * trigger interrupts when they receive PME messages from the 2086 * devices below. So PME poll is used for PCIe devices too. 2087 */ 2088 2089 if (dev->pme_poll) { 2090 struct pci_pme_device *pme_dev; 2091 if (enable) { 2092 pme_dev = kmalloc(sizeof(struct pci_pme_device), 2093 GFP_KERNEL); 2094 if (!pme_dev) { 2095 pci_warn(dev, "can't enable PME#\n"); 2096 return; 2097 } 2098 pme_dev->dev = dev; 2099 mutex_lock(&pci_pme_list_mutex); 2100 list_add(&pme_dev->list, &pci_pme_list); 2101 if (list_is_singular(&pci_pme_list)) 2102 queue_delayed_work(system_freezable_wq, 2103 &pci_pme_work, 2104 msecs_to_jiffies(PME_TIMEOUT)); 2105 mutex_unlock(&pci_pme_list_mutex); 2106 } else { 2107 mutex_lock(&pci_pme_list_mutex); 2108 list_for_each_entry(pme_dev, &pci_pme_list, list) { 2109 if (pme_dev->dev == dev) { 2110 list_del(&pme_dev->list); 2111 kfree(pme_dev); 2112 break; 2113 } 2114 } 2115 mutex_unlock(&pci_pme_list_mutex); 2116 } 2117 } 2118 2119 pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled"); 2120 } 2121 EXPORT_SYMBOL(pci_pme_active); 2122 2123 /** 2124 * __pci_enable_wake - enable PCI device as wakeup event source 2125 * @dev: PCI device affected 2126 * @state: PCI state from which device will issue wakeup events 2127 * @enable: True to enable event generation; false to disable 2128 * 2129 * This enables the device as a wakeup event source, or disables it. 2130 * When such events involves platform-specific hooks, those hooks are 2131 * called automatically by this routine. 2132 * 2133 * Devices with legacy power management (no standard PCI PM capabilities) 2134 * always require such platform hooks. 2135 * 2136 * RETURN VALUE: 2137 * 0 is returned on success 2138 * -EINVAL is returned if device is not supposed to wake up the system 2139 * Error code depending on the platform is returned if both the platform and 2140 * the native mechanism fail to enable the generation of wake-up events 2141 */ 2142 static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable) 2143 { 2144 int ret = 0; 2145 2146 /* 2147 * Bridges can only signal wakeup on behalf of subordinate devices, 2148 * but that is set up elsewhere, so skip them. 2149 */ 2150 if (pci_has_subordinate(dev)) 2151 return 0; 2152 2153 /* Don't do the same thing twice in a row for one device. */ 2154 if (!!enable == !!dev->wakeup_prepared) 2155 return 0; 2156 2157 /* 2158 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don 2159 * Anderson we should be doing PME# wake enable followed by ACPI wake 2160 * enable. To disable wake-up we call the platform first, for symmetry. 2161 */ 2162 2163 if (enable) { 2164 int error; 2165 2166 if (pci_pme_capable(dev, state)) 2167 pci_pme_active(dev, true); 2168 else 2169 ret = 1; 2170 error = platform_pci_set_wakeup(dev, true); 2171 if (ret) 2172 ret = error; 2173 if (!ret) 2174 dev->wakeup_prepared = true; 2175 } else { 2176 platform_pci_set_wakeup(dev, false); 2177 pci_pme_active(dev, false); 2178 dev->wakeup_prepared = false; 2179 } 2180 2181 return ret; 2182 } 2183 2184 /** 2185 * pci_enable_wake - change wakeup settings for a PCI device 2186 * @pci_dev: Target device 2187 * @state: PCI state from which device will issue wakeup events 2188 * @enable: Whether or not to enable event generation 2189 * 2190 * If @enable is set, check device_may_wakeup() for the device before calling 2191 * __pci_enable_wake() for it. 2192 */ 2193 int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable) 2194 { 2195 if (enable && !device_may_wakeup(&pci_dev->dev)) 2196 return -EINVAL; 2197 2198 return __pci_enable_wake(pci_dev, state, enable); 2199 } 2200 EXPORT_SYMBOL(pci_enable_wake); 2201 2202 /** 2203 * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold 2204 * @dev: PCI device to prepare 2205 * @enable: True to enable wake-up event generation; false to disable 2206 * 2207 * Many drivers want the device to wake up the system from D3_hot or D3_cold 2208 * and this function allows them to set that up cleanly - pci_enable_wake() 2209 * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI 2210 * ordering constraints. 2211 * 2212 * This function only returns error code if the device is not allowed to wake 2213 * up the system from sleep or it is not capable of generating PME# from both 2214 * D3_hot and D3_cold and the platform is unable to enable wake-up power for it. 2215 */ 2216 int pci_wake_from_d3(struct pci_dev *dev, bool enable) 2217 { 2218 return pci_pme_capable(dev, PCI_D3cold) ? 2219 pci_enable_wake(dev, PCI_D3cold, enable) : 2220 pci_enable_wake(dev, PCI_D3hot, enable); 2221 } 2222 EXPORT_SYMBOL(pci_wake_from_d3); 2223 2224 /** 2225 * pci_target_state - find an appropriate low power state for a given PCI dev 2226 * @dev: PCI device 2227 * @wakeup: Whether or not wakeup functionality will be enabled for the device. 2228 * 2229 * Use underlying platform code to find a supported low power state for @dev. 2230 * If the platform can't manage @dev, return the deepest state from which it 2231 * can generate wake events, based on any available PME info. 2232 */ 2233 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup) 2234 { 2235 pci_power_t target_state = PCI_D3hot; 2236 2237 if (platform_pci_power_manageable(dev)) { 2238 /* 2239 * Call the platform to find the target state for the device. 2240 */ 2241 pci_power_t state = platform_pci_choose_state(dev); 2242 2243 switch (state) { 2244 case PCI_POWER_ERROR: 2245 case PCI_UNKNOWN: 2246 break; 2247 case PCI_D1: 2248 case PCI_D2: 2249 if (pci_no_d1d2(dev)) 2250 break; 2251 /* else: fall through */ 2252 default: 2253 target_state = state; 2254 } 2255 2256 return target_state; 2257 } 2258 2259 if (!dev->pm_cap) 2260 target_state = PCI_D0; 2261 2262 /* 2263 * If the device is in D3cold even though it's not power-manageable by 2264 * the platform, it may have been powered down by non-standard means. 2265 * Best to let it slumber. 2266 */ 2267 if (dev->current_state == PCI_D3cold) 2268 target_state = PCI_D3cold; 2269 2270 if (wakeup) { 2271 /* 2272 * Find the deepest state from which the device can generate 2273 * PME#. 2274 */ 2275 if (dev->pme_support) { 2276 while (target_state 2277 && !(dev->pme_support & (1 << target_state))) 2278 target_state--; 2279 } 2280 } 2281 2282 return target_state; 2283 } 2284 2285 /** 2286 * pci_prepare_to_sleep - prepare PCI device for system-wide transition into a sleep state 2287 * @dev: Device to handle. 2288 * 2289 * Choose the power state appropriate for the device depending on whether 2290 * it can wake up the system and/or is power manageable by the platform 2291 * (PCI_D3hot is the default) and put the device into that state. 2292 */ 2293 int pci_prepare_to_sleep(struct pci_dev *dev) 2294 { 2295 bool wakeup = device_may_wakeup(&dev->dev); 2296 pci_power_t target_state = pci_target_state(dev, wakeup); 2297 int error; 2298 2299 if (target_state == PCI_POWER_ERROR) 2300 return -EIO; 2301 2302 pci_enable_wake(dev, target_state, wakeup); 2303 2304 error = pci_set_power_state(dev, target_state); 2305 2306 if (error) 2307 pci_enable_wake(dev, target_state, false); 2308 2309 return error; 2310 } 2311 EXPORT_SYMBOL(pci_prepare_to_sleep); 2312 2313 /** 2314 * pci_back_from_sleep - turn PCI device on during system-wide transition into working state 2315 * @dev: Device to handle. 2316 * 2317 * Disable device's system wake-up capability and put it into D0. 2318 */ 2319 int pci_back_from_sleep(struct pci_dev *dev) 2320 { 2321 pci_enable_wake(dev, PCI_D0, false); 2322 return pci_set_power_state(dev, PCI_D0); 2323 } 2324 EXPORT_SYMBOL(pci_back_from_sleep); 2325 2326 /** 2327 * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend. 2328 * @dev: PCI device being suspended. 2329 * 2330 * Prepare @dev to generate wake-up events at run time and put it into a low 2331 * power state. 2332 */ 2333 int pci_finish_runtime_suspend(struct pci_dev *dev) 2334 { 2335 pci_power_t target_state; 2336 int error; 2337 2338 target_state = pci_target_state(dev, device_can_wakeup(&dev->dev)); 2339 if (target_state == PCI_POWER_ERROR) 2340 return -EIO; 2341 2342 dev->runtime_d3cold = target_state == PCI_D3cold; 2343 2344 __pci_enable_wake(dev, target_state, pci_dev_run_wake(dev)); 2345 2346 error = pci_set_power_state(dev, target_state); 2347 2348 if (error) { 2349 pci_enable_wake(dev, target_state, false); 2350 dev->runtime_d3cold = false; 2351 } 2352 2353 return error; 2354 } 2355 2356 /** 2357 * pci_dev_run_wake - Check if device can generate run-time wake-up events. 2358 * @dev: Device to check. 2359 * 2360 * Return true if the device itself is capable of generating wake-up events 2361 * (through the platform or using the native PCIe PME) or if the device supports 2362 * PME and one of its upstream bridges can generate wake-up events. 2363 */ 2364 bool pci_dev_run_wake(struct pci_dev *dev) 2365 { 2366 struct pci_bus *bus = dev->bus; 2367 2368 if (!dev->pme_support) 2369 return false; 2370 2371 /* PME-capable in principle, but not from the target power state */ 2372 if (!pci_pme_capable(dev, pci_target_state(dev, true))) 2373 return false; 2374 2375 if (device_can_wakeup(&dev->dev)) 2376 return true; 2377 2378 while (bus->parent) { 2379 struct pci_dev *bridge = bus->self; 2380 2381 if (device_can_wakeup(&bridge->dev)) 2382 return true; 2383 2384 bus = bus->parent; 2385 } 2386 2387 /* We have reached the root bus. */ 2388 if (bus->bridge) 2389 return device_can_wakeup(bus->bridge); 2390 2391 return false; 2392 } 2393 EXPORT_SYMBOL_GPL(pci_dev_run_wake); 2394 2395 /** 2396 * pci_dev_keep_suspended - Check if the device can stay in the suspended state. 2397 * @pci_dev: Device to check. 2398 * 2399 * Return 'true' if the device is runtime-suspended, it doesn't have to be 2400 * reconfigured due to wakeup settings difference between system and runtime 2401 * suspend and the current power state of it is suitable for the upcoming 2402 * (system) transition. 2403 * 2404 * If the device is not configured for system wakeup, disable PME for it before 2405 * returning 'true' to prevent it from waking up the system unnecessarily. 2406 */ 2407 bool pci_dev_keep_suspended(struct pci_dev *pci_dev) 2408 { 2409 struct device *dev = &pci_dev->dev; 2410 bool wakeup = device_may_wakeup(dev); 2411 2412 if (!pm_runtime_suspended(dev) 2413 || pci_target_state(pci_dev, wakeup) != pci_dev->current_state 2414 || platform_pci_need_resume(pci_dev)) 2415 return false; 2416 2417 /* 2418 * At this point the device is good to go unless it's been configured 2419 * to generate PME at the runtime suspend time, but it is not supposed 2420 * to wake up the system. In that case, simply disable PME for it 2421 * (it will have to be re-enabled on exit from system resume). 2422 * 2423 * If the device's power state is D3cold and the platform check above 2424 * hasn't triggered, the device's configuration is suitable and we don't 2425 * need to manipulate it at all. 2426 */ 2427 spin_lock_irq(&dev->power.lock); 2428 2429 if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold && 2430 !wakeup) 2431 __pci_pme_active(pci_dev, false); 2432 2433 spin_unlock_irq(&dev->power.lock); 2434 return true; 2435 } 2436 2437 /** 2438 * pci_dev_complete_resume - Finalize resume from system sleep for a device. 2439 * @pci_dev: Device to handle. 2440 * 2441 * If the device is runtime suspended and wakeup-capable, enable PME for it as 2442 * it might have been disabled during the prepare phase of system suspend if 2443 * the device was not configured for system wakeup. 2444 */ 2445 void pci_dev_complete_resume(struct pci_dev *pci_dev) 2446 { 2447 struct device *dev = &pci_dev->dev; 2448 2449 if (!pci_dev_run_wake(pci_dev)) 2450 return; 2451 2452 spin_lock_irq(&dev->power.lock); 2453 2454 if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold) 2455 __pci_pme_active(pci_dev, true); 2456 2457 spin_unlock_irq(&dev->power.lock); 2458 } 2459 2460 void pci_config_pm_runtime_get(struct pci_dev *pdev) 2461 { 2462 struct device *dev = &pdev->dev; 2463 struct device *parent = dev->parent; 2464 2465 if (parent) 2466 pm_runtime_get_sync(parent); 2467 pm_runtime_get_noresume(dev); 2468 /* 2469 * pdev->current_state is set to PCI_D3cold during suspending, 2470 * so wait until suspending completes 2471 */ 2472 pm_runtime_barrier(dev); 2473 /* 2474 * Only need to resume devices in D3cold, because config 2475 * registers are still accessible for devices suspended but 2476 * not in D3cold. 2477 */ 2478 if (pdev->current_state == PCI_D3cold) 2479 pm_runtime_resume(dev); 2480 } 2481 2482 void pci_config_pm_runtime_put(struct pci_dev *pdev) 2483 { 2484 struct device *dev = &pdev->dev; 2485 struct device *parent = dev->parent; 2486 2487 pm_runtime_put(dev); 2488 if (parent) 2489 pm_runtime_put_sync(parent); 2490 } 2491 2492 /** 2493 * pci_bridge_d3_possible - Is it possible to put the bridge into D3 2494 * @bridge: Bridge to check 2495 * 2496 * This function checks if it is possible to move the bridge to D3. 2497 * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt. 2498 */ 2499 bool pci_bridge_d3_possible(struct pci_dev *bridge) 2500 { 2501 if (!pci_is_pcie(bridge)) 2502 return false; 2503 2504 switch (pci_pcie_type(bridge)) { 2505 case PCI_EXP_TYPE_ROOT_PORT: 2506 case PCI_EXP_TYPE_UPSTREAM: 2507 case PCI_EXP_TYPE_DOWNSTREAM: 2508 if (pci_bridge_d3_disable) 2509 return false; 2510 2511 /* 2512 * Hotplug ports handled by firmware in System Management Mode 2513 * may not be put into D3 by the OS (Thunderbolt on non-Macs). 2514 */ 2515 if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge)) 2516 return false; 2517 2518 if (pci_bridge_d3_force) 2519 return true; 2520 2521 /* Even the oldest 2010 Thunderbolt controller supports D3. */ 2522 if (bridge->is_thunderbolt) 2523 return true; 2524 2525 /* 2526 * Hotplug ports handled natively by the OS were not validated 2527 * by vendors for runtime D3 at least until 2018 because there 2528 * was no OS support. 2529 */ 2530 if (bridge->is_hotplug_bridge) 2531 return false; 2532 2533 /* 2534 * It should be safe to put PCIe ports from 2015 or newer 2535 * to D3. 2536 */ 2537 if (dmi_get_bios_year() >= 2015) 2538 return true; 2539 break; 2540 } 2541 2542 return false; 2543 } 2544 2545 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) 2546 { 2547 bool *d3cold_ok = data; 2548 2549 if (/* The device needs to be allowed to go D3cold ... */ 2550 dev->no_d3cold || !dev->d3cold_allowed || 2551 2552 /* ... and if it is wakeup capable to do so from D3cold. */ 2553 (device_may_wakeup(&dev->dev) && 2554 !pci_pme_capable(dev, PCI_D3cold)) || 2555 2556 /* If it is a bridge it must be allowed to go to D3. */ 2557 !pci_power_manageable(dev)) 2558 2559 *d3cold_ok = false; 2560 2561 return !*d3cold_ok; 2562 } 2563 2564 /* 2565 * pci_bridge_d3_update - Update bridge D3 capabilities 2566 * @dev: PCI device which is changed 2567 * 2568 * Update upstream bridge PM capabilities accordingly depending on if the 2569 * device PM configuration was changed or the device is being removed. The 2570 * change is also propagated upstream. 2571 */ 2572 void pci_bridge_d3_update(struct pci_dev *dev) 2573 { 2574 bool remove = !device_is_registered(&dev->dev); 2575 struct pci_dev *bridge; 2576 bool d3cold_ok = true; 2577 2578 bridge = pci_upstream_bridge(dev); 2579 if (!bridge || !pci_bridge_d3_possible(bridge)) 2580 return; 2581 2582 /* 2583 * If D3 is currently allowed for the bridge, removing one of its 2584 * children won't change that. 2585 */ 2586 if (remove && bridge->bridge_d3) 2587 return; 2588 2589 /* 2590 * If D3 is currently allowed for the bridge and a child is added or 2591 * changed, disallowance of D3 can only be caused by that child, so 2592 * we only need to check that single device, not any of its siblings. 2593 * 2594 * If D3 is currently not allowed for the bridge, checking the device 2595 * first may allow us to skip checking its siblings. 2596 */ 2597 if (!remove) 2598 pci_dev_check_d3cold(dev, &d3cold_ok); 2599 2600 /* 2601 * If D3 is currently not allowed for the bridge, this may be caused 2602 * either by the device being changed/removed or any of its siblings, 2603 * so we need to go through all children to find out if one of them 2604 * continues to block D3. 2605 */ 2606 if (d3cold_ok && !bridge->bridge_d3) 2607 pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold, 2608 &d3cold_ok); 2609 2610 if (bridge->bridge_d3 != d3cold_ok) { 2611 bridge->bridge_d3 = d3cold_ok; 2612 /* Propagate change to upstream bridges */ 2613 pci_bridge_d3_update(bridge); 2614 } 2615 } 2616 2617 /** 2618 * pci_d3cold_enable - Enable D3cold for device 2619 * @dev: PCI device to handle 2620 * 2621 * This function can be used in drivers to enable D3cold from the device 2622 * they handle. It also updates upstream PCI bridge PM capabilities 2623 * accordingly. 2624 */ 2625 void pci_d3cold_enable(struct pci_dev *dev) 2626 { 2627 if (dev->no_d3cold) { 2628 dev->no_d3cold = false; 2629 pci_bridge_d3_update(dev); 2630 } 2631 } 2632 EXPORT_SYMBOL_GPL(pci_d3cold_enable); 2633 2634 /** 2635 * pci_d3cold_disable - Disable D3cold for device 2636 * @dev: PCI device to handle 2637 * 2638 * This function can be used in drivers to disable D3cold from the device 2639 * they handle. It also updates upstream PCI bridge PM capabilities 2640 * accordingly. 2641 */ 2642 void pci_d3cold_disable(struct pci_dev *dev) 2643 { 2644 if (!dev->no_d3cold) { 2645 dev->no_d3cold = true; 2646 pci_bridge_d3_update(dev); 2647 } 2648 } 2649 EXPORT_SYMBOL_GPL(pci_d3cold_disable); 2650 2651 /** 2652 * pci_pm_init - Initialize PM functions of given PCI device 2653 * @dev: PCI device to handle. 2654 */ 2655 void pci_pm_init(struct pci_dev *dev) 2656 { 2657 int pm; 2658 u16 pmc; 2659 2660 pm_runtime_forbid(&dev->dev); 2661 pm_runtime_set_active(&dev->dev); 2662 pm_runtime_enable(&dev->dev); 2663 device_enable_async_suspend(&dev->dev); 2664 dev->wakeup_prepared = false; 2665 2666 dev->pm_cap = 0; 2667 dev->pme_support = 0; 2668 2669 /* find PCI PM capability in list */ 2670 pm = pci_find_capability(dev, PCI_CAP_ID_PM); 2671 if (!pm) 2672 return; 2673 /* Check device's ability to generate PME# */ 2674 pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc); 2675 2676 if ((pmc & PCI_PM_CAP_VER_MASK) > 3) { 2677 pci_err(dev, "unsupported PM cap regs version (%u)\n", 2678 pmc & PCI_PM_CAP_VER_MASK); 2679 return; 2680 } 2681 2682 dev->pm_cap = pm; 2683 dev->d3_delay = PCI_PM_D3_WAIT; 2684 dev->d3cold_delay = PCI_PM_D3COLD_WAIT; 2685 dev->bridge_d3 = pci_bridge_d3_possible(dev); 2686 dev->d3cold_allowed = true; 2687 2688 dev->d1_support = false; 2689 dev->d2_support = false; 2690 if (!pci_no_d1d2(dev)) { 2691 if (pmc & PCI_PM_CAP_D1) 2692 dev->d1_support = true; 2693 if (pmc & PCI_PM_CAP_D2) 2694 dev->d2_support = true; 2695 2696 if (dev->d1_support || dev->d2_support) 2697 pci_printk(KERN_DEBUG, dev, "supports%s%s\n", 2698 dev->d1_support ? " D1" : "", 2699 dev->d2_support ? " D2" : ""); 2700 } 2701 2702 pmc &= PCI_PM_CAP_PME_MASK; 2703 if (pmc) { 2704 pci_printk(KERN_DEBUG, dev, "PME# supported from%s%s%s%s%s\n", 2705 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "", 2706 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "", 2707 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "", 2708 (pmc & PCI_PM_CAP_PME_D3) ? " D3hot" : "", 2709 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : ""); 2710 dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT; 2711 dev->pme_poll = true; 2712 /* 2713 * Make device's PM flags reflect the wake-up capability, but 2714 * let the user space enable it to wake up the system as needed. 2715 */ 2716 device_set_wakeup_capable(&dev->dev, true); 2717 /* Disable the PME# generation functionality */ 2718 pci_pme_active(dev, false); 2719 } 2720 } 2721 2722 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) 2723 { 2724 unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI; 2725 2726 switch (prop) { 2727 case PCI_EA_P_MEM: 2728 case PCI_EA_P_VF_MEM: 2729 flags |= IORESOURCE_MEM; 2730 break; 2731 case PCI_EA_P_MEM_PREFETCH: 2732 case PCI_EA_P_VF_MEM_PREFETCH: 2733 flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH; 2734 break; 2735 case PCI_EA_P_IO: 2736 flags |= IORESOURCE_IO; 2737 break; 2738 default: 2739 return 0; 2740 } 2741 2742 return flags; 2743 } 2744 2745 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei, 2746 u8 prop) 2747 { 2748 if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO) 2749 return &dev->resource[bei]; 2750 #ifdef CONFIG_PCI_IOV 2751 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 && 2752 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH)) 2753 return &dev->resource[PCI_IOV_RESOURCES + 2754 bei - PCI_EA_BEI_VF_BAR0]; 2755 #endif 2756 else if (bei == PCI_EA_BEI_ROM) 2757 return &dev->resource[PCI_ROM_RESOURCE]; 2758 else 2759 return NULL; 2760 } 2761 2762 /* Read an Enhanced Allocation (EA) entry */ 2763 static int pci_ea_read(struct pci_dev *dev, int offset) 2764 { 2765 struct resource *res; 2766 int ent_size, ent_offset = offset; 2767 resource_size_t start, end; 2768 unsigned long flags; 2769 u32 dw0, bei, base, max_offset; 2770 u8 prop; 2771 bool support_64 = (sizeof(resource_size_t) >= 8); 2772 2773 pci_read_config_dword(dev, ent_offset, &dw0); 2774 ent_offset += 4; 2775 2776 /* Entry size field indicates DWORDs after 1st */ 2777 ent_size = ((dw0 & PCI_EA_ES) + 1) << 2; 2778 2779 if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */ 2780 goto out; 2781 2782 bei = (dw0 & PCI_EA_BEI) >> 4; 2783 prop = (dw0 & PCI_EA_PP) >> 8; 2784 2785 /* 2786 * If the Property is in the reserved range, try the Secondary 2787 * Property instead. 2788 */ 2789 if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED) 2790 prop = (dw0 & PCI_EA_SP) >> 16; 2791 if (prop > PCI_EA_P_BRIDGE_IO) 2792 goto out; 2793 2794 res = pci_ea_get_resource(dev, bei, prop); 2795 if (!res) { 2796 pci_err(dev, "Unsupported EA entry BEI: %u\n", bei); 2797 goto out; 2798 } 2799 2800 flags = pci_ea_flags(dev, prop); 2801 if (!flags) { 2802 pci_err(dev, "Unsupported EA properties: %#x\n", prop); 2803 goto out; 2804 } 2805 2806 /* Read Base */ 2807 pci_read_config_dword(dev, ent_offset, &base); 2808 start = (base & PCI_EA_FIELD_MASK); 2809 ent_offset += 4; 2810 2811 /* Read MaxOffset */ 2812 pci_read_config_dword(dev, ent_offset, &max_offset); 2813 ent_offset += 4; 2814 2815 /* Read Base MSBs (if 64-bit entry) */ 2816 if (base & PCI_EA_IS_64) { 2817 u32 base_upper; 2818 2819 pci_read_config_dword(dev, ent_offset, &base_upper); 2820 ent_offset += 4; 2821 2822 flags |= IORESOURCE_MEM_64; 2823 2824 /* entry starts above 32-bit boundary, can't use */ 2825 if (!support_64 && base_upper) 2826 goto out; 2827 2828 if (support_64) 2829 start |= ((u64)base_upper << 32); 2830 } 2831 2832 end = start + (max_offset | 0x03); 2833 2834 /* Read MaxOffset MSBs (if 64-bit entry) */ 2835 if (max_offset & PCI_EA_IS_64) { 2836 u32 max_offset_upper; 2837 2838 pci_read_config_dword(dev, ent_offset, &max_offset_upper); 2839 ent_offset += 4; 2840 2841 flags |= IORESOURCE_MEM_64; 2842 2843 /* entry too big, can't use */ 2844 if (!support_64 && max_offset_upper) 2845 goto out; 2846 2847 if (support_64) 2848 end += ((u64)max_offset_upper << 32); 2849 } 2850 2851 if (end < start) { 2852 pci_err(dev, "EA Entry crosses address boundary\n"); 2853 goto out; 2854 } 2855 2856 if (ent_size != ent_offset - offset) { 2857 pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n", 2858 ent_size, ent_offset - offset); 2859 goto out; 2860 } 2861 2862 res->name = pci_name(dev); 2863 res->start = start; 2864 res->end = end; 2865 res->flags = flags; 2866 2867 if (bei <= PCI_EA_BEI_BAR5) 2868 pci_printk(KERN_DEBUG, dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 2869 bei, res, prop); 2870 else if (bei == PCI_EA_BEI_ROM) 2871 pci_printk(KERN_DEBUG, dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n", 2872 res, prop); 2873 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5) 2874 pci_printk(KERN_DEBUG, dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 2875 bei - PCI_EA_BEI_VF_BAR0, res, prop); 2876 else 2877 pci_printk(KERN_DEBUG, dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n", 2878 bei, res, prop); 2879 2880 out: 2881 return offset + ent_size; 2882 } 2883 2884 /* Enhanced Allocation Initialization */ 2885 void pci_ea_init(struct pci_dev *dev) 2886 { 2887 int ea; 2888 u8 num_ent; 2889 int offset; 2890 int i; 2891 2892 /* find PCI EA capability in list */ 2893 ea = pci_find_capability(dev, PCI_CAP_ID_EA); 2894 if (!ea) 2895 return; 2896 2897 /* determine the number of entries */ 2898 pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT, 2899 &num_ent); 2900 num_ent &= PCI_EA_NUM_ENT_MASK; 2901 2902 offset = ea + PCI_EA_FIRST_ENT; 2903 2904 /* Skip DWORD 2 for type 1 functions */ 2905 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 2906 offset += 4; 2907 2908 /* parse each EA entry */ 2909 for (i = 0; i < num_ent; ++i) 2910 offset = pci_ea_read(dev, offset); 2911 } 2912 2913 static void pci_add_saved_cap(struct pci_dev *pci_dev, 2914 struct pci_cap_saved_state *new_cap) 2915 { 2916 hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space); 2917 } 2918 2919 /** 2920 * _pci_add_cap_save_buffer - allocate buffer for saving given 2921 * capability registers 2922 * @dev: the PCI device 2923 * @cap: the capability to allocate the buffer for 2924 * @extended: Standard or Extended capability ID 2925 * @size: requested size of the buffer 2926 */ 2927 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap, 2928 bool extended, unsigned int size) 2929 { 2930 int pos; 2931 struct pci_cap_saved_state *save_state; 2932 2933 if (extended) 2934 pos = pci_find_ext_capability(dev, cap); 2935 else 2936 pos = pci_find_capability(dev, cap); 2937 2938 if (!pos) 2939 return 0; 2940 2941 save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL); 2942 if (!save_state) 2943 return -ENOMEM; 2944 2945 save_state->cap.cap_nr = cap; 2946 save_state->cap.cap_extended = extended; 2947 save_state->cap.size = size; 2948 pci_add_saved_cap(dev, save_state); 2949 2950 return 0; 2951 } 2952 2953 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size) 2954 { 2955 return _pci_add_cap_save_buffer(dev, cap, false, size); 2956 } 2957 2958 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size) 2959 { 2960 return _pci_add_cap_save_buffer(dev, cap, true, size); 2961 } 2962 2963 /** 2964 * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities 2965 * @dev: the PCI device 2966 */ 2967 void pci_allocate_cap_save_buffers(struct pci_dev *dev) 2968 { 2969 int error; 2970 2971 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP, 2972 PCI_EXP_SAVE_REGS * sizeof(u16)); 2973 if (error) 2974 pci_err(dev, "unable to preallocate PCI Express save buffer\n"); 2975 2976 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16)); 2977 if (error) 2978 pci_err(dev, "unable to preallocate PCI-X save buffer\n"); 2979 2980 pci_allocate_vc_save_buffers(dev); 2981 } 2982 2983 void pci_free_cap_save_buffers(struct pci_dev *dev) 2984 { 2985 struct pci_cap_saved_state *tmp; 2986 struct hlist_node *n; 2987 2988 hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next) 2989 kfree(tmp); 2990 } 2991 2992 /** 2993 * pci_configure_ari - enable or disable ARI forwarding 2994 * @dev: the PCI device 2995 * 2996 * If @dev and its upstream bridge both support ARI, enable ARI in the 2997 * bridge. Otherwise, disable ARI in the bridge. 2998 */ 2999 void pci_configure_ari(struct pci_dev *dev) 3000 { 3001 u32 cap; 3002 struct pci_dev *bridge; 3003 3004 if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn) 3005 return; 3006 3007 bridge = dev->bus->self; 3008 if (!bridge) 3009 return; 3010 3011 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3012 if (!(cap & PCI_EXP_DEVCAP2_ARI)) 3013 return; 3014 3015 if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) { 3016 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, 3017 PCI_EXP_DEVCTL2_ARI); 3018 bridge->ari_enabled = 1; 3019 } else { 3020 pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2, 3021 PCI_EXP_DEVCTL2_ARI); 3022 bridge->ari_enabled = 0; 3023 } 3024 } 3025 3026 static int pci_acs_enable; 3027 3028 /** 3029 * pci_request_acs - ask for ACS to be enabled if supported 3030 */ 3031 void pci_request_acs(void) 3032 { 3033 pci_acs_enable = 1; 3034 } 3035 3036 static const char *disable_acs_redir_param; 3037 3038 /** 3039 * pci_disable_acs_redir - disable ACS redirect capabilities 3040 * @dev: the PCI device 3041 * 3042 * For only devices specified in the disable_acs_redir parameter. 3043 */ 3044 static void pci_disable_acs_redir(struct pci_dev *dev) 3045 { 3046 int ret = 0; 3047 const char *p; 3048 int pos; 3049 u16 ctrl; 3050 3051 if (!disable_acs_redir_param) 3052 return; 3053 3054 p = disable_acs_redir_param; 3055 while (*p) { 3056 ret = pci_dev_str_match(dev, p, &p); 3057 if (ret < 0) { 3058 pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n", 3059 disable_acs_redir_param); 3060 3061 break; 3062 } else if (ret == 1) { 3063 /* Found a match */ 3064 break; 3065 } 3066 3067 if (*p != ';' && *p != ',') { 3068 /* End of param or invalid format */ 3069 break; 3070 } 3071 p++; 3072 } 3073 3074 if (ret != 1) 3075 return; 3076 3077 if (!pci_dev_specific_disable_acs_redir(dev)) 3078 return; 3079 3080 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); 3081 if (!pos) { 3082 pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n"); 3083 return; 3084 } 3085 3086 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 3087 3088 /* P2P Request & Completion Redirect */ 3089 ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC); 3090 3091 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 3092 3093 pci_info(dev, "disabled ACS redirect\n"); 3094 } 3095 3096 /** 3097 * pci_std_enable_acs - enable ACS on devices using standard ACS capabilites 3098 * @dev: the PCI device 3099 */ 3100 static void pci_std_enable_acs(struct pci_dev *dev) 3101 { 3102 int pos; 3103 u16 cap; 3104 u16 ctrl; 3105 3106 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); 3107 if (!pos) 3108 return; 3109 3110 pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap); 3111 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 3112 3113 /* Source Validation */ 3114 ctrl |= (cap & PCI_ACS_SV); 3115 3116 /* P2P Request Redirect */ 3117 ctrl |= (cap & PCI_ACS_RR); 3118 3119 /* P2P Completion Redirect */ 3120 ctrl |= (cap & PCI_ACS_CR); 3121 3122 /* Upstream Forwarding */ 3123 ctrl |= (cap & PCI_ACS_UF); 3124 3125 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 3126 } 3127 3128 /** 3129 * pci_enable_acs - enable ACS if hardware support it 3130 * @dev: the PCI device 3131 */ 3132 void pci_enable_acs(struct pci_dev *dev) 3133 { 3134 if (!pci_acs_enable) 3135 goto disable_acs_redir; 3136 3137 if (!pci_dev_specific_enable_acs(dev)) 3138 goto disable_acs_redir; 3139 3140 pci_std_enable_acs(dev); 3141 3142 disable_acs_redir: 3143 /* 3144 * Note: pci_disable_acs_redir() must be called even if ACS was not 3145 * enabled by the kernel because it may have been enabled by 3146 * platform firmware. So if we are told to disable it, we should 3147 * always disable it after setting the kernel's default 3148 * preferences. 3149 */ 3150 pci_disable_acs_redir(dev); 3151 } 3152 3153 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags) 3154 { 3155 int pos; 3156 u16 cap, ctrl; 3157 3158 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ACS); 3159 if (!pos) 3160 return false; 3161 3162 /* 3163 * Except for egress control, capabilities are either required 3164 * or only required if controllable. Features missing from the 3165 * capability field can therefore be assumed as hard-wired enabled. 3166 */ 3167 pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap); 3168 acs_flags &= (cap | PCI_ACS_EC); 3169 3170 pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl); 3171 return (ctrl & acs_flags) == acs_flags; 3172 } 3173 3174 /** 3175 * pci_acs_enabled - test ACS against required flags for a given device 3176 * @pdev: device to test 3177 * @acs_flags: required PCI ACS flags 3178 * 3179 * Return true if the device supports the provided flags. Automatically 3180 * filters out flags that are not implemented on multifunction devices. 3181 * 3182 * Note that this interface checks the effective ACS capabilities of the 3183 * device rather than the actual capabilities. For instance, most single 3184 * function endpoints are not required to support ACS because they have no 3185 * opportunity for peer-to-peer access. We therefore return 'true' 3186 * regardless of whether the device exposes an ACS capability. This makes 3187 * it much easier for callers of this function to ignore the actual type 3188 * or topology of the device when testing ACS support. 3189 */ 3190 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags) 3191 { 3192 int ret; 3193 3194 ret = pci_dev_specific_acs_enabled(pdev, acs_flags); 3195 if (ret >= 0) 3196 return ret > 0; 3197 3198 /* 3199 * Conventional PCI and PCI-X devices never support ACS, either 3200 * effectively or actually. The shared bus topology implies that 3201 * any device on the bus can receive or snoop DMA. 3202 */ 3203 if (!pci_is_pcie(pdev)) 3204 return false; 3205 3206 switch (pci_pcie_type(pdev)) { 3207 /* 3208 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec, 3209 * but since their primary interface is PCI/X, we conservatively 3210 * handle them as we would a non-PCIe device. 3211 */ 3212 case PCI_EXP_TYPE_PCIE_BRIDGE: 3213 /* 3214 * PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never 3215 * applicable... must never implement an ACS Extended Capability...". 3216 * This seems arbitrary, but we take a conservative interpretation 3217 * of this statement. 3218 */ 3219 case PCI_EXP_TYPE_PCI_BRIDGE: 3220 case PCI_EXP_TYPE_RC_EC: 3221 return false; 3222 /* 3223 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should 3224 * implement ACS in order to indicate their peer-to-peer capabilities, 3225 * regardless of whether they are single- or multi-function devices. 3226 */ 3227 case PCI_EXP_TYPE_DOWNSTREAM: 3228 case PCI_EXP_TYPE_ROOT_PORT: 3229 return pci_acs_flags_enabled(pdev, acs_flags); 3230 /* 3231 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be 3232 * implemented by the remaining PCIe types to indicate peer-to-peer 3233 * capabilities, but only when they are part of a multifunction 3234 * device. The footnote for section 6.12 indicates the specific 3235 * PCIe types included here. 3236 */ 3237 case PCI_EXP_TYPE_ENDPOINT: 3238 case PCI_EXP_TYPE_UPSTREAM: 3239 case PCI_EXP_TYPE_LEG_END: 3240 case PCI_EXP_TYPE_RC_END: 3241 if (!pdev->multifunction) 3242 break; 3243 3244 return pci_acs_flags_enabled(pdev, acs_flags); 3245 } 3246 3247 /* 3248 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable 3249 * to single function devices with the exception of downstream ports. 3250 */ 3251 return true; 3252 } 3253 3254 /** 3255 * pci_acs_path_enable - test ACS flags from start to end in a hierarchy 3256 * @start: starting downstream device 3257 * @end: ending upstream device or NULL to search to the root bus 3258 * @acs_flags: required flags 3259 * 3260 * Walk up a device tree from start to end testing PCI ACS support. If 3261 * any step along the way does not support the required flags, return false. 3262 */ 3263 bool pci_acs_path_enabled(struct pci_dev *start, 3264 struct pci_dev *end, u16 acs_flags) 3265 { 3266 struct pci_dev *pdev, *parent = start; 3267 3268 do { 3269 pdev = parent; 3270 3271 if (!pci_acs_enabled(pdev, acs_flags)) 3272 return false; 3273 3274 if (pci_is_root_bus(pdev->bus)) 3275 return (end == NULL); 3276 3277 parent = pdev->bus->self; 3278 } while (pdev != end); 3279 3280 return true; 3281 } 3282 3283 /** 3284 * pci_rebar_find_pos - find position of resize ctrl reg for BAR 3285 * @pdev: PCI device 3286 * @bar: BAR to find 3287 * 3288 * Helper to find the position of the ctrl register for a BAR. 3289 * Returns -ENOTSUPP if resizable BARs are not supported at all. 3290 * Returns -ENOENT if no ctrl register for the BAR could be found. 3291 */ 3292 static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) 3293 { 3294 unsigned int pos, nbars, i; 3295 u32 ctrl; 3296 3297 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 3298 if (!pos) 3299 return -ENOTSUPP; 3300 3301 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3302 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 3303 PCI_REBAR_CTRL_NBAR_SHIFT; 3304 3305 for (i = 0; i < nbars; i++, pos += 8) { 3306 int bar_idx; 3307 3308 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3309 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 3310 if (bar_idx == bar) 3311 return pos; 3312 } 3313 3314 return -ENOENT; 3315 } 3316 3317 /** 3318 * pci_rebar_get_possible_sizes - get possible sizes for BAR 3319 * @pdev: PCI device 3320 * @bar: BAR to query 3321 * 3322 * Get the possible sizes of a resizable BAR as bitmask defined in the spec 3323 * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable. 3324 */ 3325 u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar) 3326 { 3327 int pos; 3328 u32 cap; 3329 3330 pos = pci_rebar_find_pos(pdev, bar); 3331 if (pos < 0) 3332 return 0; 3333 3334 pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap); 3335 return (cap & PCI_REBAR_CAP_SIZES) >> 4; 3336 } 3337 3338 /** 3339 * pci_rebar_get_current_size - get the current size of a BAR 3340 * @pdev: PCI device 3341 * @bar: BAR to set size to 3342 * 3343 * Read the size of a BAR from the resizable BAR config. 3344 * Returns size if found or negative error code. 3345 */ 3346 int pci_rebar_get_current_size(struct pci_dev *pdev, int bar) 3347 { 3348 int pos; 3349 u32 ctrl; 3350 3351 pos = pci_rebar_find_pos(pdev, bar); 3352 if (pos < 0) 3353 return pos; 3354 3355 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3356 return (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT; 3357 } 3358 3359 /** 3360 * pci_rebar_set_size - set a new size for a BAR 3361 * @pdev: PCI device 3362 * @bar: BAR to set size to 3363 * @size: new size as defined in the spec (0=1MB, 19=512GB) 3364 * 3365 * Set the new size of a BAR as defined in the spec. 3366 * Returns zero if resizing was successful, error code otherwise. 3367 */ 3368 int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size) 3369 { 3370 int pos; 3371 u32 ctrl; 3372 3373 pos = pci_rebar_find_pos(pdev, bar); 3374 if (pos < 0) 3375 return pos; 3376 3377 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3378 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 3379 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 3380 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 3381 return 0; 3382 } 3383 3384 /** 3385 * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port 3386 * @dev: the PCI device 3387 * @cap_mask: mask of desired AtomicOp sizes, including one or more of: 3388 * PCI_EXP_DEVCAP2_ATOMIC_COMP32 3389 * PCI_EXP_DEVCAP2_ATOMIC_COMP64 3390 * PCI_EXP_DEVCAP2_ATOMIC_COMP128 3391 * 3392 * Return 0 if all upstream bridges support AtomicOp routing, egress 3393 * blocking is disabled on all upstream ports, and the root port supports 3394 * the requested completion capabilities (32-bit, 64-bit and/or 128-bit 3395 * AtomicOp completion), or negative otherwise. 3396 */ 3397 int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) 3398 { 3399 struct pci_bus *bus = dev->bus; 3400 struct pci_dev *bridge; 3401 u32 cap, ctl2; 3402 3403 if (!pci_is_pcie(dev)) 3404 return -EINVAL; 3405 3406 /* 3407 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be 3408 * AtomicOp requesters. For now, we only support endpoints as 3409 * requesters and root ports as completers. No endpoints as 3410 * completers, and no peer-to-peer. 3411 */ 3412 3413 switch (pci_pcie_type(dev)) { 3414 case PCI_EXP_TYPE_ENDPOINT: 3415 case PCI_EXP_TYPE_LEG_END: 3416 case PCI_EXP_TYPE_RC_END: 3417 break; 3418 default: 3419 return -EINVAL; 3420 } 3421 3422 while (bus->parent) { 3423 bridge = bus->self; 3424 3425 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3426 3427 switch (pci_pcie_type(bridge)) { 3428 /* Ensure switch ports support AtomicOp routing */ 3429 case PCI_EXP_TYPE_UPSTREAM: 3430 case PCI_EXP_TYPE_DOWNSTREAM: 3431 if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) 3432 return -EINVAL; 3433 break; 3434 3435 /* Ensure root port supports all the sizes we care about */ 3436 case PCI_EXP_TYPE_ROOT_PORT: 3437 if ((cap & cap_mask) != cap_mask) 3438 return -EINVAL; 3439 break; 3440 } 3441 3442 /* Ensure upstream ports don't block AtomicOps on egress */ 3443 if (!bridge->has_secondary_link) { 3444 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, 3445 &ctl2); 3446 if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK) 3447 return -EINVAL; 3448 } 3449 3450 bus = bus->parent; 3451 } 3452 3453 pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, 3454 PCI_EXP_DEVCTL2_ATOMIC_REQ); 3455 return 0; 3456 } 3457 EXPORT_SYMBOL(pci_enable_atomic_ops_to_root); 3458 3459 /** 3460 * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge 3461 * @dev: the PCI device 3462 * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) 3463 * 3464 * Perform INTx swizzling for a device behind one level of bridge. This is 3465 * required by section 9.1 of the PCI-to-PCI bridge specification for devices 3466 * behind bridges on add-in cards. For devices with ARI enabled, the slot 3467 * number is always 0 (see the Implementation Note in section 2.2.8.1 of 3468 * the PCI Express Base Specification, Revision 2.1) 3469 */ 3470 u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin) 3471 { 3472 int slot; 3473 3474 if (pci_ari_enabled(dev->bus)) 3475 slot = 0; 3476 else 3477 slot = PCI_SLOT(dev->devfn); 3478 3479 return (((pin - 1) + slot) % 4) + 1; 3480 } 3481 3482 int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge) 3483 { 3484 u8 pin; 3485 3486 pin = dev->pin; 3487 if (!pin) 3488 return -1; 3489 3490 while (!pci_is_root_bus(dev->bus)) { 3491 pin = pci_swizzle_interrupt_pin(dev, pin); 3492 dev = dev->bus->self; 3493 } 3494 *bridge = dev; 3495 return pin; 3496 } 3497 3498 /** 3499 * pci_common_swizzle - swizzle INTx all the way to root bridge 3500 * @dev: the PCI device 3501 * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD) 3502 * 3503 * Perform INTx swizzling for a device. This traverses through all PCI-to-PCI 3504 * bridges all the way up to a PCI root bus. 3505 */ 3506 u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp) 3507 { 3508 u8 pin = *pinp; 3509 3510 while (!pci_is_root_bus(dev->bus)) { 3511 pin = pci_swizzle_interrupt_pin(dev, pin); 3512 dev = dev->bus->self; 3513 } 3514 *pinp = pin; 3515 return PCI_SLOT(dev->devfn); 3516 } 3517 EXPORT_SYMBOL_GPL(pci_common_swizzle); 3518 3519 /** 3520 * pci_release_region - Release a PCI bar 3521 * @pdev: PCI device whose resources were previously reserved by pci_request_region 3522 * @bar: BAR to release 3523 * 3524 * Releases the PCI I/O and memory resources previously reserved by a 3525 * successful call to pci_request_region. Call this function only 3526 * after all use of the PCI regions has ceased. 3527 */ 3528 void pci_release_region(struct pci_dev *pdev, int bar) 3529 { 3530 struct pci_devres *dr; 3531 3532 if (pci_resource_len(pdev, bar) == 0) 3533 return; 3534 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) 3535 release_region(pci_resource_start(pdev, bar), 3536 pci_resource_len(pdev, bar)); 3537 else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) 3538 release_mem_region(pci_resource_start(pdev, bar), 3539 pci_resource_len(pdev, bar)); 3540 3541 dr = find_pci_dr(pdev); 3542 if (dr) 3543 dr->region_mask &= ~(1 << bar); 3544 } 3545 EXPORT_SYMBOL(pci_release_region); 3546 3547 /** 3548 * __pci_request_region - Reserved PCI I/O and memory resource 3549 * @pdev: PCI device whose resources are to be reserved 3550 * @bar: BAR to be reserved 3551 * @res_name: Name to be associated with resource. 3552 * @exclusive: whether the region access is exclusive or not 3553 * 3554 * Mark the PCI region associated with PCI device @pdev BR @bar as 3555 * being reserved by owner @res_name. Do not access any 3556 * address inside the PCI regions unless this call returns 3557 * successfully. 3558 * 3559 * If @exclusive is set, then the region is marked so that userspace 3560 * is explicitly not allowed to map the resource via /dev/mem or 3561 * sysfs MMIO access. 3562 * 3563 * Returns 0 on success, or %EBUSY on error. A warning 3564 * message is also printed on failure. 3565 */ 3566 static int __pci_request_region(struct pci_dev *pdev, int bar, 3567 const char *res_name, int exclusive) 3568 { 3569 struct pci_devres *dr; 3570 3571 if (pci_resource_len(pdev, bar) == 0) 3572 return 0; 3573 3574 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) { 3575 if (!request_region(pci_resource_start(pdev, bar), 3576 pci_resource_len(pdev, bar), res_name)) 3577 goto err_out; 3578 } else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) { 3579 if (!__request_mem_region(pci_resource_start(pdev, bar), 3580 pci_resource_len(pdev, bar), res_name, 3581 exclusive)) 3582 goto err_out; 3583 } 3584 3585 dr = find_pci_dr(pdev); 3586 if (dr) 3587 dr->region_mask |= 1 << bar; 3588 3589 return 0; 3590 3591 err_out: 3592 pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar, 3593 &pdev->resource[bar]); 3594 return -EBUSY; 3595 } 3596 3597 /** 3598 * pci_request_region - Reserve PCI I/O and memory resource 3599 * @pdev: PCI device whose resources are to be reserved 3600 * @bar: BAR to be reserved 3601 * @res_name: Name to be associated with resource 3602 * 3603 * Mark the PCI region associated with PCI device @pdev BAR @bar as 3604 * being reserved by owner @res_name. Do not access any 3605 * address inside the PCI regions unless this call returns 3606 * successfully. 3607 * 3608 * Returns 0 on success, or %EBUSY on error. A warning 3609 * message is also printed on failure. 3610 */ 3611 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 3612 { 3613 return __pci_request_region(pdev, bar, res_name, 0); 3614 } 3615 EXPORT_SYMBOL(pci_request_region); 3616 3617 /** 3618 * pci_request_region_exclusive - Reserved PCI I/O and memory resource 3619 * @pdev: PCI device whose resources are to be reserved 3620 * @bar: BAR to be reserved 3621 * @res_name: Name to be associated with resource. 3622 * 3623 * Mark the PCI region associated with PCI device @pdev BR @bar as 3624 * being reserved by owner @res_name. Do not access any 3625 * address inside the PCI regions unless this call returns 3626 * successfully. 3627 * 3628 * Returns 0 on success, or %EBUSY on error. A warning 3629 * message is also printed on failure. 3630 * 3631 * The key difference that _exclusive makes it that userspace is 3632 * explicitly not allowed to map the resource via /dev/mem or 3633 * sysfs. 3634 */ 3635 int pci_request_region_exclusive(struct pci_dev *pdev, int bar, 3636 const char *res_name) 3637 { 3638 return __pci_request_region(pdev, bar, res_name, IORESOURCE_EXCLUSIVE); 3639 } 3640 EXPORT_SYMBOL(pci_request_region_exclusive); 3641 3642 /** 3643 * pci_release_selected_regions - Release selected PCI I/O and memory resources 3644 * @pdev: PCI device whose resources were previously reserved 3645 * @bars: Bitmask of BARs to be released 3646 * 3647 * Release selected PCI I/O and memory resources previously reserved. 3648 * Call this function only after all use of the PCI regions has ceased. 3649 */ 3650 void pci_release_selected_regions(struct pci_dev *pdev, int bars) 3651 { 3652 int i; 3653 3654 for (i = 0; i < 6; i++) 3655 if (bars & (1 << i)) 3656 pci_release_region(pdev, i); 3657 } 3658 EXPORT_SYMBOL(pci_release_selected_regions); 3659 3660 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars, 3661 const char *res_name, int excl) 3662 { 3663 int i; 3664 3665 for (i = 0; i < 6; i++) 3666 if (bars & (1 << i)) 3667 if (__pci_request_region(pdev, i, res_name, excl)) 3668 goto err_out; 3669 return 0; 3670 3671 err_out: 3672 while (--i >= 0) 3673 if (bars & (1 << i)) 3674 pci_release_region(pdev, i); 3675 3676 return -EBUSY; 3677 } 3678 3679 3680 /** 3681 * pci_request_selected_regions - Reserve selected PCI I/O and memory resources 3682 * @pdev: PCI device whose resources are to be reserved 3683 * @bars: Bitmask of BARs to be requested 3684 * @res_name: Name to be associated with resource 3685 */ 3686 int pci_request_selected_regions(struct pci_dev *pdev, int bars, 3687 const char *res_name) 3688 { 3689 return __pci_request_selected_regions(pdev, bars, res_name, 0); 3690 } 3691 EXPORT_SYMBOL(pci_request_selected_regions); 3692 3693 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars, 3694 const char *res_name) 3695 { 3696 return __pci_request_selected_regions(pdev, bars, res_name, 3697 IORESOURCE_EXCLUSIVE); 3698 } 3699 EXPORT_SYMBOL(pci_request_selected_regions_exclusive); 3700 3701 /** 3702 * pci_release_regions - Release reserved PCI I/O and memory resources 3703 * @pdev: PCI device whose resources were previously reserved by pci_request_regions 3704 * 3705 * Releases all PCI I/O and memory resources previously reserved by a 3706 * successful call to pci_request_regions. Call this function only 3707 * after all use of the PCI regions has ceased. 3708 */ 3709 3710 void pci_release_regions(struct pci_dev *pdev) 3711 { 3712 pci_release_selected_regions(pdev, (1 << 6) - 1); 3713 } 3714 EXPORT_SYMBOL(pci_release_regions); 3715 3716 /** 3717 * pci_request_regions - Reserved PCI I/O and memory resources 3718 * @pdev: PCI device whose resources are to be reserved 3719 * @res_name: Name to be associated with resource. 3720 * 3721 * Mark all PCI regions associated with PCI device @pdev as 3722 * being reserved by owner @res_name. Do not access any 3723 * address inside the PCI regions unless this call returns 3724 * successfully. 3725 * 3726 * Returns 0 on success, or %EBUSY on error. A warning 3727 * message is also printed on failure. 3728 */ 3729 int pci_request_regions(struct pci_dev *pdev, const char *res_name) 3730 { 3731 return pci_request_selected_regions(pdev, ((1 << 6) - 1), res_name); 3732 } 3733 EXPORT_SYMBOL(pci_request_regions); 3734 3735 /** 3736 * pci_request_regions_exclusive - Reserved PCI I/O and memory resources 3737 * @pdev: PCI device whose resources are to be reserved 3738 * @res_name: Name to be associated with resource. 3739 * 3740 * Mark all PCI regions associated with PCI device @pdev as 3741 * being reserved by owner @res_name. Do not access any 3742 * address inside the PCI regions unless this call returns 3743 * successfully. 3744 * 3745 * pci_request_regions_exclusive() will mark the region so that 3746 * /dev/mem and the sysfs MMIO access will not be allowed. 3747 * 3748 * Returns 0 on success, or %EBUSY on error. A warning 3749 * message is also printed on failure. 3750 */ 3751 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name) 3752 { 3753 return pci_request_selected_regions_exclusive(pdev, 3754 ((1 << 6) - 1), res_name); 3755 } 3756 EXPORT_SYMBOL(pci_request_regions_exclusive); 3757 3758 /* 3759 * Record the PCI IO range (expressed as CPU physical address + size). 3760 * Return a negative value if an error has occured, zero otherwise 3761 */ 3762 int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr, 3763 resource_size_t size) 3764 { 3765 int ret = 0; 3766 #ifdef PCI_IOBASE 3767 struct logic_pio_hwaddr *range; 3768 3769 if (!size || addr + size < addr) 3770 return -EINVAL; 3771 3772 range = kzalloc(sizeof(*range), GFP_ATOMIC); 3773 if (!range) 3774 return -ENOMEM; 3775 3776 range->fwnode = fwnode; 3777 range->size = size; 3778 range->hw_start = addr; 3779 range->flags = LOGIC_PIO_CPU_MMIO; 3780 3781 ret = logic_pio_register_range(range); 3782 if (ret) 3783 kfree(range); 3784 #endif 3785 3786 return ret; 3787 } 3788 3789 phys_addr_t pci_pio_to_address(unsigned long pio) 3790 { 3791 phys_addr_t address = (phys_addr_t)OF_BAD_ADDR; 3792 3793 #ifdef PCI_IOBASE 3794 if (pio >= MMIO_UPPER_LIMIT) 3795 return address; 3796 3797 address = logic_pio_to_hwaddr(pio); 3798 #endif 3799 3800 return address; 3801 } 3802 3803 unsigned long __weak pci_address_to_pio(phys_addr_t address) 3804 { 3805 #ifdef PCI_IOBASE 3806 return logic_pio_trans_cpuaddr(address); 3807 #else 3808 if (address > IO_SPACE_LIMIT) 3809 return (unsigned long)-1; 3810 3811 return (unsigned long) address; 3812 #endif 3813 } 3814 3815 /** 3816 * pci_remap_iospace - Remap the memory mapped I/O space 3817 * @res: Resource describing the I/O space 3818 * @phys_addr: physical address of range to be mapped 3819 * 3820 * Remap the memory mapped I/O space described by the @res 3821 * and the CPU physical address @phys_addr into virtual address space. 3822 * Only architectures that have memory mapped IO functions defined 3823 * (and the PCI_IOBASE value defined) should call this function. 3824 */ 3825 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) 3826 { 3827 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 3828 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 3829 3830 if (!(res->flags & IORESOURCE_IO)) 3831 return -EINVAL; 3832 3833 if (res->end > IO_SPACE_LIMIT) 3834 return -EINVAL; 3835 3836 return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr, 3837 pgprot_device(PAGE_KERNEL)); 3838 #else 3839 /* this architecture does not have memory mapped I/O space, 3840 so this function should never be called */ 3841 WARN_ONCE(1, "This architecture does not support memory mapped I/O\n"); 3842 return -ENODEV; 3843 #endif 3844 } 3845 EXPORT_SYMBOL(pci_remap_iospace); 3846 3847 /** 3848 * pci_unmap_iospace - Unmap the memory mapped I/O space 3849 * @res: resource to be unmapped 3850 * 3851 * Unmap the CPU virtual address @res from virtual address space. 3852 * Only architectures that have memory mapped IO functions defined 3853 * (and the PCI_IOBASE value defined) should call this function. 3854 */ 3855 void pci_unmap_iospace(struct resource *res) 3856 { 3857 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 3858 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 3859 3860 unmap_kernel_range(vaddr, resource_size(res)); 3861 #endif 3862 } 3863 EXPORT_SYMBOL(pci_unmap_iospace); 3864 3865 static void devm_pci_unmap_iospace(struct device *dev, void *ptr) 3866 { 3867 struct resource **res = ptr; 3868 3869 pci_unmap_iospace(*res); 3870 } 3871 3872 /** 3873 * devm_pci_remap_iospace - Managed pci_remap_iospace() 3874 * @dev: Generic device to remap IO address for 3875 * @res: Resource describing the I/O space 3876 * @phys_addr: physical address of range to be mapped 3877 * 3878 * Managed pci_remap_iospace(). Map is automatically unmapped on driver 3879 * detach. 3880 */ 3881 int devm_pci_remap_iospace(struct device *dev, const struct resource *res, 3882 phys_addr_t phys_addr) 3883 { 3884 const struct resource **ptr; 3885 int error; 3886 3887 ptr = devres_alloc(devm_pci_unmap_iospace, sizeof(*ptr), GFP_KERNEL); 3888 if (!ptr) 3889 return -ENOMEM; 3890 3891 error = pci_remap_iospace(res, phys_addr); 3892 if (error) { 3893 devres_free(ptr); 3894 } else { 3895 *ptr = res; 3896 devres_add(dev, ptr); 3897 } 3898 3899 return error; 3900 } 3901 EXPORT_SYMBOL(devm_pci_remap_iospace); 3902 3903 /** 3904 * devm_pci_remap_cfgspace - Managed pci_remap_cfgspace() 3905 * @dev: Generic device to remap IO address for 3906 * @offset: Resource address to map 3907 * @size: Size of map 3908 * 3909 * Managed pci_remap_cfgspace(). Map is automatically unmapped on driver 3910 * detach. 3911 */ 3912 void __iomem *devm_pci_remap_cfgspace(struct device *dev, 3913 resource_size_t offset, 3914 resource_size_t size) 3915 { 3916 void __iomem **ptr, *addr; 3917 3918 ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL); 3919 if (!ptr) 3920 return NULL; 3921 3922 addr = pci_remap_cfgspace(offset, size); 3923 if (addr) { 3924 *ptr = addr; 3925 devres_add(dev, ptr); 3926 } else 3927 devres_free(ptr); 3928 3929 return addr; 3930 } 3931 EXPORT_SYMBOL(devm_pci_remap_cfgspace); 3932 3933 /** 3934 * devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource 3935 * @dev: generic device to handle the resource for 3936 * @res: configuration space resource to be handled 3937 * 3938 * Checks that a resource is a valid memory region, requests the memory 3939 * region and ioremaps with pci_remap_cfgspace() API that ensures the 3940 * proper PCI configuration space memory attributes are guaranteed. 3941 * 3942 * All operations are managed and will be undone on driver detach. 3943 * 3944 * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code 3945 * on failure. Usage example:: 3946 * 3947 * res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3948 * base = devm_pci_remap_cfg_resource(&pdev->dev, res); 3949 * if (IS_ERR(base)) 3950 * return PTR_ERR(base); 3951 */ 3952 void __iomem *devm_pci_remap_cfg_resource(struct device *dev, 3953 struct resource *res) 3954 { 3955 resource_size_t size; 3956 const char *name; 3957 void __iomem *dest_ptr; 3958 3959 BUG_ON(!dev); 3960 3961 if (!res || resource_type(res) != IORESOURCE_MEM) { 3962 dev_err(dev, "invalid resource\n"); 3963 return IOMEM_ERR_PTR(-EINVAL); 3964 } 3965 3966 size = resource_size(res); 3967 name = res->name ?: dev_name(dev); 3968 3969 if (!devm_request_mem_region(dev, res->start, size, name)) { 3970 dev_err(dev, "can't request region for resource %pR\n", res); 3971 return IOMEM_ERR_PTR(-EBUSY); 3972 } 3973 3974 dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size); 3975 if (!dest_ptr) { 3976 dev_err(dev, "ioremap failed for resource %pR\n", res); 3977 devm_release_mem_region(dev, res->start, size); 3978 dest_ptr = IOMEM_ERR_PTR(-ENOMEM); 3979 } 3980 3981 return dest_ptr; 3982 } 3983 EXPORT_SYMBOL(devm_pci_remap_cfg_resource); 3984 3985 static void __pci_set_master(struct pci_dev *dev, bool enable) 3986 { 3987 u16 old_cmd, cmd; 3988 3989 pci_read_config_word(dev, PCI_COMMAND, &old_cmd); 3990 if (enable) 3991 cmd = old_cmd | PCI_COMMAND_MASTER; 3992 else 3993 cmd = old_cmd & ~PCI_COMMAND_MASTER; 3994 if (cmd != old_cmd) { 3995 pci_dbg(dev, "%s bus mastering\n", 3996 enable ? "enabling" : "disabling"); 3997 pci_write_config_word(dev, PCI_COMMAND, cmd); 3998 } 3999 dev->is_busmaster = enable; 4000 } 4001 4002 /** 4003 * pcibios_setup - process "pci=" kernel boot arguments 4004 * @str: string used to pass in "pci=" kernel boot arguments 4005 * 4006 * Process kernel boot arguments. This is the default implementation. 4007 * Architecture specific implementations can override this as necessary. 4008 */ 4009 char * __weak __init pcibios_setup(char *str) 4010 { 4011 return str; 4012 } 4013 4014 /** 4015 * pcibios_set_master - enable PCI bus-mastering for device dev 4016 * @dev: the PCI device to enable 4017 * 4018 * Enables PCI bus-mastering for the device. This is the default 4019 * implementation. Architecture specific implementations can override 4020 * this if necessary. 4021 */ 4022 void __weak pcibios_set_master(struct pci_dev *dev) 4023 { 4024 u8 lat; 4025 4026 /* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */ 4027 if (pci_is_pcie(dev)) 4028 return; 4029 4030 pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat); 4031 if (lat < 16) 4032 lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency; 4033 else if (lat > pcibios_max_latency) 4034 lat = pcibios_max_latency; 4035 else 4036 return; 4037 4038 pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat); 4039 } 4040 4041 /** 4042 * pci_set_master - enables bus-mastering for device dev 4043 * @dev: the PCI device to enable 4044 * 4045 * Enables bus-mastering on the device and calls pcibios_set_master() 4046 * to do the needed arch specific settings. 4047 */ 4048 void pci_set_master(struct pci_dev *dev) 4049 { 4050 __pci_set_master(dev, true); 4051 pcibios_set_master(dev); 4052 } 4053 EXPORT_SYMBOL(pci_set_master); 4054 4055 /** 4056 * pci_clear_master - disables bus-mastering for device dev 4057 * @dev: the PCI device to disable 4058 */ 4059 void pci_clear_master(struct pci_dev *dev) 4060 { 4061 __pci_set_master(dev, false); 4062 } 4063 EXPORT_SYMBOL(pci_clear_master); 4064 4065 /** 4066 * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed 4067 * @dev: the PCI device for which MWI is to be enabled 4068 * 4069 * Helper function for pci_set_mwi. 4070 * Originally copied from drivers/net/acenic.c. 4071 * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>. 4072 * 4073 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4074 */ 4075 int pci_set_cacheline_size(struct pci_dev *dev) 4076 { 4077 u8 cacheline_size; 4078 4079 if (!pci_cache_line_size) 4080 return -EINVAL; 4081 4082 /* Validate current setting: the PCI_CACHE_LINE_SIZE must be 4083 equal to or multiple of the right value. */ 4084 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4085 if (cacheline_size >= pci_cache_line_size && 4086 (cacheline_size % pci_cache_line_size) == 0) 4087 return 0; 4088 4089 /* Write the correct value. */ 4090 pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size); 4091 /* Read it back. */ 4092 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4093 if (cacheline_size == pci_cache_line_size) 4094 return 0; 4095 4096 pci_printk(KERN_DEBUG, dev, "cache line size of %d is not supported\n", 4097 pci_cache_line_size << 2); 4098 4099 return -EINVAL; 4100 } 4101 EXPORT_SYMBOL_GPL(pci_set_cacheline_size); 4102 4103 /** 4104 * pci_set_mwi - enables memory-write-invalidate PCI transaction 4105 * @dev: the PCI device for which MWI is enabled 4106 * 4107 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4108 * 4109 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4110 */ 4111 int pci_set_mwi(struct pci_dev *dev) 4112 { 4113 #ifdef PCI_DISABLE_MWI 4114 return 0; 4115 #else 4116 int rc; 4117 u16 cmd; 4118 4119 rc = pci_set_cacheline_size(dev); 4120 if (rc) 4121 return rc; 4122 4123 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4124 if (!(cmd & PCI_COMMAND_INVALIDATE)) { 4125 pci_dbg(dev, "enabling Mem-Wr-Inval\n"); 4126 cmd |= PCI_COMMAND_INVALIDATE; 4127 pci_write_config_word(dev, PCI_COMMAND, cmd); 4128 } 4129 return 0; 4130 #endif 4131 } 4132 EXPORT_SYMBOL(pci_set_mwi); 4133 4134 /** 4135 * pcim_set_mwi - a device-managed pci_set_mwi() 4136 * @dev: the PCI device for which MWI is enabled 4137 * 4138 * Managed pci_set_mwi(). 4139 * 4140 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4141 */ 4142 int pcim_set_mwi(struct pci_dev *dev) 4143 { 4144 struct pci_devres *dr; 4145 4146 dr = find_pci_dr(dev); 4147 if (!dr) 4148 return -ENOMEM; 4149 4150 dr->mwi = 1; 4151 return pci_set_mwi(dev); 4152 } 4153 EXPORT_SYMBOL(pcim_set_mwi); 4154 4155 /** 4156 * pci_try_set_mwi - enables memory-write-invalidate PCI transaction 4157 * @dev: the PCI device for which MWI is enabled 4158 * 4159 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4160 * Callers are not required to check the return value. 4161 * 4162 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4163 */ 4164 int pci_try_set_mwi(struct pci_dev *dev) 4165 { 4166 #ifdef PCI_DISABLE_MWI 4167 return 0; 4168 #else 4169 return pci_set_mwi(dev); 4170 #endif 4171 } 4172 EXPORT_SYMBOL(pci_try_set_mwi); 4173 4174 /** 4175 * pci_clear_mwi - disables Memory-Write-Invalidate for device dev 4176 * @dev: the PCI device to disable 4177 * 4178 * Disables PCI Memory-Write-Invalidate transaction on the device 4179 */ 4180 void pci_clear_mwi(struct pci_dev *dev) 4181 { 4182 #ifndef PCI_DISABLE_MWI 4183 u16 cmd; 4184 4185 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4186 if (cmd & PCI_COMMAND_INVALIDATE) { 4187 cmd &= ~PCI_COMMAND_INVALIDATE; 4188 pci_write_config_word(dev, PCI_COMMAND, cmd); 4189 } 4190 #endif 4191 } 4192 EXPORT_SYMBOL(pci_clear_mwi); 4193 4194 /** 4195 * pci_intx - enables/disables PCI INTx for device dev 4196 * @pdev: the PCI device to operate on 4197 * @enable: boolean: whether to enable or disable PCI INTx 4198 * 4199 * Enables/disables PCI INTx for device dev 4200 */ 4201 void pci_intx(struct pci_dev *pdev, int enable) 4202 { 4203 u16 pci_command, new; 4204 4205 pci_read_config_word(pdev, PCI_COMMAND, &pci_command); 4206 4207 if (enable) 4208 new = pci_command & ~PCI_COMMAND_INTX_DISABLE; 4209 else 4210 new = pci_command | PCI_COMMAND_INTX_DISABLE; 4211 4212 if (new != pci_command) { 4213 struct pci_devres *dr; 4214 4215 pci_write_config_word(pdev, PCI_COMMAND, new); 4216 4217 dr = find_pci_dr(pdev); 4218 if (dr && !dr->restore_intx) { 4219 dr->restore_intx = 1; 4220 dr->orig_intx = !enable; 4221 } 4222 } 4223 } 4224 EXPORT_SYMBOL_GPL(pci_intx); 4225 4226 static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask) 4227 { 4228 struct pci_bus *bus = dev->bus; 4229 bool mask_updated = true; 4230 u32 cmd_status_dword; 4231 u16 origcmd, newcmd; 4232 unsigned long flags; 4233 bool irq_pending; 4234 4235 /* 4236 * We do a single dword read to retrieve both command and status. 4237 * Document assumptions that make this possible. 4238 */ 4239 BUILD_BUG_ON(PCI_COMMAND % 4); 4240 BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS); 4241 4242 raw_spin_lock_irqsave(&pci_lock, flags); 4243 4244 bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword); 4245 4246 irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT; 4247 4248 /* 4249 * Check interrupt status register to see whether our device 4250 * triggered the interrupt (when masking) or the next IRQ is 4251 * already pending (when unmasking). 4252 */ 4253 if (mask != irq_pending) { 4254 mask_updated = false; 4255 goto done; 4256 } 4257 4258 origcmd = cmd_status_dword; 4259 newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE; 4260 if (mask) 4261 newcmd |= PCI_COMMAND_INTX_DISABLE; 4262 if (newcmd != origcmd) 4263 bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd); 4264 4265 done: 4266 raw_spin_unlock_irqrestore(&pci_lock, flags); 4267 4268 return mask_updated; 4269 } 4270 4271 /** 4272 * pci_check_and_mask_intx - mask INTx on pending interrupt 4273 * @dev: the PCI device to operate on 4274 * 4275 * Check if the device dev has its INTx line asserted, mask it and 4276 * return true in that case. False is returned if no interrupt was 4277 * pending. 4278 */ 4279 bool pci_check_and_mask_intx(struct pci_dev *dev) 4280 { 4281 return pci_check_and_set_intx_mask(dev, true); 4282 } 4283 EXPORT_SYMBOL_GPL(pci_check_and_mask_intx); 4284 4285 /** 4286 * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending 4287 * @dev: the PCI device to operate on 4288 * 4289 * Check if the device dev has its INTx line asserted, unmask it if not 4290 * and return true. False is returned and the mask remains active if 4291 * there was still an interrupt pending. 4292 */ 4293 bool pci_check_and_unmask_intx(struct pci_dev *dev) 4294 { 4295 return pci_check_and_set_intx_mask(dev, false); 4296 } 4297 EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx); 4298 4299 /** 4300 * pci_wait_for_pending_transaction - waits for pending transaction 4301 * @dev: the PCI device to operate on 4302 * 4303 * Return 0 if transaction is pending 1 otherwise. 4304 */ 4305 int pci_wait_for_pending_transaction(struct pci_dev *dev) 4306 { 4307 if (!pci_is_pcie(dev)) 4308 return 1; 4309 4310 return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA, 4311 PCI_EXP_DEVSTA_TRPND); 4312 } 4313 EXPORT_SYMBOL(pci_wait_for_pending_transaction); 4314 4315 static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) 4316 { 4317 int delay = 1; 4318 u32 id; 4319 4320 /* 4321 * After reset, the device should not silently discard config 4322 * requests, but it may still indicate that it needs more time by 4323 * responding to them with CRS completions. The Root Port will 4324 * generally synthesize ~0 data to complete the read (except when 4325 * CRS SV is enabled and the read was for the Vendor ID; in that 4326 * case it synthesizes 0x0001 data). 4327 * 4328 * Wait for the device to return a non-CRS completion. Read the 4329 * Command register instead of Vendor ID so we don't have to 4330 * contend with the CRS SV value. 4331 */ 4332 pci_read_config_dword(dev, PCI_COMMAND, &id); 4333 while (id == ~0) { 4334 if (delay > timeout) { 4335 pci_warn(dev, "not ready %dms after %s; giving up\n", 4336 delay - 1, reset_type); 4337 return -ENOTTY; 4338 } 4339 4340 if (delay > 1000) 4341 pci_info(dev, "not ready %dms after %s; waiting\n", 4342 delay - 1, reset_type); 4343 4344 msleep(delay); 4345 delay *= 2; 4346 pci_read_config_dword(dev, PCI_COMMAND, &id); 4347 } 4348 4349 if (delay > 1000) 4350 pci_info(dev, "ready %dms after %s\n", delay - 1, 4351 reset_type); 4352 4353 return 0; 4354 } 4355 4356 /** 4357 * pcie_has_flr - check if a device supports function level resets 4358 * @dev: device to check 4359 * 4360 * Returns true if the device advertises support for PCIe function level 4361 * resets. 4362 */ 4363 bool pcie_has_flr(struct pci_dev *dev) 4364 { 4365 u32 cap; 4366 4367 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4368 return false; 4369 4370 pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); 4371 return cap & PCI_EXP_DEVCAP_FLR; 4372 } 4373 EXPORT_SYMBOL_GPL(pcie_has_flr); 4374 4375 /** 4376 * pcie_flr - initiate a PCIe function level reset 4377 * @dev: device to reset 4378 * 4379 * Initiate a function level reset on @dev. The caller should ensure the 4380 * device supports FLR before calling this function, e.g. by using the 4381 * pcie_has_flr() helper. 4382 */ 4383 int pcie_flr(struct pci_dev *dev) 4384 { 4385 if (!pci_wait_for_pending_transaction(dev)) 4386 pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n"); 4387 4388 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); 4389 4390 /* 4391 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within 4392 * 100ms, but may silently discard requests while the FLR is in 4393 * progress. Wait 100ms before trying to access the device. 4394 */ 4395 msleep(100); 4396 4397 return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS); 4398 } 4399 EXPORT_SYMBOL_GPL(pcie_flr); 4400 4401 static int pci_af_flr(struct pci_dev *dev, int probe) 4402 { 4403 int pos; 4404 u8 cap; 4405 4406 pos = pci_find_capability(dev, PCI_CAP_ID_AF); 4407 if (!pos) 4408 return -ENOTTY; 4409 4410 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4411 return -ENOTTY; 4412 4413 pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); 4414 if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR)) 4415 return -ENOTTY; 4416 4417 if (probe) 4418 return 0; 4419 4420 /* 4421 * Wait for Transaction Pending bit to clear. A word-aligned test 4422 * is used, so we use the conrol offset rather than status and shift 4423 * the test bit to match. 4424 */ 4425 if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL, 4426 PCI_AF_STATUS_TP << 8)) 4427 pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n"); 4428 4429 pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR); 4430 4431 /* 4432 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006, 4433 * updated 27 July 2006; a device must complete an FLR within 4434 * 100ms, but may silently discard requests while the FLR is in 4435 * progress. Wait 100ms before trying to access the device. 4436 */ 4437 msleep(100); 4438 4439 return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS); 4440 } 4441 4442 /** 4443 * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0. 4444 * @dev: Device to reset. 4445 * @probe: If set, only check if the device can be reset this way. 4446 * 4447 * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is 4448 * unset, it will be reinitialized internally when going from PCI_D3hot to 4449 * PCI_D0. If that's the case and the device is not in a low-power state 4450 * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset. 4451 * 4452 * NOTE: This causes the caller to sleep for twice the device power transition 4453 * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms 4454 * by default (i.e. unless the @dev's d3_delay field has a different value). 4455 * Moreover, only devices in D0 can be reset by this function. 4456 */ 4457 static int pci_pm_reset(struct pci_dev *dev, int probe) 4458 { 4459 u16 csr; 4460 4461 if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET) 4462 return -ENOTTY; 4463 4464 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr); 4465 if (csr & PCI_PM_CTRL_NO_SOFT_RESET) 4466 return -ENOTTY; 4467 4468 if (probe) 4469 return 0; 4470 4471 if (dev->current_state != PCI_D0) 4472 return -EINVAL; 4473 4474 csr &= ~PCI_PM_CTRL_STATE_MASK; 4475 csr |= PCI_D3hot; 4476 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4477 pci_dev_d3_sleep(dev); 4478 4479 csr &= ~PCI_PM_CTRL_STATE_MASK; 4480 csr |= PCI_D0; 4481 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4482 pci_dev_d3_sleep(dev); 4483 4484 return pci_dev_wait(dev, "PM D3->D0", PCIE_RESET_READY_POLL_MS); 4485 } 4486 /** 4487 * pcie_wait_for_link - Wait until link is active or inactive 4488 * @pdev: Bridge device 4489 * @active: waiting for active or inactive? 4490 * 4491 * Use this to wait till link becomes active or inactive. 4492 */ 4493 bool pcie_wait_for_link(struct pci_dev *pdev, bool active) 4494 { 4495 int timeout = 1000; 4496 bool ret; 4497 u16 lnk_status; 4498 4499 for (;;) { 4500 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status); 4501 ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA); 4502 if (ret == active) 4503 return true; 4504 if (timeout <= 0) 4505 break; 4506 msleep(10); 4507 timeout -= 10; 4508 } 4509 4510 pci_info(pdev, "Data Link Layer Link Active not %s in 1000 msec\n", 4511 active ? "set" : "cleared"); 4512 4513 return false; 4514 } 4515 4516 void pci_reset_secondary_bus(struct pci_dev *dev) 4517 { 4518 u16 ctrl; 4519 4520 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl); 4521 ctrl |= PCI_BRIDGE_CTL_BUS_RESET; 4522 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 4523 4524 /* 4525 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double 4526 * this to 2ms to ensure that we meet the minimum requirement. 4527 */ 4528 msleep(2); 4529 4530 ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; 4531 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 4532 4533 /* 4534 * Trhfa for conventional PCI is 2^25 clock cycles. 4535 * Assuming a minimum 33MHz clock this results in a 1s 4536 * delay before we can consider subordinate devices to 4537 * be re-initialized. PCIe has some ways to shorten this, 4538 * but we don't make use of them yet. 4539 */ 4540 ssleep(1); 4541 } 4542 4543 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) 4544 { 4545 pci_reset_secondary_bus(dev); 4546 } 4547 4548 /** 4549 * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge. 4550 * @dev: Bridge device 4551 * 4552 * Use the bridge control register to assert reset on the secondary bus. 4553 * Devices on the secondary bus are left in power-on state. 4554 */ 4555 int pci_bridge_secondary_bus_reset(struct pci_dev *dev) 4556 { 4557 pcibios_reset_secondary_bus(dev); 4558 4559 return pci_dev_wait(dev, "bus reset", PCIE_RESET_READY_POLL_MS); 4560 } 4561 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset); 4562 4563 static int pci_parent_bus_reset(struct pci_dev *dev, int probe) 4564 { 4565 struct pci_dev *pdev; 4566 4567 if (pci_is_root_bus(dev->bus) || dev->subordinate || 4568 !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 4569 return -ENOTTY; 4570 4571 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 4572 if (pdev != dev) 4573 return -ENOTTY; 4574 4575 if (probe) 4576 return 0; 4577 4578 return pci_bridge_secondary_bus_reset(dev->bus->self); 4579 } 4580 4581 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe) 4582 { 4583 int rc = -ENOTTY; 4584 4585 if (!hotplug || !try_module_get(hotplug->ops->owner)) 4586 return rc; 4587 4588 if (hotplug->ops->reset_slot) 4589 rc = hotplug->ops->reset_slot(hotplug, probe); 4590 4591 module_put(hotplug->ops->owner); 4592 4593 return rc; 4594 } 4595 4596 static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe) 4597 { 4598 struct pci_dev *pdev; 4599 4600 if (dev->subordinate || !dev->slot || 4601 dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 4602 return -ENOTTY; 4603 4604 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 4605 if (pdev != dev && pdev->slot == dev->slot) 4606 return -ENOTTY; 4607 4608 return pci_reset_hotplug_slot(dev->slot->hotplug, probe); 4609 } 4610 4611 static void pci_dev_lock(struct pci_dev *dev) 4612 { 4613 pci_cfg_access_lock(dev); 4614 /* block PM suspend, driver probe, etc. */ 4615 device_lock(&dev->dev); 4616 } 4617 4618 /* Return 1 on successful lock, 0 on contention */ 4619 static int pci_dev_trylock(struct pci_dev *dev) 4620 { 4621 if (pci_cfg_access_trylock(dev)) { 4622 if (device_trylock(&dev->dev)) 4623 return 1; 4624 pci_cfg_access_unlock(dev); 4625 } 4626 4627 return 0; 4628 } 4629 4630 static void pci_dev_unlock(struct pci_dev *dev) 4631 { 4632 device_unlock(&dev->dev); 4633 pci_cfg_access_unlock(dev); 4634 } 4635 4636 static void pci_dev_save_and_disable(struct pci_dev *dev) 4637 { 4638 const struct pci_error_handlers *err_handler = 4639 dev->driver ? dev->driver->err_handler : NULL; 4640 4641 /* 4642 * dev->driver->err_handler->reset_prepare() is protected against 4643 * races with ->remove() by the device lock, which must be held by 4644 * the caller. 4645 */ 4646 if (err_handler && err_handler->reset_prepare) 4647 err_handler->reset_prepare(dev); 4648 4649 /* 4650 * Wake-up device prior to save. PM registers default to D0 after 4651 * reset and a simple register restore doesn't reliably return 4652 * to a non-D0 state anyway. 4653 */ 4654 pci_set_power_state(dev, PCI_D0); 4655 4656 pci_save_state(dev); 4657 /* 4658 * Disable the device by clearing the Command register, except for 4659 * INTx-disable which is set. This not only disables MMIO and I/O port 4660 * BARs, but also prevents the device from being Bus Master, preventing 4661 * DMA from the device including MSI/MSI-X interrupts. For PCI 2.3 4662 * compliant devices, INTx-disable prevents legacy interrupts. 4663 */ 4664 pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 4665 } 4666 4667 static void pci_dev_restore(struct pci_dev *dev) 4668 { 4669 const struct pci_error_handlers *err_handler = 4670 dev->driver ? dev->driver->err_handler : NULL; 4671 4672 pci_restore_state(dev); 4673 4674 /* 4675 * dev->driver->err_handler->reset_done() is protected against 4676 * races with ->remove() by the device lock, which must be held by 4677 * the caller. 4678 */ 4679 if (err_handler && err_handler->reset_done) 4680 err_handler->reset_done(dev); 4681 } 4682 4683 /** 4684 * __pci_reset_function_locked - reset a PCI device function while holding 4685 * the @dev mutex lock. 4686 * @dev: PCI device to reset 4687 * 4688 * Some devices allow an individual function to be reset without affecting 4689 * other functions in the same device. The PCI device must be responsive 4690 * to PCI config space in order to use this function. 4691 * 4692 * The device function is presumed to be unused and the caller is holding 4693 * the device mutex lock when this function is called. 4694 * Resetting the device will make the contents of PCI configuration space 4695 * random, so any caller of this must be prepared to reinitialise the 4696 * device including MSI, bus mastering, BARs, decoding IO and memory spaces, 4697 * etc. 4698 * 4699 * Returns 0 if the device function was successfully reset or negative if the 4700 * device doesn't support resetting a single function. 4701 */ 4702 int __pci_reset_function_locked(struct pci_dev *dev) 4703 { 4704 int rc; 4705 4706 might_sleep(); 4707 4708 /* 4709 * A reset method returns -ENOTTY if it doesn't support this device 4710 * and we should try the next method. 4711 * 4712 * If it returns 0 (success), we're finished. If it returns any 4713 * other error, we're also finished: this indicates that further 4714 * reset mechanisms might be broken on the device. 4715 */ 4716 rc = pci_dev_specific_reset(dev, 0); 4717 if (rc != -ENOTTY) 4718 return rc; 4719 if (pcie_has_flr(dev)) { 4720 rc = pcie_flr(dev); 4721 if (rc != -ENOTTY) 4722 return rc; 4723 } 4724 rc = pci_af_flr(dev, 0); 4725 if (rc != -ENOTTY) 4726 return rc; 4727 rc = pci_pm_reset(dev, 0); 4728 if (rc != -ENOTTY) 4729 return rc; 4730 rc = pci_dev_reset_slot_function(dev, 0); 4731 if (rc != -ENOTTY) 4732 return rc; 4733 return pci_parent_bus_reset(dev, 0); 4734 } 4735 EXPORT_SYMBOL_GPL(__pci_reset_function_locked); 4736 4737 /** 4738 * pci_probe_reset_function - check whether the device can be safely reset 4739 * @dev: PCI device to reset 4740 * 4741 * Some devices allow an individual function to be reset without affecting 4742 * other functions in the same device. The PCI device must be responsive 4743 * to PCI config space in order to use this function. 4744 * 4745 * Returns 0 if the device function can be reset or negative if the 4746 * device doesn't support resetting a single function. 4747 */ 4748 int pci_probe_reset_function(struct pci_dev *dev) 4749 { 4750 int rc; 4751 4752 might_sleep(); 4753 4754 rc = pci_dev_specific_reset(dev, 1); 4755 if (rc != -ENOTTY) 4756 return rc; 4757 if (pcie_has_flr(dev)) 4758 return 0; 4759 rc = pci_af_flr(dev, 1); 4760 if (rc != -ENOTTY) 4761 return rc; 4762 rc = pci_pm_reset(dev, 1); 4763 if (rc != -ENOTTY) 4764 return rc; 4765 rc = pci_dev_reset_slot_function(dev, 1); 4766 if (rc != -ENOTTY) 4767 return rc; 4768 4769 return pci_parent_bus_reset(dev, 1); 4770 } 4771 4772 /** 4773 * pci_reset_function - quiesce and reset a PCI device function 4774 * @dev: PCI device to reset 4775 * 4776 * Some devices allow an individual function to be reset without affecting 4777 * other functions in the same device. The PCI device must be responsive 4778 * to PCI config space in order to use this function. 4779 * 4780 * This function does not just reset the PCI portion of a device, but 4781 * clears all the state associated with the device. This function differs 4782 * from __pci_reset_function_locked() in that it saves and restores device state 4783 * over the reset and takes the PCI device lock. 4784 * 4785 * Returns 0 if the device function was successfully reset or negative if the 4786 * device doesn't support resetting a single function. 4787 */ 4788 int pci_reset_function(struct pci_dev *dev) 4789 { 4790 int rc; 4791 4792 if (!dev->reset_fn) 4793 return -ENOTTY; 4794 4795 pci_dev_lock(dev); 4796 pci_dev_save_and_disable(dev); 4797 4798 rc = __pci_reset_function_locked(dev); 4799 4800 pci_dev_restore(dev); 4801 pci_dev_unlock(dev); 4802 4803 return rc; 4804 } 4805 EXPORT_SYMBOL_GPL(pci_reset_function); 4806 4807 /** 4808 * pci_reset_function_locked - quiesce and reset a PCI device function 4809 * @dev: PCI device to reset 4810 * 4811 * Some devices allow an individual function to be reset without affecting 4812 * other functions in the same device. The PCI device must be responsive 4813 * to PCI config space in order to use this function. 4814 * 4815 * This function does not just reset the PCI portion of a device, but 4816 * clears all the state associated with the device. This function differs 4817 * from __pci_reset_function_locked() in that it saves and restores device state 4818 * over the reset. It also differs from pci_reset_function() in that it 4819 * requires the PCI device lock to be held. 4820 * 4821 * Returns 0 if the device function was successfully reset or negative if the 4822 * device doesn't support resetting a single function. 4823 */ 4824 int pci_reset_function_locked(struct pci_dev *dev) 4825 { 4826 int rc; 4827 4828 if (!dev->reset_fn) 4829 return -ENOTTY; 4830 4831 pci_dev_save_and_disable(dev); 4832 4833 rc = __pci_reset_function_locked(dev); 4834 4835 pci_dev_restore(dev); 4836 4837 return rc; 4838 } 4839 EXPORT_SYMBOL_GPL(pci_reset_function_locked); 4840 4841 /** 4842 * pci_try_reset_function - quiesce and reset a PCI device function 4843 * @dev: PCI device to reset 4844 * 4845 * Same as above, except return -EAGAIN if unable to lock device. 4846 */ 4847 int pci_try_reset_function(struct pci_dev *dev) 4848 { 4849 int rc; 4850 4851 if (!dev->reset_fn) 4852 return -ENOTTY; 4853 4854 if (!pci_dev_trylock(dev)) 4855 return -EAGAIN; 4856 4857 pci_dev_save_and_disable(dev); 4858 rc = __pci_reset_function_locked(dev); 4859 pci_dev_restore(dev); 4860 pci_dev_unlock(dev); 4861 4862 return rc; 4863 } 4864 EXPORT_SYMBOL_GPL(pci_try_reset_function); 4865 4866 /* Do any devices on or below this bus prevent a bus reset? */ 4867 static bool pci_bus_resetable(struct pci_bus *bus) 4868 { 4869 struct pci_dev *dev; 4870 4871 4872 if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 4873 return false; 4874 4875 list_for_each_entry(dev, &bus->devices, bus_list) { 4876 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 4877 (dev->subordinate && !pci_bus_resetable(dev->subordinate))) 4878 return false; 4879 } 4880 4881 return true; 4882 } 4883 4884 /* Lock devices from the top of the tree down */ 4885 static void pci_bus_lock(struct pci_bus *bus) 4886 { 4887 struct pci_dev *dev; 4888 4889 list_for_each_entry(dev, &bus->devices, bus_list) { 4890 pci_dev_lock(dev); 4891 if (dev->subordinate) 4892 pci_bus_lock(dev->subordinate); 4893 } 4894 } 4895 4896 /* Unlock devices from the bottom of the tree up */ 4897 static void pci_bus_unlock(struct pci_bus *bus) 4898 { 4899 struct pci_dev *dev; 4900 4901 list_for_each_entry(dev, &bus->devices, bus_list) { 4902 if (dev->subordinate) 4903 pci_bus_unlock(dev->subordinate); 4904 pci_dev_unlock(dev); 4905 } 4906 } 4907 4908 /* Return 1 on successful lock, 0 on contention */ 4909 static int pci_bus_trylock(struct pci_bus *bus) 4910 { 4911 struct pci_dev *dev; 4912 4913 list_for_each_entry(dev, &bus->devices, bus_list) { 4914 if (!pci_dev_trylock(dev)) 4915 goto unlock; 4916 if (dev->subordinate) { 4917 if (!pci_bus_trylock(dev->subordinate)) { 4918 pci_dev_unlock(dev); 4919 goto unlock; 4920 } 4921 } 4922 } 4923 return 1; 4924 4925 unlock: 4926 list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) { 4927 if (dev->subordinate) 4928 pci_bus_unlock(dev->subordinate); 4929 pci_dev_unlock(dev); 4930 } 4931 return 0; 4932 } 4933 4934 /* Do any devices on or below this slot prevent a bus reset? */ 4935 static bool pci_slot_resetable(struct pci_slot *slot) 4936 { 4937 struct pci_dev *dev; 4938 4939 if (slot->bus->self && 4940 (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 4941 return false; 4942 4943 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 4944 if (!dev->slot || dev->slot != slot) 4945 continue; 4946 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 4947 (dev->subordinate && !pci_bus_resetable(dev->subordinate))) 4948 return false; 4949 } 4950 4951 return true; 4952 } 4953 4954 /* Lock devices from the top of the tree down */ 4955 static void pci_slot_lock(struct pci_slot *slot) 4956 { 4957 struct pci_dev *dev; 4958 4959 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 4960 if (!dev->slot || dev->slot != slot) 4961 continue; 4962 pci_dev_lock(dev); 4963 if (dev->subordinate) 4964 pci_bus_lock(dev->subordinate); 4965 } 4966 } 4967 4968 /* Unlock devices from the bottom of the tree up */ 4969 static void pci_slot_unlock(struct pci_slot *slot) 4970 { 4971 struct pci_dev *dev; 4972 4973 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 4974 if (!dev->slot || dev->slot != slot) 4975 continue; 4976 if (dev->subordinate) 4977 pci_bus_unlock(dev->subordinate); 4978 pci_dev_unlock(dev); 4979 } 4980 } 4981 4982 /* Return 1 on successful lock, 0 on contention */ 4983 static int pci_slot_trylock(struct pci_slot *slot) 4984 { 4985 struct pci_dev *dev; 4986 4987 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 4988 if (!dev->slot || dev->slot != slot) 4989 continue; 4990 if (!pci_dev_trylock(dev)) 4991 goto unlock; 4992 if (dev->subordinate) { 4993 if (!pci_bus_trylock(dev->subordinate)) { 4994 pci_dev_unlock(dev); 4995 goto unlock; 4996 } 4997 } 4998 } 4999 return 1; 5000 5001 unlock: 5002 list_for_each_entry_continue_reverse(dev, 5003 &slot->bus->devices, bus_list) { 5004 if (!dev->slot || dev->slot != slot) 5005 continue; 5006 if (dev->subordinate) 5007 pci_bus_unlock(dev->subordinate); 5008 pci_dev_unlock(dev); 5009 } 5010 return 0; 5011 } 5012 5013 /* Save and disable devices from the top of the tree down */ 5014 static void pci_bus_save_and_disable(struct pci_bus *bus) 5015 { 5016 struct pci_dev *dev; 5017 5018 list_for_each_entry(dev, &bus->devices, bus_list) { 5019 pci_dev_lock(dev); 5020 pci_dev_save_and_disable(dev); 5021 pci_dev_unlock(dev); 5022 if (dev->subordinate) 5023 pci_bus_save_and_disable(dev->subordinate); 5024 } 5025 } 5026 5027 /* 5028 * Restore devices from top of the tree down - parent bridges need to be 5029 * restored before we can get to subordinate devices. 5030 */ 5031 static void pci_bus_restore(struct pci_bus *bus) 5032 { 5033 struct pci_dev *dev; 5034 5035 list_for_each_entry(dev, &bus->devices, bus_list) { 5036 pci_dev_lock(dev); 5037 pci_dev_restore(dev); 5038 pci_dev_unlock(dev); 5039 if (dev->subordinate) 5040 pci_bus_restore(dev->subordinate); 5041 } 5042 } 5043 5044 /* Save and disable devices from the top of the tree down */ 5045 static void pci_slot_save_and_disable(struct pci_slot *slot) 5046 { 5047 struct pci_dev *dev; 5048 5049 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5050 if (!dev->slot || dev->slot != slot) 5051 continue; 5052 pci_dev_save_and_disable(dev); 5053 if (dev->subordinate) 5054 pci_bus_save_and_disable(dev->subordinate); 5055 } 5056 } 5057 5058 /* 5059 * Restore devices from top of the tree down - parent bridges need to be 5060 * restored before we can get to subordinate devices. 5061 */ 5062 static void pci_slot_restore(struct pci_slot *slot) 5063 { 5064 struct pci_dev *dev; 5065 5066 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5067 if (!dev->slot || dev->slot != slot) 5068 continue; 5069 pci_dev_lock(dev); 5070 pci_dev_restore(dev); 5071 pci_dev_unlock(dev); 5072 if (dev->subordinate) 5073 pci_bus_restore(dev->subordinate); 5074 } 5075 } 5076 5077 static int pci_slot_reset(struct pci_slot *slot, int probe) 5078 { 5079 int rc; 5080 5081 if (!slot || !pci_slot_resetable(slot)) 5082 return -ENOTTY; 5083 5084 if (!probe) 5085 pci_slot_lock(slot); 5086 5087 might_sleep(); 5088 5089 rc = pci_reset_hotplug_slot(slot->hotplug, probe); 5090 5091 if (!probe) 5092 pci_slot_unlock(slot); 5093 5094 return rc; 5095 } 5096 5097 /** 5098 * pci_probe_reset_slot - probe whether a PCI slot can be reset 5099 * @slot: PCI slot to probe 5100 * 5101 * Return 0 if slot can be reset, negative if a slot reset is not supported. 5102 */ 5103 int pci_probe_reset_slot(struct pci_slot *slot) 5104 { 5105 return pci_slot_reset(slot, 1); 5106 } 5107 EXPORT_SYMBOL_GPL(pci_probe_reset_slot); 5108 5109 /** 5110 * __pci_reset_slot - Try to reset a PCI slot 5111 * @slot: PCI slot to reset 5112 * 5113 * A PCI bus may host multiple slots, each slot may support a reset mechanism 5114 * independent of other slots. For instance, some slots may support slot power 5115 * control. In the case of a 1:1 bus to slot architecture, this function may 5116 * wrap the bus reset to avoid spurious slot related events such as hotplug. 5117 * Generally a slot reset should be attempted before a bus reset. All of the 5118 * function of the slot and any subordinate buses behind the slot are reset 5119 * through this function. PCI config space of all devices in the slot and 5120 * behind the slot is saved before and restored after reset. 5121 * 5122 * Same as above except return -EAGAIN if the slot cannot be locked 5123 */ 5124 static int __pci_reset_slot(struct pci_slot *slot) 5125 { 5126 int rc; 5127 5128 rc = pci_slot_reset(slot, 1); 5129 if (rc) 5130 return rc; 5131 5132 pci_slot_save_and_disable(slot); 5133 5134 if (pci_slot_trylock(slot)) { 5135 might_sleep(); 5136 rc = pci_reset_hotplug_slot(slot->hotplug, 0); 5137 pci_slot_unlock(slot); 5138 } else 5139 rc = -EAGAIN; 5140 5141 pci_slot_restore(slot); 5142 5143 return rc; 5144 } 5145 5146 static int pci_bus_reset(struct pci_bus *bus, int probe) 5147 { 5148 int ret; 5149 5150 if (!bus->self || !pci_bus_resetable(bus)) 5151 return -ENOTTY; 5152 5153 if (probe) 5154 return 0; 5155 5156 pci_bus_lock(bus); 5157 5158 might_sleep(); 5159 5160 ret = pci_bridge_secondary_bus_reset(bus->self); 5161 5162 pci_bus_unlock(bus); 5163 5164 return ret; 5165 } 5166 5167 /** 5168 * pci_probe_reset_bus - probe whether a PCI bus can be reset 5169 * @bus: PCI bus to probe 5170 * 5171 * Return 0 if bus can be reset, negative if a bus reset is not supported. 5172 */ 5173 int pci_probe_reset_bus(struct pci_bus *bus) 5174 { 5175 return pci_bus_reset(bus, 1); 5176 } 5177 EXPORT_SYMBOL_GPL(pci_probe_reset_bus); 5178 5179 /** 5180 * __pci_reset_bus - Try to reset a PCI bus 5181 * @bus: top level PCI bus to reset 5182 * 5183 * Same as above except return -EAGAIN if the bus cannot be locked 5184 */ 5185 static int __pci_reset_bus(struct pci_bus *bus) 5186 { 5187 int rc; 5188 5189 rc = pci_bus_reset(bus, 1); 5190 if (rc) 5191 return rc; 5192 5193 pci_bus_save_and_disable(bus); 5194 5195 if (pci_bus_trylock(bus)) { 5196 might_sleep(); 5197 rc = pci_bridge_secondary_bus_reset(bus->self); 5198 pci_bus_unlock(bus); 5199 } else 5200 rc = -EAGAIN; 5201 5202 pci_bus_restore(bus); 5203 5204 return rc; 5205 } 5206 5207 /** 5208 * pci_reset_bus - Try to reset a PCI bus 5209 * @pdev: top level PCI device to reset via slot/bus 5210 * 5211 * Same as above except return -EAGAIN if the bus cannot be locked 5212 */ 5213 int pci_reset_bus(struct pci_dev *pdev) 5214 { 5215 return (!pci_probe_reset_slot(pdev->slot)) ? 5216 __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus); 5217 } 5218 EXPORT_SYMBOL_GPL(pci_reset_bus); 5219 5220 /** 5221 * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count 5222 * @dev: PCI device to query 5223 * 5224 * Returns mmrbc: maximum designed memory read count in bytes 5225 * or appropriate error value. 5226 */ 5227 int pcix_get_max_mmrbc(struct pci_dev *dev) 5228 { 5229 int cap; 5230 u32 stat; 5231 5232 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5233 if (!cap) 5234 return -EINVAL; 5235 5236 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 5237 return -EINVAL; 5238 5239 return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21); 5240 } 5241 EXPORT_SYMBOL(pcix_get_max_mmrbc); 5242 5243 /** 5244 * pcix_get_mmrbc - get PCI-X maximum memory read byte count 5245 * @dev: PCI device to query 5246 * 5247 * Returns mmrbc: maximum memory read count in bytes 5248 * or appropriate error value. 5249 */ 5250 int pcix_get_mmrbc(struct pci_dev *dev) 5251 { 5252 int cap; 5253 u16 cmd; 5254 5255 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5256 if (!cap) 5257 return -EINVAL; 5258 5259 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 5260 return -EINVAL; 5261 5262 return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2); 5263 } 5264 EXPORT_SYMBOL(pcix_get_mmrbc); 5265 5266 /** 5267 * pcix_set_mmrbc - set PCI-X maximum memory read byte count 5268 * @dev: PCI device to query 5269 * @mmrbc: maximum memory read count in bytes 5270 * valid values are 512, 1024, 2048, 4096 5271 * 5272 * If possible sets maximum memory read byte count, some bridges have erratas 5273 * that prevent this. 5274 */ 5275 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc) 5276 { 5277 int cap; 5278 u32 stat, v, o; 5279 u16 cmd; 5280 5281 if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc)) 5282 return -EINVAL; 5283 5284 v = ffs(mmrbc) - 10; 5285 5286 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5287 if (!cap) 5288 return -EINVAL; 5289 5290 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 5291 return -EINVAL; 5292 5293 if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21) 5294 return -E2BIG; 5295 5296 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 5297 return -EINVAL; 5298 5299 o = (cmd & PCI_X_CMD_MAX_READ) >> 2; 5300 if (o != v) { 5301 if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC)) 5302 return -EIO; 5303 5304 cmd &= ~PCI_X_CMD_MAX_READ; 5305 cmd |= v << 2; 5306 if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd)) 5307 return -EIO; 5308 } 5309 return 0; 5310 } 5311 EXPORT_SYMBOL(pcix_set_mmrbc); 5312 5313 /** 5314 * pcie_get_readrq - get PCI Express read request size 5315 * @dev: PCI device to query 5316 * 5317 * Returns maximum memory read request in bytes 5318 * or appropriate error value. 5319 */ 5320 int pcie_get_readrq(struct pci_dev *dev) 5321 { 5322 u16 ctl; 5323 5324 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 5325 5326 return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12); 5327 } 5328 EXPORT_SYMBOL(pcie_get_readrq); 5329 5330 /** 5331 * pcie_set_readrq - set PCI Express maximum memory read request 5332 * @dev: PCI device to query 5333 * @rq: maximum memory read count in bytes 5334 * valid values are 128, 256, 512, 1024, 2048, 4096 5335 * 5336 * If possible sets maximum memory read request in bytes 5337 */ 5338 int pcie_set_readrq(struct pci_dev *dev, int rq) 5339 { 5340 u16 v; 5341 5342 if (rq < 128 || rq > 4096 || !is_power_of_2(rq)) 5343 return -EINVAL; 5344 5345 /* 5346 * If using the "performance" PCIe config, we clamp the 5347 * read rq size to the max packet size to prevent the 5348 * host bridge generating requests larger than we can 5349 * cope with 5350 */ 5351 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) { 5352 int mps = pcie_get_mps(dev); 5353 5354 if (mps < rq) 5355 rq = mps; 5356 } 5357 5358 v = (ffs(rq) - 8) << 12; 5359 5360 return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 5361 PCI_EXP_DEVCTL_READRQ, v); 5362 } 5363 EXPORT_SYMBOL(pcie_set_readrq); 5364 5365 /** 5366 * pcie_get_mps - get PCI Express maximum payload size 5367 * @dev: PCI device to query 5368 * 5369 * Returns maximum payload size in bytes 5370 */ 5371 int pcie_get_mps(struct pci_dev *dev) 5372 { 5373 u16 ctl; 5374 5375 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 5376 5377 return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5); 5378 } 5379 EXPORT_SYMBOL(pcie_get_mps); 5380 5381 /** 5382 * pcie_set_mps - set PCI Express maximum payload size 5383 * @dev: PCI device to query 5384 * @mps: maximum payload size in bytes 5385 * valid values are 128, 256, 512, 1024, 2048, 4096 5386 * 5387 * If possible sets maximum payload size 5388 */ 5389 int pcie_set_mps(struct pci_dev *dev, int mps) 5390 { 5391 u16 v; 5392 5393 if (mps < 128 || mps > 4096 || !is_power_of_2(mps)) 5394 return -EINVAL; 5395 5396 v = ffs(mps) - 8; 5397 if (v > dev->pcie_mpss) 5398 return -EINVAL; 5399 v <<= 5; 5400 5401 return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 5402 PCI_EXP_DEVCTL_PAYLOAD, v); 5403 } 5404 EXPORT_SYMBOL(pcie_set_mps); 5405 5406 /** 5407 * pcie_bandwidth_available - determine minimum link settings of a PCIe 5408 * device and its bandwidth limitation 5409 * @dev: PCI device to query 5410 * @limiting_dev: storage for device causing the bandwidth limitation 5411 * @speed: storage for speed of limiting device 5412 * @width: storage for width of limiting device 5413 * 5414 * Walk up the PCI device chain and find the point where the minimum 5415 * bandwidth is available. Return the bandwidth available there and (if 5416 * limiting_dev, speed, and width pointers are supplied) information about 5417 * that point. The bandwidth returned is in Mb/s, i.e., megabits/second of 5418 * raw bandwidth. 5419 */ 5420 u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, 5421 enum pci_bus_speed *speed, 5422 enum pcie_link_width *width) 5423 { 5424 u16 lnksta; 5425 enum pci_bus_speed next_speed; 5426 enum pcie_link_width next_width; 5427 u32 bw, next_bw; 5428 5429 if (speed) 5430 *speed = PCI_SPEED_UNKNOWN; 5431 if (width) 5432 *width = PCIE_LNK_WIDTH_UNKNOWN; 5433 5434 bw = 0; 5435 5436 while (dev) { 5437 pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); 5438 5439 next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS]; 5440 next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >> 5441 PCI_EXP_LNKSTA_NLW_SHIFT; 5442 5443 next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed); 5444 5445 /* Check if current device limits the total bandwidth */ 5446 if (!bw || next_bw <= bw) { 5447 bw = next_bw; 5448 5449 if (limiting_dev) 5450 *limiting_dev = dev; 5451 if (speed) 5452 *speed = next_speed; 5453 if (width) 5454 *width = next_width; 5455 } 5456 5457 dev = pci_upstream_bridge(dev); 5458 } 5459 5460 return bw; 5461 } 5462 EXPORT_SYMBOL(pcie_bandwidth_available); 5463 5464 /** 5465 * pcie_get_speed_cap - query for the PCI device's link speed capability 5466 * @dev: PCI device to query 5467 * 5468 * Query the PCI device speed capability. Return the maximum link speed 5469 * supported by the device. 5470 */ 5471 enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev) 5472 { 5473 u32 lnkcap2, lnkcap; 5474 5475 /* 5476 * PCIe r4.0 sec 7.5.3.18 recommends using the Supported Link 5477 * Speeds Vector in Link Capabilities 2 when supported, falling 5478 * back to Max Link Speed in Link Capabilities otherwise. 5479 */ 5480 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2); 5481 if (lnkcap2) { /* PCIe r3.0-compliant */ 5482 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_16_0GB) 5483 return PCIE_SPEED_16_0GT; 5484 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) 5485 return PCIE_SPEED_8_0GT; 5486 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) 5487 return PCIE_SPEED_5_0GT; 5488 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) 5489 return PCIE_SPEED_2_5GT; 5490 return PCI_SPEED_UNKNOWN; 5491 } 5492 5493 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 5494 if (lnkcap) { 5495 if (lnkcap & PCI_EXP_LNKCAP_SLS_16_0GB) 5496 return PCIE_SPEED_16_0GT; 5497 else if (lnkcap & PCI_EXP_LNKCAP_SLS_8_0GB) 5498 return PCIE_SPEED_8_0GT; 5499 else if (lnkcap & PCI_EXP_LNKCAP_SLS_5_0GB) 5500 return PCIE_SPEED_5_0GT; 5501 else if (lnkcap & PCI_EXP_LNKCAP_SLS_2_5GB) 5502 return PCIE_SPEED_2_5GT; 5503 } 5504 5505 return PCI_SPEED_UNKNOWN; 5506 } 5507 EXPORT_SYMBOL(pcie_get_speed_cap); 5508 5509 /** 5510 * pcie_get_width_cap - query for the PCI device's link width capability 5511 * @dev: PCI device to query 5512 * 5513 * Query the PCI device width capability. Return the maximum link width 5514 * supported by the device. 5515 */ 5516 enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev) 5517 { 5518 u32 lnkcap; 5519 5520 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 5521 if (lnkcap) 5522 return (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4; 5523 5524 return PCIE_LNK_WIDTH_UNKNOWN; 5525 } 5526 EXPORT_SYMBOL(pcie_get_width_cap); 5527 5528 /** 5529 * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability 5530 * @dev: PCI device 5531 * @speed: storage for link speed 5532 * @width: storage for link width 5533 * 5534 * Calculate a PCI device's link bandwidth by querying for its link speed 5535 * and width, multiplying them, and applying encoding overhead. The result 5536 * is in Mb/s, i.e., megabits/second of raw bandwidth. 5537 */ 5538 u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed, 5539 enum pcie_link_width *width) 5540 { 5541 *speed = pcie_get_speed_cap(dev); 5542 *width = pcie_get_width_cap(dev); 5543 5544 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 5545 return 0; 5546 5547 return *width * PCIE_SPEED2MBS_ENC(*speed); 5548 } 5549 5550 /** 5551 * __pcie_print_link_status - Report the PCI device's link speed and width 5552 * @dev: PCI device to query 5553 * @verbose: Print info even when enough bandwidth is available 5554 * 5555 * If the available bandwidth at the device is less than the device is 5556 * capable of, report the device's maximum possible bandwidth and the 5557 * upstream link that limits its performance. If @verbose, always print 5558 * the available bandwidth, even if the device isn't constrained. 5559 */ 5560 void __pcie_print_link_status(struct pci_dev *dev, bool verbose) 5561 { 5562 enum pcie_link_width width, width_cap; 5563 enum pci_bus_speed speed, speed_cap; 5564 struct pci_dev *limiting_dev = NULL; 5565 u32 bw_avail, bw_cap; 5566 5567 bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap); 5568 bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width); 5569 5570 if (bw_avail >= bw_cap && verbose) 5571 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n", 5572 bw_cap / 1000, bw_cap % 1000, 5573 PCIE_SPEED2STR(speed_cap), width_cap); 5574 else if (bw_avail < bw_cap) 5575 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n", 5576 bw_avail / 1000, bw_avail % 1000, 5577 PCIE_SPEED2STR(speed), width, 5578 limiting_dev ? pci_name(limiting_dev) : "<unknown>", 5579 bw_cap / 1000, bw_cap % 1000, 5580 PCIE_SPEED2STR(speed_cap), width_cap); 5581 } 5582 5583 /** 5584 * pcie_print_link_status - Report the PCI device's link speed and width 5585 * @dev: PCI device to query 5586 * 5587 * Report the available bandwidth at the device. 5588 */ 5589 void pcie_print_link_status(struct pci_dev *dev) 5590 { 5591 __pcie_print_link_status(dev, true); 5592 } 5593 EXPORT_SYMBOL(pcie_print_link_status); 5594 5595 /** 5596 * pci_select_bars - Make BAR mask from the type of resource 5597 * @dev: the PCI device for which BAR mask is made 5598 * @flags: resource type mask to be selected 5599 * 5600 * This helper routine makes bar mask from the type of resource. 5601 */ 5602 int pci_select_bars(struct pci_dev *dev, unsigned long flags) 5603 { 5604 int i, bars = 0; 5605 for (i = 0; i < PCI_NUM_RESOURCES; i++) 5606 if (pci_resource_flags(dev, i) & flags) 5607 bars |= (1 << i); 5608 return bars; 5609 } 5610 EXPORT_SYMBOL(pci_select_bars); 5611 5612 /* Some architectures require additional programming to enable VGA */ 5613 static arch_set_vga_state_t arch_set_vga_state; 5614 5615 void __init pci_register_set_vga_state(arch_set_vga_state_t func) 5616 { 5617 arch_set_vga_state = func; /* NULL disables */ 5618 } 5619 5620 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode, 5621 unsigned int command_bits, u32 flags) 5622 { 5623 if (arch_set_vga_state) 5624 return arch_set_vga_state(dev, decode, command_bits, 5625 flags); 5626 return 0; 5627 } 5628 5629 /** 5630 * pci_set_vga_state - set VGA decode state on device and parents if requested 5631 * @dev: the PCI device 5632 * @decode: true = enable decoding, false = disable decoding 5633 * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY 5634 * @flags: traverse ancestors and change bridges 5635 * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE 5636 */ 5637 int pci_set_vga_state(struct pci_dev *dev, bool decode, 5638 unsigned int command_bits, u32 flags) 5639 { 5640 struct pci_bus *bus; 5641 struct pci_dev *bridge; 5642 u16 cmd; 5643 int rc; 5644 5645 WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY))); 5646 5647 /* ARCH specific VGA enables */ 5648 rc = pci_set_vga_state_arch(dev, decode, command_bits, flags); 5649 if (rc) 5650 return rc; 5651 5652 if (flags & PCI_VGA_STATE_CHANGE_DECODES) { 5653 pci_read_config_word(dev, PCI_COMMAND, &cmd); 5654 if (decode == true) 5655 cmd |= command_bits; 5656 else 5657 cmd &= ~command_bits; 5658 pci_write_config_word(dev, PCI_COMMAND, cmd); 5659 } 5660 5661 if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE)) 5662 return 0; 5663 5664 bus = dev->bus; 5665 while (bus) { 5666 bridge = bus->self; 5667 if (bridge) { 5668 pci_read_config_word(bridge, PCI_BRIDGE_CONTROL, 5669 &cmd); 5670 if (decode == true) 5671 cmd |= PCI_BRIDGE_CTL_VGA; 5672 else 5673 cmd &= ~PCI_BRIDGE_CTL_VGA; 5674 pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, 5675 cmd); 5676 } 5677 bus = bus->parent; 5678 } 5679 return 0; 5680 } 5681 5682 /** 5683 * pci_add_dma_alias - Add a DMA devfn alias for a device 5684 * @dev: the PCI device for which alias is added 5685 * @devfn: alias slot and function 5686 * 5687 * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask 5688 * which is used to program permissible bus-devfn source addresses for DMA 5689 * requests in an IOMMU. These aliases factor into IOMMU group creation 5690 * and are useful for devices generating DMA requests beyond or different 5691 * from their logical bus-devfn. Examples include device quirks where the 5692 * device simply uses the wrong devfn, as well as non-transparent bridges 5693 * where the alias may be a proxy for devices in another domain. 5694 * 5695 * IOMMU group creation is performed during device discovery or addition, 5696 * prior to any potential DMA mapping and therefore prior to driver probing 5697 * (especially for userspace assigned devices where IOMMU group definition 5698 * cannot be left as a userspace activity). DMA aliases should therefore 5699 * be configured via quirks, such as the PCI fixup header quirk. 5700 */ 5701 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn) 5702 { 5703 if (!dev->dma_alias_mask) 5704 dev->dma_alias_mask = kcalloc(BITS_TO_LONGS(U8_MAX), 5705 sizeof(long), GFP_KERNEL); 5706 if (!dev->dma_alias_mask) { 5707 pci_warn(dev, "Unable to allocate DMA alias mask\n"); 5708 return; 5709 } 5710 5711 set_bit(devfn, dev->dma_alias_mask); 5712 pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n", 5713 PCI_SLOT(devfn), PCI_FUNC(devfn)); 5714 } 5715 5716 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) 5717 { 5718 return (dev1->dma_alias_mask && 5719 test_bit(dev2->devfn, dev1->dma_alias_mask)) || 5720 (dev2->dma_alias_mask && 5721 test_bit(dev1->devfn, dev2->dma_alias_mask)); 5722 } 5723 5724 bool pci_device_is_present(struct pci_dev *pdev) 5725 { 5726 u32 v; 5727 5728 if (pci_dev_is_disconnected(pdev)) 5729 return false; 5730 return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0); 5731 } 5732 EXPORT_SYMBOL_GPL(pci_device_is_present); 5733 5734 void pci_ignore_hotplug(struct pci_dev *dev) 5735 { 5736 struct pci_dev *bridge = dev->bus->self; 5737 5738 dev->ignore_hotplug = 1; 5739 /* Propagate the "ignore hotplug" setting to the parent bridge. */ 5740 if (bridge) 5741 bridge->ignore_hotplug = 1; 5742 } 5743 EXPORT_SYMBOL_GPL(pci_ignore_hotplug); 5744 5745 resource_size_t __weak pcibios_default_alignment(void) 5746 { 5747 return 0; 5748 } 5749 5750 #define RESOURCE_ALIGNMENT_PARAM_SIZE COMMAND_LINE_SIZE 5751 static char resource_alignment_param[RESOURCE_ALIGNMENT_PARAM_SIZE] = {0}; 5752 static DEFINE_SPINLOCK(resource_alignment_lock); 5753 5754 /** 5755 * pci_specified_resource_alignment - get resource alignment specified by user. 5756 * @dev: the PCI device to get 5757 * @resize: whether or not to change resources' size when reassigning alignment 5758 * 5759 * RETURNS: Resource alignment if it is specified. 5760 * Zero if it is not specified. 5761 */ 5762 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev, 5763 bool *resize) 5764 { 5765 int align_order, count; 5766 resource_size_t align = pcibios_default_alignment(); 5767 const char *p; 5768 int ret; 5769 5770 spin_lock(&resource_alignment_lock); 5771 p = resource_alignment_param; 5772 if (!*p && !align) 5773 goto out; 5774 if (pci_has_flag(PCI_PROBE_ONLY)) { 5775 align = 0; 5776 pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n"); 5777 goto out; 5778 } 5779 5780 while (*p) { 5781 count = 0; 5782 if (sscanf(p, "%d%n", &align_order, &count) == 1 && 5783 p[count] == '@') { 5784 p += count + 1; 5785 } else { 5786 align_order = -1; 5787 } 5788 5789 ret = pci_dev_str_match(dev, p, &p); 5790 if (ret == 1) { 5791 *resize = true; 5792 if (align_order == -1) 5793 align = PAGE_SIZE; 5794 else 5795 align = 1 << align_order; 5796 break; 5797 } else if (ret < 0) { 5798 pr_err("PCI: Can't parse resource_alignment parameter: %s\n", 5799 p); 5800 break; 5801 } 5802 5803 if (*p != ';' && *p != ',') { 5804 /* End of param or invalid format */ 5805 break; 5806 } 5807 p++; 5808 } 5809 out: 5810 spin_unlock(&resource_alignment_lock); 5811 return align; 5812 } 5813 5814 static void pci_request_resource_alignment(struct pci_dev *dev, int bar, 5815 resource_size_t align, bool resize) 5816 { 5817 struct resource *r = &dev->resource[bar]; 5818 resource_size_t size; 5819 5820 if (!(r->flags & IORESOURCE_MEM)) 5821 return; 5822 5823 if (r->flags & IORESOURCE_PCI_FIXED) { 5824 pci_info(dev, "BAR%d %pR: ignoring requested alignment %#llx\n", 5825 bar, r, (unsigned long long)align); 5826 return; 5827 } 5828 5829 size = resource_size(r); 5830 if (size >= align) 5831 return; 5832 5833 /* 5834 * Increase the alignment of the resource. There are two ways we 5835 * can do this: 5836 * 5837 * 1) Increase the size of the resource. BARs are aligned on their 5838 * size, so when we reallocate space for this resource, we'll 5839 * allocate it with the larger alignment. This also prevents 5840 * assignment of any other BARs inside the alignment region, so 5841 * if we're requesting page alignment, this means no other BARs 5842 * will share the page. 5843 * 5844 * The disadvantage is that this makes the resource larger than 5845 * the hardware BAR, which may break drivers that compute things 5846 * based on the resource size, e.g., to find registers at a 5847 * fixed offset before the end of the BAR. 5848 * 5849 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and 5850 * set r->start to the desired alignment. By itself this 5851 * doesn't prevent other BARs being put inside the alignment 5852 * region, but if we realign *every* resource of every device in 5853 * the system, none of them will share an alignment region. 5854 * 5855 * When the user has requested alignment for only some devices via 5856 * the "pci=resource_alignment" argument, "resize" is true and we 5857 * use the first method. Otherwise we assume we're aligning all 5858 * devices and we use the second. 5859 */ 5860 5861 pci_info(dev, "BAR%d %pR: requesting alignment to %#llx\n", 5862 bar, r, (unsigned long long)align); 5863 5864 if (resize) { 5865 r->start = 0; 5866 r->end = align - 1; 5867 } else { 5868 r->flags &= ~IORESOURCE_SIZEALIGN; 5869 r->flags |= IORESOURCE_STARTALIGN; 5870 r->start = align; 5871 r->end = r->start + size - 1; 5872 } 5873 r->flags |= IORESOURCE_UNSET; 5874 } 5875 5876 /* 5877 * This function disables memory decoding and releases memory resources 5878 * of the device specified by kernel's boot parameter 'pci=resource_alignment='. 5879 * It also rounds up size to specified alignment. 5880 * Later on, the kernel will assign page-aligned memory resource back 5881 * to the device. 5882 */ 5883 void pci_reassigndev_resource_alignment(struct pci_dev *dev) 5884 { 5885 int i; 5886 struct resource *r; 5887 resource_size_t align; 5888 u16 command; 5889 bool resize = false; 5890 5891 /* 5892 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec 5893 * 3.4.1.11. Their resources are allocated from the space 5894 * described by the VF BARx register in the PF's SR-IOV capability. 5895 * We can't influence their alignment here. 5896 */ 5897 if (dev->is_virtfn) 5898 return; 5899 5900 /* check if specified PCI is target device to reassign */ 5901 align = pci_specified_resource_alignment(dev, &resize); 5902 if (!align) 5903 return; 5904 5905 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL && 5906 (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) { 5907 pci_warn(dev, "Can't reassign resources to host bridge\n"); 5908 return; 5909 } 5910 5911 pci_read_config_word(dev, PCI_COMMAND, &command); 5912 command &= ~PCI_COMMAND_MEMORY; 5913 pci_write_config_word(dev, PCI_COMMAND, command); 5914 5915 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 5916 pci_request_resource_alignment(dev, i, align, resize); 5917 5918 /* 5919 * Need to disable bridge's resource window, 5920 * to enable the kernel to reassign new resource 5921 * window later on. 5922 */ 5923 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE && 5924 (dev->class >> 8) == PCI_CLASS_BRIDGE_PCI) { 5925 for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) { 5926 r = &dev->resource[i]; 5927 if (!(r->flags & IORESOURCE_MEM)) 5928 continue; 5929 r->flags |= IORESOURCE_UNSET; 5930 r->end = resource_size(r) - 1; 5931 r->start = 0; 5932 } 5933 pci_disable_bridge_window(dev); 5934 } 5935 } 5936 5937 static ssize_t pci_set_resource_alignment_param(const char *buf, size_t count) 5938 { 5939 if (count > RESOURCE_ALIGNMENT_PARAM_SIZE - 1) 5940 count = RESOURCE_ALIGNMENT_PARAM_SIZE - 1; 5941 spin_lock(&resource_alignment_lock); 5942 strncpy(resource_alignment_param, buf, count); 5943 resource_alignment_param[count] = '\0'; 5944 spin_unlock(&resource_alignment_lock); 5945 return count; 5946 } 5947 5948 static ssize_t pci_get_resource_alignment_param(char *buf, size_t size) 5949 { 5950 size_t count; 5951 spin_lock(&resource_alignment_lock); 5952 count = snprintf(buf, size, "%s", resource_alignment_param); 5953 spin_unlock(&resource_alignment_lock); 5954 return count; 5955 } 5956 5957 static ssize_t pci_resource_alignment_show(struct bus_type *bus, char *buf) 5958 { 5959 return pci_get_resource_alignment_param(buf, PAGE_SIZE); 5960 } 5961 5962 static ssize_t pci_resource_alignment_store(struct bus_type *bus, 5963 const char *buf, size_t count) 5964 { 5965 return pci_set_resource_alignment_param(buf, count); 5966 } 5967 5968 static BUS_ATTR(resource_alignment, 0644, pci_resource_alignment_show, 5969 pci_resource_alignment_store); 5970 5971 static int __init pci_resource_alignment_sysfs_init(void) 5972 { 5973 return bus_create_file(&pci_bus_type, 5974 &bus_attr_resource_alignment); 5975 } 5976 late_initcall(pci_resource_alignment_sysfs_init); 5977 5978 static void pci_no_domains(void) 5979 { 5980 #ifdef CONFIG_PCI_DOMAINS 5981 pci_domains_supported = 0; 5982 #endif 5983 } 5984 5985 #ifdef CONFIG_PCI_DOMAINS_GENERIC 5986 static atomic_t __domain_nr = ATOMIC_INIT(-1); 5987 5988 static int pci_get_new_domain_nr(void) 5989 { 5990 return atomic_inc_return(&__domain_nr); 5991 } 5992 5993 static int of_pci_bus_find_domain_nr(struct device *parent) 5994 { 5995 static int use_dt_domains = -1; 5996 int domain = -1; 5997 5998 if (parent) 5999 domain = of_get_pci_domain_nr(parent->of_node); 6000 /* 6001 * Check DT domain and use_dt_domains values. 6002 * 6003 * If DT domain property is valid (domain >= 0) and 6004 * use_dt_domains != 0, the DT assignment is valid since this means 6005 * we have not previously allocated a domain number by using 6006 * pci_get_new_domain_nr(); we should also update use_dt_domains to 6007 * 1, to indicate that we have just assigned a domain number from 6008 * DT. 6009 * 6010 * If DT domain property value is not valid (ie domain < 0), and we 6011 * have not previously assigned a domain number from DT 6012 * (use_dt_domains != 1) we should assign a domain number by 6013 * using the: 6014 * 6015 * pci_get_new_domain_nr() 6016 * 6017 * API and update the use_dt_domains value to keep track of method we 6018 * are using to assign domain numbers (use_dt_domains = 0). 6019 * 6020 * All other combinations imply we have a platform that is trying 6021 * to mix domain numbers obtained from DT and pci_get_new_domain_nr(), 6022 * which is a recipe for domain mishandling and it is prevented by 6023 * invalidating the domain value (domain = -1) and printing a 6024 * corresponding error. 6025 */ 6026 if (domain >= 0 && use_dt_domains) { 6027 use_dt_domains = 1; 6028 } else if (domain < 0 && use_dt_domains != 1) { 6029 use_dt_domains = 0; 6030 domain = pci_get_new_domain_nr(); 6031 } else { 6032 if (parent) 6033 pr_err("Node %pOF has ", parent->of_node); 6034 pr_err("Inconsistent \"linux,pci-domain\" property in DT\n"); 6035 domain = -1; 6036 } 6037 6038 return domain; 6039 } 6040 6041 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent) 6042 { 6043 return acpi_disabled ? of_pci_bus_find_domain_nr(parent) : 6044 acpi_pci_bus_find_domain_nr(bus); 6045 } 6046 #endif 6047 6048 /** 6049 * pci_ext_cfg_avail - can we access extended PCI config space? 6050 * 6051 * Returns 1 if we can access PCI extended config space (offsets 6052 * greater than 0xff). This is the default implementation. Architecture 6053 * implementations can override this. 6054 */ 6055 int __weak pci_ext_cfg_avail(void) 6056 { 6057 return 1; 6058 } 6059 6060 void __weak pci_fixup_cardbus(struct pci_bus *bus) 6061 { 6062 } 6063 EXPORT_SYMBOL(pci_fixup_cardbus); 6064 6065 static int __init pci_setup(char *str) 6066 { 6067 while (str) { 6068 char *k = strchr(str, ','); 6069 if (k) 6070 *k++ = 0; 6071 if (*str && (str = pcibios_setup(str)) && *str) { 6072 if (!strcmp(str, "nomsi")) { 6073 pci_no_msi(); 6074 } else if (!strncmp(str, "noats", 5)) { 6075 pr_info("PCIe: ATS is disabled\n"); 6076 pcie_ats_disabled = true; 6077 } else if (!strcmp(str, "noaer")) { 6078 pci_no_aer(); 6079 } else if (!strcmp(str, "earlydump")) { 6080 pci_early_dump = true; 6081 } else if (!strncmp(str, "realloc=", 8)) { 6082 pci_realloc_get_opt(str + 8); 6083 } else if (!strncmp(str, "realloc", 7)) { 6084 pci_realloc_get_opt("on"); 6085 } else if (!strcmp(str, "nodomains")) { 6086 pci_no_domains(); 6087 } else if (!strncmp(str, "noari", 5)) { 6088 pcie_ari_disabled = true; 6089 } else if (!strncmp(str, "cbiosize=", 9)) { 6090 pci_cardbus_io_size = memparse(str + 9, &str); 6091 } else if (!strncmp(str, "cbmemsize=", 10)) { 6092 pci_cardbus_mem_size = memparse(str + 10, &str); 6093 } else if (!strncmp(str, "resource_alignment=", 19)) { 6094 pci_set_resource_alignment_param(str + 19, 6095 strlen(str + 19)); 6096 } else if (!strncmp(str, "ecrc=", 5)) { 6097 pcie_ecrc_get_policy(str + 5); 6098 } else if (!strncmp(str, "hpiosize=", 9)) { 6099 pci_hotplug_io_size = memparse(str + 9, &str); 6100 } else if (!strncmp(str, "hpmemsize=", 10)) { 6101 pci_hotplug_mem_size = memparse(str + 10, &str); 6102 } else if (!strncmp(str, "hpbussize=", 10)) { 6103 pci_hotplug_bus_size = 6104 simple_strtoul(str + 10, &str, 0); 6105 if (pci_hotplug_bus_size > 0xff) 6106 pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 6107 } else if (!strncmp(str, "pcie_bus_tune_off", 17)) { 6108 pcie_bus_config = PCIE_BUS_TUNE_OFF; 6109 } else if (!strncmp(str, "pcie_bus_safe", 13)) { 6110 pcie_bus_config = PCIE_BUS_SAFE; 6111 } else if (!strncmp(str, "pcie_bus_perf", 13)) { 6112 pcie_bus_config = PCIE_BUS_PERFORMANCE; 6113 } else if (!strncmp(str, "pcie_bus_peer2peer", 18)) { 6114 pcie_bus_config = PCIE_BUS_PEER2PEER; 6115 } else if (!strncmp(str, "pcie_scan_all", 13)) { 6116 pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS); 6117 } else if (!strncmp(str, "disable_acs_redir=", 18)) { 6118 disable_acs_redir_param = str + 18; 6119 } else { 6120 printk(KERN_ERR "PCI: Unknown option `%s'\n", 6121 str); 6122 } 6123 } 6124 str = k; 6125 } 6126 return 0; 6127 } 6128 early_param("pci", pci_setup); 6129