1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Bus Services, see include/linux/pci.h for further explanation. 4 * 5 * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter, 6 * David Mosberger-Tang 7 * 8 * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz> 9 */ 10 11 #include <linux/acpi.h> 12 #include <linux/kernel.h> 13 #include <linux/delay.h> 14 #include <linux/dmi.h> 15 #include <linux/init.h> 16 #include <linux/msi.h> 17 #include <linux/of.h> 18 #include <linux/pci.h> 19 #include <linux/pm.h> 20 #include <linux/slab.h> 21 #include <linux/module.h> 22 #include <linux/spinlock.h> 23 #include <linux/string.h> 24 #include <linux/log2.h> 25 #include <linux/logic_pio.h> 26 #include <linux/pm_wakeup.h> 27 #include <linux/interrupt.h> 28 #include <linux/device.h> 29 #include <linux/pm_runtime.h> 30 #include <linux/pci_hotplug.h> 31 #include <linux/vmalloc.h> 32 #include <asm/dma.h> 33 #include <linux/aer.h> 34 #include <linux/bitfield.h> 35 #include "pci.h" 36 37 DEFINE_MUTEX(pci_slot_mutex); 38 39 const char *pci_power_names[] = { 40 "error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown", 41 }; 42 EXPORT_SYMBOL_GPL(pci_power_names); 43 44 #ifdef CONFIG_X86_32 45 int isa_dma_bridge_buggy; 46 EXPORT_SYMBOL(isa_dma_bridge_buggy); 47 #endif 48 49 int pci_pci_problems; 50 EXPORT_SYMBOL(pci_pci_problems); 51 52 unsigned int pci_pm_d3hot_delay; 53 54 static void pci_pme_list_scan(struct work_struct *work); 55 56 static LIST_HEAD(pci_pme_list); 57 static DEFINE_MUTEX(pci_pme_list_mutex); 58 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan); 59 60 struct pci_pme_device { 61 struct list_head list; 62 struct pci_dev *dev; 63 }; 64 65 #define PME_TIMEOUT 1000 /* How long between PME checks */ 66 67 /* 68 * Following exit from Conventional Reset, devices must be ready within 1 sec 69 * (PCIe r6.0 sec 6.6.1). A D3cold to D0 transition implies a Conventional 70 * Reset (PCIe r6.0 sec 5.8). 71 */ 72 #define PCI_RESET_WAIT 1000 /* msec */ 73 74 /* 75 * Devices may extend the 1 sec period through Request Retry Status 76 * completions (PCIe r6.0 sec 2.3.1). The spec does not provide an upper 77 * limit, but 60 sec ought to be enough for any device to become 78 * responsive. 79 */ 80 #define PCIE_RESET_READY_POLL_MS 60000 /* msec */ 81 82 static void pci_dev_d3_sleep(struct pci_dev *dev) 83 { 84 unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay); 85 unsigned int upper; 86 87 if (delay_ms) { 88 /* Use a 20% upper bound, 1ms minimum */ 89 upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U); 90 usleep_range(delay_ms * USEC_PER_MSEC, 91 (delay_ms + upper) * USEC_PER_MSEC); 92 } 93 } 94 95 bool pci_reset_supported(struct pci_dev *dev) 96 { 97 return dev->reset_methods[0] != 0; 98 } 99 100 #ifdef CONFIG_PCI_DOMAINS 101 int pci_domains_supported = 1; 102 #endif 103 104 #define DEFAULT_CARDBUS_IO_SIZE (256) 105 #define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024) 106 /* pci=cbmemsize=nnM,cbiosize=nn can override this */ 107 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE; 108 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE; 109 110 #define DEFAULT_HOTPLUG_IO_SIZE (256) 111 #define DEFAULT_HOTPLUG_MMIO_SIZE (2*1024*1024) 112 #define DEFAULT_HOTPLUG_MMIO_PREF_SIZE (2*1024*1024) 113 /* hpiosize=nn can override this */ 114 unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE; 115 /* 116 * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size, 117 * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size; 118 * pci=hpmemsize=nnM overrides both 119 */ 120 unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE; 121 unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE; 122 123 #define DEFAULT_HOTPLUG_BUS_SIZE 1 124 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 125 126 127 /* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */ 128 #ifdef CONFIG_PCIE_BUS_TUNE_OFF 129 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF; 130 #elif defined CONFIG_PCIE_BUS_SAFE 131 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE; 132 #elif defined CONFIG_PCIE_BUS_PERFORMANCE 133 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE; 134 #elif defined CONFIG_PCIE_BUS_PEER2PEER 135 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER; 136 #else 137 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT; 138 #endif 139 140 /* 141 * The default CLS is used if arch didn't set CLS explicitly and not 142 * all pci devices agree on the same value. Arch can override either 143 * the dfl or actual value as it sees fit. Don't forget this is 144 * measured in 32-bit words, not bytes. 145 */ 146 u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2; 147 u8 pci_cache_line_size; 148 149 /* 150 * If we set up a device for bus mastering, we need to check the latency 151 * timer as certain BIOSes forget to set it properly. 152 */ 153 unsigned int pcibios_max_latency = 255; 154 155 /* If set, the PCIe ARI capability will not be used. */ 156 static bool pcie_ari_disabled; 157 158 /* If set, the PCIe ATS capability will not be used. */ 159 static bool pcie_ats_disabled; 160 161 /* If set, the PCI config space of each device is printed during boot. */ 162 bool pci_early_dump; 163 164 bool pci_ats_disabled(void) 165 { 166 return pcie_ats_disabled; 167 } 168 EXPORT_SYMBOL_GPL(pci_ats_disabled); 169 170 /* Disable bridge_d3 for all PCIe ports */ 171 static bool pci_bridge_d3_disable; 172 /* Force bridge_d3 for all PCIe ports */ 173 static bool pci_bridge_d3_force; 174 175 static int __init pcie_port_pm_setup(char *str) 176 { 177 if (!strcmp(str, "off")) 178 pci_bridge_d3_disable = true; 179 else if (!strcmp(str, "force")) 180 pci_bridge_d3_force = true; 181 return 1; 182 } 183 __setup("pcie_port_pm=", pcie_port_pm_setup); 184 185 /** 186 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children 187 * @bus: pointer to PCI bus structure to search 188 * 189 * Given a PCI bus, returns the highest PCI bus number present in the set 190 * including the given PCI bus and its list of child PCI buses. 191 */ 192 unsigned char pci_bus_max_busnr(struct pci_bus *bus) 193 { 194 struct pci_bus *tmp; 195 unsigned char max, n; 196 197 max = bus->busn_res.end; 198 list_for_each_entry(tmp, &bus->children, node) { 199 n = pci_bus_max_busnr(tmp); 200 if (n > max) 201 max = n; 202 } 203 return max; 204 } 205 EXPORT_SYMBOL_GPL(pci_bus_max_busnr); 206 207 /** 208 * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS 209 * @pdev: the PCI device 210 * 211 * Returns error bits set in PCI_STATUS and clears them. 212 */ 213 int pci_status_get_and_clear_errors(struct pci_dev *pdev) 214 { 215 u16 status; 216 int ret; 217 218 ret = pci_read_config_word(pdev, PCI_STATUS, &status); 219 if (ret != PCIBIOS_SUCCESSFUL) 220 return -EIO; 221 222 status &= PCI_STATUS_ERROR_BITS; 223 if (status) 224 pci_write_config_word(pdev, PCI_STATUS, status); 225 226 return status; 227 } 228 EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors); 229 230 #ifdef CONFIG_HAS_IOMEM 231 static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar, 232 bool write_combine) 233 { 234 struct resource *res = &pdev->resource[bar]; 235 resource_size_t start = res->start; 236 resource_size_t size = resource_size(res); 237 238 /* 239 * Make sure the BAR is actually a memory resource, not an IO resource 240 */ 241 if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) { 242 pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res); 243 return NULL; 244 } 245 246 if (write_combine) 247 return ioremap_wc(start, size); 248 249 return ioremap(start, size); 250 } 251 252 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) 253 { 254 return __pci_ioremap_resource(pdev, bar, false); 255 } 256 EXPORT_SYMBOL_GPL(pci_ioremap_bar); 257 258 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar) 259 { 260 return __pci_ioremap_resource(pdev, bar, true); 261 } 262 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar); 263 #endif 264 265 /** 266 * pci_dev_str_match_path - test if a path string matches a device 267 * @dev: the PCI device to test 268 * @path: string to match the device against 269 * @endptr: pointer to the string after the match 270 * 271 * Test if a string (typically from a kernel parameter) formatted as a 272 * path of device/function addresses matches a PCI device. The string must 273 * be of the form: 274 * 275 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 276 * 277 * A path for a device can be obtained using 'lspci -t'. Using a path 278 * is more robust against bus renumbering than using only a single bus, 279 * device and function address. 280 * 281 * Returns 1 if the string matches the device, 0 if it does not and 282 * a negative error code if it fails to parse the string. 283 */ 284 static int pci_dev_str_match_path(struct pci_dev *dev, const char *path, 285 const char **endptr) 286 { 287 int ret; 288 unsigned int seg, bus, slot, func; 289 char *wpath, *p; 290 char end; 291 292 *endptr = strchrnul(path, ';'); 293 294 wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC); 295 if (!wpath) 296 return -ENOMEM; 297 298 while (1) { 299 p = strrchr(wpath, '/'); 300 if (!p) 301 break; 302 ret = sscanf(p, "/%x.%x%c", &slot, &func, &end); 303 if (ret != 2) { 304 ret = -EINVAL; 305 goto free_and_exit; 306 } 307 308 if (dev->devfn != PCI_DEVFN(slot, func)) { 309 ret = 0; 310 goto free_and_exit; 311 } 312 313 /* 314 * Note: we don't need to get a reference to the upstream 315 * bridge because we hold a reference to the top level 316 * device which should hold a reference to the bridge, 317 * and so on. 318 */ 319 dev = pci_upstream_bridge(dev); 320 if (!dev) { 321 ret = 0; 322 goto free_and_exit; 323 } 324 325 *p = 0; 326 } 327 328 ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot, 329 &func, &end); 330 if (ret != 4) { 331 seg = 0; 332 ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end); 333 if (ret != 3) { 334 ret = -EINVAL; 335 goto free_and_exit; 336 } 337 } 338 339 ret = (seg == pci_domain_nr(dev->bus) && 340 bus == dev->bus->number && 341 dev->devfn == PCI_DEVFN(slot, func)); 342 343 free_and_exit: 344 kfree(wpath); 345 return ret; 346 } 347 348 /** 349 * pci_dev_str_match - test if a string matches a device 350 * @dev: the PCI device to test 351 * @p: string to match the device against 352 * @endptr: pointer to the string after the match 353 * 354 * Test if a string (typically from a kernel parameter) matches a specified 355 * PCI device. The string may be of one of the following formats: 356 * 357 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 358 * pci:<vendor>:<device>[:<subvendor>:<subdevice>] 359 * 360 * The first format specifies a PCI bus/device/function address which 361 * may change if new hardware is inserted, if motherboard firmware changes, 362 * or due to changes caused in kernel parameters. If the domain is 363 * left unspecified, it is taken to be 0. In order to be robust against 364 * bus renumbering issues, a path of PCI device/function numbers may be used 365 * to address the specific device. The path for a device can be determined 366 * through the use of 'lspci -t'. 367 * 368 * The second format matches devices using IDs in the configuration 369 * space which may match multiple devices in the system. A value of 0 370 * for any field will match all devices. (Note: this differs from 371 * in-kernel code that uses PCI_ANY_ID which is ~0; this is for 372 * legacy reasons and convenience so users don't have to specify 373 * FFFFFFFFs on the command line.) 374 * 375 * Returns 1 if the string matches the device, 0 if it does not and 376 * a negative error code if the string cannot be parsed. 377 */ 378 static int pci_dev_str_match(struct pci_dev *dev, const char *p, 379 const char **endptr) 380 { 381 int ret; 382 int count; 383 unsigned short vendor, device, subsystem_vendor, subsystem_device; 384 385 if (strncmp(p, "pci:", 4) == 0) { 386 /* PCI vendor/device (subvendor/subdevice) IDs are specified */ 387 p += 4; 388 ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device, 389 &subsystem_vendor, &subsystem_device, &count); 390 if (ret != 4) { 391 ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count); 392 if (ret != 2) 393 return -EINVAL; 394 395 subsystem_vendor = 0; 396 subsystem_device = 0; 397 } 398 399 p += count; 400 401 if ((!vendor || vendor == dev->vendor) && 402 (!device || device == dev->device) && 403 (!subsystem_vendor || 404 subsystem_vendor == dev->subsystem_vendor) && 405 (!subsystem_device || 406 subsystem_device == dev->subsystem_device)) 407 goto found; 408 } else { 409 /* 410 * PCI Bus, Device, Function IDs are specified 411 * (optionally, may include a path of devfns following it) 412 */ 413 ret = pci_dev_str_match_path(dev, p, &p); 414 if (ret < 0) 415 return ret; 416 else if (ret) 417 goto found; 418 } 419 420 *endptr = p; 421 return 0; 422 423 found: 424 *endptr = p; 425 return 1; 426 } 427 428 static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn, 429 u8 pos, int cap, int *ttl) 430 { 431 u8 id; 432 u16 ent; 433 434 pci_bus_read_config_byte(bus, devfn, pos, &pos); 435 436 while ((*ttl)--) { 437 if (pos < 0x40) 438 break; 439 pos &= ~3; 440 pci_bus_read_config_word(bus, devfn, pos, &ent); 441 442 id = ent & 0xff; 443 if (id == 0xff) 444 break; 445 if (id == cap) 446 return pos; 447 pos = (ent >> 8); 448 } 449 return 0; 450 } 451 452 static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, 453 u8 pos, int cap) 454 { 455 int ttl = PCI_FIND_CAP_TTL; 456 457 return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl); 458 } 459 460 u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) 461 { 462 return __pci_find_next_cap(dev->bus, dev->devfn, 463 pos + PCI_CAP_LIST_NEXT, cap); 464 } 465 EXPORT_SYMBOL_GPL(pci_find_next_capability); 466 467 static u8 __pci_bus_find_cap_start(struct pci_bus *bus, 468 unsigned int devfn, u8 hdr_type) 469 { 470 u16 status; 471 472 pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status); 473 if (!(status & PCI_STATUS_CAP_LIST)) 474 return 0; 475 476 switch (hdr_type) { 477 case PCI_HEADER_TYPE_NORMAL: 478 case PCI_HEADER_TYPE_BRIDGE: 479 return PCI_CAPABILITY_LIST; 480 case PCI_HEADER_TYPE_CARDBUS: 481 return PCI_CB_CAPABILITY_LIST; 482 } 483 484 return 0; 485 } 486 487 /** 488 * pci_find_capability - query for devices' capabilities 489 * @dev: PCI device to query 490 * @cap: capability code 491 * 492 * Tell if a device supports a given PCI capability. 493 * Returns the address of the requested capability structure within the 494 * device's PCI configuration space or 0 in case the device does not 495 * support it. Possible values for @cap include: 496 * 497 * %PCI_CAP_ID_PM Power Management 498 * %PCI_CAP_ID_AGP Accelerated Graphics Port 499 * %PCI_CAP_ID_VPD Vital Product Data 500 * %PCI_CAP_ID_SLOTID Slot Identification 501 * %PCI_CAP_ID_MSI Message Signalled Interrupts 502 * %PCI_CAP_ID_CHSWP CompactPCI HotSwap 503 * %PCI_CAP_ID_PCIX PCI-X 504 * %PCI_CAP_ID_EXP PCI Express 505 */ 506 u8 pci_find_capability(struct pci_dev *dev, int cap) 507 { 508 u8 pos; 509 510 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 511 if (pos) 512 pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap); 513 514 return pos; 515 } 516 EXPORT_SYMBOL(pci_find_capability); 517 518 /** 519 * pci_bus_find_capability - query for devices' capabilities 520 * @bus: the PCI bus to query 521 * @devfn: PCI device to query 522 * @cap: capability code 523 * 524 * Like pci_find_capability() but works for PCI devices that do not have a 525 * pci_dev structure set up yet. 526 * 527 * Returns the address of the requested capability structure within the 528 * device's PCI configuration space or 0 in case the device does not 529 * support it. 530 */ 531 u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap) 532 { 533 u8 hdr_type, pos; 534 535 pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type); 536 537 pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f); 538 if (pos) 539 pos = __pci_find_next_cap(bus, devfn, pos, cap); 540 541 return pos; 542 } 543 EXPORT_SYMBOL(pci_bus_find_capability); 544 545 /** 546 * pci_find_next_ext_capability - Find an extended capability 547 * @dev: PCI device to query 548 * @start: address at which to start looking (0 to start at beginning of list) 549 * @cap: capability code 550 * 551 * Returns the address of the next matching extended capability structure 552 * within the device's PCI configuration space or 0 if the device does 553 * not support it. Some capabilities can occur several times, e.g., the 554 * vendor-specific capability, and this provides a way to find them all. 555 */ 556 u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) 557 { 558 u32 header; 559 int ttl; 560 u16 pos = PCI_CFG_SPACE_SIZE; 561 562 /* minimum 8 bytes per capability */ 563 ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; 564 565 if (dev->cfg_size <= PCI_CFG_SPACE_SIZE) 566 return 0; 567 568 if (start) 569 pos = start; 570 571 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 572 return 0; 573 574 /* 575 * If we have no capabilities, this is indicated by cap ID, 576 * cap version and next pointer all being 0. 577 */ 578 if (header == 0) 579 return 0; 580 581 while (ttl-- > 0) { 582 if (PCI_EXT_CAP_ID(header) == cap && pos != start) 583 return pos; 584 585 pos = PCI_EXT_CAP_NEXT(header); 586 if (pos < PCI_CFG_SPACE_SIZE) 587 break; 588 589 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 590 break; 591 } 592 593 return 0; 594 } 595 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); 596 597 /** 598 * pci_find_ext_capability - Find an extended capability 599 * @dev: PCI device to query 600 * @cap: capability code 601 * 602 * Returns the address of the requested extended capability structure 603 * within the device's PCI configuration space or 0 if the device does 604 * not support it. Possible values for @cap include: 605 * 606 * %PCI_EXT_CAP_ID_ERR Advanced Error Reporting 607 * %PCI_EXT_CAP_ID_VC Virtual Channel 608 * %PCI_EXT_CAP_ID_DSN Device Serial Number 609 * %PCI_EXT_CAP_ID_PWR Power Budgeting 610 */ 611 u16 pci_find_ext_capability(struct pci_dev *dev, int cap) 612 { 613 return pci_find_next_ext_capability(dev, 0, cap); 614 } 615 EXPORT_SYMBOL_GPL(pci_find_ext_capability); 616 617 /** 618 * pci_get_dsn - Read and return the 8-byte Device Serial Number 619 * @dev: PCI device to query 620 * 621 * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial 622 * Number. 623 * 624 * Returns the DSN, or zero if the capability does not exist. 625 */ 626 u64 pci_get_dsn(struct pci_dev *dev) 627 { 628 u32 dword; 629 u64 dsn; 630 int pos; 631 632 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN); 633 if (!pos) 634 return 0; 635 636 /* 637 * The Device Serial Number is two dwords offset 4 bytes from the 638 * capability position. The specification says that the first dword is 639 * the lower half, and the second dword is the upper half. 640 */ 641 pos += 4; 642 pci_read_config_dword(dev, pos, &dword); 643 dsn = (u64)dword; 644 pci_read_config_dword(dev, pos + 4, &dword); 645 dsn |= ((u64)dword) << 32; 646 647 return dsn; 648 } 649 EXPORT_SYMBOL_GPL(pci_get_dsn); 650 651 static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) 652 { 653 int rc, ttl = PCI_FIND_CAP_TTL; 654 u8 cap, mask; 655 656 if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST) 657 mask = HT_3BIT_CAP_MASK; 658 else 659 mask = HT_5BIT_CAP_MASK; 660 661 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos, 662 PCI_CAP_ID_HT, &ttl); 663 while (pos) { 664 rc = pci_read_config_byte(dev, pos + 3, &cap); 665 if (rc != PCIBIOS_SUCCESSFUL) 666 return 0; 667 668 if ((cap & mask) == ht_cap) 669 return pos; 670 671 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, 672 pos + PCI_CAP_LIST_NEXT, 673 PCI_CAP_ID_HT, &ttl); 674 } 675 676 return 0; 677 } 678 679 /** 680 * pci_find_next_ht_capability - query a device's HyperTransport capabilities 681 * @dev: PCI device to query 682 * @pos: Position from which to continue searching 683 * @ht_cap: HyperTransport capability code 684 * 685 * To be used in conjunction with pci_find_ht_capability() to search for 686 * all capabilities matching @ht_cap. @pos should always be a value returned 687 * from pci_find_ht_capability(). 688 * 689 * NB. To be 100% safe against broken PCI devices, the caller should take 690 * steps to avoid an infinite loop. 691 */ 692 u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap) 693 { 694 return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap); 695 } 696 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability); 697 698 /** 699 * pci_find_ht_capability - query a device's HyperTransport capabilities 700 * @dev: PCI device to query 701 * @ht_cap: HyperTransport capability code 702 * 703 * Tell if a device supports a given HyperTransport capability. 704 * Returns an address within the device's PCI configuration space 705 * or 0 in case the device does not support the request capability. 706 * The address points to the PCI capability, of type PCI_CAP_ID_HT, 707 * which has a HyperTransport capability matching @ht_cap. 708 */ 709 u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap) 710 { 711 u8 pos; 712 713 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 714 if (pos) 715 pos = __pci_find_next_ht_cap(dev, pos, ht_cap); 716 717 return pos; 718 } 719 EXPORT_SYMBOL_GPL(pci_find_ht_capability); 720 721 /** 722 * pci_find_vsec_capability - Find a vendor-specific extended capability 723 * @dev: PCI device to query 724 * @vendor: Vendor ID for which capability is defined 725 * @cap: Vendor-specific capability ID 726 * 727 * If @dev has Vendor ID @vendor, search for a VSEC capability with 728 * VSEC ID @cap. If found, return the capability offset in 729 * config space; otherwise return 0. 730 */ 731 u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap) 732 { 733 u16 vsec = 0; 734 u32 header; 735 736 if (vendor != dev->vendor) 737 return 0; 738 739 while ((vsec = pci_find_next_ext_capability(dev, vsec, 740 PCI_EXT_CAP_ID_VNDR))) { 741 if (pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, 742 &header) == PCIBIOS_SUCCESSFUL && 743 PCI_VNDR_HEADER_ID(header) == cap) 744 return vsec; 745 } 746 747 return 0; 748 } 749 EXPORT_SYMBOL_GPL(pci_find_vsec_capability); 750 751 /** 752 * pci_find_dvsec_capability - Find DVSEC for vendor 753 * @dev: PCI device to query 754 * @vendor: Vendor ID to match for the DVSEC 755 * @dvsec: Designated Vendor-specific capability ID 756 * 757 * If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability 758 * offset in config space; otherwise return 0. 759 */ 760 u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec) 761 { 762 int pos; 763 764 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC); 765 if (!pos) 766 return 0; 767 768 while (pos) { 769 u16 v, id; 770 771 pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v); 772 pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id); 773 if (vendor == v && dvsec == id) 774 return pos; 775 776 pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC); 777 } 778 779 return 0; 780 } 781 EXPORT_SYMBOL_GPL(pci_find_dvsec_capability); 782 783 /** 784 * pci_find_parent_resource - return resource region of parent bus of given 785 * region 786 * @dev: PCI device structure contains resources to be searched 787 * @res: child resource record for which parent is sought 788 * 789 * For given resource region of given device, return the resource region of 790 * parent bus the given region is contained in. 791 */ 792 struct resource *pci_find_parent_resource(const struct pci_dev *dev, 793 struct resource *res) 794 { 795 const struct pci_bus *bus = dev->bus; 796 struct resource *r; 797 798 pci_bus_for_each_resource(bus, r) { 799 if (!r) 800 continue; 801 if (resource_contains(r, res)) { 802 803 /* 804 * If the window is prefetchable but the BAR is 805 * not, the allocator made a mistake. 806 */ 807 if (r->flags & IORESOURCE_PREFETCH && 808 !(res->flags & IORESOURCE_PREFETCH)) 809 return NULL; 810 811 /* 812 * If we're below a transparent bridge, there may 813 * be both a positively-decoded aperture and a 814 * subtractively-decoded region that contain the BAR. 815 * We want the positively-decoded one, so this depends 816 * on pci_bus_for_each_resource() giving us those 817 * first. 818 */ 819 return r; 820 } 821 } 822 return NULL; 823 } 824 EXPORT_SYMBOL(pci_find_parent_resource); 825 826 /** 827 * pci_find_resource - Return matching PCI device resource 828 * @dev: PCI device to query 829 * @res: Resource to look for 830 * 831 * Goes over standard PCI resources (BARs) and checks if the given resource 832 * is partially or fully contained in any of them. In that case the 833 * matching resource is returned, %NULL otherwise. 834 */ 835 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res) 836 { 837 int i; 838 839 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 840 struct resource *r = &dev->resource[i]; 841 842 if (r->start && resource_contains(r, res)) 843 return r; 844 } 845 846 return NULL; 847 } 848 EXPORT_SYMBOL(pci_find_resource); 849 850 /** 851 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos 852 * @dev: the PCI device to operate on 853 * @pos: config space offset of status word 854 * @mask: mask of bit(s) to care about in status word 855 * 856 * Return 1 when mask bit(s) in status word clear, 0 otherwise. 857 */ 858 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask) 859 { 860 int i; 861 862 /* Wait for Transaction Pending bit clean */ 863 for (i = 0; i < 4; i++) { 864 u16 status; 865 if (i) 866 msleep((1 << (i - 1)) * 100); 867 868 pci_read_config_word(dev, pos, &status); 869 if (!(status & mask)) 870 return 1; 871 } 872 873 return 0; 874 } 875 876 static int pci_acs_enable; 877 878 /** 879 * pci_request_acs - ask for ACS to be enabled if supported 880 */ 881 void pci_request_acs(void) 882 { 883 pci_acs_enable = 1; 884 } 885 886 static const char *disable_acs_redir_param; 887 888 /** 889 * pci_disable_acs_redir - disable ACS redirect capabilities 890 * @dev: the PCI device 891 * 892 * For only devices specified in the disable_acs_redir parameter. 893 */ 894 static void pci_disable_acs_redir(struct pci_dev *dev) 895 { 896 int ret = 0; 897 const char *p; 898 int pos; 899 u16 ctrl; 900 901 if (!disable_acs_redir_param) 902 return; 903 904 p = disable_acs_redir_param; 905 while (*p) { 906 ret = pci_dev_str_match(dev, p, &p); 907 if (ret < 0) { 908 pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n", 909 disable_acs_redir_param); 910 911 break; 912 } else if (ret == 1) { 913 /* Found a match */ 914 break; 915 } 916 917 if (*p != ';' && *p != ',') { 918 /* End of param or invalid format */ 919 break; 920 } 921 p++; 922 } 923 924 if (ret != 1) 925 return; 926 927 if (!pci_dev_specific_disable_acs_redir(dev)) 928 return; 929 930 pos = dev->acs_cap; 931 if (!pos) { 932 pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n"); 933 return; 934 } 935 936 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 937 938 /* P2P Request & Completion Redirect */ 939 ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC); 940 941 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 942 943 pci_info(dev, "disabled ACS redirect\n"); 944 } 945 946 /** 947 * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities 948 * @dev: the PCI device 949 */ 950 static void pci_std_enable_acs(struct pci_dev *dev) 951 { 952 int pos; 953 u16 cap; 954 u16 ctrl; 955 956 pos = dev->acs_cap; 957 if (!pos) 958 return; 959 960 pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap); 961 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 962 963 /* Source Validation */ 964 ctrl |= (cap & PCI_ACS_SV); 965 966 /* P2P Request Redirect */ 967 ctrl |= (cap & PCI_ACS_RR); 968 969 /* P2P Completion Redirect */ 970 ctrl |= (cap & PCI_ACS_CR); 971 972 /* Upstream Forwarding */ 973 ctrl |= (cap & PCI_ACS_UF); 974 975 /* Enable Translation Blocking for external devices and noats */ 976 if (pci_ats_disabled() || dev->external_facing || dev->untrusted) 977 ctrl |= (cap & PCI_ACS_TB); 978 979 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 980 } 981 982 /** 983 * pci_enable_acs - enable ACS if hardware support it 984 * @dev: the PCI device 985 */ 986 static void pci_enable_acs(struct pci_dev *dev) 987 { 988 if (!pci_acs_enable) 989 goto disable_acs_redir; 990 991 if (!pci_dev_specific_enable_acs(dev)) 992 goto disable_acs_redir; 993 994 pci_std_enable_acs(dev); 995 996 disable_acs_redir: 997 /* 998 * Note: pci_disable_acs_redir() must be called even if ACS was not 999 * enabled by the kernel because it may have been enabled by 1000 * platform firmware. So if we are told to disable it, we should 1001 * always disable it after setting the kernel's default 1002 * preferences. 1003 */ 1004 pci_disable_acs_redir(dev); 1005 } 1006 1007 /** 1008 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up) 1009 * @dev: PCI device to have its BARs restored 1010 * 1011 * Restore the BAR values for a given device, so as to make it 1012 * accessible by its driver. 1013 */ 1014 static void pci_restore_bars(struct pci_dev *dev) 1015 { 1016 int i; 1017 1018 for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) 1019 pci_update_resource(dev, i); 1020 } 1021 1022 static inline bool platform_pci_power_manageable(struct pci_dev *dev) 1023 { 1024 if (pci_use_mid_pm()) 1025 return true; 1026 1027 return acpi_pci_power_manageable(dev); 1028 } 1029 1030 static inline int platform_pci_set_power_state(struct pci_dev *dev, 1031 pci_power_t t) 1032 { 1033 if (pci_use_mid_pm()) 1034 return mid_pci_set_power_state(dev, t); 1035 1036 return acpi_pci_set_power_state(dev, t); 1037 } 1038 1039 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev) 1040 { 1041 if (pci_use_mid_pm()) 1042 return mid_pci_get_power_state(dev); 1043 1044 return acpi_pci_get_power_state(dev); 1045 } 1046 1047 static inline void platform_pci_refresh_power_state(struct pci_dev *dev) 1048 { 1049 if (!pci_use_mid_pm()) 1050 acpi_pci_refresh_power_state(dev); 1051 } 1052 1053 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev) 1054 { 1055 if (pci_use_mid_pm()) 1056 return PCI_POWER_ERROR; 1057 1058 return acpi_pci_choose_state(dev); 1059 } 1060 1061 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable) 1062 { 1063 if (pci_use_mid_pm()) 1064 return PCI_POWER_ERROR; 1065 1066 return acpi_pci_wakeup(dev, enable); 1067 } 1068 1069 static inline bool platform_pci_need_resume(struct pci_dev *dev) 1070 { 1071 if (pci_use_mid_pm()) 1072 return false; 1073 1074 return acpi_pci_need_resume(dev); 1075 } 1076 1077 static inline bool platform_pci_bridge_d3(struct pci_dev *dev) 1078 { 1079 if (pci_use_mid_pm()) 1080 return false; 1081 1082 return acpi_pci_bridge_d3(dev); 1083 } 1084 1085 /** 1086 * pci_update_current_state - Read power state of given device and cache it 1087 * @dev: PCI device to handle. 1088 * @state: State to cache in case the device doesn't have the PM capability 1089 * 1090 * The power state is read from the PMCSR register, which however is 1091 * inaccessible in D3cold. The platform firmware is therefore queried first 1092 * to detect accessibility of the register. In case the platform firmware 1093 * reports an incorrect state or the device isn't power manageable by the 1094 * platform at all, we try to detect D3cold by testing accessibility of the 1095 * vendor ID in config space. 1096 */ 1097 void pci_update_current_state(struct pci_dev *dev, pci_power_t state) 1098 { 1099 if (platform_pci_get_power_state(dev) == PCI_D3cold) { 1100 dev->current_state = PCI_D3cold; 1101 } else if (dev->pm_cap) { 1102 u16 pmcsr; 1103 1104 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1105 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1106 dev->current_state = PCI_D3cold; 1107 return; 1108 } 1109 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1110 } else { 1111 dev->current_state = state; 1112 } 1113 } 1114 1115 /** 1116 * pci_refresh_power_state - Refresh the given device's power state data 1117 * @dev: Target PCI device. 1118 * 1119 * Ask the platform to refresh the devices power state information and invoke 1120 * pci_update_current_state() to update its current PCI power state. 1121 */ 1122 void pci_refresh_power_state(struct pci_dev *dev) 1123 { 1124 platform_pci_refresh_power_state(dev); 1125 pci_update_current_state(dev, dev->current_state); 1126 } 1127 1128 /** 1129 * pci_platform_power_transition - Use platform to change device power state 1130 * @dev: PCI device to handle. 1131 * @state: State to put the device into. 1132 */ 1133 int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state) 1134 { 1135 int error; 1136 1137 error = platform_pci_set_power_state(dev, state); 1138 if (!error) 1139 pci_update_current_state(dev, state); 1140 else if (!dev->pm_cap) /* Fall back to PCI_D0 */ 1141 dev->current_state = PCI_D0; 1142 1143 return error; 1144 } 1145 EXPORT_SYMBOL_GPL(pci_platform_power_transition); 1146 1147 static int pci_resume_one(struct pci_dev *pci_dev, void *ign) 1148 { 1149 pm_request_resume(&pci_dev->dev); 1150 return 0; 1151 } 1152 1153 /** 1154 * pci_resume_bus - Walk given bus and runtime resume devices on it 1155 * @bus: Top bus of the subtree to walk. 1156 */ 1157 void pci_resume_bus(struct pci_bus *bus) 1158 { 1159 if (bus) 1160 pci_walk_bus(bus, pci_resume_one, NULL); 1161 } 1162 1163 static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) 1164 { 1165 int delay = 1; 1166 bool retrain = false; 1167 struct pci_dev *bridge; 1168 1169 if (pci_is_pcie(dev)) { 1170 bridge = pci_upstream_bridge(dev); 1171 if (bridge) 1172 retrain = true; 1173 } 1174 1175 /* 1176 * After reset, the device should not silently discard config 1177 * requests, but it may still indicate that it needs more time by 1178 * responding to them with CRS completions. The Root Port will 1179 * generally synthesize ~0 (PCI_ERROR_RESPONSE) data to complete 1180 * the read (except when CRS SV is enabled and the read was for the 1181 * Vendor ID; in that case it synthesizes 0x0001 data). 1182 * 1183 * Wait for the device to return a non-CRS completion. Read the 1184 * Command register instead of Vendor ID so we don't have to 1185 * contend with the CRS SV value. 1186 */ 1187 for (;;) { 1188 u32 id; 1189 1190 pci_read_config_dword(dev, PCI_COMMAND, &id); 1191 if (!PCI_POSSIBLE_ERROR(id)) 1192 break; 1193 1194 if (delay > timeout) { 1195 pci_warn(dev, "not ready %dms after %s; giving up\n", 1196 delay - 1, reset_type); 1197 return -ENOTTY; 1198 } 1199 1200 if (delay > PCI_RESET_WAIT) { 1201 if (retrain) { 1202 retrain = false; 1203 if (pcie_failed_link_retrain(bridge)) { 1204 delay = 1; 1205 continue; 1206 } 1207 } 1208 pci_info(dev, "not ready %dms after %s; waiting\n", 1209 delay - 1, reset_type); 1210 } 1211 1212 msleep(delay); 1213 delay *= 2; 1214 } 1215 1216 if (delay > PCI_RESET_WAIT) 1217 pci_info(dev, "ready %dms after %s\n", delay - 1, 1218 reset_type); 1219 1220 return 0; 1221 } 1222 1223 /** 1224 * pci_power_up - Put the given device into D0 1225 * @dev: PCI device to power up 1226 * 1227 * On success, return 0 or 1, depending on whether or not it is necessary to 1228 * restore the device's BARs subsequently (1 is returned in that case). 1229 * 1230 * On failure, return a negative error code. Always return failure if @dev 1231 * lacks a Power Management Capability, even if the platform was able to 1232 * put the device in D0 via non-PCI means. 1233 */ 1234 int pci_power_up(struct pci_dev *dev) 1235 { 1236 bool need_restore; 1237 pci_power_t state; 1238 u16 pmcsr; 1239 1240 platform_pci_set_power_state(dev, PCI_D0); 1241 1242 if (!dev->pm_cap) { 1243 state = platform_pci_get_power_state(dev); 1244 if (state == PCI_UNKNOWN) 1245 dev->current_state = PCI_D0; 1246 else 1247 dev->current_state = state; 1248 1249 return -EIO; 1250 } 1251 1252 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1253 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1254 pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n", 1255 pci_power_name(dev->current_state)); 1256 dev->current_state = PCI_D3cold; 1257 return -EIO; 1258 } 1259 1260 state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1261 1262 need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) && 1263 !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 1264 1265 if (state == PCI_D0) 1266 goto end; 1267 1268 /* 1269 * Force the entire word to 0. This doesn't affect PME_Status, disables 1270 * PME_En, and sets PowerState to 0. 1271 */ 1272 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0); 1273 1274 /* Mandatory transition delays; see PCI PM 1.2. */ 1275 if (state == PCI_D3hot) 1276 pci_dev_d3_sleep(dev); 1277 else if (state == PCI_D2) 1278 udelay(PCI_PM_D2_DELAY); 1279 1280 end: 1281 dev->current_state = PCI_D0; 1282 if (need_restore) 1283 return 1; 1284 1285 return 0; 1286 } 1287 1288 /** 1289 * pci_set_full_power_state - Put a PCI device into D0 and update its state 1290 * @dev: PCI device to power up 1291 * 1292 * Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register 1293 * to confirm the state change, restore its BARs if they might be lost and 1294 * reconfigure ASPM in accordance with the new power state. 1295 * 1296 * If pci_restore_state() is going to be called right after a power state change 1297 * to D0, it is more efficient to use pci_power_up() directly instead of this 1298 * function. 1299 */ 1300 static int pci_set_full_power_state(struct pci_dev *dev) 1301 { 1302 u16 pmcsr; 1303 int ret; 1304 1305 ret = pci_power_up(dev); 1306 if (ret < 0) { 1307 if (dev->current_state == PCI_D0) 1308 return 0; 1309 1310 return ret; 1311 } 1312 1313 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1314 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1315 if (dev->current_state != PCI_D0) { 1316 pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n", 1317 pci_power_name(dev->current_state)); 1318 } else if (ret > 0) { 1319 /* 1320 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT 1321 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning 1322 * from D3hot to D0 _may_ perform an internal reset, thereby 1323 * going to "D0 Uninitialized" rather than "D0 Initialized". 1324 * For example, at least some versions of the 3c905B and the 1325 * 3c556B exhibit this behaviour. 1326 * 1327 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave 1328 * devices in a D3hot state at boot. Consequently, we need to 1329 * restore at least the BARs so that the device will be 1330 * accessible to its driver. 1331 */ 1332 pci_restore_bars(dev); 1333 } 1334 1335 return 0; 1336 } 1337 1338 /** 1339 * __pci_dev_set_current_state - Set current state of a PCI device 1340 * @dev: Device to handle 1341 * @data: pointer to state to be set 1342 */ 1343 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data) 1344 { 1345 pci_power_t state = *(pci_power_t *)data; 1346 1347 dev->current_state = state; 1348 return 0; 1349 } 1350 1351 /** 1352 * pci_bus_set_current_state - Walk given bus and set current state of devices 1353 * @bus: Top bus of the subtree to walk. 1354 * @state: state to be set 1355 */ 1356 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state) 1357 { 1358 if (bus) 1359 pci_walk_bus(bus, __pci_dev_set_current_state, &state); 1360 } 1361 1362 /** 1363 * pci_set_low_power_state - Put a PCI device into a low-power state. 1364 * @dev: PCI device to handle. 1365 * @state: PCI power state (D1, D2, D3hot) to put the device into. 1366 * 1367 * Use the device's PCI_PM_CTRL register to put it into a low-power state. 1368 * 1369 * RETURN VALUE: 1370 * -EINVAL if the requested state is invalid. 1371 * -EIO if device does not support PCI PM or its PM capabilities register has a 1372 * wrong version, or device doesn't support the requested state. 1373 * 0 if device already is in the requested state. 1374 * 0 if device's power state has been successfully changed. 1375 */ 1376 static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state) 1377 { 1378 u16 pmcsr; 1379 1380 if (!dev->pm_cap) 1381 return -EIO; 1382 1383 /* 1384 * Validate transition: We can enter D0 from any state, but if 1385 * we're already in a low-power state, we can only go deeper. E.g., 1386 * we can go from D1 to D3, but we can't go directly from D3 to D1; 1387 * we'd have to go from D3 to D0, then to D1. 1388 */ 1389 if (dev->current_state <= PCI_D3cold && dev->current_state > state) { 1390 pci_dbg(dev, "Invalid power transition (from %s to %s)\n", 1391 pci_power_name(dev->current_state), 1392 pci_power_name(state)); 1393 return -EINVAL; 1394 } 1395 1396 /* Check if this device supports the desired state */ 1397 if ((state == PCI_D1 && !dev->d1_support) 1398 || (state == PCI_D2 && !dev->d2_support)) 1399 return -EIO; 1400 1401 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1402 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1403 pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n", 1404 pci_power_name(dev->current_state), 1405 pci_power_name(state)); 1406 dev->current_state = PCI_D3cold; 1407 return -EIO; 1408 } 1409 1410 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 1411 pmcsr |= state; 1412 1413 /* Enter specified state */ 1414 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 1415 1416 /* Mandatory power management transition delays; see PCI PM 1.2. */ 1417 if (state == PCI_D3hot) 1418 pci_dev_d3_sleep(dev); 1419 else if (state == PCI_D2) 1420 udelay(PCI_PM_D2_DELAY); 1421 1422 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1423 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1424 if (dev->current_state != state) 1425 pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n", 1426 pci_power_name(dev->current_state), 1427 pci_power_name(state)); 1428 1429 return 0; 1430 } 1431 1432 /** 1433 * pci_set_power_state - Set the power state of a PCI device 1434 * @dev: PCI device to handle. 1435 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. 1436 * 1437 * Transition a device to a new power state, using the platform firmware and/or 1438 * the device's PCI PM registers. 1439 * 1440 * RETURN VALUE: 1441 * -EINVAL if the requested state is invalid. 1442 * -EIO if device does not support PCI PM or its PM capabilities register has a 1443 * wrong version, or device doesn't support the requested state. 1444 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported. 1445 * 0 if device already is in the requested state. 1446 * 0 if the transition is to D3 but D3 is not supported. 1447 * 0 if device's power state has been successfully changed. 1448 */ 1449 int pci_set_power_state(struct pci_dev *dev, pci_power_t state) 1450 { 1451 int error; 1452 1453 /* Bound the state we're entering */ 1454 if (state > PCI_D3cold) 1455 state = PCI_D3cold; 1456 else if (state < PCI_D0) 1457 state = PCI_D0; 1458 else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev)) 1459 1460 /* 1461 * If the device or the parent bridge do not support PCI 1462 * PM, ignore the request if we're doing anything other 1463 * than putting it into D0 (which would only happen on 1464 * boot). 1465 */ 1466 return 0; 1467 1468 /* Check if we're already there */ 1469 if (dev->current_state == state) 1470 return 0; 1471 1472 if (state == PCI_D0) 1473 return pci_set_full_power_state(dev); 1474 1475 /* 1476 * This device is quirked not to be put into D3, so don't put it in 1477 * D3 1478 */ 1479 if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3)) 1480 return 0; 1481 1482 if (state == PCI_D3cold) { 1483 /* 1484 * To put the device in D3cold, put it into D3hot in the native 1485 * way, then put it into D3cold using platform ops. 1486 */ 1487 error = pci_set_low_power_state(dev, PCI_D3hot); 1488 1489 if (pci_platform_power_transition(dev, PCI_D3cold)) 1490 return error; 1491 1492 /* Powering off a bridge may power off the whole hierarchy */ 1493 if (dev->current_state == PCI_D3cold) 1494 pci_bus_set_current_state(dev->subordinate, PCI_D3cold); 1495 } else { 1496 error = pci_set_low_power_state(dev, state); 1497 1498 if (pci_platform_power_transition(dev, state)) 1499 return error; 1500 } 1501 1502 return 0; 1503 } 1504 EXPORT_SYMBOL(pci_set_power_state); 1505 1506 #define PCI_EXP_SAVE_REGS 7 1507 1508 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev, 1509 u16 cap, bool extended) 1510 { 1511 struct pci_cap_saved_state *tmp; 1512 1513 hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) { 1514 if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap) 1515 return tmp; 1516 } 1517 return NULL; 1518 } 1519 1520 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap) 1521 { 1522 return _pci_find_saved_cap(dev, cap, false); 1523 } 1524 1525 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap) 1526 { 1527 return _pci_find_saved_cap(dev, cap, true); 1528 } 1529 1530 static int pci_save_pcie_state(struct pci_dev *dev) 1531 { 1532 int i = 0; 1533 struct pci_cap_saved_state *save_state; 1534 u16 *cap; 1535 1536 if (!pci_is_pcie(dev)) 1537 return 0; 1538 1539 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1540 if (!save_state) { 1541 pci_err(dev, "buffer not found in %s\n", __func__); 1542 return -ENOMEM; 1543 } 1544 1545 cap = (u16 *)&save_state->cap.data[0]; 1546 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]); 1547 pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]); 1548 pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]); 1549 pcie_capability_read_word(dev, PCI_EXP_RTCTL, &cap[i++]); 1550 pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]); 1551 pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]); 1552 pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]); 1553 1554 return 0; 1555 } 1556 1557 void pci_bridge_reconfigure_ltr(struct pci_dev *dev) 1558 { 1559 #ifdef CONFIG_PCIEASPM 1560 struct pci_dev *bridge; 1561 u32 ctl; 1562 1563 bridge = pci_upstream_bridge(dev); 1564 if (bridge && bridge->ltr_path) { 1565 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, &ctl); 1566 if (!(ctl & PCI_EXP_DEVCTL2_LTR_EN)) { 1567 pci_dbg(bridge, "re-enabling LTR\n"); 1568 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, 1569 PCI_EXP_DEVCTL2_LTR_EN); 1570 } 1571 } 1572 #endif 1573 } 1574 1575 static void pci_restore_pcie_state(struct pci_dev *dev) 1576 { 1577 int i = 0; 1578 struct pci_cap_saved_state *save_state; 1579 u16 *cap; 1580 1581 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1582 if (!save_state) 1583 return; 1584 1585 /* 1586 * Downstream ports reset the LTR enable bit when link goes down. 1587 * Check and re-configure the bit here before restoring device. 1588 * PCIe r5.0, sec 7.5.3.16. 1589 */ 1590 pci_bridge_reconfigure_ltr(dev); 1591 1592 cap = (u16 *)&save_state->cap.data[0]; 1593 pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]); 1594 pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]); 1595 pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]); 1596 pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]); 1597 pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]); 1598 pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]); 1599 pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]); 1600 } 1601 1602 static int pci_save_pcix_state(struct pci_dev *dev) 1603 { 1604 int pos; 1605 struct pci_cap_saved_state *save_state; 1606 1607 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1608 if (!pos) 1609 return 0; 1610 1611 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1612 if (!save_state) { 1613 pci_err(dev, "buffer not found in %s\n", __func__); 1614 return -ENOMEM; 1615 } 1616 1617 pci_read_config_word(dev, pos + PCI_X_CMD, 1618 (u16 *)save_state->cap.data); 1619 1620 return 0; 1621 } 1622 1623 static void pci_restore_pcix_state(struct pci_dev *dev) 1624 { 1625 int i = 0, pos; 1626 struct pci_cap_saved_state *save_state; 1627 u16 *cap; 1628 1629 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1630 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1631 if (!save_state || !pos) 1632 return; 1633 cap = (u16 *)&save_state->cap.data[0]; 1634 1635 pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]); 1636 } 1637 1638 static void pci_save_ltr_state(struct pci_dev *dev) 1639 { 1640 int ltr; 1641 struct pci_cap_saved_state *save_state; 1642 u32 *cap; 1643 1644 if (!pci_is_pcie(dev)) 1645 return; 1646 1647 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR); 1648 if (!ltr) 1649 return; 1650 1651 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR); 1652 if (!save_state) { 1653 pci_err(dev, "no suspend buffer for LTR; ASPM issues possible after resume\n"); 1654 return; 1655 } 1656 1657 /* Some broken devices only support dword access to LTR */ 1658 cap = &save_state->cap.data[0]; 1659 pci_read_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap); 1660 } 1661 1662 static void pci_restore_ltr_state(struct pci_dev *dev) 1663 { 1664 struct pci_cap_saved_state *save_state; 1665 int ltr; 1666 u32 *cap; 1667 1668 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR); 1669 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR); 1670 if (!save_state || !ltr) 1671 return; 1672 1673 /* Some broken devices only support dword access to LTR */ 1674 cap = &save_state->cap.data[0]; 1675 pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap); 1676 } 1677 1678 /** 1679 * pci_save_state - save the PCI configuration space of a device before 1680 * suspending 1681 * @dev: PCI device that we're dealing with 1682 */ 1683 int pci_save_state(struct pci_dev *dev) 1684 { 1685 int i; 1686 /* XXX: 100% dword access ok here? */ 1687 for (i = 0; i < 16; i++) { 1688 pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]); 1689 pci_dbg(dev, "save config %#04x: %#010x\n", 1690 i * 4, dev->saved_config_space[i]); 1691 } 1692 dev->state_saved = true; 1693 1694 i = pci_save_pcie_state(dev); 1695 if (i != 0) 1696 return i; 1697 1698 i = pci_save_pcix_state(dev); 1699 if (i != 0) 1700 return i; 1701 1702 pci_save_ltr_state(dev); 1703 pci_save_dpc_state(dev); 1704 pci_save_aer_state(dev); 1705 pci_save_ptm_state(dev); 1706 return pci_save_vc_state(dev); 1707 } 1708 EXPORT_SYMBOL(pci_save_state); 1709 1710 static void pci_restore_config_dword(struct pci_dev *pdev, int offset, 1711 u32 saved_val, int retry, bool force) 1712 { 1713 u32 val; 1714 1715 pci_read_config_dword(pdev, offset, &val); 1716 if (!force && val == saved_val) 1717 return; 1718 1719 for (;;) { 1720 pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n", 1721 offset, val, saved_val); 1722 pci_write_config_dword(pdev, offset, saved_val); 1723 if (retry-- <= 0) 1724 return; 1725 1726 pci_read_config_dword(pdev, offset, &val); 1727 if (val == saved_val) 1728 return; 1729 1730 mdelay(1); 1731 } 1732 } 1733 1734 static void pci_restore_config_space_range(struct pci_dev *pdev, 1735 int start, int end, int retry, 1736 bool force) 1737 { 1738 int index; 1739 1740 for (index = end; index >= start; index--) 1741 pci_restore_config_dword(pdev, 4 * index, 1742 pdev->saved_config_space[index], 1743 retry, force); 1744 } 1745 1746 static void pci_restore_config_space(struct pci_dev *pdev) 1747 { 1748 if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) { 1749 pci_restore_config_space_range(pdev, 10, 15, 0, false); 1750 /* Restore BARs before the command register. */ 1751 pci_restore_config_space_range(pdev, 4, 9, 10, false); 1752 pci_restore_config_space_range(pdev, 0, 3, 0, false); 1753 } else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 1754 pci_restore_config_space_range(pdev, 12, 15, 0, false); 1755 1756 /* 1757 * Force rewriting of prefetch registers to avoid S3 resume 1758 * issues on Intel PCI bridges that occur when these 1759 * registers are not explicitly written. 1760 */ 1761 pci_restore_config_space_range(pdev, 9, 11, 0, true); 1762 pci_restore_config_space_range(pdev, 0, 8, 0, false); 1763 } else { 1764 pci_restore_config_space_range(pdev, 0, 15, 0, false); 1765 } 1766 } 1767 1768 static void pci_restore_rebar_state(struct pci_dev *pdev) 1769 { 1770 unsigned int pos, nbars, i; 1771 u32 ctrl; 1772 1773 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 1774 if (!pos) 1775 return; 1776 1777 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1778 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 1779 PCI_REBAR_CTRL_NBAR_SHIFT; 1780 1781 for (i = 0; i < nbars; i++, pos += 8) { 1782 struct resource *res; 1783 int bar_idx, size; 1784 1785 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1786 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 1787 res = pdev->resource + bar_idx; 1788 size = pci_rebar_bytes_to_size(resource_size(res)); 1789 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 1790 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 1791 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 1792 } 1793 } 1794 1795 /** 1796 * pci_restore_state - Restore the saved state of a PCI device 1797 * @dev: PCI device that we're dealing with 1798 */ 1799 void pci_restore_state(struct pci_dev *dev) 1800 { 1801 if (!dev->state_saved) 1802 return; 1803 1804 /* 1805 * Restore max latencies (in the LTR capability) before enabling 1806 * LTR itself (in the PCIe capability). 1807 */ 1808 pci_restore_ltr_state(dev); 1809 1810 pci_restore_pcie_state(dev); 1811 pci_restore_pasid_state(dev); 1812 pci_restore_pri_state(dev); 1813 pci_restore_ats_state(dev); 1814 pci_restore_vc_state(dev); 1815 pci_restore_rebar_state(dev); 1816 pci_restore_dpc_state(dev); 1817 pci_restore_ptm_state(dev); 1818 1819 pci_aer_clear_status(dev); 1820 pci_restore_aer_state(dev); 1821 1822 pci_restore_config_space(dev); 1823 1824 pci_restore_pcix_state(dev); 1825 pci_restore_msi_state(dev); 1826 1827 /* Restore ACS and IOV configuration state */ 1828 pci_enable_acs(dev); 1829 pci_restore_iov_state(dev); 1830 1831 dev->state_saved = false; 1832 } 1833 EXPORT_SYMBOL(pci_restore_state); 1834 1835 struct pci_saved_state { 1836 u32 config_space[16]; 1837 struct pci_cap_saved_data cap[]; 1838 }; 1839 1840 /** 1841 * pci_store_saved_state - Allocate and return an opaque struct containing 1842 * the device saved state. 1843 * @dev: PCI device that we're dealing with 1844 * 1845 * Return NULL if no state or error. 1846 */ 1847 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev) 1848 { 1849 struct pci_saved_state *state; 1850 struct pci_cap_saved_state *tmp; 1851 struct pci_cap_saved_data *cap; 1852 size_t size; 1853 1854 if (!dev->state_saved) 1855 return NULL; 1856 1857 size = sizeof(*state) + sizeof(struct pci_cap_saved_data); 1858 1859 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) 1860 size += sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1861 1862 state = kzalloc(size, GFP_KERNEL); 1863 if (!state) 1864 return NULL; 1865 1866 memcpy(state->config_space, dev->saved_config_space, 1867 sizeof(state->config_space)); 1868 1869 cap = state->cap; 1870 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) { 1871 size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1872 memcpy(cap, &tmp->cap, len); 1873 cap = (struct pci_cap_saved_data *)((u8 *)cap + len); 1874 } 1875 /* Empty cap_save terminates list */ 1876 1877 return state; 1878 } 1879 EXPORT_SYMBOL_GPL(pci_store_saved_state); 1880 1881 /** 1882 * pci_load_saved_state - Reload the provided save state into struct pci_dev. 1883 * @dev: PCI device that we're dealing with 1884 * @state: Saved state returned from pci_store_saved_state() 1885 */ 1886 int pci_load_saved_state(struct pci_dev *dev, 1887 struct pci_saved_state *state) 1888 { 1889 struct pci_cap_saved_data *cap; 1890 1891 dev->state_saved = false; 1892 1893 if (!state) 1894 return 0; 1895 1896 memcpy(dev->saved_config_space, state->config_space, 1897 sizeof(state->config_space)); 1898 1899 cap = state->cap; 1900 while (cap->size) { 1901 struct pci_cap_saved_state *tmp; 1902 1903 tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended); 1904 if (!tmp || tmp->cap.size != cap->size) 1905 return -EINVAL; 1906 1907 memcpy(tmp->cap.data, cap->data, tmp->cap.size); 1908 cap = (struct pci_cap_saved_data *)((u8 *)cap + 1909 sizeof(struct pci_cap_saved_data) + cap->size); 1910 } 1911 1912 dev->state_saved = true; 1913 return 0; 1914 } 1915 EXPORT_SYMBOL_GPL(pci_load_saved_state); 1916 1917 /** 1918 * pci_load_and_free_saved_state - Reload the save state pointed to by state, 1919 * and free the memory allocated for it. 1920 * @dev: PCI device that we're dealing with 1921 * @state: Pointer to saved state returned from pci_store_saved_state() 1922 */ 1923 int pci_load_and_free_saved_state(struct pci_dev *dev, 1924 struct pci_saved_state **state) 1925 { 1926 int ret = pci_load_saved_state(dev, *state); 1927 kfree(*state); 1928 *state = NULL; 1929 return ret; 1930 } 1931 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state); 1932 1933 int __weak pcibios_enable_device(struct pci_dev *dev, int bars) 1934 { 1935 return pci_enable_resources(dev, bars); 1936 } 1937 1938 static int do_pci_enable_device(struct pci_dev *dev, int bars) 1939 { 1940 int err; 1941 struct pci_dev *bridge; 1942 u16 cmd; 1943 u8 pin; 1944 1945 err = pci_set_power_state(dev, PCI_D0); 1946 if (err < 0 && err != -EIO) 1947 return err; 1948 1949 bridge = pci_upstream_bridge(dev); 1950 if (bridge) 1951 pcie_aspm_powersave_config_link(bridge); 1952 1953 err = pcibios_enable_device(dev, bars); 1954 if (err < 0) 1955 return err; 1956 pci_fixup_device(pci_fixup_enable, dev); 1957 1958 if (dev->msi_enabled || dev->msix_enabled) 1959 return 0; 1960 1961 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin); 1962 if (pin) { 1963 pci_read_config_word(dev, PCI_COMMAND, &cmd); 1964 if (cmd & PCI_COMMAND_INTX_DISABLE) 1965 pci_write_config_word(dev, PCI_COMMAND, 1966 cmd & ~PCI_COMMAND_INTX_DISABLE); 1967 } 1968 1969 return 0; 1970 } 1971 1972 /** 1973 * pci_reenable_device - Resume abandoned device 1974 * @dev: PCI device to be resumed 1975 * 1976 * NOTE: This function is a backend of pci_default_resume() and is not supposed 1977 * to be called by normal code, write proper resume handler and use it instead. 1978 */ 1979 int pci_reenable_device(struct pci_dev *dev) 1980 { 1981 if (pci_is_enabled(dev)) 1982 return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1); 1983 return 0; 1984 } 1985 EXPORT_SYMBOL(pci_reenable_device); 1986 1987 static void pci_enable_bridge(struct pci_dev *dev) 1988 { 1989 struct pci_dev *bridge; 1990 int retval; 1991 1992 bridge = pci_upstream_bridge(dev); 1993 if (bridge) 1994 pci_enable_bridge(bridge); 1995 1996 if (pci_is_enabled(dev)) { 1997 if (!dev->is_busmaster) 1998 pci_set_master(dev); 1999 return; 2000 } 2001 2002 retval = pci_enable_device(dev); 2003 if (retval) 2004 pci_err(dev, "Error enabling bridge (%d), continuing\n", 2005 retval); 2006 pci_set_master(dev); 2007 } 2008 2009 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) 2010 { 2011 struct pci_dev *bridge; 2012 int err; 2013 int i, bars = 0; 2014 2015 /* 2016 * Power state could be unknown at this point, either due to a fresh 2017 * boot or a device removal call. So get the current power state 2018 * so that things like MSI message writing will behave as expected 2019 * (e.g. if the device really is in D0 at enable time). 2020 */ 2021 pci_update_current_state(dev, dev->current_state); 2022 2023 if (atomic_inc_return(&dev->enable_cnt) > 1) 2024 return 0; /* already enabled */ 2025 2026 bridge = pci_upstream_bridge(dev); 2027 if (bridge) 2028 pci_enable_bridge(bridge); 2029 2030 /* only skip sriov related */ 2031 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 2032 if (dev->resource[i].flags & flags) 2033 bars |= (1 << i); 2034 for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++) 2035 if (dev->resource[i].flags & flags) 2036 bars |= (1 << i); 2037 2038 err = do_pci_enable_device(dev, bars); 2039 if (err < 0) 2040 atomic_dec(&dev->enable_cnt); 2041 return err; 2042 } 2043 2044 /** 2045 * pci_enable_device_io - Initialize a device for use with IO space 2046 * @dev: PCI device to be initialized 2047 * 2048 * Initialize device before it's used by a driver. Ask low-level code 2049 * to enable I/O resources. Wake up the device if it was suspended. 2050 * Beware, this function can fail. 2051 */ 2052 int pci_enable_device_io(struct pci_dev *dev) 2053 { 2054 return pci_enable_device_flags(dev, IORESOURCE_IO); 2055 } 2056 EXPORT_SYMBOL(pci_enable_device_io); 2057 2058 /** 2059 * pci_enable_device_mem - Initialize a device for use with Memory space 2060 * @dev: PCI device to be initialized 2061 * 2062 * Initialize device before it's used by a driver. Ask low-level code 2063 * to enable Memory resources. Wake up the device if it was suspended. 2064 * Beware, this function can fail. 2065 */ 2066 int pci_enable_device_mem(struct pci_dev *dev) 2067 { 2068 return pci_enable_device_flags(dev, IORESOURCE_MEM); 2069 } 2070 EXPORT_SYMBOL(pci_enable_device_mem); 2071 2072 /** 2073 * pci_enable_device - Initialize device before it's used by a driver. 2074 * @dev: PCI device to be initialized 2075 * 2076 * Initialize device before it's used by a driver. Ask low-level code 2077 * to enable I/O and memory. Wake up the device if it was suspended. 2078 * Beware, this function can fail. 2079 * 2080 * Note we don't actually enable the device many times if we call 2081 * this function repeatedly (we just increment the count). 2082 */ 2083 int pci_enable_device(struct pci_dev *dev) 2084 { 2085 return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO); 2086 } 2087 EXPORT_SYMBOL(pci_enable_device); 2088 2089 /* 2090 * Managed PCI resources. This manages device on/off, INTx/MSI/MSI-X 2091 * on/off and BAR regions. pci_dev itself records MSI/MSI-X status, so 2092 * there's no need to track it separately. pci_devres is initialized 2093 * when a device is enabled using managed PCI device enable interface. 2094 */ 2095 struct pci_devres { 2096 unsigned int enabled:1; 2097 unsigned int pinned:1; 2098 unsigned int orig_intx:1; 2099 unsigned int restore_intx:1; 2100 unsigned int mwi:1; 2101 u32 region_mask; 2102 }; 2103 2104 static void pcim_release(struct device *gendev, void *res) 2105 { 2106 struct pci_dev *dev = to_pci_dev(gendev); 2107 struct pci_devres *this = res; 2108 int i; 2109 2110 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) 2111 if (this->region_mask & (1 << i)) 2112 pci_release_region(dev, i); 2113 2114 if (this->mwi) 2115 pci_clear_mwi(dev); 2116 2117 if (this->restore_intx) 2118 pci_intx(dev, this->orig_intx); 2119 2120 if (this->enabled && !this->pinned) 2121 pci_disable_device(dev); 2122 } 2123 2124 static struct pci_devres *get_pci_dr(struct pci_dev *pdev) 2125 { 2126 struct pci_devres *dr, *new_dr; 2127 2128 dr = devres_find(&pdev->dev, pcim_release, NULL, NULL); 2129 if (dr) 2130 return dr; 2131 2132 new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL); 2133 if (!new_dr) 2134 return NULL; 2135 return devres_get(&pdev->dev, new_dr, NULL, NULL); 2136 } 2137 2138 static struct pci_devres *find_pci_dr(struct pci_dev *pdev) 2139 { 2140 if (pci_is_managed(pdev)) 2141 return devres_find(&pdev->dev, pcim_release, NULL, NULL); 2142 return NULL; 2143 } 2144 2145 /** 2146 * pcim_enable_device - Managed pci_enable_device() 2147 * @pdev: PCI device to be initialized 2148 * 2149 * Managed pci_enable_device(). 2150 */ 2151 int pcim_enable_device(struct pci_dev *pdev) 2152 { 2153 struct pci_devres *dr; 2154 int rc; 2155 2156 dr = get_pci_dr(pdev); 2157 if (unlikely(!dr)) 2158 return -ENOMEM; 2159 if (dr->enabled) 2160 return 0; 2161 2162 rc = pci_enable_device(pdev); 2163 if (!rc) { 2164 pdev->is_managed = 1; 2165 dr->enabled = 1; 2166 } 2167 return rc; 2168 } 2169 EXPORT_SYMBOL(pcim_enable_device); 2170 2171 /** 2172 * pcim_pin_device - Pin managed PCI device 2173 * @pdev: PCI device to pin 2174 * 2175 * Pin managed PCI device @pdev. Pinned device won't be disabled on 2176 * driver detach. @pdev must have been enabled with 2177 * pcim_enable_device(). 2178 */ 2179 void pcim_pin_device(struct pci_dev *pdev) 2180 { 2181 struct pci_devres *dr; 2182 2183 dr = find_pci_dr(pdev); 2184 WARN_ON(!dr || !dr->enabled); 2185 if (dr) 2186 dr->pinned = 1; 2187 } 2188 EXPORT_SYMBOL(pcim_pin_device); 2189 2190 /* 2191 * pcibios_device_add - provide arch specific hooks when adding device dev 2192 * @dev: the PCI device being added 2193 * 2194 * Permits the platform to provide architecture specific functionality when 2195 * devices are added. This is the default implementation. Architecture 2196 * implementations can override this. 2197 */ 2198 int __weak pcibios_device_add(struct pci_dev *dev) 2199 { 2200 return 0; 2201 } 2202 2203 /** 2204 * pcibios_release_device - provide arch specific hooks when releasing 2205 * device dev 2206 * @dev: the PCI device being released 2207 * 2208 * Permits the platform to provide architecture specific functionality when 2209 * devices are released. This is the default implementation. Architecture 2210 * implementations can override this. 2211 */ 2212 void __weak pcibios_release_device(struct pci_dev *dev) {} 2213 2214 /** 2215 * pcibios_disable_device - disable arch specific PCI resources for device dev 2216 * @dev: the PCI device to disable 2217 * 2218 * Disables architecture specific PCI resources for the device. This 2219 * is the default implementation. Architecture implementations can 2220 * override this. 2221 */ 2222 void __weak pcibios_disable_device(struct pci_dev *dev) {} 2223 2224 /** 2225 * pcibios_penalize_isa_irq - penalize an ISA IRQ 2226 * @irq: ISA IRQ to penalize 2227 * @active: IRQ active or not 2228 * 2229 * Permits the platform to provide architecture-specific functionality when 2230 * penalizing ISA IRQs. This is the default implementation. Architecture 2231 * implementations can override this. 2232 */ 2233 void __weak pcibios_penalize_isa_irq(int irq, int active) {} 2234 2235 static void do_pci_disable_device(struct pci_dev *dev) 2236 { 2237 u16 pci_command; 2238 2239 pci_read_config_word(dev, PCI_COMMAND, &pci_command); 2240 if (pci_command & PCI_COMMAND_MASTER) { 2241 pci_command &= ~PCI_COMMAND_MASTER; 2242 pci_write_config_word(dev, PCI_COMMAND, pci_command); 2243 } 2244 2245 pcibios_disable_device(dev); 2246 } 2247 2248 /** 2249 * pci_disable_enabled_device - Disable device without updating enable_cnt 2250 * @dev: PCI device to disable 2251 * 2252 * NOTE: This function is a backend of PCI power management routines and is 2253 * not supposed to be called drivers. 2254 */ 2255 void pci_disable_enabled_device(struct pci_dev *dev) 2256 { 2257 if (pci_is_enabled(dev)) 2258 do_pci_disable_device(dev); 2259 } 2260 2261 /** 2262 * pci_disable_device - Disable PCI device after use 2263 * @dev: PCI device to be disabled 2264 * 2265 * Signal to the system that the PCI device is not in use by the system 2266 * anymore. This only involves disabling PCI bus-mastering, if active. 2267 * 2268 * Note we don't actually disable the device until all callers of 2269 * pci_enable_device() have called pci_disable_device(). 2270 */ 2271 void pci_disable_device(struct pci_dev *dev) 2272 { 2273 struct pci_devres *dr; 2274 2275 dr = find_pci_dr(dev); 2276 if (dr) 2277 dr->enabled = 0; 2278 2279 dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0, 2280 "disabling already-disabled device"); 2281 2282 if (atomic_dec_return(&dev->enable_cnt) != 0) 2283 return; 2284 2285 do_pci_disable_device(dev); 2286 2287 dev->is_busmaster = 0; 2288 } 2289 EXPORT_SYMBOL(pci_disable_device); 2290 2291 /** 2292 * pcibios_set_pcie_reset_state - set reset state for device dev 2293 * @dev: the PCIe device reset 2294 * @state: Reset state to enter into 2295 * 2296 * Set the PCIe reset state for the device. This is the default 2297 * implementation. Architecture implementations can override this. 2298 */ 2299 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev, 2300 enum pcie_reset_state state) 2301 { 2302 return -EINVAL; 2303 } 2304 2305 /** 2306 * pci_set_pcie_reset_state - set reset state for device dev 2307 * @dev: the PCIe device reset 2308 * @state: Reset state to enter into 2309 * 2310 * Sets the PCI reset state for the device. 2311 */ 2312 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 2313 { 2314 return pcibios_set_pcie_reset_state(dev, state); 2315 } 2316 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state); 2317 2318 #ifdef CONFIG_PCIEAER 2319 void pcie_clear_device_status(struct pci_dev *dev) 2320 { 2321 u16 sta; 2322 2323 pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta); 2324 pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta); 2325 } 2326 #endif 2327 2328 /** 2329 * pcie_clear_root_pme_status - Clear root port PME interrupt status. 2330 * @dev: PCIe root port or event collector. 2331 */ 2332 void pcie_clear_root_pme_status(struct pci_dev *dev) 2333 { 2334 pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME); 2335 } 2336 2337 /** 2338 * pci_check_pme_status - Check if given device has generated PME. 2339 * @dev: Device to check. 2340 * 2341 * Check the PME status of the device and if set, clear it and clear PME enable 2342 * (if set). Return 'true' if PME status and PME enable were both set or 2343 * 'false' otherwise. 2344 */ 2345 bool pci_check_pme_status(struct pci_dev *dev) 2346 { 2347 int pmcsr_pos; 2348 u16 pmcsr; 2349 bool ret = false; 2350 2351 if (!dev->pm_cap) 2352 return false; 2353 2354 pmcsr_pos = dev->pm_cap + PCI_PM_CTRL; 2355 pci_read_config_word(dev, pmcsr_pos, &pmcsr); 2356 if (!(pmcsr & PCI_PM_CTRL_PME_STATUS)) 2357 return false; 2358 2359 /* Clear PME status. */ 2360 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2361 if (pmcsr & PCI_PM_CTRL_PME_ENABLE) { 2362 /* Disable PME to avoid interrupt flood. */ 2363 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2364 ret = true; 2365 } 2366 2367 pci_write_config_word(dev, pmcsr_pos, pmcsr); 2368 2369 return ret; 2370 } 2371 2372 /** 2373 * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set. 2374 * @dev: Device to handle. 2375 * @pme_poll_reset: Whether or not to reset the device's pme_poll flag. 2376 * 2377 * Check if @dev has generated PME and queue a resume request for it in that 2378 * case. 2379 */ 2380 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset) 2381 { 2382 if (pme_poll_reset && dev->pme_poll) 2383 dev->pme_poll = false; 2384 2385 if (pci_check_pme_status(dev)) { 2386 pci_wakeup_event(dev); 2387 pm_request_resume(&dev->dev); 2388 } 2389 return 0; 2390 } 2391 2392 /** 2393 * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary. 2394 * @bus: Top bus of the subtree to walk. 2395 */ 2396 void pci_pme_wakeup_bus(struct pci_bus *bus) 2397 { 2398 if (bus) 2399 pci_walk_bus(bus, pci_pme_wakeup, (void *)true); 2400 } 2401 2402 2403 /** 2404 * pci_pme_capable - check the capability of PCI device to generate PME# 2405 * @dev: PCI device to handle. 2406 * @state: PCI state from which device will issue PME#. 2407 */ 2408 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state) 2409 { 2410 if (!dev->pm_cap) 2411 return false; 2412 2413 return !!(dev->pme_support & (1 << state)); 2414 } 2415 EXPORT_SYMBOL(pci_pme_capable); 2416 2417 static void pci_pme_list_scan(struct work_struct *work) 2418 { 2419 struct pci_pme_device *pme_dev, *n; 2420 2421 mutex_lock(&pci_pme_list_mutex); 2422 list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 2423 struct pci_dev *pdev = pme_dev->dev; 2424 2425 if (pdev->pme_poll) { 2426 struct pci_dev *bridge = pdev->bus->self; 2427 struct device *dev = &pdev->dev; 2428 int pm_status; 2429 2430 /* 2431 * If bridge is in low power state, the 2432 * configuration space of subordinate devices 2433 * may be not accessible 2434 */ 2435 if (bridge && bridge->current_state != PCI_D0) 2436 continue; 2437 2438 /* 2439 * If the device is in a low power state it 2440 * should not be polled either. 2441 */ 2442 pm_status = pm_runtime_get_if_active(dev, true); 2443 if (!pm_status) 2444 continue; 2445 2446 if (pdev->current_state != PCI_D3cold) 2447 pci_pme_wakeup(pdev, NULL); 2448 2449 if (pm_status > 0) 2450 pm_runtime_put(dev); 2451 } else { 2452 list_del(&pme_dev->list); 2453 kfree(pme_dev); 2454 } 2455 } 2456 if (!list_empty(&pci_pme_list)) 2457 queue_delayed_work(system_freezable_wq, &pci_pme_work, 2458 msecs_to_jiffies(PME_TIMEOUT)); 2459 mutex_unlock(&pci_pme_list_mutex); 2460 } 2461 2462 static void __pci_pme_active(struct pci_dev *dev, bool enable) 2463 { 2464 u16 pmcsr; 2465 2466 if (!dev->pme_support) 2467 return; 2468 2469 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2470 /* Clear PME_Status by writing 1 to it and enable PME# */ 2471 pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE; 2472 if (!enable) 2473 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2474 2475 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2476 } 2477 2478 /** 2479 * pci_pme_restore - Restore PME configuration after config space restore. 2480 * @dev: PCI device to update. 2481 */ 2482 void pci_pme_restore(struct pci_dev *dev) 2483 { 2484 u16 pmcsr; 2485 2486 if (!dev->pme_support) 2487 return; 2488 2489 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2490 if (dev->wakeup_prepared) { 2491 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 2492 pmcsr &= ~PCI_PM_CTRL_PME_STATUS; 2493 } else { 2494 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2495 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2496 } 2497 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2498 } 2499 2500 /** 2501 * pci_pme_active - enable or disable PCI device's PME# function 2502 * @dev: PCI device to handle. 2503 * @enable: 'true' to enable PME# generation; 'false' to disable it. 2504 * 2505 * The caller must verify that the device is capable of generating PME# before 2506 * calling this function with @enable equal to 'true'. 2507 */ 2508 void pci_pme_active(struct pci_dev *dev, bool enable) 2509 { 2510 __pci_pme_active(dev, enable); 2511 2512 /* 2513 * PCI (as opposed to PCIe) PME requires that the device have 2514 * its PME# line hooked up correctly. Not all hardware vendors 2515 * do this, so the PME never gets delivered and the device 2516 * remains asleep. The easiest way around this is to 2517 * periodically walk the list of suspended devices and check 2518 * whether any have their PME flag set. The assumption is that 2519 * we'll wake up often enough anyway that this won't be a huge 2520 * hit, and the power savings from the devices will still be a 2521 * win. 2522 * 2523 * Although PCIe uses in-band PME message instead of PME# line 2524 * to report PME, PME does not work for some PCIe devices in 2525 * reality. For example, there are devices that set their PME 2526 * status bits, but don't really bother to send a PME message; 2527 * there are PCI Express Root Ports that don't bother to 2528 * trigger interrupts when they receive PME messages from the 2529 * devices below. So PME poll is used for PCIe devices too. 2530 */ 2531 2532 if (dev->pme_poll) { 2533 struct pci_pme_device *pme_dev; 2534 if (enable) { 2535 pme_dev = kmalloc(sizeof(struct pci_pme_device), 2536 GFP_KERNEL); 2537 if (!pme_dev) { 2538 pci_warn(dev, "can't enable PME#\n"); 2539 return; 2540 } 2541 pme_dev->dev = dev; 2542 mutex_lock(&pci_pme_list_mutex); 2543 list_add(&pme_dev->list, &pci_pme_list); 2544 if (list_is_singular(&pci_pme_list)) 2545 queue_delayed_work(system_freezable_wq, 2546 &pci_pme_work, 2547 msecs_to_jiffies(PME_TIMEOUT)); 2548 mutex_unlock(&pci_pme_list_mutex); 2549 } else { 2550 mutex_lock(&pci_pme_list_mutex); 2551 list_for_each_entry(pme_dev, &pci_pme_list, list) { 2552 if (pme_dev->dev == dev) { 2553 list_del(&pme_dev->list); 2554 kfree(pme_dev); 2555 break; 2556 } 2557 } 2558 mutex_unlock(&pci_pme_list_mutex); 2559 } 2560 } 2561 2562 pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled"); 2563 } 2564 EXPORT_SYMBOL(pci_pme_active); 2565 2566 /** 2567 * __pci_enable_wake - enable PCI device as wakeup event source 2568 * @dev: PCI device affected 2569 * @state: PCI state from which device will issue wakeup events 2570 * @enable: True to enable event generation; false to disable 2571 * 2572 * This enables the device as a wakeup event source, or disables it. 2573 * When such events involves platform-specific hooks, those hooks are 2574 * called automatically by this routine. 2575 * 2576 * Devices with legacy power management (no standard PCI PM capabilities) 2577 * always require such platform hooks. 2578 * 2579 * RETURN VALUE: 2580 * 0 is returned on success 2581 * -EINVAL is returned if device is not supposed to wake up the system 2582 * Error code depending on the platform is returned if both the platform and 2583 * the native mechanism fail to enable the generation of wake-up events 2584 */ 2585 static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable) 2586 { 2587 int ret = 0; 2588 2589 /* 2590 * Bridges that are not power-manageable directly only signal 2591 * wakeup on behalf of subordinate devices which is set up 2592 * elsewhere, so skip them. However, bridges that are 2593 * power-manageable may signal wakeup for themselves (for example, 2594 * on a hotplug event) and they need to be covered here. 2595 */ 2596 if (!pci_power_manageable(dev)) 2597 return 0; 2598 2599 /* Don't do the same thing twice in a row for one device. */ 2600 if (!!enable == !!dev->wakeup_prepared) 2601 return 0; 2602 2603 /* 2604 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don 2605 * Anderson we should be doing PME# wake enable followed by ACPI wake 2606 * enable. To disable wake-up we call the platform first, for symmetry. 2607 */ 2608 2609 if (enable) { 2610 int error; 2611 2612 /* 2613 * Enable PME signaling if the device can signal PME from 2614 * D3cold regardless of whether or not it can signal PME from 2615 * the current target state, because that will allow it to 2616 * signal PME when the hierarchy above it goes into D3cold and 2617 * the device itself ends up in D3cold as a result of that. 2618 */ 2619 if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold)) 2620 pci_pme_active(dev, true); 2621 else 2622 ret = 1; 2623 error = platform_pci_set_wakeup(dev, true); 2624 if (ret) 2625 ret = error; 2626 if (!ret) 2627 dev->wakeup_prepared = true; 2628 } else { 2629 platform_pci_set_wakeup(dev, false); 2630 pci_pme_active(dev, false); 2631 dev->wakeup_prepared = false; 2632 } 2633 2634 return ret; 2635 } 2636 2637 /** 2638 * pci_enable_wake - change wakeup settings for a PCI device 2639 * @pci_dev: Target device 2640 * @state: PCI state from which device will issue wakeup events 2641 * @enable: Whether or not to enable event generation 2642 * 2643 * If @enable is set, check device_may_wakeup() for the device before calling 2644 * __pci_enable_wake() for it. 2645 */ 2646 int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable) 2647 { 2648 if (enable && !device_may_wakeup(&pci_dev->dev)) 2649 return -EINVAL; 2650 2651 return __pci_enable_wake(pci_dev, state, enable); 2652 } 2653 EXPORT_SYMBOL(pci_enable_wake); 2654 2655 /** 2656 * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold 2657 * @dev: PCI device to prepare 2658 * @enable: True to enable wake-up event generation; false to disable 2659 * 2660 * Many drivers want the device to wake up the system from D3_hot or D3_cold 2661 * and this function allows them to set that up cleanly - pci_enable_wake() 2662 * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI 2663 * ordering constraints. 2664 * 2665 * This function only returns error code if the device is not allowed to wake 2666 * up the system from sleep or it is not capable of generating PME# from both 2667 * D3_hot and D3_cold and the platform is unable to enable wake-up power for it. 2668 */ 2669 int pci_wake_from_d3(struct pci_dev *dev, bool enable) 2670 { 2671 return pci_pme_capable(dev, PCI_D3cold) ? 2672 pci_enable_wake(dev, PCI_D3cold, enable) : 2673 pci_enable_wake(dev, PCI_D3hot, enable); 2674 } 2675 EXPORT_SYMBOL(pci_wake_from_d3); 2676 2677 /** 2678 * pci_target_state - find an appropriate low power state for a given PCI dev 2679 * @dev: PCI device 2680 * @wakeup: Whether or not wakeup functionality will be enabled for the device. 2681 * 2682 * Use underlying platform code to find a supported low power state for @dev. 2683 * If the platform can't manage @dev, return the deepest state from which it 2684 * can generate wake events, based on any available PME info. 2685 */ 2686 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup) 2687 { 2688 if (platform_pci_power_manageable(dev)) { 2689 /* 2690 * Call the platform to find the target state for the device. 2691 */ 2692 pci_power_t state = platform_pci_choose_state(dev); 2693 2694 switch (state) { 2695 case PCI_POWER_ERROR: 2696 case PCI_UNKNOWN: 2697 return PCI_D3hot; 2698 2699 case PCI_D1: 2700 case PCI_D2: 2701 if (pci_no_d1d2(dev)) 2702 return PCI_D3hot; 2703 } 2704 2705 return state; 2706 } 2707 2708 /* 2709 * If the device is in D3cold even though it's not power-manageable by 2710 * the platform, it may have been powered down by non-standard means. 2711 * Best to let it slumber. 2712 */ 2713 if (dev->current_state == PCI_D3cold) 2714 return PCI_D3cold; 2715 else if (!dev->pm_cap) 2716 return PCI_D0; 2717 2718 if (wakeup && dev->pme_support) { 2719 pci_power_t state = PCI_D3hot; 2720 2721 /* 2722 * Find the deepest state from which the device can generate 2723 * PME#. 2724 */ 2725 while (state && !(dev->pme_support & (1 << state))) 2726 state--; 2727 2728 if (state) 2729 return state; 2730 else if (dev->pme_support & 1) 2731 return PCI_D0; 2732 } 2733 2734 return PCI_D3hot; 2735 } 2736 2737 /** 2738 * pci_prepare_to_sleep - prepare PCI device for system-wide transition 2739 * into a sleep state 2740 * @dev: Device to handle. 2741 * 2742 * Choose the power state appropriate for the device depending on whether 2743 * it can wake up the system and/or is power manageable by the platform 2744 * (PCI_D3hot is the default) and put the device into that state. 2745 */ 2746 int pci_prepare_to_sleep(struct pci_dev *dev) 2747 { 2748 bool wakeup = device_may_wakeup(&dev->dev); 2749 pci_power_t target_state = pci_target_state(dev, wakeup); 2750 int error; 2751 2752 if (target_state == PCI_POWER_ERROR) 2753 return -EIO; 2754 2755 pci_enable_wake(dev, target_state, wakeup); 2756 2757 error = pci_set_power_state(dev, target_state); 2758 2759 if (error) 2760 pci_enable_wake(dev, target_state, false); 2761 2762 return error; 2763 } 2764 EXPORT_SYMBOL(pci_prepare_to_sleep); 2765 2766 /** 2767 * pci_back_from_sleep - turn PCI device on during system-wide transition 2768 * into working state 2769 * @dev: Device to handle. 2770 * 2771 * Disable device's system wake-up capability and put it into D0. 2772 */ 2773 int pci_back_from_sleep(struct pci_dev *dev) 2774 { 2775 int ret = pci_set_power_state(dev, PCI_D0); 2776 2777 if (ret) 2778 return ret; 2779 2780 pci_enable_wake(dev, PCI_D0, false); 2781 return 0; 2782 } 2783 EXPORT_SYMBOL(pci_back_from_sleep); 2784 2785 /** 2786 * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend. 2787 * @dev: PCI device being suspended. 2788 * 2789 * Prepare @dev to generate wake-up events at run time and put it into a low 2790 * power state. 2791 */ 2792 int pci_finish_runtime_suspend(struct pci_dev *dev) 2793 { 2794 pci_power_t target_state; 2795 int error; 2796 2797 target_state = pci_target_state(dev, device_can_wakeup(&dev->dev)); 2798 if (target_state == PCI_POWER_ERROR) 2799 return -EIO; 2800 2801 __pci_enable_wake(dev, target_state, pci_dev_run_wake(dev)); 2802 2803 error = pci_set_power_state(dev, target_state); 2804 2805 if (error) 2806 pci_enable_wake(dev, target_state, false); 2807 2808 return error; 2809 } 2810 2811 /** 2812 * pci_dev_run_wake - Check if device can generate run-time wake-up events. 2813 * @dev: Device to check. 2814 * 2815 * Return true if the device itself is capable of generating wake-up events 2816 * (through the platform or using the native PCIe PME) or if the device supports 2817 * PME and one of its upstream bridges can generate wake-up events. 2818 */ 2819 bool pci_dev_run_wake(struct pci_dev *dev) 2820 { 2821 struct pci_bus *bus = dev->bus; 2822 2823 if (!dev->pme_support) 2824 return false; 2825 2826 /* PME-capable in principle, but not from the target power state */ 2827 if (!pci_pme_capable(dev, pci_target_state(dev, true))) 2828 return false; 2829 2830 if (device_can_wakeup(&dev->dev)) 2831 return true; 2832 2833 while (bus->parent) { 2834 struct pci_dev *bridge = bus->self; 2835 2836 if (device_can_wakeup(&bridge->dev)) 2837 return true; 2838 2839 bus = bus->parent; 2840 } 2841 2842 /* We have reached the root bus. */ 2843 if (bus->bridge) 2844 return device_can_wakeup(bus->bridge); 2845 2846 return false; 2847 } 2848 EXPORT_SYMBOL_GPL(pci_dev_run_wake); 2849 2850 /** 2851 * pci_dev_need_resume - Check if it is necessary to resume the device. 2852 * @pci_dev: Device to check. 2853 * 2854 * Return 'true' if the device is not runtime-suspended or it has to be 2855 * reconfigured due to wakeup settings difference between system and runtime 2856 * suspend, or the current power state of it is not suitable for the upcoming 2857 * (system-wide) transition. 2858 */ 2859 bool pci_dev_need_resume(struct pci_dev *pci_dev) 2860 { 2861 struct device *dev = &pci_dev->dev; 2862 pci_power_t target_state; 2863 2864 if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev)) 2865 return true; 2866 2867 target_state = pci_target_state(pci_dev, device_may_wakeup(dev)); 2868 2869 /* 2870 * If the earlier platform check has not triggered, D3cold is just power 2871 * removal on top of D3hot, so no need to resume the device in that 2872 * case. 2873 */ 2874 return target_state != pci_dev->current_state && 2875 target_state != PCI_D3cold && 2876 pci_dev->current_state != PCI_D3hot; 2877 } 2878 2879 /** 2880 * pci_dev_adjust_pme - Adjust PME setting for a suspended device. 2881 * @pci_dev: Device to check. 2882 * 2883 * If the device is suspended and it is not configured for system wakeup, 2884 * disable PME for it to prevent it from waking up the system unnecessarily. 2885 * 2886 * Note that if the device's power state is D3cold and the platform check in 2887 * pci_dev_need_resume() has not triggered, the device's configuration need not 2888 * be changed. 2889 */ 2890 void pci_dev_adjust_pme(struct pci_dev *pci_dev) 2891 { 2892 struct device *dev = &pci_dev->dev; 2893 2894 spin_lock_irq(&dev->power.lock); 2895 2896 if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) && 2897 pci_dev->current_state < PCI_D3cold) 2898 __pci_pme_active(pci_dev, false); 2899 2900 spin_unlock_irq(&dev->power.lock); 2901 } 2902 2903 /** 2904 * pci_dev_complete_resume - Finalize resume from system sleep for a device. 2905 * @pci_dev: Device to handle. 2906 * 2907 * If the device is runtime suspended and wakeup-capable, enable PME for it as 2908 * it might have been disabled during the prepare phase of system suspend if 2909 * the device was not configured for system wakeup. 2910 */ 2911 void pci_dev_complete_resume(struct pci_dev *pci_dev) 2912 { 2913 struct device *dev = &pci_dev->dev; 2914 2915 if (!pci_dev_run_wake(pci_dev)) 2916 return; 2917 2918 spin_lock_irq(&dev->power.lock); 2919 2920 if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold) 2921 __pci_pme_active(pci_dev, true); 2922 2923 spin_unlock_irq(&dev->power.lock); 2924 } 2925 2926 /** 2927 * pci_choose_state - Choose the power state of a PCI device. 2928 * @dev: Target PCI device. 2929 * @state: Target state for the whole system. 2930 * 2931 * Returns PCI power state suitable for @dev and @state. 2932 */ 2933 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) 2934 { 2935 if (state.event == PM_EVENT_ON) 2936 return PCI_D0; 2937 2938 return pci_target_state(dev, false); 2939 } 2940 EXPORT_SYMBOL(pci_choose_state); 2941 2942 void pci_config_pm_runtime_get(struct pci_dev *pdev) 2943 { 2944 struct device *dev = &pdev->dev; 2945 struct device *parent = dev->parent; 2946 2947 if (parent) 2948 pm_runtime_get_sync(parent); 2949 pm_runtime_get_noresume(dev); 2950 /* 2951 * pdev->current_state is set to PCI_D3cold during suspending, 2952 * so wait until suspending completes 2953 */ 2954 pm_runtime_barrier(dev); 2955 /* 2956 * Only need to resume devices in D3cold, because config 2957 * registers are still accessible for devices suspended but 2958 * not in D3cold. 2959 */ 2960 if (pdev->current_state == PCI_D3cold) 2961 pm_runtime_resume(dev); 2962 } 2963 2964 void pci_config_pm_runtime_put(struct pci_dev *pdev) 2965 { 2966 struct device *dev = &pdev->dev; 2967 struct device *parent = dev->parent; 2968 2969 pm_runtime_put(dev); 2970 if (parent) 2971 pm_runtime_put_sync(parent); 2972 } 2973 2974 static const struct dmi_system_id bridge_d3_blacklist[] = { 2975 #ifdef CONFIG_X86 2976 { 2977 /* 2978 * Gigabyte X299 root port is not marked as hotplug capable 2979 * which allows Linux to power manage it. However, this 2980 * confuses the BIOS SMI handler so don't power manage root 2981 * ports on that system. 2982 */ 2983 .ident = "X299 DESIGNARE EX-CF", 2984 .matches = { 2985 DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."), 2986 DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"), 2987 }, 2988 }, 2989 { 2990 /* 2991 * Downstream device is not accessible after putting a root port 2992 * into D3cold and back into D0 on Elo Continental Z2 board 2993 */ 2994 .ident = "Elo Continental Z2", 2995 .matches = { 2996 DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"), 2997 DMI_MATCH(DMI_BOARD_NAME, "Geminilake"), 2998 DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"), 2999 }, 3000 }, 3001 #endif 3002 { } 3003 }; 3004 3005 /** 3006 * pci_bridge_d3_possible - Is it possible to put the bridge into D3 3007 * @bridge: Bridge to check 3008 * 3009 * This function checks if it is possible to move the bridge to D3. 3010 * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt. 3011 */ 3012 bool pci_bridge_d3_possible(struct pci_dev *bridge) 3013 { 3014 if (!pci_is_pcie(bridge)) 3015 return false; 3016 3017 switch (pci_pcie_type(bridge)) { 3018 case PCI_EXP_TYPE_ROOT_PORT: 3019 case PCI_EXP_TYPE_UPSTREAM: 3020 case PCI_EXP_TYPE_DOWNSTREAM: 3021 if (pci_bridge_d3_disable) 3022 return false; 3023 3024 /* 3025 * Hotplug ports handled by firmware in System Management Mode 3026 * may not be put into D3 by the OS (Thunderbolt on non-Macs). 3027 */ 3028 if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge)) 3029 return false; 3030 3031 if (pci_bridge_d3_force) 3032 return true; 3033 3034 /* Even the oldest 2010 Thunderbolt controller supports D3. */ 3035 if (bridge->is_thunderbolt) 3036 return true; 3037 3038 /* Platform might know better if the bridge supports D3 */ 3039 if (platform_pci_bridge_d3(bridge)) 3040 return true; 3041 3042 /* 3043 * Hotplug ports handled natively by the OS were not validated 3044 * by vendors for runtime D3 at least until 2018 because there 3045 * was no OS support. 3046 */ 3047 if (bridge->is_hotplug_bridge) 3048 return false; 3049 3050 if (dmi_check_system(bridge_d3_blacklist)) 3051 return false; 3052 3053 /* 3054 * It should be safe to put PCIe ports from 2015 or newer 3055 * to D3. 3056 */ 3057 if (dmi_get_bios_year() >= 2015) 3058 return true; 3059 break; 3060 } 3061 3062 return false; 3063 } 3064 3065 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) 3066 { 3067 bool *d3cold_ok = data; 3068 3069 if (/* The device needs to be allowed to go D3cold ... */ 3070 dev->no_d3cold || !dev->d3cold_allowed || 3071 3072 /* ... and if it is wakeup capable to do so from D3cold. */ 3073 (device_may_wakeup(&dev->dev) && 3074 !pci_pme_capable(dev, PCI_D3cold)) || 3075 3076 /* If it is a bridge it must be allowed to go to D3. */ 3077 !pci_power_manageable(dev)) 3078 3079 *d3cold_ok = false; 3080 3081 return !*d3cold_ok; 3082 } 3083 3084 /* 3085 * pci_bridge_d3_update - Update bridge D3 capabilities 3086 * @dev: PCI device which is changed 3087 * 3088 * Update upstream bridge PM capabilities accordingly depending on if the 3089 * device PM configuration was changed or the device is being removed. The 3090 * change is also propagated upstream. 3091 */ 3092 void pci_bridge_d3_update(struct pci_dev *dev) 3093 { 3094 bool remove = !device_is_registered(&dev->dev); 3095 struct pci_dev *bridge; 3096 bool d3cold_ok = true; 3097 3098 bridge = pci_upstream_bridge(dev); 3099 if (!bridge || !pci_bridge_d3_possible(bridge)) 3100 return; 3101 3102 /* 3103 * If D3 is currently allowed for the bridge, removing one of its 3104 * children won't change that. 3105 */ 3106 if (remove && bridge->bridge_d3) 3107 return; 3108 3109 /* 3110 * If D3 is currently allowed for the bridge and a child is added or 3111 * changed, disallowance of D3 can only be caused by that child, so 3112 * we only need to check that single device, not any of its siblings. 3113 * 3114 * If D3 is currently not allowed for the bridge, checking the device 3115 * first may allow us to skip checking its siblings. 3116 */ 3117 if (!remove) 3118 pci_dev_check_d3cold(dev, &d3cold_ok); 3119 3120 /* 3121 * If D3 is currently not allowed for the bridge, this may be caused 3122 * either by the device being changed/removed or any of its siblings, 3123 * so we need to go through all children to find out if one of them 3124 * continues to block D3. 3125 */ 3126 if (d3cold_ok && !bridge->bridge_d3) 3127 pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold, 3128 &d3cold_ok); 3129 3130 if (bridge->bridge_d3 != d3cold_ok) { 3131 bridge->bridge_d3 = d3cold_ok; 3132 /* Propagate change to upstream bridges */ 3133 pci_bridge_d3_update(bridge); 3134 } 3135 } 3136 3137 /** 3138 * pci_d3cold_enable - Enable D3cold for device 3139 * @dev: PCI device to handle 3140 * 3141 * This function can be used in drivers to enable D3cold from the device 3142 * they handle. It also updates upstream PCI bridge PM capabilities 3143 * accordingly. 3144 */ 3145 void pci_d3cold_enable(struct pci_dev *dev) 3146 { 3147 if (dev->no_d3cold) { 3148 dev->no_d3cold = false; 3149 pci_bridge_d3_update(dev); 3150 } 3151 } 3152 EXPORT_SYMBOL_GPL(pci_d3cold_enable); 3153 3154 /** 3155 * pci_d3cold_disable - Disable D3cold for device 3156 * @dev: PCI device to handle 3157 * 3158 * This function can be used in drivers to disable D3cold from the device 3159 * they handle. It also updates upstream PCI bridge PM capabilities 3160 * accordingly. 3161 */ 3162 void pci_d3cold_disable(struct pci_dev *dev) 3163 { 3164 if (!dev->no_d3cold) { 3165 dev->no_d3cold = true; 3166 pci_bridge_d3_update(dev); 3167 } 3168 } 3169 EXPORT_SYMBOL_GPL(pci_d3cold_disable); 3170 3171 /** 3172 * pci_pm_init - Initialize PM functions of given PCI device 3173 * @dev: PCI device to handle. 3174 */ 3175 void pci_pm_init(struct pci_dev *dev) 3176 { 3177 int pm; 3178 u16 status; 3179 u16 pmc; 3180 3181 pm_runtime_forbid(&dev->dev); 3182 pm_runtime_set_active(&dev->dev); 3183 pm_runtime_enable(&dev->dev); 3184 device_enable_async_suspend(&dev->dev); 3185 dev->wakeup_prepared = false; 3186 3187 dev->pm_cap = 0; 3188 dev->pme_support = 0; 3189 3190 /* find PCI PM capability in list */ 3191 pm = pci_find_capability(dev, PCI_CAP_ID_PM); 3192 if (!pm) 3193 return; 3194 /* Check device's ability to generate PME# */ 3195 pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc); 3196 3197 if ((pmc & PCI_PM_CAP_VER_MASK) > 3) { 3198 pci_err(dev, "unsupported PM cap regs version (%u)\n", 3199 pmc & PCI_PM_CAP_VER_MASK); 3200 return; 3201 } 3202 3203 dev->pm_cap = pm; 3204 dev->d3hot_delay = PCI_PM_D3HOT_WAIT; 3205 dev->d3cold_delay = PCI_PM_D3COLD_WAIT; 3206 dev->bridge_d3 = pci_bridge_d3_possible(dev); 3207 dev->d3cold_allowed = true; 3208 3209 dev->d1_support = false; 3210 dev->d2_support = false; 3211 if (!pci_no_d1d2(dev)) { 3212 if (pmc & PCI_PM_CAP_D1) 3213 dev->d1_support = true; 3214 if (pmc & PCI_PM_CAP_D2) 3215 dev->d2_support = true; 3216 3217 if (dev->d1_support || dev->d2_support) 3218 pci_info(dev, "supports%s%s\n", 3219 dev->d1_support ? " D1" : "", 3220 dev->d2_support ? " D2" : ""); 3221 } 3222 3223 pmc &= PCI_PM_CAP_PME_MASK; 3224 if (pmc) { 3225 pci_info(dev, "PME# supported from%s%s%s%s%s\n", 3226 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "", 3227 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "", 3228 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "", 3229 (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "", 3230 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : ""); 3231 dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT; 3232 dev->pme_poll = true; 3233 /* 3234 * Make device's PM flags reflect the wake-up capability, but 3235 * let the user space enable it to wake up the system as needed. 3236 */ 3237 device_set_wakeup_capable(&dev->dev, true); 3238 /* Disable the PME# generation functionality */ 3239 pci_pme_active(dev, false); 3240 } 3241 3242 pci_read_config_word(dev, PCI_STATUS, &status); 3243 if (status & PCI_STATUS_IMM_READY) 3244 dev->imm_ready = 1; 3245 } 3246 3247 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) 3248 { 3249 unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI; 3250 3251 switch (prop) { 3252 case PCI_EA_P_MEM: 3253 case PCI_EA_P_VF_MEM: 3254 flags |= IORESOURCE_MEM; 3255 break; 3256 case PCI_EA_P_MEM_PREFETCH: 3257 case PCI_EA_P_VF_MEM_PREFETCH: 3258 flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH; 3259 break; 3260 case PCI_EA_P_IO: 3261 flags |= IORESOURCE_IO; 3262 break; 3263 default: 3264 return 0; 3265 } 3266 3267 return flags; 3268 } 3269 3270 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei, 3271 u8 prop) 3272 { 3273 if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO) 3274 return &dev->resource[bei]; 3275 #ifdef CONFIG_PCI_IOV 3276 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 && 3277 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH)) 3278 return &dev->resource[PCI_IOV_RESOURCES + 3279 bei - PCI_EA_BEI_VF_BAR0]; 3280 #endif 3281 else if (bei == PCI_EA_BEI_ROM) 3282 return &dev->resource[PCI_ROM_RESOURCE]; 3283 else 3284 return NULL; 3285 } 3286 3287 /* Read an Enhanced Allocation (EA) entry */ 3288 static int pci_ea_read(struct pci_dev *dev, int offset) 3289 { 3290 struct resource *res; 3291 int ent_size, ent_offset = offset; 3292 resource_size_t start, end; 3293 unsigned long flags; 3294 u32 dw0, bei, base, max_offset; 3295 u8 prop; 3296 bool support_64 = (sizeof(resource_size_t) >= 8); 3297 3298 pci_read_config_dword(dev, ent_offset, &dw0); 3299 ent_offset += 4; 3300 3301 /* Entry size field indicates DWORDs after 1st */ 3302 ent_size = ((dw0 & PCI_EA_ES) + 1) << 2; 3303 3304 if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */ 3305 goto out; 3306 3307 bei = (dw0 & PCI_EA_BEI) >> 4; 3308 prop = (dw0 & PCI_EA_PP) >> 8; 3309 3310 /* 3311 * If the Property is in the reserved range, try the Secondary 3312 * Property instead. 3313 */ 3314 if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED) 3315 prop = (dw0 & PCI_EA_SP) >> 16; 3316 if (prop > PCI_EA_P_BRIDGE_IO) 3317 goto out; 3318 3319 res = pci_ea_get_resource(dev, bei, prop); 3320 if (!res) { 3321 pci_err(dev, "Unsupported EA entry BEI: %u\n", bei); 3322 goto out; 3323 } 3324 3325 flags = pci_ea_flags(dev, prop); 3326 if (!flags) { 3327 pci_err(dev, "Unsupported EA properties: %#x\n", prop); 3328 goto out; 3329 } 3330 3331 /* Read Base */ 3332 pci_read_config_dword(dev, ent_offset, &base); 3333 start = (base & PCI_EA_FIELD_MASK); 3334 ent_offset += 4; 3335 3336 /* Read MaxOffset */ 3337 pci_read_config_dword(dev, ent_offset, &max_offset); 3338 ent_offset += 4; 3339 3340 /* Read Base MSBs (if 64-bit entry) */ 3341 if (base & PCI_EA_IS_64) { 3342 u32 base_upper; 3343 3344 pci_read_config_dword(dev, ent_offset, &base_upper); 3345 ent_offset += 4; 3346 3347 flags |= IORESOURCE_MEM_64; 3348 3349 /* entry starts above 32-bit boundary, can't use */ 3350 if (!support_64 && base_upper) 3351 goto out; 3352 3353 if (support_64) 3354 start |= ((u64)base_upper << 32); 3355 } 3356 3357 end = start + (max_offset | 0x03); 3358 3359 /* Read MaxOffset MSBs (if 64-bit entry) */ 3360 if (max_offset & PCI_EA_IS_64) { 3361 u32 max_offset_upper; 3362 3363 pci_read_config_dword(dev, ent_offset, &max_offset_upper); 3364 ent_offset += 4; 3365 3366 flags |= IORESOURCE_MEM_64; 3367 3368 /* entry too big, can't use */ 3369 if (!support_64 && max_offset_upper) 3370 goto out; 3371 3372 if (support_64) 3373 end += ((u64)max_offset_upper << 32); 3374 } 3375 3376 if (end < start) { 3377 pci_err(dev, "EA Entry crosses address boundary\n"); 3378 goto out; 3379 } 3380 3381 if (ent_size != ent_offset - offset) { 3382 pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n", 3383 ent_size, ent_offset - offset); 3384 goto out; 3385 } 3386 3387 res->name = pci_name(dev); 3388 res->start = start; 3389 res->end = end; 3390 res->flags = flags; 3391 3392 if (bei <= PCI_EA_BEI_BAR5) 3393 pci_info(dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 3394 bei, res, prop); 3395 else if (bei == PCI_EA_BEI_ROM) 3396 pci_info(dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n", 3397 res, prop); 3398 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5) 3399 pci_info(dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 3400 bei - PCI_EA_BEI_VF_BAR0, res, prop); 3401 else 3402 pci_info(dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n", 3403 bei, res, prop); 3404 3405 out: 3406 return offset + ent_size; 3407 } 3408 3409 /* Enhanced Allocation Initialization */ 3410 void pci_ea_init(struct pci_dev *dev) 3411 { 3412 int ea; 3413 u8 num_ent; 3414 int offset; 3415 int i; 3416 3417 /* find PCI EA capability in list */ 3418 ea = pci_find_capability(dev, PCI_CAP_ID_EA); 3419 if (!ea) 3420 return; 3421 3422 /* determine the number of entries */ 3423 pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT, 3424 &num_ent); 3425 num_ent &= PCI_EA_NUM_ENT_MASK; 3426 3427 offset = ea + PCI_EA_FIRST_ENT; 3428 3429 /* Skip DWORD 2 for type 1 functions */ 3430 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 3431 offset += 4; 3432 3433 /* parse each EA entry */ 3434 for (i = 0; i < num_ent; ++i) 3435 offset = pci_ea_read(dev, offset); 3436 } 3437 3438 static void pci_add_saved_cap(struct pci_dev *pci_dev, 3439 struct pci_cap_saved_state *new_cap) 3440 { 3441 hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space); 3442 } 3443 3444 /** 3445 * _pci_add_cap_save_buffer - allocate buffer for saving given 3446 * capability registers 3447 * @dev: the PCI device 3448 * @cap: the capability to allocate the buffer for 3449 * @extended: Standard or Extended capability ID 3450 * @size: requested size of the buffer 3451 */ 3452 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap, 3453 bool extended, unsigned int size) 3454 { 3455 int pos; 3456 struct pci_cap_saved_state *save_state; 3457 3458 if (extended) 3459 pos = pci_find_ext_capability(dev, cap); 3460 else 3461 pos = pci_find_capability(dev, cap); 3462 3463 if (!pos) 3464 return 0; 3465 3466 save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL); 3467 if (!save_state) 3468 return -ENOMEM; 3469 3470 save_state->cap.cap_nr = cap; 3471 save_state->cap.cap_extended = extended; 3472 save_state->cap.size = size; 3473 pci_add_saved_cap(dev, save_state); 3474 3475 return 0; 3476 } 3477 3478 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size) 3479 { 3480 return _pci_add_cap_save_buffer(dev, cap, false, size); 3481 } 3482 3483 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size) 3484 { 3485 return _pci_add_cap_save_buffer(dev, cap, true, size); 3486 } 3487 3488 /** 3489 * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities 3490 * @dev: the PCI device 3491 */ 3492 void pci_allocate_cap_save_buffers(struct pci_dev *dev) 3493 { 3494 int error; 3495 3496 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP, 3497 PCI_EXP_SAVE_REGS * sizeof(u16)); 3498 if (error) 3499 pci_err(dev, "unable to preallocate PCI Express save buffer\n"); 3500 3501 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16)); 3502 if (error) 3503 pci_err(dev, "unable to preallocate PCI-X save buffer\n"); 3504 3505 error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR, 3506 2 * sizeof(u16)); 3507 if (error) 3508 pci_err(dev, "unable to allocate suspend buffer for LTR\n"); 3509 3510 pci_allocate_vc_save_buffers(dev); 3511 } 3512 3513 void pci_free_cap_save_buffers(struct pci_dev *dev) 3514 { 3515 struct pci_cap_saved_state *tmp; 3516 struct hlist_node *n; 3517 3518 hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next) 3519 kfree(tmp); 3520 } 3521 3522 /** 3523 * pci_configure_ari - enable or disable ARI forwarding 3524 * @dev: the PCI device 3525 * 3526 * If @dev and its upstream bridge both support ARI, enable ARI in the 3527 * bridge. Otherwise, disable ARI in the bridge. 3528 */ 3529 void pci_configure_ari(struct pci_dev *dev) 3530 { 3531 u32 cap; 3532 struct pci_dev *bridge; 3533 3534 if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn) 3535 return; 3536 3537 bridge = dev->bus->self; 3538 if (!bridge) 3539 return; 3540 3541 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3542 if (!(cap & PCI_EXP_DEVCAP2_ARI)) 3543 return; 3544 3545 if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) { 3546 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, 3547 PCI_EXP_DEVCTL2_ARI); 3548 bridge->ari_enabled = 1; 3549 } else { 3550 pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2, 3551 PCI_EXP_DEVCTL2_ARI); 3552 bridge->ari_enabled = 0; 3553 } 3554 } 3555 3556 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags) 3557 { 3558 int pos; 3559 u16 cap, ctrl; 3560 3561 pos = pdev->acs_cap; 3562 if (!pos) 3563 return false; 3564 3565 /* 3566 * Except for egress control, capabilities are either required 3567 * or only required if controllable. Features missing from the 3568 * capability field can therefore be assumed as hard-wired enabled. 3569 */ 3570 pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap); 3571 acs_flags &= (cap | PCI_ACS_EC); 3572 3573 pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl); 3574 return (ctrl & acs_flags) == acs_flags; 3575 } 3576 3577 /** 3578 * pci_acs_enabled - test ACS against required flags for a given device 3579 * @pdev: device to test 3580 * @acs_flags: required PCI ACS flags 3581 * 3582 * Return true if the device supports the provided flags. Automatically 3583 * filters out flags that are not implemented on multifunction devices. 3584 * 3585 * Note that this interface checks the effective ACS capabilities of the 3586 * device rather than the actual capabilities. For instance, most single 3587 * function endpoints are not required to support ACS because they have no 3588 * opportunity for peer-to-peer access. We therefore return 'true' 3589 * regardless of whether the device exposes an ACS capability. This makes 3590 * it much easier for callers of this function to ignore the actual type 3591 * or topology of the device when testing ACS support. 3592 */ 3593 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags) 3594 { 3595 int ret; 3596 3597 ret = pci_dev_specific_acs_enabled(pdev, acs_flags); 3598 if (ret >= 0) 3599 return ret > 0; 3600 3601 /* 3602 * Conventional PCI and PCI-X devices never support ACS, either 3603 * effectively or actually. The shared bus topology implies that 3604 * any device on the bus can receive or snoop DMA. 3605 */ 3606 if (!pci_is_pcie(pdev)) 3607 return false; 3608 3609 switch (pci_pcie_type(pdev)) { 3610 /* 3611 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec, 3612 * but since their primary interface is PCI/X, we conservatively 3613 * handle them as we would a non-PCIe device. 3614 */ 3615 case PCI_EXP_TYPE_PCIE_BRIDGE: 3616 /* 3617 * PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never 3618 * applicable... must never implement an ACS Extended Capability...". 3619 * This seems arbitrary, but we take a conservative interpretation 3620 * of this statement. 3621 */ 3622 case PCI_EXP_TYPE_PCI_BRIDGE: 3623 case PCI_EXP_TYPE_RC_EC: 3624 return false; 3625 /* 3626 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should 3627 * implement ACS in order to indicate their peer-to-peer capabilities, 3628 * regardless of whether they are single- or multi-function devices. 3629 */ 3630 case PCI_EXP_TYPE_DOWNSTREAM: 3631 case PCI_EXP_TYPE_ROOT_PORT: 3632 return pci_acs_flags_enabled(pdev, acs_flags); 3633 /* 3634 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be 3635 * implemented by the remaining PCIe types to indicate peer-to-peer 3636 * capabilities, but only when they are part of a multifunction 3637 * device. The footnote for section 6.12 indicates the specific 3638 * PCIe types included here. 3639 */ 3640 case PCI_EXP_TYPE_ENDPOINT: 3641 case PCI_EXP_TYPE_UPSTREAM: 3642 case PCI_EXP_TYPE_LEG_END: 3643 case PCI_EXP_TYPE_RC_END: 3644 if (!pdev->multifunction) 3645 break; 3646 3647 return pci_acs_flags_enabled(pdev, acs_flags); 3648 } 3649 3650 /* 3651 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable 3652 * to single function devices with the exception of downstream ports. 3653 */ 3654 return true; 3655 } 3656 3657 /** 3658 * pci_acs_path_enabled - test ACS flags from start to end in a hierarchy 3659 * @start: starting downstream device 3660 * @end: ending upstream device or NULL to search to the root bus 3661 * @acs_flags: required flags 3662 * 3663 * Walk up a device tree from start to end testing PCI ACS support. If 3664 * any step along the way does not support the required flags, return false. 3665 */ 3666 bool pci_acs_path_enabled(struct pci_dev *start, 3667 struct pci_dev *end, u16 acs_flags) 3668 { 3669 struct pci_dev *pdev, *parent = start; 3670 3671 do { 3672 pdev = parent; 3673 3674 if (!pci_acs_enabled(pdev, acs_flags)) 3675 return false; 3676 3677 if (pci_is_root_bus(pdev->bus)) 3678 return (end == NULL); 3679 3680 parent = pdev->bus->self; 3681 } while (pdev != end); 3682 3683 return true; 3684 } 3685 3686 /** 3687 * pci_acs_init - Initialize ACS if hardware supports it 3688 * @dev: the PCI device 3689 */ 3690 void pci_acs_init(struct pci_dev *dev) 3691 { 3692 dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); 3693 3694 /* 3695 * Attempt to enable ACS regardless of capability because some Root 3696 * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have 3697 * the standard ACS capability but still support ACS via those 3698 * quirks. 3699 */ 3700 pci_enable_acs(dev); 3701 } 3702 3703 /** 3704 * pci_rebar_find_pos - find position of resize ctrl reg for BAR 3705 * @pdev: PCI device 3706 * @bar: BAR to find 3707 * 3708 * Helper to find the position of the ctrl register for a BAR. 3709 * Returns -ENOTSUPP if resizable BARs are not supported at all. 3710 * Returns -ENOENT if no ctrl register for the BAR could be found. 3711 */ 3712 static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) 3713 { 3714 unsigned int pos, nbars, i; 3715 u32 ctrl; 3716 3717 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 3718 if (!pos) 3719 return -ENOTSUPP; 3720 3721 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3722 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 3723 PCI_REBAR_CTRL_NBAR_SHIFT; 3724 3725 for (i = 0; i < nbars; i++, pos += 8) { 3726 int bar_idx; 3727 3728 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3729 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 3730 if (bar_idx == bar) 3731 return pos; 3732 } 3733 3734 return -ENOENT; 3735 } 3736 3737 /** 3738 * pci_rebar_get_possible_sizes - get possible sizes for BAR 3739 * @pdev: PCI device 3740 * @bar: BAR to query 3741 * 3742 * Get the possible sizes of a resizable BAR as bitmask defined in the spec 3743 * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable. 3744 */ 3745 u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar) 3746 { 3747 int pos; 3748 u32 cap; 3749 3750 pos = pci_rebar_find_pos(pdev, bar); 3751 if (pos < 0) 3752 return 0; 3753 3754 pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap); 3755 cap &= PCI_REBAR_CAP_SIZES; 3756 3757 /* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */ 3758 if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f && 3759 bar == 0 && cap == 0x7000) 3760 cap = 0x3f000; 3761 3762 return cap >> 4; 3763 } 3764 EXPORT_SYMBOL(pci_rebar_get_possible_sizes); 3765 3766 /** 3767 * pci_rebar_get_current_size - get the current size of a BAR 3768 * @pdev: PCI device 3769 * @bar: BAR to set size to 3770 * 3771 * Read the size of a BAR from the resizable BAR config. 3772 * Returns size if found or negative error code. 3773 */ 3774 int pci_rebar_get_current_size(struct pci_dev *pdev, int bar) 3775 { 3776 int pos; 3777 u32 ctrl; 3778 3779 pos = pci_rebar_find_pos(pdev, bar); 3780 if (pos < 0) 3781 return pos; 3782 3783 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3784 return (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT; 3785 } 3786 3787 /** 3788 * pci_rebar_set_size - set a new size for a BAR 3789 * @pdev: PCI device 3790 * @bar: BAR to set size to 3791 * @size: new size as defined in the spec (0=1MB, 19=512GB) 3792 * 3793 * Set the new size of a BAR as defined in the spec. 3794 * Returns zero if resizing was successful, error code otherwise. 3795 */ 3796 int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size) 3797 { 3798 int pos; 3799 u32 ctrl; 3800 3801 pos = pci_rebar_find_pos(pdev, bar); 3802 if (pos < 0) 3803 return pos; 3804 3805 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3806 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 3807 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 3808 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 3809 return 0; 3810 } 3811 3812 /** 3813 * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port 3814 * @dev: the PCI device 3815 * @cap_mask: mask of desired AtomicOp sizes, including one or more of: 3816 * PCI_EXP_DEVCAP2_ATOMIC_COMP32 3817 * PCI_EXP_DEVCAP2_ATOMIC_COMP64 3818 * PCI_EXP_DEVCAP2_ATOMIC_COMP128 3819 * 3820 * Return 0 if all upstream bridges support AtomicOp routing, egress 3821 * blocking is disabled on all upstream ports, and the root port supports 3822 * the requested completion capabilities (32-bit, 64-bit and/or 128-bit 3823 * AtomicOp completion), or negative otherwise. 3824 */ 3825 int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) 3826 { 3827 struct pci_bus *bus = dev->bus; 3828 struct pci_dev *bridge; 3829 u32 cap, ctl2; 3830 3831 /* 3832 * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit 3833 * in Device Control 2 is reserved in VFs and the PF value applies 3834 * to all associated VFs. 3835 */ 3836 if (dev->is_virtfn) 3837 return -EINVAL; 3838 3839 if (!pci_is_pcie(dev)) 3840 return -EINVAL; 3841 3842 /* 3843 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be 3844 * AtomicOp requesters. For now, we only support endpoints as 3845 * requesters and root ports as completers. No endpoints as 3846 * completers, and no peer-to-peer. 3847 */ 3848 3849 switch (pci_pcie_type(dev)) { 3850 case PCI_EXP_TYPE_ENDPOINT: 3851 case PCI_EXP_TYPE_LEG_END: 3852 case PCI_EXP_TYPE_RC_END: 3853 break; 3854 default: 3855 return -EINVAL; 3856 } 3857 3858 while (bus->parent) { 3859 bridge = bus->self; 3860 3861 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3862 3863 switch (pci_pcie_type(bridge)) { 3864 /* Ensure switch ports support AtomicOp routing */ 3865 case PCI_EXP_TYPE_UPSTREAM: 3866 case PCI_EXP_TYPE_DOWNSTREAM: 3867 if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) 3868 return -EINVAL; 3869 break; 3870 3871 /* Ensure root port supports all the sizes we care about */ 3872 case PCI_EXP_TYPE_ROOT_PORT: 3873 if ((cap & cap_mask) != cap_mask) 3874 return -EINVAL; 3875 break; 3876 } 3877 3878 /* Ensure upstream ports don't block AtomicOps on egress */ 3879 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) { 3880 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, 3881 &ctl2); 3882 if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK) 3883 return -EINVAL; 3884 } 3885 3886 bus = bus->parent; 3887 } 3888 3889 pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, 3890 PCI_EXP_DEVCTL2_ATOMIC_REQ); 3891 return 0; 3892 } 3893 EXPORT_SYMBOL(pci_enable_atomic_ops_to_root); 3894 3895 /** 3896 * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge 3897 * @dev: the PCI device 3898 * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) 3899 * 3900 * Perform INTx swizzling for a device behind one level of bridge. This is 3901 * required by section 9.1 of the PCI-to-PCI bridge specification for devices 3902 * behind bridges on add-in cards. For devices with ARI enabled, the slot 3903 * number is always 0 (see the Implementation Note in section 2.2.8.1 of 3904 * the PCI Express Base Specification, Revision 2.1) 3905 */ 3906 u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin) 3907 { 3908 int slot; 3909 3910 if (pci_ari_enabled(dev->bus)) 3911 slot = 0; 3912 else 3913 slot = PCI_SLOT(dev->devfn); 3914 3915 return (((pin - 1) + slot) % 4) + 1; 3916 } 3917 3918 int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge) 3919 { 3920 u8 pin; 3921 3922 pin = dev->pin; 3923 if (!pin) 3924 return -1; 3925 3926 while (!pci_is_root_bus(dev->bus)) { 3927 pin = pci_swizzle_interrupt_pin(dev, pin); 3928 dev = dev->bus->self; 3929 } 3930 *bridge = dev; 3931 return pin; 3932 } 3933 3934 /** 3935 * pci_common_swizzle - swizzle INTx all the way to root bridge 3936 * @dev: the PCI device 3937 * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD) 3938 * 3939 * Perform INTx swizzling for a device. This traverses through all PCI-to-PCI 3940 * bridges all the way up to a PCI root bus. 3941 */ 3942 u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp) 3943 { 3944 u8 pin = *pinp; 3945 3946 while (!pci_is_root_bus(dev->bus)) { 3947 pin = pci_swizzle_interrupt_pin(dev, pin); 3948 dev = dev->bus->self; 3949 } 3950 *pinp = pin; 3951 return PCI_SLOT(dev->devfn); 3952 } 3953 EXPORT_SYMBOL_GPL(pci_common_swizzle); 3954 3955 /** 3956 * pci_release_region - Release a PCI bar 3957 * @pdev: PCI device whose resources were previously reserved by 3958 * pci_request_region() 3959 * @bar: BAR to release 3960 * 3961 * Releases the PCI I/O and memory resources previously reserved by a 3962 * successful call to pci_request_region(). Call this function only 3963 * after all use of the PCI regions has ceased. 3964 */ 3965 void pci_release_region(struct pci_dev *pdev, int bar) 3966 { 3967 struct pci_devres *dr; 3968 3969 if (pci_resource_len(pdev, bar) == 0) 3970 return; 3971 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) 3972 release_region(pci_resource_start(pdev, bar), 3973 pci_resource_len(pdev, bar)); 3974 else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) 3975 release_mem_region(pci_resource_start(pdev, bar), 3976 pci_resource_len(pdev, bar)); 3977 3978 dr = find_pci_dr(pdev); 3979 if (dr) 3980 dr->region_mask &= ~(1 << bar); 3981 } 3982 EXPORT_SYMBOL(pci_release_region); 3983 3984 /** 3985 * __pci_request_region - Reserved PCI I/O and memory resource 3986 * @pdev: PCI device whose resources are to be reserved 3987 * @bar: BAR to be reserved 3988 * @res_name: Name to be associated with resource. 3989 * @exclusive: whether the region access is exclusive or not 3990 * 3991 * Mark the PCI region associated with PCI device @pdev BAR @bar as 3992 * being reserved by owner @res_name. Do not access any 3993 * address inside the PCI regions unless this call returns 3994 * successfully. 3995 * 3996 * If @exclusive is set, then the region is marked so that userspace 3997 * is explicitly not allowed to map the resource via /dev/mem or 3998 * sysfs MMIO access. 3999 * 4000 * Returns 0 on success, or %EBUSY on error. A warning 4001 * message is also printed on failure. 4002 */ 4003 static int __pci_request_region(struct pci_dev *pdev, int bar, 4004 const char *res_name, int exclusive) 4005 { 4006 struct pci_devres *dr; 4007 4008 if (pci_resource_len(pdev, bar) == 0) 4009 return 0; 4010 4011 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) { 4012 if (!request_region(pci_resource_start(pdev, bar), 4013 pci_resource_len(pdev, bar), res_name)) 4014 goto err_out; 4015 } else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) { 4016 if (!__request_mem_region(pci_resource_start(pdev, bar), 4017 pci_resource_len(pdev, bar), res_name, 4018 exclusive)) 4019 goto err_out; 4020 } 4021 4022 dr = find_pci_dr(pdev); 4023 if (dr) 4024 dr->region_mask |= 1 << bar; 4025 4026 return 0; 4027 4028 err_out: 4029 pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar, 4030 &pdev->resource[bar]); 4031 return -EBUSY; 4032 } 4033 4034 /** 4035 * pci_request_region - Reserve PCI I/O and memory resource 4036 * @pdev: PCI device whose resources are to be reserved 4037 * @bar: BAR to be reserved 4038 * @res_name: Name to be associated with resource 4039 * 4040 * Mark the PCI region associated with PCI device @pdev BAR @bar as 4041 * being reserved by owner @res_name. Do not access any 4042 * address inside the PCI regions unless this call returns 4043 * successfully. 4044 * 4045 * Returns 0 on success, or %EBUSY on error. A warning 4046 * message is also printed on failure. 4047 */ 4048 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 4049 { 4050 return __pci_request_region(pdev, bar, res_name, 0); 4051 } 4052 EXPORT_SYMBOL(pci_request_region); 4053 4054 /** 4055 * pci_release_selected_regions - Release selected PCI I/O and memory resources 4056 * @pdev: PCI device whose resources were previously reserved 4057 * @bars: Bitmask of BARs to be released 4058 * 4059 * Release selected PCI I/O and memory resources previously reserved. 4060 * Call this function only after all use of the PCI regions has ceased. 4061 */ 4062 void pci_release_selected_regions(struct pci_dev *pdev, int bars) 4063 { 4064 int i; 4065 4066 for (i = 0; i < PCI_STD_NUM_BARS; i++) 4067 if (bars & (1 << i)) 4068 pci_release_region(pdev, i); 4069 } 4070 EXPORT_SYMBOL(pci_release_selected_regions); 4071 4072 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars, 4073 const char *res_name, int excl) 4074 { 4075 int i; 4076 4077 for (i = 0; i < PCI_STD_NUM_BARS; i++) 4078 if (bars & (1 << i)) 4079 if (__pci_request_region(pdev, i, res_name, excl)) 4080 goto err_out; 4081 return 0; 4082 4083 err_out: 4084 while (--i >= 0) 4085 if (bars & (1 << i)) 4086 pci_release_region(pdev, i); 4087 4088 return -EBUSY; 4089 } 4090 4091 4092 /** 4093 * pci_request_selected_regions - Reserve selected PCI I/O and memory resources 4094 * @pdev: PCI device whose resources are to be reserved 4095 * @bars: Bitmask of BARs to be requested 4096 * @res_name: Name to be associated with resource 4097 */ 4098 int pci_request_selected_regions(struct pci_dev *pdev, int bars, 4099 const char *res_name) 4100 { 4101 return __pci_request_selected_regions(pdev, bars, res_name, 0); 4102 } 4103 EXPORT_SYMBOL(pci_request_selected_regions); 4104 4105 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars, 4106 const char *res_name) 4107 { 4108 return __pci_request_selected_regions(pdev, bars, res_name, 4109 IORESOURCE_EXCLUSIVE); 4110 } 4111 EXPORT_SYMBOL(pci_request_selected_regions_exclusive); 4112 4113 /** 4114 * pci_release_regions - Release reserved PCI I/O and memory resources 4115 * @pdev: PCI device whose resources were previously reserved by 4116 * pci_request_regions() 4117 * 4118 * Releases all PCI I/O and memory resources previously reserved by a 4119 * successful call to pci_request_regions(). Call this function only 4120 * after all use of the PCI regions has ceased. 4121 */ 4122 4123 void pci_release_regions(struct pci_dev *pdev) 4124 { 4125 pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1); 4126 } 4127 EXPORT_SYMBOL(pci_release_regions); 4128 4129 /** 4130 * pci_request_regions - Reserve PCI I/O and memory resources 4131 * @pdev: PCI device whose resources are to be reserved 4132 * @res_name: Name to be associated with resource. 4133 * 4134 * Mark all PCI regions associated with PCI device @pdev as 4135 * being reserved by owner @res_name. Do not access any 4136 * address inside the PCI regions unless this call returns 4137 * successfully. 4138 * 4139 * Returns 0 on success, or %EBUSY on error. A warning 4140 * message is also printed on failure. 4141 */ 4142 int pci_request_regions(struct pci_dev *pdev, const char *res_name) 4143 { 4144 return pci_request_selected_regions(pdev, 4145 ((1 << PCI_STD_NUM_BARS) - 1), res_name); 4146 } 4147 EXPORT_SYMBOL(pci_request_regions); 4148 4149 /** 4150 * pci_request_regions_exclusive - Reserve PCI I/O and memory resources 4151 * @pdev: PCI device whose resources are to be reserved 4152 * @res_name: Name to be associated with resource. 4153 * 4154 * Mark all PCI regions associated with PCI device @pdev as being reserved 4155 * by owner @res_name. Do not access any address inside the PCI regions 4156 * unless this call returns successfully. 4157 * 4158 * pci_request_regions_exclusive() will mark the region so that /dev/mem 4159 * and the sysfs MMIO access will not be allowed. 4160 * 4161 * Returns 0 on success, or %EBUSY on error. A warning message is also 4162 * printed on failure. 4163 */ 4164 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name) 4165 { 4166 return pci_request_selected_regions_exclusive(pdev, 4167 ((1 << PCI_STD_NUM_BARS) - 1), res_name); 4168 } 4169 EXPORT_SYMBOL(pci_request_regions_exclusive); 4170 4171 /* 4172 * Record the PCI IO range (expressed as CPU physical address + size). 4173 * Return a negative value if an error has occurred, zero otherwise 4174 */ 4175 int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr, 4176 resource_size_t size) 4177 { 4178 int ret = 0; 4179 #ifdef PCI_IOBASE 4180 struct logic_pio_hwaddr *range; 4181 4182 if (!size || addr + size < addr) 4183 return -EINVAL; 4184 4185 range = kzalloc(sizeof(*range), GFP_ATOMIC); 4186 if (!range) 4187 return -ENOMEM; 4188 4189 range->fwnode = fwnode; 4190 range->size = size; 4191 range->hw_start = addr; 4192 range->flags = LOGIC_PIO_CPU_MMIO; 4193 4194 ret = logic_pio_register_range(range); 4195 if (ret) 4196 kfree(range); 4197 4198 /* Ignore duplicates due to deferred probing */ 4199 if (ret == -EEXIST) 4200 ret = 0; 4201 #endif 4202 4203 return ret; 4204 } 4205 4206 phys_addr_t pci_pio_to_address(unsigned long pio) 4207 { 4208 #ifdef PCI_IOBASE 4209 if (pio < MMIO_UPPER_LIMIT) 4210 return logic_pio_to_hwaddr(pio); 4211 #endif 4212 4213 return (phys_addr_t) OF_BAD_ADDR; 4214 } 4215 EXPORT_SYMBOL_GPL(pci_pio_to_address); 4216 4217 unsigned long __weak pci_address_to_pio(phys_addr_t address) 4218 { 4219 #ifdef PCI_IOBASE 4220 return logic_pio_trans_cpuaddr(address); 4221 #else 4222 if (address > IO_SPACE_LIMIT) 4223 return (unsigned long)-1; 4224 4225 return (unsigned long) address; 4226 #endif 4227 } 4228 4229 /** 4230 * pci_remap_iospace - Remap the memory mapped I/O space 4231 * @res: Resource describing the I/O space 4232 * @phys_addr: physical address of range to be mapped 4233 * 4234 * Remap the memory mapped I/O space described by the @res and the CPU 4235 * physical address @phys_addr into virtual address space. Only 4236 * architectures that have memory mapped IO functions defined (and the 4237 * PCI_IOBASE value defined) should call this function. 4238 */ 4239 #ifndef pci_remap_iospace 4240 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) 4241 { 4242 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 4243 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4244 4245 if (!(res->flags & IORESOURCE_IO)) 4246 return -EINVAL; 4247 4248 if (res->end > IO_SPACE_LIMIT) 4249 return -EINVAL; 4250 4251 return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr, 4252 pgprot_device(PAGE_KERNEL)); 4253 #else 4254 /* 4255 * This architecture does not have memory mapped I/O space, 4256 * so this function should never be called 4257 */ 4258 WARN_ONCE(1, "This architecture does not support memory mapped I/O\n"); 4259 return -ENODEV; 4260 #endif 4261 } 4262 EXPORT_SYMBOL(pci_remap_iospace); 4263 #endif 4264 4265 /** 4266 * pci_unmap_iospace - Unmap the memory mapped I/O space 4267 * @res: resource to be unmapped 4268 * 4269 * Unmap the CPU virtual address @res from virtual address space. Only 4270 * architectures that have memory mapped IO functions defined (and the 4271 * PCI_IOBASE value defined) should call this function. 4272 */ 4273 void pci_unmap_iospace(struct resource *res) 4274 { 4275 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 4276 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4277 4278 vunmap_range(vaddr, vaddr + resource_size(res)); 4279 #endif 4280 } 4281 EXPORT_SYMBOL(pci_unmap_iospace); 4282 4283 static void devm_pci_unmap_iospace(struct device *dev, void *ptr) 4284 { 4285 struct resource **res = ptr; 4286 4287 pci_unmap_iospace(*res); 4288 } 4289 4290 /** 4291 * devm_pci_remap_iospace - Managed pci_remap_iospace() 4292 * @dev: Generic device to remap IO address for 4293 * @res: Resource describing the I/O space 4294 * @phys_addr: physical address of range to be mapped 4295 * 4296 * Managed pci_remap_iospace(). Map is automatically unmapped on driver 4297 * detach. 4298 */ 4299 int devm_pci_remap_iospace(struct device *dev, const struct resource *res, 4300 phys_addr_t phys_addr) 4301 { 4302 const struct resource **ptr; 4303 int error; 4304 4305 ptr = devres_alloc(devm_pci_unmap_iospace, sizeof(*ptr), GFP_KERNEL); 4306 if (!ptr) 4307 return -ENOMEM; 4308 4309 error = pci_remap_iospace(res, phys_addr); 4310 if (error) { 4311 devres_free(ptr); 4312 } else { 4313 *ptr = res; 4314 devres_add(dev, ptr); 4315 } 4316 4317 return error; 4318 } 4319 EXPORT_SYMBOL(devm_pci_remap_iospace); 4320 4321 /** 4322 * devm_pci_remap_cfgspace - Managed pci_remap_cfgspace() 4323 * @dev: Generic device to remap IO address for 4324 * @offset: Resource address to map 4325 * @size: Size of map 4326 * 4327 * Managed pci_remap_cfgspace(). Map is automatically unmapped on driver 4328 * detach. 4329 */ 4330 void __iomem *devm_pci_remap_cfgspace(struct device *dev, 4331 resource_size_t offset, 4332 resource_size_t size) 4333 { 4334 void __iomem **ptr, *addr; 4335 4336 ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL); 4337 if (!ptr) 4338 return NULL; 4339 4340 addr = pci_remap_cfgspace(offset, size); 4341 if (addr) { 4342 *ptr = addr; 4343 devres_add(dev, ptr); 4344 } else 4345 devres_free(ptr); 4346 4347 return addr; 4348 } 4349 EXPORT_SYMBOL(devm_pci_remap_cfgspace); 4350 4351 /** 4352 * devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource 4353 * @dev: generic device to handle the resource for 4354 * @res: configuration space resource to be handled 4355 * 4356 * Checks that a resource is a valid memory region, requests the memory 4357 * region and ioremaps with pci_remap_cfgspace() API that ensures the 4358 * proper PCI configuration space memory attributes are guaranteed. 4359 * 4360 * All operations are managed and will be undone on driver detach. 4361 * 4362 * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code 4363 * on failure. Usage example:: 4364 * 4365 * res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4366 * base = devm_pci_remap_cfg_resource(&pdev->dev, res); 4367 * if (IS_ERR(base)) 4368 * return PTR_ERR(base); 4369 */ 4370 void __iomem *devm_pci_remap_cfg_resource(struct device *dev, 4371 struct resource *res) 4372 { 4373 resource_size_t size; 4374 const char *name; 4375 void __iomem *dest_ptr; 4376 4377 BUG_ON(!dev); 4378 4379 if (!res || resource_type(res) != IORESOURCE_MEM) { 4380 dev_err(dev, "invalid resource\n"); 4381 return IOMEM_ERR_PTR(-EINVAL); 4382 } 4383 4384 size = resource_size(res); 4385 4386 if (res->name) 4387 name = devm_kasprintf(dev, GFP_KERNEL, "%s %s", dev_name(dev), 4388 res->name); 4389 else 4390 name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL); 4391 if (!name) 4392 return IOMEM_ERR_PTR(-ENOMEM); 4393 4394 if (!devm_request_mem_region(dev, res->start, size, name)) { 4395 dev_err(dev, "can't request region for resource %pR\n", res); 4396 return IOMEM_ERR_PTR(-EBUSY); 4397 } 4398 4399 dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size); 4400 if (!dest_ptr) { 4401 dev_err(dev, "ioremap failed for resource %pR\n", res); 4402 devm_release_mem_region(dev, res->start, size); 4403 dest_ptr = IOMEM_ERR_PTR(-ENOMEM); 4404 } 4405 4406 return dest_ptr; 4407 } 4408 EXPORT_SYMBOL(devm_pci_remap_cfg_resource); 4409 4410 static void __pci_set_master(struct pci_dev *dev, bool enable) 4411 { 4412 u16 old_cmd, cmd; 4413 4414 pci_read_config_word(dev, PCI_COMMAND, &old_cmd); 4415 if (enable) 4416 cmd = old_cmd | PCI_COMMAND_MASTER; 4417 else 4418 cmd = old_cmd & ~PCI_COMMAND_MASTER; 4419 if (cmd != old_cmd) { 4420 pci_dbg(dev, "%s bus mastering\n", 4421 enable ? "enabling" : "disabling"); 4422 pci_write_config_word(dev, PCI_COMMAND, cmd); 4423 } 4424 dev->is_busmaster = enable; 4425 } 4426 4427 /** 4428 * pcibios_setup - process "pci=" kernel boot arguments 4429 * @str: string used to pass in "pci=" kernel boot arguments 4430 * 4431 * Process kernel boot arguments. This is the default implementation. 4432 * Architecture specific implementations can override this as necessary. 4433 */ 4434 char * __weak __init pcibios_setup(char *str) 4435 { 4436 return str; 4437 } 4438 4439 /** 4440 * pcibios_set_master - enable PCI bus-mastering for device dev 4441 * @dev: the PCI device to enable 4442 * 4443 * Enables PCI bus-mastering for the device. This is the default 4444 * implementation. Architecture specific implementations can override 4445 * this if necessary. 4446 */ 4447 void __weak pcibios_set_master(struct pci_dev *dev) 4448 { 4449 u8 lat; 4450 4451 /* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */ 4452 if (pci_is_pcie(dev)) 4453 return; 4454 4455 pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat); 4456 if (lat < 16) 4457 lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency; 4458 else if (lat > pcibios_max_latency) 4459 lat = pcibios_max_latency; 4460 else 4461 return; 4462 4463 pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat); 4464 } 4465 4466 /** 4467 * pci_set_master - enables bus-mastering for device dev 4468 * @dev: the PCI device to enable 4469 * 4470 * Enables bus-mastering on the device and calls pcibios_set_master() 4471 * to do the needed arch specific settings. 4472 */ 4473 void pci_set_master(struct pci_dev *dev) 4474 { 4475 __pci_set_master(dev, true); 4476 pcibios_set_master(dev); 4477 } 4478 EXPORT_SYMBOL(pci_set_master); 4479 4480 /** 4481 * pci_clear_master - disables bus-mastering for device dev 4482 * @dev: the PCI device to disable 4483 */ 4484 void pci_clear_master(struct pci_dev *dev) 4485 { 4486 __pci_set_master(dev, false); 4487 } 4488 EXPORT_SYMBOL(pci_clear_master); 4489 4490 /** 4491 * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed 4492 * @dev: the PCI device for which MWI is to be enabled 4493 * 4494 * Helper function for pci_set_mwi. 4495 * Originally copied from drivers/net/acenic.c. 4496 * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>. 4497 * 4498 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4499 */ 4500 int pci_set_cacheline_size(struct pci_dev *dev) 4501 { 4502 u8 cacheline_size; 4503 4504 if (!pci_cache_line_size) 4505 return -EINVAL; 4506 4507 /* Validate current setting: the PCI_CACHE_LINE_SIZE must be 4508 equal to or multiple of the right value. */ 4509 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4510 if (cacheline_size >= pci_cache_line_size && 4511 (cacheline_size % pci_cache_line_size) == 0) 4512 return 0; 4513 4514 /* Write the correct value. */ 4515 pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size); 4516 /* Read it back. */ 4517 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4518 if (cacheline_size == pci_cache_line_size) 4519 return 0; 4520 4521 pci_dbg(dev, "cache line size of %d is not supported\n", 4522 pci_cache_line_size << 2); 4523 4524 return -EINVAL; 4525 } 4526 EXPORT_SYMBOL_GPL(pci_set_cacheline_size); 4527 4528 /** 4529 * pci_set_mwi - enables memory-write-invalidate PCI transaction 4530 * @dev: the PCI device for which MWI is enabled 4531 * 4532 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4533 * 4534 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4535 */ 4536 int pci_set_mwi(struct pci_dev *dev) 4537 { 4538 #ifdef PCI_DISABLE_MWI 4539 return 0; 4540 #else 4541 int rc; 4542 u16 cmd; 4543 4544 rc = pci_set_cacheline_size(dev); 4545 if (rc) 4546 return rc; 4547 4548 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4549 if (!(cmd & PCI_COMMAND_INVALIDATE)) { 4550 pci_dbg(dev, "enabling Mem-Wr-Inval\n"); 4551 cmd |= PCI_COMMAND_INVALIDATE; 4552 pci_write_config_word(dev, PCI_COMMAND, cmd); 4553 } 4554 return 0; 4555 #endif 4556 } 4557 EXPORT_SYMBOL(pci_set_mwi); 4558 4559 /** 4560 * pcim_set_mwi - a device-managed pci_set_mwi() 4561 * @dev: the PCI device for which MWI is enabled 4562 * 4563 * Managed pci_set_mwi(). 4564 * 4565 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4566 */ 4567 int pcim_set_mwi(struct pci_dev *dev) 4568 { 4569 struct pci_devres *dr; 4570 4571 dr = find_pci_dr(dev); 4572 if (!dr) 4573 return -ENOMEM; 4574 4575 dr->mwi = 1; 4576 return pci_set_mwi(dev); 4577 } 4578 EXPORT_SYMBOL(pcim_set_mwi); 4579 4580 /** 4581 * pci_try_set_mwi - enables memory-write-invalidate PCI transaction 4582 * @dev: the PCI device for which MWI is enabled 4583 * 4584 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4585 * Callers are not required to check the return value. 4586 * 4587 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4588 */ 4589 int pci_try_set_mwi(struct pci_dev *dev) 4590 { 4591 #ifdef PCI_DISABLE_MWI 4592 return 0; 4593 #else 4594 return pci_set_mwi(dev); 4595 #endif 4596 } 4597 EXPORT_SYMBOL(pci_try_set_mwi); 4598 4599 /** 4600 * pci_clear_mwi - disables Memory-Write-Invalidate for device dev 4601 * @dev: the PCI device to disable 4602 * 4603 * Disables PCI Memory-Write-Invalidate transaction on the device 4604 */ 4605 void pci_clear_mwi(struct pci_dev *dev) 4606 { 4607 #ifndef PCI_DISABLE_MWI 4608 u16 cmd; 4609 4610 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4611 if (cmd & PCI_COMMAND_INVALIDATE) { 4612 cmd &= ~PCI_COMMAND_INVALIDATE; 4613 pci_write_config_word(dev, PCI_COMMAND, cmd); 4614 } 4615 #endif 4616 } 4617 EXPORT_SYMBOL(pci_clear_mwi); 4618 4619 /** 4620 * pci_disable_parity - disable parity checking for device 4621 * @dev: the PCI device to operate on 4622 * 4623 * Disable parity checking for device @dev 4624 */ 4625 void pci_disable_parity(struct pci_dev *dev) 4626 { 4627 u16 cmd; 4628 4629 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4630 if (cmd & PCI_COMMAND_PARITY) { 4631 cmd &= ~PCI_COMMAND_PARITY; 4632 pci_write_config_word(dev, PCI_COMMAND, cmd); 4633 } 4634 } 4635 4636 /** 4637 * pci_intx - enables/disables PCI INTx for device dev 4638 * @pdev: the PCI device to operate on 4639 * @enable: boolean: whether to enable or disable PCI INTx 4640 * 4641 * Enables/disables PCI INTx for device @pdev 4642 */ 4643 void pci_intx(struct pci_dev *pdev, int enable) 4644 { 4645 u16 pci_command, new; 4646 4647 pci_read_config_word(pdev, PCI_COMMAND, &pci_command); 4648 4649 if (enable) 4650 new = pci_command & ~PCI_COMMAND_INTX_DISABLE; 4651 else 4652 new = pci_command | PCI_COMMAND_INTX_DISABLE; 4653 4654 if (new != pci_command) { 4655 struct pci_devres *dr; 4656 4657 pci_write_config_word(pdev, PCI_COMMAND, new); 4658 4659 dr = find_pci_dr(pdev); 4660 if (dr && !dr->restore_intx) { 4661 dr->restore_intx = 1; 4662 dr->orig_intx = !enable; 4663 } 4664 } 4665 } 4666 EXPORT_SYMBOL_GPL(pci_intx); 4667 4668 static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask) 4669 { 4670 struct pci_bus *bus = dev->bus; 4671 bool mask_updated = true; 4672 u32 cmd_status_dword; 4673 u16 origcmd, newcmd; 4674 unsigned long flags; 4675 bool irq_pending; 4676 4677 /* 4678 * We do a single dword read to retrieve both command and status. 4679 * Document assumptions that make this possible. 4680 */ 4681 BUILD_BUG_ON(PCI_COMMAND % 4); 4682 BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS); 4683 4684 raw_spin_lock_irqsave(&pci_lock, flags); 4685 4686 bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword); 4687 4688 irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT; 4689 4690 /* 4691 * Check interrupt status register to see whether our device 4692 * triggered the interrupt (when masking) or the next IRQ is 4693 * already pending (when unmasking). 4694 */ 4695 if (mask != irq_pending) { 4696 mask_updated = false; 4697 goto done; 4698 } 4699 4700 origcmd = cmd_status_dword; 4701 newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE; 4702 if (mask) 4703 newcmd |= PCI_COMMAND_INTX_DISABLE; 4704 if (newcmd != origcmd) 4705 bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd); 4706 4707 done: 4708 raw_spin_unlock_irqrestore(&pci_lock, flags); 4709 4710 return mask_updated; 4711 } 4712 4713 /** 4714 * pci_check_and_mask_intx - mask INTx on pending interrupt 4715 * @dev: the PCI device to operate on 4716 * 4717 * Check if the device dev has its INTx line asserted, mask it and return 4718 * true in that case. False is returned if no interrupt was pending. 4719 */ 4720 bool pci_check_and_mask_intx(struct pci_dev *dev) 4721 { 4722 return pci_check_and_set_intx_mask(dev, true); 4723 } 4724 EXPORT_SYMBOL_GPL(pci_check_and_mask_intx); 4725 4726 /** 4727 * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending 4728 * @dev: the PCI device to operate on 4729 * 4730 * Check if the device dev has its INTx line asserted, unmask it if not and 4731 * return true. False is returned and the mask remains active if there was 4732 * still an interrupt pending. 4733 */ 4734 bool pci_check_and_unmask_intx(struct pci_dev *dev) 4735 { 4736 return pci_check_and_set_intx_mask(dev, false); 4737 } 4738 EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx); 4739 4740 /** 4741 * pci_wait_for_pending_transaction - wait for pending transaction 4742 * @dev: the PCI device to operate on 4743 * 4744 * Return 0 if transaction is pending 1 otherwise. 4745 */ 4746 int pci_wait_for_pending_transaction(struct pci_dev *dev) 4747 { 4748 if (!pci_is_pcie(dev)) 4749 return 1; 4750 4751 return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA, 4752 PCI_EXP_DEVSTA_TRPND); 4753 } 4754 EXPORT_SYMBOL(pci_wait_for_pending_transaction); 4755 4756 /** 4757 * pcie_flr - initiate a PCIe function level reset 4758 * @dev: device to reset 4759 * 4760 * Initiate a function level reset unconditionally on @dev without 4761 * checking any flags and DEVCAP 4762 */ 4763 int pcie_flr(struct pci_dev *dev) 4764 { 4765 if (!pci_wait_for_pending_transaction(dev)) 4766 pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n"); 4767 4768 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); 4769 4770 if (dev->imm_ready) 4771 return 0; 4772 4773 /* 4774 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within 4775 * 100ms, but may silently discard requests while the FLR is in 4776 * progress. Wait 100ms before trying to access the device. 4777 */ 4778 msleep(100); 4779 4780 return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS); 4781 } 4782 EXPORT_SYMBOL_GPL(pcie_flr); 4783 4784 /** 4785 * pcie_reset_flr - initiate a PCIe function level reset 4786 * @dev: device to reset 4787 * @probe: if true, return 0 if device can be reset this way 4788 * 4789 * Initiate a function level reset on @dev. 4790 */ 4791 int pcie_reset_flr(struct pci_dev *dev, bool probe) 4792 { 4793 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4794 return -ENOTTY; 4795 4796 if (!(dev->devcap & PCI_EXP_DEVCAP_FLR)) 4797 return -ENOTTY; 4798 4799 if (probe) 4800 return 0; 4801 4802 return pcie_flr(dev); 4803 } 4804 EXPORT_SYMBOL_GPL(pcie_reset_flr); 4805 4806 static int pci_af_flr(struct pci_dev *dev, bool probe) 4807 { 4808 int pos; 4809 u8 cap; 4810 4811 pos = pci_find_capability(dev, PCI_CAP_ID_AF); 4812 if (!pos) 4813 return -ENOTTY; 4814 4815 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4816 return -ENOTTY; 4817 4818 pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); 4819 if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR)) 4820 return -ENOTTY; 4821 4822 if (probe) 4823 return 0; 4824 4825 /* 4826 * Wait for Transaction Pending bit to clear. A word-aligned test 4827 * is used, so we use the control offset rather than status and shift 4828 * the test bit to match. 4829 */ 4830 if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL, 4831 PCI_AF_STATUS_TP << 8)) 4832 pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n"); 4833 4834 pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR); 4835 4836 if (dev->imm_ready) 4837 return 0; 4838 4839 /* 4840 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006, 4841 * updated 27 July 2006; a device must complete an FLR within 4842 * 100ms, but may silently discard requests while the FLR is in 4843 * progress. Wait 100ms before trying to access the device. 4844 */ 4845 msleep(100); 4846 4847 return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS); 4848 } 4849 4850 /** 4851 * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0. 4852 * @dev: Device to reset. 4853 * @probe: if true, return 0 if the device can be reset this way. 4854 * 4855 * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is 4856 * unset, it will be reinitialized internally when going from PCI_D3hot to 4857 * PCI_D0. If that's the case and the device is not in a low-power state 4858 * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset. 4859 * 4860 * NOTE: This causes the caller to sleep for twice the device power transition 4861 * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms 4862 * by default (i.e. unless the @dev's d3hot_delay field has a different value). 4863 * Moreover, only devices in D0 can be reset by this function. 4864 */ 4865 static int pci_pm_reset(struct pci_dev *dev, bool probe) 4866 { 4867 u16 csr; 4868 4869 if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET) 4870 return -ENOTTY; 4871 4872 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr); 4873 if (csr & PCI_PM_CTRL_NO_SOFT_RESET) 4874 return -ENOTTY; 4875 4876 if (probe) 4877 return 0; 4878 4879 if (dev->current_state != PCI_D0) 4880 return -EINVAL; 4881 4882 csr &= ~PCI_PM_CTRL_STATE_MASK; 4883 csr |= PCI_D3hot; 4884 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4885 pci_dev_d3_sleep(dev); 4886 4887 csr &= ~PCI_PM_CTRL_STATE_MASK; 4888 csr |= PCI_D0; 4889 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4890 pci_dev_d3_sleep(dev); 4891 4892 return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS); 4893 } 4894 4895 /** 4896 * pcie_wait_for_link_status - Wait for link status change 4897 * @pdev: Device whose link to wait for. 4898 * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE. 4899 * @active: Waiting for active or inactive? 4900 * 4901 * Return 0 if successful, or -ETIMEDOUT if status has not changed within 4902 * PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds. 4903 */ 4904 static int pcie_wait_for_link_status(struct pci_dev *pdev, 4905 bool use_lt, bool active) 4906 { 4907 u16 lnksta_mask, lnksta_match; 4908 unsigned long end_jiffies; 4909 u16 lnksta; 4910 4911 lnksta_mask = use_lt ? PCI_EXP_LNKSTA_LT : PCI_EXP_LNKSTA_DLLLA; 4912 lnksta_match = active ? lnksta_mask : 0; 4913 4914 end_jiffies = jiffies + msecs_to_jiffies(PCIE_LINK_RETRAIN_TIMEOUT_MS); 4915 do { 4916 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); 4917 if ((lnksta & lnksta_mask) == lnksta_match) 4918 return 0; 4919 msleep(1); 4920 } while (time_before(jiffies, end_jiffies)); 4921 4922 return -ETIMEDOUT; 4923 } 4924 4925 /** 4926 * pcie_retrain_link - Request a link retrain and wait for it to complete 4927 * @pdev: Device whose link to retrain. 4928 * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE, for status. 4929 * 4930 * Retrain completion status is retrieved from the Link Status Register 4931 * according to @use_lt. It is not verified whether the use of the DLLLA 4932 * bit is valid. 4933 * 4934 * Return 0 if successful, or -ETIMEDOUT if training has not completed 4935 * within PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds. 4936 */ 4937 int pcie_retrain_link(struct pci_dev *pdev, bool use_lt) 4938 { 4939 int rc; 4940 4941 /* 4942 * Ensure the updated LNKCTL parameters are used during link 4943 * training by checking that there is no ongoing link training to 4944 * avoid LTSSM race as recommended in Implementation Note at the 4945 * end of PCIe r6.0.1 sec 7.5.3.7. 4946 */ 4947 rc = pcie_wait_for_link_status(pdev, use_lt, !use_lt); 4948 if (rc) 4949 return rc; 4950 4951 pcie_capability_set_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL); 4952 if (pdev->clear_retrain_link) { 4953 /* 4954 * Due to an erratum in some devices the Retrain Link bit 4955 * needs to be cleared again manually to allow the link 4956 * training to succeed. 4957 */ 4958 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL); 4959 } 4960 4961 return pcie_wait_for_link_status(pdev, use_lt, !use_lt); 4962 } 4963 4964 /** 4965 * pcie_wait_for_link_delay - Wait until link is active or inactive 4966 * @pdev: Bridge device 4967 * @active: waiting for active or inactive? 4968 * @delay: Delay to wait after link has become active (in ms) 4969 * 4970 * Use this to wait till link becomes active or inactive. 4971 */ 4972 static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active, 4973 int delay) 4974 { 4975 int rc; 4976 4977 /* 4978 * Some controllers might not implement link active reporting. In this 4979 * case, we wait for 1000 ms + any delay requested by the caller. 4980 */ 4981 if (!pdev->link_active_reporting) { 4982 msleep(PCIE_LINK_RETRAIN_TIMEOUT_MS + delay); 4983 return true; 4984 } 4985 4986 /* 4987 * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms, 4988 * after which we should expect an link active if the reset was 4989 * successful. If so, software must wait a minimum 100ms before sending 4990 * configuration requests to devices downstream this port. 4991 * 4992 * If the link fails to activate, either the device was physically 4993 * removed or the link is permanently failed. 4994 */ 4995 if (active) 4996 msleep(20); 4997 rc = pcie_wait_for_link_status(pdev, false, active); 4998 if (active) { 4999 if (rc) 5000 rc = pcie_failed_link_retrain(pdev); 5001 if (rc) 5002 return false; 5003 5004 msleep(delay); 5005 return true; 5006 } 5007 5008 if (rc) 5009 return false; 5010 5011 return true; 5012 } 5013 5014 /** 5015 * pcie_wait_for_link - Wait until link is active or inactive 5016 * @pdev: Bridge device 5017 * @active: waiting for active or inactive? 5018 * 5019 * Use this to wait till link becomes active or inactive. 5020 */ 5021 bool pcie_wait_for_link(struct pci_dev *pdev, bool active) 5022 { 5023 return pcie_wait_for_link_delay(pdev, active, 100); 5024 } 5025 5026 /* 5027 * Find maximum D3cold delay required by all the devices on the bus. The 5028 * spec says 100 ms, but firmware can lower it and we allow drivers to 5029 * increase it as well. 5030 * 5031 * Called with @pci_bus_sem locked for reading. 5032 */ 5033 static int pci_bus_max_d3cold_delay(const struct pci_bus *bus) 5034 { 5035 const struct pci_dev *pdev; 5036 int min_delay = 100; 5037 int max_delay = 0; 5038 5039 list_for_each_entry(pdev, &bus->devices, bus_list) { 5040 if (pdev->d3cold_delay < min_delay) 5041 min_delay = pdev->d3cold_delay; 5042 if (pdev->d3cold_delay > max_delay) 5043 max_delay = pdev->d3cold_delay; 5044 } 5045 5046 return max(min_delay, max_delay); 5047 } 5048 5049 /** 5050 * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible 5051 * @dev: PCI bridge 5052 * @reset_type: reset type in human-readable form 5053 * 5054 * Handle necessary delays before access to the devices on the secondary 5055 * side of the bridge are permitted after D3cold to D0 transition 5056 * or Conventional Reset. 5057 * 5058 * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For 5059 * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section 5060 * 4.3.2. 5061 * 5062 * Return 0 on success or -ENOTTY if the first device on the secondary bus 5063 * failed to become accessible. 5064 */ 5065 int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type) 5066 { 5067 struct pci_dev *child; 5068 int delay; 5069 5070 if (pci_dev_is_disconnected(dev)) 5071 return 0; 5072 5073 if (!pci_is_bridge(dev)) 5074 return 0; 5075 5076 down_read(&pci_bus_sem); 5077 5078 /* 5079 * We only deal with devices that are present currently on the bus. 5080 * For any hot-added devices the access delay is handled in pciehp 5081 * board_added(). In case of ACPI hotplug the firmware is expected 5082 * to configure the devices before OS is notified. 5083 */ 5084 if (!dev->subordinate || list_empty(&dev->subordinate->devices)) { 5085 up_read(&pci_bus_sem); 5086 return 0; 5087 } 5088 5089 /* Take d3cold_delay requirements into account */ 5090 delay = pci_bus_max_d3cold_delay(dev->subordinate); 5091 if (!delay) { 5092 up_read(&pci_bus_sem); 5093 return 0; 5094 } 5095 5096 child = list_first_entry(&dev->subordinate->devices, struct pci_dev, 5097 bus_list); 5098 up_read(&pci_bus_sem); 5099 5100 /* 5101 * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before 5102 * accessing the device after reset (that is 1000 ms + 100 ms). 5103 */ 5104 if (!pci_is_pcie(dev)) { 5105 pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay); 5106 msleep(1000 + delay); 5107 return 0; 5108 } 5109 5110 /* 5111 * For PCIe downstream and root ports that do not support speeds 5112 * greater than 5 GT/s need to wait minimum 100 ms. For higher 5113 * speeds (gen3) we need to wait first for the data link layer to 5114 * become active. 5115 * 5116 * However, 100 ms is the minimum and the PCIe spec says the 5117 * software must allow at least 1s before it can determine that the 5118 * device that did not respond is a broken device. Also device can 5119 * take longer than that to respond if it indicates so through Request 5120 * Retry Status completions. 5121 * 5122 * Therefore we wait for 100 ms and check for the device presence 5123 * until the timeout expires. 5124 */ 5125 if (!pcie_downstream_port(dev)) 5126 return 0; 5127 5128 if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) { 5129 u16 status; 5130 5131 pci_dbg(dev, "waiting %d ms for downstream link\n", delay); 5132 msleep(delay); 5133 5134 if (!pci_dev_wait(child, reset_type, PCI_RESET_WAIT - delay)) 5135 return 0; 5136 5137 /* 5138 * If the port supports active link reporting we now check 5139 * whether the link is active and if not bail out early with 5140 * the assumption that the device is not present anymore. 5141 */ 5142 if (!dev->link_active_reporting) 5143 return -ENOTTY; 5144 5145 pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &status); 5146 if (!(status & PCI_EXP_LNKSTA_DLLLA)) 5147 return -ENOTTY; 5148 5149 return pci_dev_wait(child, reset_type, 5150 PCIE_RESET_READY_POLL_MS - PCI_RESET_WAIT); 5151 } 5152 5153 pci_dbg(dev, "waiting %d ms for downstream link, after activation\n", 5154 delay); 5155 if (!pcie_wait_for_link_delay(dev, true, delay)) { 5156 /* Did not train, no need to wait any further */ 5157 pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n"); 5158 return -ENOTTY; 5159 } 5160 5161 return pci_dev_wait(child, reset_type, 5162 PCIE_RESET_READY_POLL_MS - delay); 5163 } 5164 5165 void pci_reset_secondary_bus(struct pci_dev *dev) 5166 { 5167 u16 ctrl; 5168 5169 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl); 5170 ctrl |= PCI_BRIDGE_CTL_BUS_RESET; 5171 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 5172 5173 /* 5174 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double 5175 * this to 2ms to ensure that we meet the minimum requirement. 5176 */ 5177 msleep(2); 5178 5179 ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; 5180 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 5181 } 5182 5183 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) 5184 { 5185 pci_reset_secondary_bus(dev); 5186 } 5187 5188 /** 5189 * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge. 5190 * @dev: Bridge device 5191 * 5192 * Use the bridge control register to assert reset on the secondary bus. 5193 * Devices on the secondary bus are left in power-on state. 5194 */ 5195 int pci_bridge_secondary_bus_reset(struct pci_dev *dev) 5196 { 5197 pcibios_reset_secondary_bus(dev); 5198 5199 return pci_bridge_wait_for_secondary_bus(dev, "bus reset"); 5200 } 5201 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset); 5202 5203 static int pci_parent_bus_reset(struct pci_dev *dev, bool probe) 5204 { 5205 struct pci_dev *pdev; 5206 5207 if (pci_is_root_bus(dev->bus) || dev->subordinate || 5208 !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 5209 return -ENOTTY; 5210 5211 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 5212 if (pdev != dev) 5213 return -ENOTTY; 5214 5215 if (probe) 5216 return 0; 5217 5218 return pci_bridge_secondary_bus_reset(dev->bus->self); 5219 } 5220 5221 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe) 5222 { 5223 int rc = -ENOTTY; 5224 5225 if (!hotplug || !try_module_get(hotplug->owner)) 5226 return rc; 5227 5228 if (hotplug->ops->reset_slot) 5229 rc = hotplug->ops->reset_slot(hotplug, probe); 5230 5231 module_put(hotplug->owner); 5232 5233 return rc; 5234 } 5235 5236 static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe) 5237 { 5238 if (dev->multifunction || dev->subordinate || !dev->slot || 5239 dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 5240 return -ENOTTY; 5241 5242 return pci_reset_hotplug_slot(dev->slot->hotplug, probe); 5243 } 5244 5245 static int pci_reset_bus_function(struct pci_dev *dev, bool probe) 5246 { 5247 int rc; 5248 5249 rc = pci_dev_reset_slot_function(dev, probe); 5250 if (rc != -ENOTTY) 5251 return rc; 5252 return pci_parent_bus_reset(dev, probe); 5253 } 5254 5255 void pci_dev_lock(struct pci_dev *dev) 5256 { 5257 /* block PM suspend, driver probe, etc. */ 5258 device_lock(&dev->dev); 5259 pci_cfg_access_lock(dev); 5260 } 5261 EXPORT_SYMBOL_GPL(pci_dev_lock); 5262 5263 /* Return 1 on successful lock, 0 on contention */ 5264 int pci_dev_trylock(struct pci_dev *dev) 5265 { 5266 if (device_trylock(&dev->dev)) { 5267 if (pci_cfg_access_trylock(dev)) 5268 return 1; 5269 device_unlock(&dev->dev); 5270 } 5271 5272 return 0; 5273 } 5274 EXPORT_SYMBOL_GPL(pci_dev_trylock); 5275 5276 void pci_dev_unlock(struct pci_dev *dev) 5277 { 5278 pci_cfg_access_unlock(dev); 5279 device_unlock(&dev->dev); 5280 } 5281 EXPORT_SYMBOL_GPL(pci_dev_unlock); 5282 5283 static void pci_dev_save_and_disable(struct pci_dev *dev) 5284 { 5285 const struct pci_error_handlers *err_handler = 5286 dev->driver ? dev->driver->err_handler : NULL; 5287 5288 /* 5289 * dev->driver->err_handler->reset_prepare() is protected against 5290 * races with ->remove() by the device lock, which must be held by 5291 * the caller. 5292 */ 5293 if (err_handler && err_handler->reset_prepare) 5294 err_handler->reset_prepare(dev); 5295 5296 /* 5297 * Wake-up device prior to save. PM registers default to D0 after 5298 * reset and a simple register restore doesn't reliably return 5299 * to a non-D0 state anyway. 5300 */ 5301 pci_set_power_state(dev, PCI_D0); 5302 5303 pci_save_state(dev); 5304 /* 5305 * Disable the device by clearing the Command register, except for 5306 * INTx-disable which is set. This not only disables MMIO and I/O port 5307 * BARs, but also prevents the device from being Bus Master, preventing 5308 * DMA from the device including MSI/MSI-X interrupts. For PCI 2.3 5309 * compliant devices, INTx-disable prevents legacy interrupts. 5310 */ 5311 pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 5312 } 5313 5314 static void pci_dev_restore(struct pci_dev *dev) 5315 { 5316 const struct pci_error_handlers *err_handler = 5317 dev->driver ? dev->driver->err_handler : NULL; 5318 5319 pci_restore_state(dev); 5320 5321 /* 5322 * dev->driver->err_handler->reset_done() is protected against 5323 * races with ->remove() by the device lock, which must be held by 5324 * the caller. 5325 */ 5326 if (err_handler && err_handler->reset_done) 5327 err_handler->reset_done(dev); 5328 } 5329 5330 /* dev->reset_methods[] is a 0-terminated list of indices into this array */ 5331 static const struct pci_reset_fn_method pci_reset_fn_methods[] = { 5332 { }, 5333 { pci_dev_specific_reset, .name = "device_specific" }, 5334 { pci_dev_acpi_reset, .name = "acpi" }, 5335 { pcie_reset_flr, .name = "flr" }, 5336 { pci_af_flr, .name = "af_flr" }, 5337 { pci_pm_reset, .name = "pm" }, 5338 { pci_reset_bus_function, .name = "bus" }, 5339 }; 5340 5341 static ssize_t reset_method_show(struct device *dev, 5342 struct device_attribute *attr, char *buf) 5343 { 5344 struct pci_dev *pdev = to_pci_dev(dev); 5345 ssize_t len = 0; 5346 int i, m; 5347 5348 for (i = 0; i < PCI_NUM_RESET_METHODS; i++) { 5349 m = pdev->reset_methods[i]; 5350 if (!m) 5351 break; 5352 5353 len += sysfs_emit_at(buf, len, "%s%s", len ? " " : "", 5354 pci_reset_fn_methods[m].name); 5355 } 5356 5357 if (len) 5358 len += sysfs_emit_at(buf, len, "\n"); 5359 5360 return len; 5361 } 5362 5363 static int reset_method_lookup(const char *name) 5364 { 5365 int m; 5366 5367 for (m = 1; m < PCI_NUM_RESET_METHODS; m++) { 5368 if (sysfs_streq(name, pci_reset_fn_methods[m].name)) 5369 return m; 5370 } 5371 5372 return 0; /* not found */ 5373 } 5374 5375 static ssize_t reset_method_store(struct device *dev, 5376 struct device_attribute *attr, 5377 const char *buf, size_t count) 5378 { 5379 struct pci_dev *pdev = to_pci_dev(dev); 5380 char *options, *name; 5381 int m, n; 5382 u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 }; 5383 5384 if (sysfs_streq(buf, "")) { 5385 pdev->reset_methods[0] = 0; 5386 pci_warn(pdev, "All device reset methods disabled by user"); 5387 return count; 5388 } 5389 5390 if (sysfs_streq(buf, "default")) { 5391 pci_init_reset_methods(pdev); 5392 return count; 5393 } 5394 5395 options = kstrndup(buf, count, GFP_KERNEL); 5396 if (!options) 5397 return -ENOMEM; 5398 5399 n = 0; 5400 while ((name = strsep(&options, " ")) != NULL) { 5401 if (sysfs_streq(name, "")) 5402 continue; 5403 5404 name = strim(name); 5405 5406 m = reset_method_lookup(name); 5407 if (!m) { 5408 pci_err(pdev, "Invalid reset method '%s'", name); 5409 goto error; 5410 } 5411 5412 if (pci_reset_fn_methods[m].reset_fn(pdev, PCI_RESET_PROBE)) { 5413 pci_err(pdev, "Unsupported reset method '%s'", name); 5414 goto error; 5415 } 5416 5417 if (n == PCI_NUM_RESET_METHODS - 1) { 5418 pci_err(pdev, "Too many reset methods\n"); 5419 goto error; 5420 } 5421 5422 reset_methods[n++] = m; 5423 } 5424 5425 reset_methods[n] = 0; 5426 5427 /* Warn if dev-specific supported but not highest priority */ 5428 if (pci_reset_fn_methods[1].reset_fn(pdev, PCI_RESET_PROBE) == 0 && 5429 reset_methods[0] != 1) 5430 pci_warn(pdev, "Device-specific reset disabled/de-prioritized by user"); 5431 memcpy(pdev->reset_methods, reset_methods, sizeof(pdev->reset_methods)); 5432 kfree(options); 5433 return count; 5434 5435 error: 5436 /* Leave previous methods unchanged */ 5437 kfree(options); 5438 return -EINVAL; 5439 } 5440 static DEVICE_ATTR_RW(reset_method); 5441 5442 static struct attribute *pci_dev_reset_method_attrs[] = { 5443 &dev_attr_reset_method.attr, 5444 NULL, 5445 }; 5446 5447 static umode_t pci_dev_reset_method_attr_is_visible(struct kobject *kobj, 5448 struct attribute *a, int n) 5449 { 5450 struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); 5451 5452 if (!pci_reset_supported(pdev)) 5453 return 0; 5454 5455 return a->mode; 5456 } 5457 5458 const struct attribute_group pci_dev_reset_method_attr_group = { 5459 .attrs = pci_dev_reset_method_attrs, 5460 .is_visible = pci_dev_reset_method_attr_is_visible, 5461 }; 5462 5463 /** 5464 * __pci_reset_function_locked - reset a PCI device function while holding 5465 * the @dev mutex lock. 5466 * @dev: PCI device to reset 5467 * 5468 * Some devices allow an individual function to be reset without affecting 5469 * other functions in the same device. The PCI device must be responsive 5470 * to PCI config space in order to use this function. 5471 * 5472 * The device function is presumed to be unused and the caller is holding 5473 * the device mutex lock when this function is called. 5474 * 5475 * Resetting the device will make the contents of PCI configuration space 5476 * random, so any caller of this must be prepared to reinitialise the 5477 * device including MSI, bus mastering, BARs, decoding IO and memory spaces, 5478 * etc. 5479 * 5480 * Returns 0 if the device function was successfully reset or negative if the 5481 * device doesn't support resetting a single function. 5482 */ 5483 int __pci_reset_function_locked(struct pci_dev *dev) 5484 { 5485 int i, m, rc; 5486 5487 might_sleep(); 5488 5489 /* 5490 * A reset method returns -ENOTTY if it doesn't support this device and 5491 * we should try the next method. 5492 * 5493 * If it returns 0 (success), we're finished. If it returns any other 5494 * error, we're also finished: this indicates that further reset 5495 * mechanisms might be broken on the device. 5496 */ 5497 for (i = 0; i < PCI_NUM_RESET_METHODS; i++) { 5498 m = dev->reset_methods[i]; 5499 if (!m) 5500 return -ENOTTY; 5501 5502 rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET); 5503 if (!rc) 5504 return 0; 5505 if (rc != -ENOTTY) 5506 return rc; 5507 } 5508 5509 return -ENOTTY; 5510 } 5511 EXPORT_SYMBOL_GPL(__pci_reset_function_locked); 5512 5513 /** 5514 * pci_init_reset_methods - check whether device can be safely reset 5515 * and store supported reset mechanisms. 5516 * @dev: PCI device to check for reset mechanisms 5517 * 5518 * Some devices allow an individual function to be reset without affecting 5519 * other functions in the same device. The PCI device must be in D0-D3hot 5520 * state. 5521 * 5522 * Stores reset mechanisms supported by device in reset_methods byte array 5523 * which is a member of struct pci_dev. 5524 */ 5525 void pci_init_reset_methods(struct pci_dev *dev) 5526 { 5527 int m, i, rc; 5528 5529 BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS); 5530 5531 might_sleep(); 5532 5533 i = 0; 5534 for (m = 1; m < PCI_NUM_RESET_METHODS; m++) { 5535 rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE); 5536 if (!rc) 5537 dev->reset_methods[i++] = m; 5538 else if (rc != -ENOTTY) 5539 break; 5540 } 5541 5542 dev->reset_methods[i] = 0; 5543 } 5544 5545 /** 5546 * pci_reset_function - quiesce and reset a PCI device function 5547 * @dev: PCI device to reset 5548 * 5549 * Some devices allow an individual function to be reset without affecting 5550 * other functions in the same device. The PCI device must be responsive 5551 * to PCI config space in order to use this function. 5552 * 5553 * This function does not just reset the PCI portion of a device, but 5554 * clears all the state associated with the device. This function differs 5555 * from __pci_reset_function_locked() in that it saves and restores device state 5556 * over the reset and takes the PCI device lock. 5557 * 5558 * Returns 0 if the device function was successfully reset or negative if the 5559 * device doesn't support resetting a single function. 5560 */ 5561 int pci_reset_function(struct pci_dev *dev) 5562 { 5563 int rc; 5564 5565 if (!pci_reset_supported(dev)) 5566 return -ENOTTY; 5567 5568 pci_dev_lock(dev); 5569 pci_dev_save_and_disable(dev); 5570 5571 rc = __pci_reset_function_locked(dev); 5572 5573 pci_dev_restore(dev); 5574 pci_dev_unlock(dev); 5575 5576 return rc; 5577 } 5578 EXPORT_SYMBOL_GPL(pci_reset_function); 5579 5580 /** 5581 * pci_reset_function_locked - quiesce and reset a PCI device function 5582 * @dev: PCI device to reset 5583 * 5584 * Some devices allow an individual function to be reset without affecting 5585 * other functions in the same device. The PCI device must be responsive 5586 * to PCI config space in order to use this function. 5587 * 5588 * This function does not just reset the PCI portion of a device, but 5589 * clears all the state associated with the device. This function differs 5590 * from __pci_reset_function_locked() in that it saves and restores device state 5591 * over the reset. It also differs from pci_reset_function() in that it 5592 * requires the PCI device lock to be held. 5593 * 5594 * Returns 0 if the device function was successfully reset or negative if the 5595 * device doesn't support resetting a single function. 5596 */ 5597 int pci_reset_function_locked(struct pci_dev *dev) 5598 { 5599 int rc; 5600 5601 if (!pci_reset_supported(dev)) 5602 return -ENOTTY; 5603 5604 pci_dev_save_and_disable(dev); 5605 5606 rc = __pci_reset_function_locked(dev); 5607 5608 pci_dev_restore(dev); 5609 5610 return rc; 5611 } 5612 EXPORT_SYMBOL_GPL(pci_reset_function_locked); 5613 5614 /** 5615 * pci_try_reset_function - quiesce and reset a PCI device function 5616 * @dev: PCI device to reset 5617 * 5618 * Same as above, except return -EAGAIN if unable to lock device. 5619 */ 5620 int pci_try_reset_function(struct pci_dev *dev) 5621 { 5622 int rc; 5623 5624 if (!pci_reset_supported(dev)) 5625 return -ENOTTY; 5626 5627 if (!pci_dev_trylock(dev)) 5628 return -EAGAIN; 5629 5630 pci_dev_save_and_disable(dev); 5631 rc = __pci_reset_function_locked(dev); 5632 pci_dev_restore(dev); 5633 pci_dev_unlock(dev); 5634 5635 return rc; 5636 } 5637 EXPORT_SYMBOL_GPL(pci_try_reset_function); 5638 5639 /* Do any devices on or below this bus prevent a bus reset? */ 5640 static bool pci_bus_resettable(struct pci_bus *bus) 5641 { 5642 struct pci_dev *dev; 5643 5644 5645 if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5646 return false; 5647 5648 list_for_each_entry(dev, &bus->devices, bus_list) { 5649 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5650 (dev->subordinate && !pci_bus_resettable(dev->subordinate))) 5651 return false; 5652 } 5653 5654 return true; 5655 } 5656 5657 /* Lock devices from the top of the tree down */ 5658 static void pci_bus_lock(struct pci_bus *bus) 5659 { 5660 struct pci_dev *dev; 5661 5662 list_for_each_entry(dev, &bus->devices, bus_list) { 5663 pci_dev_lock(dev); 5664 if (dev->subordinate) 5665 pci_bus_lock(dev->subordinate); 5666 } 5667 } 5668 5669 /* Unlock devices from the bottom of the tree up */ 5670 static void pci_bus_unlock(struct pci_bus *bus) 5671 { 5672 struct pci_dev *dev; 5673 5674 list_for_each_entry(dev, &bus->devices, bus_list) { 5675 if (dev->subordinate) 5676 pci_bus_unlock(dev->subordinate); 5677 pci_dev_unlock(dev); 5678 } 5679 } 5680 5681 /* Return 1 on successful lock, 0 on contention */ 5682 static int pci_bus_trylock(struct pci_bus *bus) 5683 { 5684 struct pci_dev *dev; 5685 5686 list_for_each_entry(dev, &bus->devices, bus_list) { 5687 if (!pci_dev_trylock(dev)) 5688 goto unlock; 5689 if (dev->subordinate) { 5690 if (!pci_bus_trylock(dev->subordinate)) { 5691 pci_dev_unlock(dev); 5692 goto unlock; 5693 } 5694 } 5695 } 5696 return 1; 5697 5698 unlock: 5699 list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) { 5700 if (dev->subordinate) 5701 pci_bus_unlock(dev->subordinate); 5702 pci_dev_unlock(dev); 5703 } 5704 return 0; 5705 } 5706 5707 /* Do any devices on or below this slot prevent a bus reset? */ 5708 static bool pci_slot_resettable(struct pci_slot *slot) 5709 { 5710 struct pci_dev *dev; 5711 5712 if (slot->bus->self && 5713 (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5714 return false; 5715 5716 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5717 if (!dev->slot || dev->slot != slot) 5718 continue; 5719 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5720 (dev->subordinate && !pci_bus_resettable(dev->subordinate))) 5721 return false; 5722 } 5723 5724 return true; 5725 } 5726 5727 /* Lock devices from the top of the tree down */ 5728 static void pci_slot_lock(struct pci_slot *slot) 5729 { 5730 struct pci_dev *dev; 5731 5732 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5733 if (!dev->slot || dev->slot != slot) 5734 continue; 5735 pci_dev_lock(dev); 5736 if (dev->subordinate) 5737 pci_bus_lock(dev->subordinate); 5738 } 5739 } 5740 5741 /* Unlock devices from the bottom of the tree up */ 5742 static void pci_slot_unlock(struct pci_slot *slot) 5743 { 5744 struct pci_dev *dev; 5745 5746 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5747 if (!dev->slot || dev->slot != slot) 5748 continue; 5749 if (dev->subordinate) 5750 pci_bus_unlock(dev->subordinate); 5751 pci_dev_unlock(dev); 5752 } 5753 } 5754 5755 /* Return 1 on successful lock, 0 on contention */ 5756 static int pci_slot_trylock(struct pci_slot *slot) 5757 { 5758 struct pci_dev *dev; 5759 5760 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5761 if (!dev->slot || dev->slot != slot) 5762 continue; 5763 if (!pci_dev_trylock(dev)) 5764 goto unlock; 5765 if (dev->subordinate) { 5766 if (!pci_bus_trylock(dev->subordinate)) { 5767 pci_dev_unlock(dev); 5768 goto unlock; 5769 } 5770 } 5771 } 5772 return 1; 5773 5774 unlock: 5775 list_for_each_entry_continue_reverse(dev, 5776 &slot->bus->devices, bus_list) { 5777 if (!dev->slot || dev->slot != slot) 5778 continue; 5779 if (dev->subordinate) 5780 pci_bus_unlock(dev->subordinate); 5781 pci_dev_unlock(dev); 5782 } 5783 return 0; 5784 } 5785 5786 /* 5787 * Save and disable devices from the top of the tree down while holding 5788 * the @dev mutex lock for the entire tree. 5789 */ 5790 static void pci_bus_save_and_disable_locked(struct pci_bus *bus) 5791 { 5792 struct pci_dev *dev; 5793 5794 list_for_each_entry(dev, &bus->devices, bus_list) { 5795 pci_dev_save_and_disable(dev); 5796 if (dev->subordinate) 5797 pci_bus_save_and_disable_locked(dev->subordinate); 5798 } 5799 } 5800 5801 /* 5802 * Restore devices from top of the tree down while holding @dev mutex lock 5803 * for the entire tree. Parent bridges need to be restored before we can 5804 * get to subordinate devices. 5805 */ 5806 static void pci_bus_restore_locked(struct pci_bus *bus) 5807 { 5808 struct pci_dev *dev; 5809 5810 list_for_each_entry(dev, &bus->devices, bus_list) { 5811 pci_dev_restore(dev); 5812 if (dev->subordinate) 5813 pci_bus_restore_locked(dev->subordinate); 5814 } 5815 } 5816 5817 /* 5818 * Save and disable devices from the top of the tree down while holding 5819 * the @dev mutex lock for the entire tree. 5820 */ 5821 static void pci_slot_save_and_disable_locked(struct pci_slot *slot) 5822 { 5823 struct pci_dev *dev; 5824 5825 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5826 if (!dev->slot || dev->slot != slot) 5827 continue; 5828 pci_dev_save_and_disable(dev); 5829 if (dev->subordinate) 5830 pci_bus_save_and_disable_locked(dev->subordinate); 5831 } 5832 } 5833 5834 /* 5835 * Restore devices from top of the tree down while holding @dev mutex lock 5836 * for the entire tree. Parent bridges need to be restored before we can 5837 * get to subordinate devices. 5838 */ 5839 static void pci_slot_restore_locked(struct pci_slot *slot) 5840 { 5841 struct pci_dev *dev; 5842 5843 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5844 if (!dev->slot || dev->slot != slot) 5845 continue; 5846 pci_dev_restore(dev); 5847 if (dev->subordinate) 5848 pci_bus_restore_locked(dev->subordinate); 5849 } 5850 } 5851 5852 static int pci_slot_reset(struct pci_slot *slot, bool probe) 5853 { 5854 int rc; 5855 5856 if (!slot || !pci_slot_resettable(slot)) 5857 return -ENOTTY; 5858 5859 if (!probe) 5860 pci_slot_lock(slot); 5861 5862 might_sleep(); 5863 5864 rc = pci_reset_hotplug_slot(slot->hotplug, probe); 5865 5866 if (!probe) 5867 pci_slot_unlock(slot); 5868 5869 return rc; 5870 } 5871 5872 /** 5873 * pci_probe_reset_slot - probe whether a PCI slot can be reset 5874 * @slot: PCI slot to probe 5875 * 5876 * Return 0 if slot can be reset, negative if a slot reset is not supported. 5877 */ 5878 int pci_probe_reset_slot(struct pci_slot *slot) 5879 { 5880 return pci_slot_reset(slot, PCI_RESET_PROBE); 5881 } 5882 EXPORT_SYMBOL_GPL(pci_probe_reset_slot); 5883 5884 /** 5885 * __pci_reset_slot - Try to reset a PCI slot 5886 * @slot: PCI slot to reset 5887 * 5888 * A PCI bus may host multiple slots, each slot may support a reset mechanism 5889 * independent of other slots. For instance, some slots may support slot power 5890 * control. In the case of a 1:1 bus to slot architecture, this function may 5891 * wrap the bus reset to avoid spurious slot related events such as hotplug. 5892 * Generally a slot reset should be attempted before a bus reset. All of the 5893 * function of the slot and any subordinate buses behind the slot are reset 5894 * through this function. PCI config space of all devices in the slot and 5895 * behind the slot is saved before and restored after reset. 5896 * 5897 * Same as above except return -EAGAIN if the slot cannot be locked 5898 */ 5899 static int __pci_reset_slot(struct pci_slot *slot) 5900 { 5901 int rc; 5902 5903 rc = pci_slot_reset(slot, PCI_RESET_PROBE); 5904 if (rc) 5905 return rc; 5906 5907 if (pci_slot_trylock(slot)) { 5908 pci_slot_save_and_disable_locked(slot); 5909 might_sleep(); 5910 rc = pci_reset_hotplug_slot(slot->hotplug, PCI_RESET_DO_RESET); 5911 pci_slot_restore_locked(slot); 5912 pci_slot_unlock(slot); 5913 } else 5914 rc = -EAGAIN; 5915 5916 return rc; 5917 } 5918 5919 static int pci_bus_reset(struct pci_bus *bus, bool probe) 5920 { 5921 int ret; 5922 5923 if (!bus->self || !pci_bus_resettable(bus)) 5924 return -ENOTTY; 5925 5926 if (probe) 5927 return 0; 5928 5929 pci_bus_lock(bus); 5930 5931 might_sleep(); 5932 5933 ret = pci_bridge_secondary_bus_reset(bus->self); 5934 5935 pci_bus_unlock(bus); 5936 5937 return ret; 5938 } 5939 5940 /** 5941 * pci_bus_error_reset - reset the bridge's subordinate bus 5942 * @bridge: The parent device that connects to the bus to reset 5943 * 5944 * This function will first try to reset the slots on this bus if the method is 5945 * available. If slot reset fails or is not available, this will fall back to a 5946 * secondary bus reset. 5947 */ 5948 int pci_bus_error_reset(struct pci_dev *bridge) 5949 { 5950 struct pci_bus *bus = bridge->subordinate; 5951 struct pci_slot *slot; 5952 5953 if (!bus) 5954 return -ENOTTY; 5955 5956 mutex_lock(&pci_slot_mutex); 5957 if (list_empty(&bus->slots)) 5958 goto bus_reset; 5959 5960 list_for_each_entry(slot, &bus->slots, list) 5961 if (pci_probe_reset_slot(slot)) 5962 goto bus_reset; 5963 5964 list_for_each_entry(slot, &bus->slots, list) 5965 if (pci_slot_reset(slot, PCI_RESET_DO_RESET)) 5966 goto bus_reset; 5967 5968 mutex_unlock(&pci_slot_mutex); 5969 return 0; 5970 bus_reset: 5971 mutex_unlock(&pci_slot_mutex); 5972 return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET); 5973 } 5974 5975 /** 5976 * pci_probe_reset_bus - probe whether a PCI bus can be reset 5977 * @bus: PCI bus to probe 5978 * 5979 * Return 0 if bus can be reset, negative if a bus reset is not supported. 5980 */ 5981 int pci_probe_reset_bus(struct pci_bus *bus) 5982 { 5983 return pci_bus_reset(bus, PCI_RESET_PROBE); 5984 } 5985 EXPORT_SYMBOL_GPL(pci_probe_reset_bus); 5986 5987 /** 5988 * __pci_reset_bus - Try to reset a PCI bus 5989 * @bus: top level PCI bus to reset 5990 * 5991 * Same as above except return -EAGAIN if the bus cannot be locked 5992 */ 5993 static int __pci_reset_bus(struct pci_bus *bus) 5994 { 5995 int rc; 5996 5997 rc = pci_bus_reset(bus, PCI_RESET_PROBE); 5998 if (rc) 5999 return rc; 6000 6001 if (pci_bus_trylock(bus)) { 6002 pci_bus_save_and_disable_locked(bus); 6003 might_sleep(); 6004 rc = pci_bridge_secondary_bus_reset(bus->self); 6005 pci_bus_restore_locked(bus); 6006 pci_bus_unlock(bus); 6007 } else 6008 rc = -EAGAIN; 6009 6010 return rc; 6011 } 6012 6013 /** 6014 * pci_reset_bus - Try to reset a PCI bus 6015 * @pdev: top level PCI device to reset via slot/bus 6016 * 6017 * Same as above except return -EAGAIN if the bus cannot be locked 6018 */ 6019 int pci_reset_bus(struct pci_dev *pdev) 6020 { 6021 return (!pci_probe_reset_slot(pdev->slot)) ? 6022 __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus); 6023 } 6024 EXPORT_SYMBOL_GPL(pci_reset_bus); 6025 6026 /** 6027 * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count 6028 * @dev: PCI device to query 6029 * 6030 * Returns mmrbc: maximum designed memory read count in bytes or 6031 * appropriate error value. 6032 */ 6033 int pcix_get_max_mmrbc(struct pci_dev *dev) 6034 { 6035 int cap; 6036 u32 stat; 6037 6038 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 6039 if (!cap) 6040 return -EINVAL; 6041 6042 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 6043 return -EINVAL; 6044 6045 return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21); 6046 } 6047 EXPORT_SYMBOL(pcix_get_max_mmrbc); 6048 6049 /** 6050 * pcix_get_mmrbc - get PCI-X maximum memory read byte count 6051 * @dev: PCI device to query 6052 * 6053 * Returns mmrbc: maximum memory read count in bytes or appropriate error 6054 * value. 6055 */ 6056 int pcix_get_mmrbc(struct pci_dev *dev) 6057 { 6058 int cap; 6059 u16 cmd; 6060 6061 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 6062 if (!cap) 6063 return -EINVAL; 6064 6065 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 6066 return -EINVAL; 6067 6068 return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2); 6069 } 6070 EXPORT_SYMBOL(pcix_get_mmrbc); 6071 6072 /** 6073 * pcix_set_mmrbc - set PCI-X maximum memory read byte count 6074 * @dev: PCI device to query 6075 * @mmrbc: maximum memory read count in bytes 6076 * valid values are 512, 1024, 2048, 4096 6077 * 6078 * If possible sets maximum memory read byte count, some bridges have errata 6079 * that prevent this. 6080 */ 6081 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc) 6082 { 6083 int cap; 6084 u32 stat, v, o; 6085 u16 cmd; 6086 6087 if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc)) 6088 return -EINVAL; 6089 6090 v = ffs(mmrbc) - 10; 6091 6092 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 6093 if (!cap) 6094 return -EINVAL; 6095 6096 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 6097 return -EINVAL; 6098 6099 if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21) 6100 return -E2BIG; 6101 6102 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 6103 return -EINVAL; 6104 6105 o = (cmd & PCI_X_CMD_MAX_READ) >> 2; 6106 if (o != v) { 6107 if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC)) 6108 return -EIO; 6109 6110 cmd &= ~PCI_X_CMD_MAX_READ; 6111 cmd |= v << 2; 6112 if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd)) 6113 return -EIO; 6114 } 6115 return 0; 6116 } 6117 EXPORT_SYMBOL(pcix_set_mmrbc); 6118 6119 /** 6120 * pcie_get_readrq - get PCI Express read request size 6121 * @dev: PCI device to query 6122 * 6123 * Returns maximum memory read request in bytes or appropriate error value. 6124 */ 6125 int pcie_get_readrq(struct pci_dev *dev) 6126 { 6127 u16 ctl; 6128 6129 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 6130 6131 return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12); 6132 } 6133 EXPORT_SYMBOL(pcie_get_readrq); 6134 6135 /** 6136 * pcie_set_readrq - set PCI Express maximum memory read request 6137 * @dev: PCI device to query 6138 * @rq: maximum memory read count in bytes 6139 * valid values are 128, 256, 512, 1024, 2048, 4096 6140 * 6141 * If possible sets maximum memory read request in bytes 6142 */ 6143 int pcie_set_readrq(struct pci_dev *dev, int rq) 6144 { 6145 u16 v; 6146 int ret; 6147 struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); 6148 6149 if (rq < 128 || rq > 4096 || !is_power_of_2(rq)) 6150 return -EINVAL; 6151 6152 /* 6153 * If using the "performance" PCIe config, we clamp the read rq 6154 * size to the max packet size to keep the host bridge from 6155 * generating requests larger than we can cope with. 6156 */ 6157 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) { 6158 int mps = pcie_get_mps(dev); 6159 6160 if (mps < rq) 6161 rq = mps; 6162 } 6163 6164 v = (ffs(rq) - 8) << 12; 6165 6166 if (bridge->no_inc_mrrs) { 6167 int max_mrrs = pcie_get_readrq(dev); 6168 6169 if (rq > max_mrrs) { 6170 pci_info(dev, "can't set Max_Read_Request_Size to %d; max is %d\n", rq, max_mrrs); 6171 return -EINVAL; 6172 } 6173 } 6174 6175 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 6176 PCI_EXP_DEVCTL_READRQ, v); 6177 6178 return pcibios_err_to_errno(ret); 6179 } 6180 EXPORT_SYMBOL(pcie_set_readrq); 6181 6182 /** 6183 * pcie_get_mps - get PCI Express maximum payload size 6184 * @dev: PCI device to query 6185 * 6186 * Returns maximum payload size in bytes 6187 */ 6188 int pcie_get_mps(struct pci_dev *dev) 6189 { 6190 u16 ctl; 6191 6192 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 6193 6194 return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5); 6195 } 6196 EXPORT_SYMBOL(pcie_get_mps); 6197 6198 /** 6199 * pcie_set_mps - set PCI Express maximum payload size 6200 * @dev: PCI device to query 6201 * @mps: maximum payload size in bytes 6202 * valid values are 128, 256, 512, 1024, 2048, 4096 6203 * 6204 * If possible sets maximum payload size 6205 */ 6206 int pcie_set_mps(struct pci_dev *dev, int mps) 6207 { 6208 u16 v; 6209 int ret; 6210 6211 if (mps < 128 || mps > 4096 || !is_power_of_2(mps)) 6212 return -EINVAL; 6213 6214 v = ffs(mps) - 8; 6215 if (v > dev->pcie_mpss) 6216 return -EINVAL; 6217 v <<= 5; 6218 6219 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 6220 PCI_EXP_DEVCTL_PAYLOAD, v); 6221 6222 return pcibios_err_to_errno(ret); 6223 } 6224 EXPORT_SYMBOL(pcie_set_mps); 6225 6226 /** 6227 * pcie_bandwidth_available - determine minimum link settings of a PCIe 6228 * device and its bandwidth limitation 6229 * @dev: PCI device to query 6230 * @limiting_dev: storage for device causing the bandwidth limitation 6231 * @speed: storage for speed of limiting device 6232 * @width: storage for width of limiting device 6233 * 6234 * Walk up the PCI device chain and find the point where the minimum 6235 * bandwidth is available. Return the bandwidth available there and (if 6236 * limiting_dev, speed, and width pointers are supplied) information about 6237 * that point. The bandwidth returned is in Mb/s, i.e., megabits/second of 6238 * raw bandwidth. 6239 */ 6240 u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, 6241 enum pci_bus_speed *speed, 6242 enum pcie_link_width *width) 6243 { 6244 u16 lnksta; 6245 enum pci_bus_speed next_speed; 6246 enum pcie_link_width next_width; 6247 u32 bw, next_bw; 6248 6249 if (speed) 6250 *speed = PCI_SPEED_UNKNOWN; 6251 if (width) 6252 *width = PCIE_LNK_WIDTH_UNKNOWN; 6253 6254 bw = 0; 6255 6256 while (dev) { 6257 pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); 6258 6259 next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS]; 6260 next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >> 6261 PCI_EXP_LNKSTA_NLW_SHIFT; 6262 6263 next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed); 6264 6265 /* Check if current device limits the total bandwidth */ 6266 if (!bw || next_bw <= bw) { 6267 bw = next_bw; 6268 6269 if (limiting_dev) 6270 *limiting_dev = dev; 6271 if (speed) 6272 *speed = next_speed; 6273 if (width) 6274 *width = next_width; 6275 } 6276 6277 dev = pci_upstream_bridge(dev); 6278 } 6279 6280 return bw; 6281 } 6282 EXPORT_SYMBOL(pcie_bandwidth_available); 6283 6284 /** 6285 * pcie_get_speed_cap - query for the PCI device's link speed capability 6286 * @dev: PCI device to query 6287 * 6288 * Query the PCI device speed capability. Return the maximum link speed 6289 * supported by the device. 6290 */ 6291 enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev) 6292 { 6293 u32 lnkcap2, lnkcap; 6294 6295 /* 6296 * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. The 6297 * implementation note there recommends using the Supported Link 6298 * Speeds Vector in Link Capabilities 2 when supported. 6299 * 6300 * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software 6301 * should use the Supported Link Speeds field in Link Capabilities, 6302 * where only 2.5 GT/s and 5.0 GT/s speeds were defined. 6303 */ 6304 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2); 6305 6306 /* PCIe r3.0-compliant */ 6307 if (lnkcap2) 6308 return PCIE_LNKCAP2_SLS2SPEED(lnkcap2); 6309 6310 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 6311 if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB) 6312 return PCIE_SPEED_5_0GT; 6313 else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB) 6314 return PCIE_SPEED_2_5GT; 6315 6316 return PCI_SPEED_UNKNOWN; 6317 } 6318 EXPORT_SYMBOL(pcie_get_speed_cap); 6319 6320 /** 6321 * pcie_get_width_cap - query for the PCI device's link width capability 6322 * @dev: PCI device to query 6323 * 6324 * Query the PCI device width capability. Return the maximum link width 6325 * supported by the device. 6326 */ 6327 enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev) 6328 { 6329 u32 lnkcap; 6330 6331 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 6332 if (lnkcap) 6333 return (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4; 6334 6335 return PCIE_LNK_WIDTH_UNKNOWN; 6336 } 6337 EXPORT_SYMBOL(pcie_get_width_cap); 6338 6339 /** 6340 * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability 6341 * @dev: PCI device 6342 * @speed: storage for link speed 6343 * @width: storage for link width 6344 * 6345 * Calculate a PCI device's link bandwidth by querying for its link speed 6346 * and width, multiplying them, and applying encoding overhead. The result 6347 * is in Mb/s, i.e., megabits/second of raw bandwidth. 6348 */ 6349 u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed, 6350 enum pcie_link_width *width) 6351 { 6352 *speed = pcie_get_speed_cap(dev); 6353 *width = pcie_get_width_cap(dev); 6354 6355 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 6356 return 0; 6357 6358 return *width * PCIE_SPEED2MBS_ENC(*speed); 6359 } 6360 6361 /** 6362 * __pcie_print_link_status - Report the PCI device's link speed and width 6363 * @dev: PCI device to query 6364 * @verbose: Print info even when enough bandwidth is available 6365 * 6366 * If the available bandwidth at the device is less than the device is 6367 * capable of, report the device's maximum possible bandwidth and the 6368 * upstream link that limits its performance. If @verbose, always print 6369 * the available bandwidth, even if the device isn't constrained. 6370 */ 6371 void __pcie_print_link_status(struct pci_dev *dev, bool verbose) 6372 { 6373 enum pcie_link_width width, width_cap; 6374 enum pci_bus_speed speed, speed_cap; 6375 struct pci_dev *limiting_dev = NULL; 6376 u32 bw_avail, bw_cap; 6377 6378 bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap); 6379 bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width); 6380 6381 if (bw_avail >= bw_cap && verbose) 6382 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n", 6383 bw_cap / 1000, bw_cap % 1000, 6384 pci_speed_string(speed_cap), width_cap); 6385 else if (bw_avail < bw_cap) 6386 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n", 6387 bw_avail / 1000, bw_avail % 1000, 6388 pci_speed_string(speed), width, 6389 limiting_dev ? pci_name(limiting_dev) : "<unknown>", 6390 bw_cap / 1000, bw_cap % 1000, 6391 pci_speed_string(speed_cap), width_cap); 6392 } 6393 6394 /** 6395 * pcie_print_link_status - Report the PCI device's link speed and width 6396 * @dev: PCI device to query 6397 * 6398 * Report the available bandwidth at the device. 6399 */ 6400 void pcie_print_link_status(struct pci_dev *dev) 6401 { 6402 __pcie_print_link_status(dev, true); 6403 } 6404 EXPORT_SYMBOL(pcie_print_link_status); 6405 6406 /** 6407 * pci_select_bars - Make BAR mask from the type of resource 6408 * @dev: the PCI device for which BAR mask is made 6409 * @flags: resource type mask to be selected 6410 * 6411 * This helper routine makes bar mask from the type of resource. 6412 */ 6413 int pci_select_bars(struct pci_dev *dev, unsigned long flags) 6414 { 6415 int i, bars = 0; 6416 for (i = 0; i < PCI_NUM_RESOURCES; i++) 6417 if (pci_resource_flags(dev, i) & flags) 6418 bars |= (1 << i); 6419 return bars; 6420 } 6421 EXPORT_SYMBOL(pci_select_bars); 6422 6423 /* Some architectures require additional programming to enable VGA */ 6424 static arch_set_vga_state_t arch_set_vga_state; 6425 6426 void __init pci_register_set_vga_state(arch_set_vga_state_t func) 6427 { 6428 arch_set_vga_state = func; /* NULL disables */ 6429 } 6430 6431 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode, 6432 unsigned int command_bits, u32 flags) 6433 { 6434 if (arch_set_vga_state) 6435 return arch_set_vga_state(dev, decode, command_bits, 6436 flags); 6437 return 0; 6438 } 6439 6440 /** 6441 * pci_set_vga_state - set VGA decode state on device and parents if requested 6442 * @dev: the PCI device 6443 * @decode: true = enable decoding, false = disable decoding 6444 * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY 6445 * @flags: traverse ancestors and change bridges 6446 * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE 6447 */ 6448 int pci_set_vga_state(struct pci_dev *dev, bool decode, 6449 unsigned int command_bits, u32 flags) 6450 { 6451 struct pci_bus *bus; 6452 struct pci_dev *bridge; 6453 u16 cmd; 6454 int rc; 6455 6456 WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY))); 6457 6458 /* ARCH specific VGA enables */ 6459 rc = pci_set_vga_state_arch(dev, decode, command_bits, flags); 6460 if (rc) 6461 return rc; 6462 6463 if (flags & PCI_VGA_STATE_CHANGE_DECODES) { 6464 pci_read_config_word(dev, PCI_COMMAND, &cmd); 6465 if (decode) 6466 cmd |= command_bits; 6467 else 6468 cmd &= ~command_bits; 6469 pci_write_config_word(dev, PCI_COMMAND, cmd); 6470 } 6471 6472 if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE)) 6473 return 0; 6474 6475 bus = dev->bus; 6476 while (bus) { 6477 bridge = bus->self; 6478 if (bridge) { 6479 pci_read_config_word(bridge, PCI_BRIDGE_CONTROL, 6480 &cmd); 6481 if (decode) 6482 cmd |= PCI_BRIDGE_CTL_VGA; 6483 else 6484 cmd &= ~PCI_BRIDGE_CTL_VGA; 6485 pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, 6486 cmd); 6487 } 6488 bus = bus->parent; 6489 } 6490 return 0; 6491 } 6492 6493 #ifdef CONFIG_ACPI 6494 bool pci_pr3_present(struct pci_dev *pdev) 6495 { 6496 struct acpi_device *adev; 6497 6498 if (acpi_disabled) 6499 return false; 6500 6501 adev = ACPI_COMPANION(&pdev->dev); 6502 if (!adev) 6503 return false; 6504 6505 return adev->power.flags.power_resources && 6506 acpi_has_method(adev->handle, "_PR3"); 6507 } 6508 EXPORT_SYMBOL_GPL(pci_pr3_present); 6509 #endif 6510 6511 /** 6512 * pci_add_dma_alias - Add a DMA devfn alias for a device 6513 * @dev: the PCI device for which alias is added 6514 * @devfn_from: alias slot and function 6515 * @nr_devfns: number of subsequent devfns to alias 6516 * 6517 * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask 6518 * which is used to program permissible bus-devfn source addresses for DMA 6519 * requests in an IOMMU. These aliases factor into IOMMU group creation 6520 * and are useful for devices generating DMA requests beyond or different 6521 * from their logical bus-devfn. Examples include device quirks where the 6522 * device simply uses the wrong devfn, as well as non-transparent bridges 6523 * where the alias may be a proxy for devices in another domain. 6524 * 6525 * IOMMU group creation is performed during device discovery or addition, 6526 * prior to any potential DMA mapping and therefore prior to driver probing 6527 * (especially for userspace assigned devices where IOMMU group definition 6528 * cannot be left as a userspace activity). DMA aliases should therefore 6529 * be configured via quirks, such as the PCI fixup header quirk. 6530 */ 6531 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, 6532 unsigned int nr_devfns) 6533 { 6534 int devfn_to; 6535 6536 nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from); 6537 devfn_to = devfn_from + nr_devfns - 1; 6538 6539 if (!dev->dma_alias_mask) 6540 dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL); 6541 if (!dev->dma_alias_mask) { 6542 pci_warn(dev, "Unable to allocate DMA alias mask\n"); 6543 return; 6544 } 6545 6546 bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns); 6547 6548 if (nr_devfns == 1) 6549 pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n", 6550 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from)); 6551 else if (nr_devfns > 1) 6552 pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n", 6553 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from), 6554 PCI_SLOT(devfn_to), PCI_FUNC(devfn_to)); 6555 } 6556 6557 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) 6558 { 6559 return (dev1->dma_alias_mask && 6560 test_bit(dev2->devfn, dev1->dma_alias_mask)) || 6561 (dev2->dma_alias_mask && 6562 test_bit(dev1->devfn, dev2->dma_alias_mask)) || 6563 pci_real_dma_dev(dev1) == dev2 || 6564 pci_real_dma_dev(dev2) == dev1; 6565 } 6566 6567 bool pci_device_is_present(struct pci_dev *pdev) 6568 { 6569 u32 v; 6570 6571 /* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */ 6572 pdev = pci_physfn(pdev); 6573 if (pci_dev_is_disconnected(pdev)) 6574 return false; 6575 return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0); 6576 } 6577 EXPORT_SYMBOL_GPL(pci_device_is_present); 6578 6579 void pci_ignore_hotplug(struct pci_dev *dev) 6580 { 6581 struct pci_dev *bridge = dev->bus->self; 6582 6583 dev->ignore_hotplug = 1; 6584 /* Propagate the "ignore hotplug" setting to the parent bridge. */ 6585 if (bridge) 6586 bridge->ignore_hotplug = 1; 6587 } 6588 EXPORT_SYMBOL_GPL(pci_ignore_hotplug); 6589 6590 /** 6591 * pci_real_dma_dev - Get PCI DMA device for PCI device 6592 * @dev: the PCI device that may have a PCI DMA alias 6593 * 6594 * Permits the platform to provide architecture-specific functionality to 6595 * devices needing to alias DMA to another PCI device on another PCI bus. If 6596 * the PCI device is on the same bus, it is recommended to use 6597 * pci_add_dma_alias(). This is the default implementation. Architecture 6598 * implementations can override this. 6599 */ 6600 struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev) 6601 { 6602 return dev; 6603 } 6604 6605 resource_size_t __weak pcibios_default_alignment(void) 6606 { 6607 return 0; 6608 } 6609 6610 /* 6611 * Arches that don't want to expose struct resource to userland as-is in 6612 * sysfs and /proc can implement their own pci_resource_to_user(). 6613 */ 6614 void __weak pci_resource_to_user(const struct pci_dev *dev, int bar, 6615 const struct resource *rsrc, 6616 resource_size_t *start, resource_size_t *end) 6617 { 6618 *start = rsrc->start; 6619 *end = rsrc->end; 6620 } 6621 6622 static char *resource_alignment_param; 6623 static DEFINE_SPINLOCK(resource_alignment_lock); 6624 6625 /** 6626 * pci_specified_resource_alignment - get resource alignment specified by user. 6627 * @dev: the PCI device to get 6628 * @resize: whether or not to change resources' size when reassigning alignment 6629 * 6630 * RETURNS: Resource alignment if it is specified. 6631 * Zero if it is not specified. 6632 */ 6633 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev, 6634 bool *resize) 6635 { 6636 int align_order, count; 6637 resource_size_t align = pcibios_default_alignment(); 6638 const char *p; 6639 int ret; 6640 6641 spin_lock(&resource_alignment_lock); 6642 p = resource_alignment_param; 6643 if (!p || !*p) 6644 goto out; 6645 if (pci_has_flag(PCI_PROBE_ONLY)) { 6646 align = 0; 6647 pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n"); 6648 goto out; 6649 } 6650 6651 while (*p) { 6652 count = 0; 6653 if (sscanf(p, "%d%n", &align_order, &count) == 1 && 6654 p[count] == '@') { 6655 p += count + 1; 6656 if (align_order > 63) { 6657 pr_err("PCI: Invalid requested alignment (order %d)\n", 6658 align_order); 6659 align_order = PAGE_SHIFT; 6660 } 6661 } else { 6662 align_order = PAGE_SHIFT; 6663 } 6664 6665 ret = pci_dev_str_match(dev, p, &p); 6666 if (ret == 1) { 6667 *resize = true; 6668 align = 1ULL << align_order; 6669 break; 6670 } else if (ret < 0) { 6671 pr_err("PCI: Can't parse resource_alignment parameter: %s\n", 6672 p); 6673 break; 6674 } 6675 6676 if (*p != ';' && *p != ',') { 6677 /* End of param or invalid format */ 6678 break; 6679 } 6680 p++; 6681 } 6682 out: 6683 spin_unlock(&resource_alignment_lock); 6684 return align; 6685 } 6686 6687 static void pci_request_resource_alignment(struct pci_dev *dev, int bar, 6688 resource_size_t align, bool resize) 6689 { 6690 struct resource *r = &dev->resource[bar]; 6691 resource_size_t size; 6692 6693 if (!(r->flags & IORESOURCE_MEM)) 6694 return; 6695 6696 if (r->flags & IORESOURCE_PCI_FIXED) { 6697 pci_info(dev, "BAR%d %pR: ignoring requested alignment %#llx\n", 6698 bar, r, (unsigned long long)align); 6699 return; 6700 } 6701 6702 size = resource_size(r); 6703 if (size >= align) 6704 return; 6705 6706 /* 6707 * Increase the alignment of the resource. There are two ways we 6708 * can do this: 6709 * 6710 * 1) Increase the size of the resource. BARs are aligned on their 6711 * size, so when we reallocate space for this resource, we'll 6712 * allocate it with the larger alignment. This also prevents 6713 * assignment of any other BARs inside the alignment region, so 6714 * if we're requesting page alignment, this means no other BARs 6715 * will share the page. 6716 * 6717 * The disadvantage is that this makes the resource larger than 6718 * the hardware BAR, which may break drivers that compute things 6719 * based on the resource size, e.g., to find registers at a 6720 * fixed offset before the end of the BAR. 6721 * 6722 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and 6723 * set r->start to the desired alignment. By itself this 6724 * doesn't prevent other BARs being put inside the alignment 6725 * region, but if we realign *every* resource of every device in 6726 * the system, none of them will share an alignment region. 6727 * 6728 * When the user has requested alignment for only some devices via 6729 * the "pci=resource_alignment" argument, "resize" is true and we 6730 * use the first method. Otherwise we assume we're aligning all 6731 * devices and we use the second. 6732 */ 6733 6734 pci_info(dev, "BAR%d %pR: requesting alignment to %#llx\n", 6735 bar, r, (unsigned long long)align); 6736 6737 if (resize) { 6738 r->start = 0; 6739 r->end = align - 1; 6740 } else { 6741 r->flags &= ~IORESOURCE_SIZEALIGN; 6742 r->flags |= IORESOURCE_STARTALIGN; 6743 r->start = align; 6744 r->end = r->start + size - 1; 6745 } 6746 r->flags |= IORESOURCE_UNSET; 6747 } 6748 6749 /* 6750 * This function disables memory decoding and releases memory resources 6751 * of the device specified by kernel's boot parameter 'pci=resource_alignment='. 6752 * It also rounds up size to specified alignment. 6753 * Later on, the kernel will assign page-aligned memory resource back 6754 * to the device. 6755 */ 6756 void pci_reassigndev_resource_alignment(struct pci_dev *dev) 6757 { 6758 int i; 6759 struct resource *r; 6760 resource_size_t align; 6761 u16 command; 6762 bool resize = false; 6763 6764 /* 6765 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec 6766 * 3.4.1.11. Their resources are allocated from the space 6767 * described by the VF BARx register in the PF's SR-IOV capability. 6768 * We can't influence their alignment here. 6769 */ 6770 if (dev->is_virtfn) 6771 return; 6772 6773 /* check if specified PCI is target device to reassign */ 6774 align = pci_specified_resource_alignment(dev, &resize); 6775 if (!align) 6776 return; 6777 6778 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL && 6779 (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) { 6780 pci_warn(dev, "Can't reassign resources to host bridge\n"); 6781 return; 6782 } 6783 6784 pci_read_config_word(dev, PCI_COMMAND, &command); 6785 command &= ~PCI_COMMAND_MEMORY; 6786 pci_write_config_word(dev, PCI_COMMAND, command); 6787 6788 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 6789 pci_request_resource_alignment(dev, i, align, resize); 6790 6791 /* 6792 * Need to disable bridge's resource window, 6793 * to enable the kernel to reassign new resource 6794 * window later on. 6795 */ 6796 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 6797 for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) { 6798 r = &dev->resource[i]; 6799 if (!(r->flags & IORESOURCE_MEM)) 6800 continue; 6801 r->flags |= IORESOURCE_UNSET; 6802 r->end = resource_size(r) - 1; 6803 r->start = 0; 6804 } 6805 pci_disable_bridge_window(dev); 6806 } 6807 } 6808 6809 static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf) 6810 { 6811 size_t count = 0; 6812 6813 spin_lock(&resource_alignment_lock); 6814 if (resource_alignment_param) 6815 count = sysfs_emit(buf, "%s\n", resource_alignment_param); 6816 spin_unlock(&resource_alignment_lock); 6817 6818 return count; 6819 } 6820 6821 static ssize_t resource_alignment_store(const struct bus_type *bus, 6822 const char *buf, size_t count) 6823 { 6824 char *param, *old, *end; 6825 6826 if (count >= (PAGE_SIZE - 1)) 6827 return -EINVAL; 6828 6829 param = kstrndup(buf, count, GFP_KERNEL); 6830 if (!param) 6831 return -ENOMEM; 6832 6833 end = strchr(param, '\n'); 6834 if (end) 6835 *end = '\0'; 6836 6837 spin_lock(&resource_alignment_lock); 6838 old = resource_alignment_param; 6839 if (strlen(param)) { 6840 resource_alignment_param = param; 6841 } else { 6842 kfree(param); 6843 resource_alignment_param = NULL; 6844 } 6845 spin_unlock(&resource_alignment_lock); 6846 6847 kfree(old); 6848 6849 return count; 6850 } 6851 6852 static BUS_ATTR_RW(resource_alignment); 6853 6854 static int __init pci_resource_alignment_sysfs_init(void) 6855 { 6856 return bus_create_file(&pci_bus_type, 6857 &bus_attr_resource_alignment); 6858 } 6859 late_initcall(pci_resource_alignment_sysfs_init); 6860 6861 static void pci_no_domains(void) 6862 { 6863 #ifdef CONFIG_PCI_DOMAINS 6864 pci_domains_supported = 0; 6865 #endif 6866 } 6867 6868 #ifdef CONFIG_PCI_DOMAINS_GENERIC 6869 static DEFINE_IDA(pci_domain_nr_static_ida); 6870 static DEFINE_IDA(pci_domain_nr_dynamic_ida); 6871 6872 static void of_pci_reserve_static_domain_nr(void) 6873 { 6874 struct device_node *np; 6875 int domain_nr; 6876 6877 for_each_node_by_type(np, "pci") { 6878 domain_nr = of_get_pci_domain_nr(np); 6879 if (domain_nr < 0) 6880 continue; 6881 /* 6882 * Permanently allocate domain_nr in dynamic_ida 6883 * to prevent it from dynamic allocation. 6884 */ 6885 ida_alloc_range(&pci_domain_nr_dynamic_ida, 6886 domain_nr, domain_nr, GFP_KERNEL); 6887 } 6888 } 6889 6890 static int of_pci_bus_find_domain_nr(struct device *parent) 6891 { 6892 static bool static_domains_reserved = false; 6893 int domain_nr; 6894 6895 /* On the first call scan device tree for static allocations. */ 6896 if (!static_domains_reserved) { 6897 of_pci_reserve_static_domain_nr(); 6898 static_domains_reserved = true; 6899 } 6900 6901 if (parent) { 6902 /* 6903 * If domain is in DT, allocate it in static IDA. This 6904 * prevents duplicate static allocations in case of errors 6905 * in DT. 6906 */ 6907 domain_nr = of_get_pci_domain_nr(parent->of_node); 6908 if (domain_nr >= 0) 6909 return ida_alloc_range(&pci_domain_nr_static_ida, 6910 domain_nr, domain_nr, 6911 GFP_KERNEL); 6912 } 6913 6914 /* 6915 * If domain was not specified in DT, choose a free ID from dynamic 6916 * allocations. All domain numbers from DT are permanently in 6917 * dynamic allocations to prevent assigning them to other DT nodes 6918 * without static domain. 6919 */ 6920 return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL); 6921 } 6922 6923 static void of_pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent) 6924 { 6925 if (bus->domain_nr < 0) 6926 return; 6927 6928 /* Release domain from IDA where it was allocated. */ 6929 if (of_get_pci_domain_nr(parent->of_node) == bus->domain_nr) 6930 ida_free(&pci_domain_nr_static_ida, bus->domain_nr); 6931 else 6932 ida_free(&pci_domain_nr_dynamic_ida, bus->domain_nr); 6933 } 6934 6935 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent) 6936 { 6937 return acpi_disabled ? of_pci_bus_find_domain_nr(parent) : 6938 acpi_pci_bus_find_domain_nr(bus); 6939 } 6940 6941 void pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent) 6942 { 6943 if (!acpi_disabled) 6944 return; 6945 of_pci_bus_release_domain_nr(bus, parent); 6946 } 6947 #endif 6948 6949 /** 6950 * pci_ext_cfg_avail - can we access extended PCI config space? 6951 * 6952 * Returns 1 if we can access PCI extended config space (offsets 6953 * greater than 0xff). This is the default implementation. Architecture 6954 * implementations can override this. 6955 */ 6956 int __weak pci_ext_cfg_avail(void) 6957 { 6958 return 1; 6959 } 6960 6961 void __weak pci_fixup_cardbus(struct pci_bus *bus) 6962 { 6963 } 6964 EXPORT_SYMBOL(pci_fixup_cardbus); 6965 6966 static int __init pci_setup(char *str) 6967 { 6968 while (str) { 6969 char *k = strchr(str, ','); 6970 if (k) 6971 *k++ = 0; 6972 if (*str && (str = pcibios_setup(str)) && *str) { 6973 if (!strcmp(str, "nomsi")) { 6974 pci_no_msi(); 6975 } else if (!strncmp(str, "noats", 5)) { 6976 pr_info("PCIe: ATS is disabled\n"); 6977 pcie_ats_disabled = true; 6978 } else if (!strcmp(str, "noaer")) { 6979 pci_no_aer(); 6980 } else if (!strcmp(str, "earlydump")) { 6981 pci_early_dump = true; 6982 } else if (!strncmp(str, "realloc=", 8)) { 6983 pci_realloc_get_opt(str + 8); 6984 } else if (!strncmp(str, "realloc", 7)) { 6985 pci_realloc_get_opt("on"); 6986 } else if (!strcmp(str, "nodomains")) { 6987 pci_no_domains(); 6988 } else if (!strncmp(str, "noari", 5)) { 6989 pcie_ari_disabled = true; 6990 } else if (!strncmp(str, "cbiosize=", 9)) { 6991 pci_cardbus_io_size = memparse(str + 9, &str); 6992 } else if (!strncmp(str, "cbmemsize=", 10)) { 6993 pci_cardbus_mem_size = memparse(str + 10, &str); 6994 } else if (!strncmp(str, "resource_alignment=", 19)) { 6995 resource_alignment_param = str + 19; 6996 } else if (!strncmp(str, "ecrc=", 5)) { 6997 pcie_ecrc_get_policy(str + 5); 6998 } else if (!strncmp(str, "hpiosize=", 9)) { 6999 pci_hotplug_io_size = memparse(str + 9, &str); 7000 } else if (!strncmp(str, "hpmmiosize=", 11)) { 7001 pci_hotplug_mmio_size = memparse(str + 11, &str); 7002 } else if (!strncmp(str, "hpmmioprefsize=", 15)) { 7003 pci_hotplug_mmio_pref_size = memparse(str + 15, &str); 7004 } else if (!strncmp(str, "hpmemsize=", 10)) { 7005 pci_hotplug_mmio_size = memparse(str + 10, &str); 7006 pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size; 7007 } else if (!strncmp(str, "hpbussize=", 10)) { 7008 pci_hotplug_bus_size = 7009 simple_strtoul(str + 10, &str, 0); 7010 if (pci_hotplug_bus_size > 0xff) 7011 pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 7012 } else if (!strncmp(str, "pcie_bus_tune_off", 17)) { 7013 pcie_bus_config = PCIE_BUS_TUNE_OFF; 7014 } else if (!strncmp(str, "pcie_bus_safe", 13)) { 7015 pcie_bus_config = PCIE_BUS_SAFE; 7016 } else if (!strncmp(str, "pcie_bus_perf", 13)) { 7017 pcie_bus_config = PCIE_BUS_PERFORMANCE; 7018 } else if (!strncmp(str, "pcie_bus_peer2peer", 18)) { 7019 pcie_bus_config = PCIE_BUS_PEER2PEER; 7020 } else if (!strncmp(str, "pcie_scan_all", 13)) { 7021 pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS); 7022 } else if (!strncmp(str, "disable_acs_redir=", 18)) { 7023 disable_acs_redir_param = str + 18; 7024 } else { 7025 pr_err("PCI: Unknown option `%s'\n", str); 7026 } 7027 } 7028 str = k; 7029 } 7030 return 0; 7031 } 7032 early_param("pci", pci_setup); 7033 7034 /* 7035 * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized 7036 * in pci_setup(), above, to point to data in the __initdata section which 7037 * will be freed after the init sequence is complete. We can't allocate memory 7038 * in pci_setup() because some architectures do not have any memory allocation 7039 * service available during an early_param() call. So we allocate memory and 7040 * copy the variable here before the init section is freed. 7041 * 7042 */ 7043 static int __init pci_realloc_setup_params(void) 7044 { 7045 resource_alignment_param = kstrdup(resource_alignment_param, 7046 GFP_KERNEL); 7047 disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL); 7048 7049 return 0; 7050 } 7051 pure_initcall(pci_realloc_setup_params); 7052