1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Bus Services, see include/linux/pci.h for further explanation. 4 * 5 * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter, 6 * David Mosberger-Tang 7 * 8 * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz> 9 */ 10 11 #include <linux/acpi.h> 12 #include <linux/kernel.h> 13 #include <linux/delay.h> 14 #include <linux/dmi.h> 15 #include <linux/init.h> 16 #include <linux/msi.h> 17 #include <linux/of.h> 18 #include <linux/pci.h> 19 #include <linux/pm.h> 20 #include <linux/slab.h> 21 #include <linux/module.h> 22 #include <linux/spinlock.h> 23 #include <linux/string.h> 24 #include <linux/log2.h> 25 #include <linux/logic_pio.h> 26 #include <linux/pm_wakeup.h> 27 #include <linux/interrupt.h> 28 #include <linux/device.h> 29 #include <linux/pm_runtime.h> 30 #include <linux/pci_hotplug.h> 31 #include <linux/vmalloc.h> 32 #include <asm/dma.h> 33 #include <linux/aer.h> 34 #include <linux/bitfield.h> 35 #include "pci.h" 36 37 DEFINE_MUTEX(pci_slot_mutex); 38 39 const char *pci_power_names[] = { 40 "error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown", 41 }; 42 EXPORT_SYMBOL_GPL(pci_power_names); 43 44 int isa_dma_bridge_buggy; 45 EXPORT_SYMBOL(isa_dma_bridge_buggy); 46 47 int pci_pci_problems; 48 EXPORT_SYMBOL(pci_pci_problems); 49 50 unsigned int pci_pm_d3hot_delay; 51 52 static void pci_pme_list_scan(struct work_struct *work); 53 54 static LIST_HEAD(pci_pme_list); 55 static DEFINE_MUTEX(pci_pme_list_mutex); 56 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan); 57 58 struct pci_pme_device { 59 struct list_head list; 60 struct pci_dev *dev; 61 }; 62 63 #define PME_TIMEOUT 1000 /* How long between PME checks */ 64 65 static void pci_dev_d3_sleep(struct pci_dev *dev) 66 { 67 unsigned int delay = dev->d3hot_delay; 68 69 if (delay < pci_pm_d3hot_delay) 70 delay = pci_pm_d3hot_delay; 71 72 if (delay) 73 msleep(delay); 74 } 75 76 bool pci_reset_supported(struct pci_dev *dev) 77 { 78 return dev->reset_methods[0] != 0; 79 } 80 81 #ifdef CONFIG_PCI_DOMAINS 82 int pci_domains_supported = 1; 83 #endif 84 85 #define DEFAULT_CARDBUS_IO_SIZE (256) 86 #define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024) 87 /* pci=cbmemsize=nnM,cbiosize=nn can override this */ 88 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE; 89 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE; 90 91 #define DEFAULT_HOTPLUG_IO_SIZE (256) 92 #define DEFAULT_HOTPLUG_MMIO_SIZE (2*1024*1024) 93 #define DEFAULT_HOTPLUG_MMIO_PREF_SIZE (2*1024*1024) 94 /* hpiosize=nn can override this */ 95 unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE; 96 /* 97 * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size, 98 * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size; 99 * pci=hpmemsize=nnM overrides both 100 */ 101 unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE; 102 unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE; 103 104 #define DEFAULT_HOTPLUG_BUS_SIZE 1 105 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 106 107 108 /* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */ 109 #ifdef CONFIG_PCIE_BUS_TUNE_OFF 110 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF; 111 #elif defined CONFIG_PCIE_BUS_SAFE 112 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE; 113 #elif defined CONFIG_PCIE_BUS_PERFORMANCE 114 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE; 115 #elif defined CONFIG_PCIE_BUS_PEER2PEER 116 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER; 117 #else 118 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT; 119 #endif 120 121 /* 122 * The default CLS is used if arch didn't set CLS explicitly and not 123 * all pci devices agree on the same value. Arch can override either 124 * the dfl or actual value as it sees fit. Don't forget this is 125 * measured in 32-bit words, not bytes. 126 */ 127 u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2; 128 u8 pci_cache_line_size; 129 130 /* 131 * If we set up a device for bus mastering, we need to check the latency 132 * timer as certain BIOSes forget to set it properly. 133 */ 134 unsigned int pcibios_max_latency = 255; 135 136 /* If set, the PCIe ARI capability will not be used. */ 137 static bool pcie_ari_disabled; 138 139 /* If set, the PCIe ATS capability will not be used. */ 140 static bool pcie_ats_disabled; 141 142 /* If set, the PCI config space of each device is printed during boot. */ 143 bool pci_early_dump; 144 145 bool pci_ats_disabled(void) 146 { 147 return pcie_ats_disabled; 148 } 149 EXPORT_SYMBOL_GPL(pci_ats_disabled); 150 151 /* Disable bridge_d3 for all PCIe ports */ 152 static bool pci_bridge_d3_disable; 153 /* Force bridge_d3 for all PCIe ports */ 154 static bool pci_bridge_d3_force; 155 156 static int __init pcie_port_pm_setup(char *str) 157 { 158 if (!strcmp(str, "off")) 159 pci_bridge_d3_disable = true; 160 else if (!strcmp(str, "force")) 161 pci_bridge_d3_force = true; 162 return 1; 163 } 164 __setup("pcie_port_pm=", pcie_port_pm_setup); 165 166 /* Time to wait after a reset for device to become responsive */ 167 #define PCIE_RESET_READY_POLL_MS 60000 168 169 /** 170 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children 171 * @bus: pointer to PCI bus structure to search 172 * 173 * Given a PCI bus, returns the highest PCI bus number present in the set 174 * including the given PCI bus and its list of child PCI buses. 175 */ 176 unsigned char pci_bus_max_busnr(struct pci_bus *bus) 177 { 178 struct pci_bus *tmp; 179 unsigned char max, n; 180 181 max = bus->busn_res.end; 182 list_for_each_entry(tmp, &bus->children, node) { 183 n = pci_bus_max_busnr(tmp); 184 if (n > max) 185 max = n; 186 } 187 return max; 188 } 189 EXPORT_SYMBOL_GPL(pci_bus_max_busnr); 190 191 /** 192 * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS 193 * @pdev: the PCI device 194 * 195 * Returns error bits set in PCI_STATUS and clears them. 196 */ 197 int pci_status_get_and_clear_errors(struct pci_dev *pdev) 198 { 199 u16 status; 200 int ret; 201 202 ret = pci_read_config_word(pdev, PCI_STATUS, &status); 203 if (ret != PCIBIOS_SUCCESSFUL) 204 return -EIO; 205 206 status &= PCI_STATUS_ERROR_BITS; 207 if (status) 208 pci_write_config_word(pdev, PCI_STATUS, status); 209 210 return status; 211 } 212 EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors); 213 214 #ifdef CONFIG_HAS_IOMEM 215 static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar, 216 bool write_combine) 217 { 218 struct resource *res = &pdev->resource[bar]; 219 resource_size_t start = res->start; 220 resource_size_t size = resource_size(res); 221 222 /* 223 * Make sure the BAR is actually a memory resource, not an IO resource 224 */ 225 if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) { 226 pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res); 227 return NULL; 228 } 229 230 if (write_combine) 231 return ioremap_wc(start, size); 232 233 return ioremap(start, size); 234 } 235 236 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) 237 { 238 return __pci_ioremap_resource(pdev, bar, false); 239 } 240 EXPORT_SYMBOL_GPL(pci_ioremap_bar); 241 242 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar) 243 { 244 return __pci_ioremap_resource(pdev, bar, true); 245 } 246 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar); 247 #endif 248 249 /** 250 * pci_dev_str_match_path - test if a path string matches a device 251 * @dev: the PCI device to test 252 * @path: string to match the device against 253 * @endptr: pointer to the string after the match 254 * 255 * Test if a string (typically from a kernel parameter) formatted as a 256 * path of device/function addresses matches a PCI device. The string must 257 * be of the form: 258 * 259 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 260 * 261 * A path for a device can be obtained using 'lspci -t'. Using a path 262 * is more robust against bus renumbering than using only a single bus, 263 * device and function address. 264 * 265 * Returns 1 if the string matches the device, 0 if it does not and 266 * a negative error code if it fails to parse the string. 267 */ 268 static int pci_dev_str_match_path(struct pci_dev *dev, const char *path, 269 const char **endptr) 270 { 271 int ret; 272 unsigned int seg, bus, slot, func; 273 char *wpath, *p; 274 char end; 275 276 *endptr = strchrnul(path, ';'); 277 278 wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC); 279 if (!wpath) 280 return -ENOMEM; 281 282 while (1) { 283 p = strrchr(wpath, '/'); 284 if (!p) 285 break; 286 ret = sscanf(p, "/%x.%x%c", &slot, &func, &end); 287 if (ret != 2) { 288 ret = -EINVAL; 289 goto free_and_exit; 290 } 291 292 if (dev->devfn != PCI_DEVFN(slot, func)) { 293 ret = 0; 294 goto free_and_exit; 295 } 296 297 /* 298 * Note: we don't need to get a reference to the upstream 299 * bridge because we hold a reference to the top level 300 * device which should hold a reference to the bridge, 301 * and so on. 302 */ 303 dev = pci_upstream_bridge(dev); 304 if (!dev) { 305 ret = 0; 306 goto free_and_exit; 307 } 308 309 *p = 0; 310 } 311 312 ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot, 313 &func, &end); 314 if (ret != 4) { 315 seg = 0; 316 ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end); 317 if (ret != 3) { 318 ret = -EINVAL; 319 goto free_and_exit; 320 } 321 } 322 323 ret = (seg == pci_domain_nr(dev->bus) && 324 bus == dev->bus->number && 325 dev->devfn == PCI_DEVFN(slot, func)); 326 327 free_and_exit: 328 kfree(wpath); 329 return ret; 330 } 331 332 /** 333 * pci_dev_str_match - test if a string matches a device 334 * @dev: the PCI device to test 335 * @p: string to match the device against 336 * @endptr: pointer to the string after the match 337 * 338 * Test if a string (typically from a kernel parameter) matches a specified 339 * PCI device. The string may be of one of the following formats: 340 * 341 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 342 * pci:<vendor>:<device>[:<subvendor>:<subdevice>] 343 * 344 * The first format specifies a PCI bus/device/function address which 345 * may change if new hardware is inserted, if motherboard firmware changes, 346 * or due to changes caused in kernel parameters. If the domain is 347 * left unspecified, it is taken to be 0. In order to be robust against 348 * bus renumbering issues, a path of PCI device/function numbers may be used 349 * to address the specific device. The path for a device can be determined 350 * through the use of 'lspci -t'. 351 * 352 * The second format matches devices using IDs in the configuration 353 * space which may match multiple devices in the system. A value of 0 354 * for any field will match all devices. (Note: this differs from 355 * in-kernel code that uses PCI_ANY_ID which is ~0; this is for 356 * legacy reasons and convenience so users don't have to specify 357 * FFFFFFFFs on the command line.) 358 * 359 * Returns 1 if the string matches the device, 0 if it does not and 360 * a negative error code if the string cannot be parsed. 361 */ 362 static int pci_dev_str_match(struct pci_dev *dev, const char *p, 363 const char **endptr) 364 { 365 int ret; 366 int count; 367 unsigned short vendor, device, subsystem_vendor, subsystem_device; 368 369 if (strncmp(p, "pci:", 4) == 0) { 370 /* PCI vendor/device (subvendor/subdevice) IDs are specified */ 371 p += 4; 372 ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device, 373 &subsystem_vendor, &subsystem_device, &count); 374 if (ret != 4) { 375 ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count); 376 if (ret != 2) 377 return -EINVAL; 378 379 subsystem_vendor = 0; 380 subsystem_device = 0; 381 } 382 383 p += count; 384 385 if ((!vendor || vendor == dev->vendor) && 386 (!device || device == dev->device) && 387 (!subsystem_vendor || 388 subsystem_vendor == dev->subsystem_vendor) && 389 (!subsystem_device || 390 subsystem_device == dev->subsystem_device)) 391 goto found; 392 } else { 393 /* 394 * PCI Bus, Device, Function IDs are specified 395 * (optionally, may include a path of devfns following it) 396 */ 397 ret = pci_dev_str_match_path(dev, p, &p); 398 if (ret < 0) 399 return ret; 400 else if (ret) 401 goto found; 402 } 403 404 *endptr = p; 405 return 0; 406 407 found: 408 *endptr = p; 409 return 1; 410 } 411 412 static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn, 413 u8 pos, int cap, int *ttl) 414 { 415 u8 id; 416 u16 ent; 417 418 pci_bus_read_config_byte(bus, devfn, pos, &pos); 419 420 while ((*ttl)--) { 421 if (pos < 0x40) 422 break; 423 pos &= ~3; 424 pci_bus_read_config_word(bus, devfn, pos, &ent); 425 426 id = ent & 0xff; 427 if (id == 0xff) 428 break; 429 if (id == cap) 430 return pos; 431 pos = (ent >> 8); 432 } 433 return 0; 434 } 435 436 static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, 437 u8 pos, int cap) 438 { 439 int ttl = PCI_FIND_CAP_TTL; 440 441 return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl); 442 } 443 444 u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) 445 { 446 return __pci_find_next_cap(dev->bus, dev->devfn, 447 pos + PCI_CAP_LIST_NEXT, cap); 448 } 449 EXPORT_SYMBOL_GPL(pci_find_next_capability); 450 451 static u8 __pci_bus_find_cap_start(struct pci_bus *bus, 452 unsigned int devfn, u8 hdr_type) 453 { 454 u16 status; 455 456 pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status); 457 if (!(status & PCI_STATUS_CAP_LIST)) 458 return 0; 459 460 switch (hdr_type) { 461 case PCI_HEADER_TYPE_NORMAL: 462 case PCI_HEADER_TYPE_BRIDGE: 463 return PCI_CAPABILITY_LIST; 464 case PCI_HEADER_TYPE_CARDBUS: 465 return PCI_CB_CAPABILITY_LIST; 466 } 467 468 return 0; 469 } 470 471 /** 472 * pci_find_capability - query for devices' capabilities 473 * @dev: PCI device to query 474 * @cap: capability code 475 * 476 * Tell if a device supports a given PCI capability. 477 * Returns the address of the requested capability structure within the 478 * device's PCI configuration space or 0 in case the device does not 479 * support it. Possible values for @cap include: 480 * 481 * %PCI_CAP_ID_PM Power Management 482 * %PCI_CAP_ID_AGP Accelerated Graphics Port 483 * %PCI_CAP_ID_VPD Vital Product Data 484 * %PCI_CAP_ID_SLOTID Slot Identification 485 * %PCI_CAP_ID_MSI Message Signalled Interrupts 486 * %PCI_CAP_ID_CHSWP CompactPCI HotSwap 487 * %PCI_CAP_ID_PCIX PCI-X 488 * %PCI_CAP_ID_EXP PCI Express 489 */ 490 u8 pci_find_capability(struct pci_dev *dev, int cap) 491 { 492 u8 pos; 493 494 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 495 if (pos) 496 pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap); 497 498 return pos; 499 } 500 EXPORT_SYMBOL(pci_find_capability); 501 502 /** 503 * pci_bus_find_capability - query for devices' capabilities 504 * @bus: the PCI bus to query 505 * @devfn: PCI device to query 506 * @cap: capability code 507 * 508 * Like pci_find_capability() but works for PCI devices that do not have a 509 * pci_dev structure set up yet. 510 * 511 * Returns the address of the requested capability structure within the 512 * device's PCI configuration space or 0 in case the device does not 513 * support it. 514 */ 515 u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap) 516 { 517 u8 hdr_type, pos; 518 519 pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type); 520 521 pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f); 522 if (pos) 523 pos = __pci_find_next_cap(bus, devfn, pos, cap); 524 525 return pos; 526 } 527 EXPORT_SYMBOL(pci_bus_find_capability); 528 529 /** 530 * pci_find_next_ext_capability - Find an extended capability 531 * @dev: PCI device to query 532 * @start: address at which to start looking (0 to start at beginning of list) 533 * @cap: capability code 534 * 535 * Returns the address of the next matching extended capability structure 536 * within the device's PCI configuration space or 0 if the device does 537 * not support it. Some capabilities can occur several times, e.g., the 538 * vendor-specific capability, and this provides a way to find them all. 539 */ 540 u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) 541 { 542 u32 header; 543 int ttl; 544 u16 pos = PCI_CFG_SPACE_SIZE; 545 546 /* minimum 8 bytes per capability */ 547 ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; 548 549 if (dev->cfg_size <= PCI_CFG_SPACE_SIZE) 550 return 0; 551 552 if (start) 553 pos = start; 554 555 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 556 return 0; 557 558 /* 559 * If we have no capabilities, this is indicated by cap ID, 560 * cap version and next pointer all being 0. 561 */ 562 if (header == 0) 563 return 0; 564 565 while (ttl-- > 0) { 566 if (PCI_EXT_CAP_ID(header) == cap && pos != start) 567 return pos; 568 569 pos = PCI_EXT_CAP_NEXT(header); 570 if (pos < PCI_CFG_SPACE_SIZE) 571 break; 572 573 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 574 break; 575 } 576 577 return 0; 578 } 579 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); 580 581 /** 582 * pci_find_ext_capability - Find an extended capability 583 * @dev: PCI device to query 584 * @cap: capability code 585 * 586 * Returns the address of the requested extended capability structure 587 * within the device's PCI configuration space or 0 if the device does 588 * not support it. Possible values for @cap include: 589 * 590 * %PCI_EXT_CAP_ID_ERR Advanced Error Reporting 591 * %PCI_EXT_CAP_ID_VC Virtual Channel 592 * %PCI_EXT_CAP_ID_DSN Device Serial Number 593 * %PCI_EXT_CAP_ID_PWR Power Budgeting 594 */ 595 u16 pci_find_ext_capability(struct pci_dev *dev, int cap) 596 { 597 return pci_find_next_ext_capability(dev, 0, cap); 598 } 599 EXPORT_SYMBOL_GPL(pci_find_ext_capability); 600 601 /** 602 * pci_get_dsn - Read and return the 8-byte Device Serial Number 603 * @dev: PCI device to query 604 * 605 * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial 606 * Number. 607 * 608 * Returns the DSN, or zero if the capability does not exist. 609 */ 610 u64 pci_get_dsn(struct pci_dev *dev) 611 { 612 u32 dword; 613 u64 dsn; 614 int pos; 615 616 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN); 617 if (!pos) 618 return 0; 619 620 /* 621 * The Device Serial Number is two dwords offset 4 bytes from the 622 * capability position. The specification says that the first dword is 623 * the lower half, and the second dword is the upper half. 624 */ 625 pos += 4; 626 pci_read_config_dword(dev, pos, &dword); 627 dsn = (u64)dword; 628 pci_read_config_dword(dev, pos + 4, &dword); 629 dsn |= ((u64)dword) << 32; 630 631 return dsn; 632 } 633 EXPORT_SYMBOL_GPL(pci_get_dsn); 634 635 static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) 636 { 637 int rc, ttl = PCI_FIND_CAP_TTL; 638 u8 cap, mask; 639 640 if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST) 641 mask = HT_3BIT_CAP_MASK; 642 else 643 mask = HT_5BIT_CAP_MASK; 644 645 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos, 646 PCI_CAP_ID_HT, &ttl); 647 while (pos) { 648 rc = pci_read_config_byte(dev, pos + 3, &cap); 649 if (rc != PCIBIOS_SUCCESSFUL) 650 return 0; 651 652 if ((cap & mask) == ht_cap) 653 return pos; 654 655 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, 656 pos + PCI_CAP_LIST_NEXT, 657 PCI_CAP_ID_HT, &ttl); 658 } 659 660 return 0; 661 } 662 663 /** 664 * pci_find_next_ht_capability - query a device's HyperTransport capabilities 665 * @dev: PCI device to query 666 * @pos: Position from which to continue searching 667 * @ht_cap: HyperTransport capability code 668 * 669 * To be used in conjunction with pci_find_ht_capability() to search for 670 * all capabilities matching @ht_cap. @pos should always be a value returned 671 * from pci_find_ht_capability(). 672 * 673 * NB. To be 100% safe against broken PCI devices, the caller should take 674 * steps to avoid an infinite loop. 675 */ 676 u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap) 677 { 678 return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap); 679 } 680 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability); 681 682 /** 683 * pci_find_ht_capability - query a device's HyperTransport capabilities 684 * @dev: PCI device to query 685 * @ht_cap: HyperTransport capability code 686 * 687 * Tell if a device supports a given HyperTransport capability. 688 * Returns an address within the device's PCI configuration space 689 * or 0 in case the device does not support the request capability. 690 * The address points to the PCI capability, of type PCI_CAP_ID_HT, 691 * which has a HyperTransport capability matching @ht_cap. 692 */ 693 u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap) 694 { 695 u8 pos; 696 697 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 698 if (pos) 699 pos = __pci_find_next_ht_cap(dev, pos, ht_cap); 700 701 return pos; 702 } 703 EXPORT_SYMBOL_GPL(pci_find_ht_capability); 704 705 /** 706 * pci_find_vsec_capability - Find a vendor-specific extended capability 707 * @dev: PCI device to query 708 * @vendor: Vendor ID for which capability is defined 709 * @cap: Vendor-specific capability ID 710 * 711 * If @dev has Vendor ID @vendor, search for a VSEC capability with 712 * VSEC ID @cap. If found, return the capability offset in 713 * config space; otherwise return 0. 714 */ 715 u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap) 716 { 717 u16 vsec = 0; 718 u32 header; 719 720 if (vendor != dev->vendor) 721 return 0; 722 723 while ((vsec = pci_find_next_ext_capability(dev, vsec, 724 PCI_EXT_CAP_ID_VNDR))) { 725 if (pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, 726 &header) == PCIBIOS_SUCCESSFUL && 727 PCI_VNDR_HEADER_ID(header) == cap) 728 return vsec; 729 } 730 731 return 0; 732 } 733 EXPORT_SYMBOL_GPL(pci_find_vsec_capability); 734 735 /** 736 * pci_find_dvsec_capability - Find DVSEC for vendor 737 * @dev: PCI device to query 738 * @vendor: Vendor ID to match for the DVSEC 739 * @dvsec: Designated Vendor-specific capability ID 740 * 741 * If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability 742 * offset in config space; otherwise return 0. 743 */ 744 u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec) 745 { 746 int pos; 747 748 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC); 749 if (!pos) 750 return 0; 751 752 while (pos) { 753 u16 v, id; 754 755 pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v); 756 pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id); 757 if (vendor == v && dvsec == id) 758 return pos; 759 760 pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC); 761 } 762 763 return 0; 764 } 765 EXPORT_SYMBOL_GPL(pci_find_dvsec_capability); 766 767 /** 768 * pci_find_parent_resource - return resource region of parent bus of given 769 * region 770 * @dev: PCI device structure contains resources to be searched 771 * @res: child resource record for which parent is sought 772 * 773 * For given resource region of given device, return the resource region of 774 * parent bus the given region is contained in. 775 */ 776 struct resource *pci_find_parent_resource(const struct pci_dev *dev, 777 struct resource *res) 778 { 779 const struct pci_bus *bus = dev->bus; 780 struct resource *r; 781 int i; 782 783 pci_bus_for_each_resource(bus, r, i) { 784 if (!r) 785 continue; 786 if (resource_contains(r, res)) { 787 788 /* 789 * If the window is prefetchable but the BAR is 790 * not, the allocator made a mistake. 791 */ 792 if (r->flags & IORESOURCE_PREFETCH && 793 !(res->flags & IORESOURCE_PREFETCH)) 794 return NULL; 795 796 /* 797 * If we're below a transparent bridge, there may 798 * be both a positively-decoded aperture and a 799 * subtractively-decoded region that contain the BAR. 800 * We want the positively-decoded one, so this depends 801 * on pci_bus_for_each_resource() giving us those 802 * first. 803 */ 804 return r; 805 } 806 } 807 return NULL; 808 } 809 EXPORT_SYMBOL(pci_find_parent_resource); 810 811 /** 812 * pci_find_resource - Return matching PCI device resource 813 * @dev: PCI device to query 814 * @res: Resource to look for 815 * 816 * Goes over standard PCI resources (BARs) and checks if the given resource 817 * is partially or fully contained in any of them. In that case the 818 * matching resource is returned, %NULL otherwise. 819 */ 820 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res) 821 { 822 int i; 823 824 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 825 struct resource *r = &dev->resource[i]; 826 827 if (r->start && resource_contains(r, res)) 828 return r; 829 } 830 831 return NULL; 832 } 833 EXPORT_SYMBOL(pci_find_resource); 834 835 /** 836 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos 837 * @dev: the PCI device to operate on 838 * @pos: config space offset of status word 839 * @mask: mask of bit(s) to care about in status word 840 * 841 * Return 1 when mask bit(s) in status word clear, 0 otherwise. 842 */ 843 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask) 844 { 845 int i; 846 847 /* Wait for Transaction Pending bit clean */ 848 for (i = 0; i < 4; i++) { 849 u16 status; 850 if (i) 851 msleep((1 << (i - 1)) * 100); 852 853 pci_read_config_word(dev, pos, &status); 854 if (!(status & mask)) 855 return 1; 856 } 857 858 return 0; 859 } 860 861 static int pci_acs_enable; 862 863 /** 864 * pci_request_acs - ask for ACS to be enabled if supported 865 */ 866 void pci_request_acs(void) 867 { 868 pci_acs_enable = 1; 869 } 870 871 static const char *disable_acs_redir_param; 872 873 /** 874 * pci_disable_acs_redir - disable ACS redirect capabilities 875 * @dev: the PCI device 876 * 877 * For only devices specified in the disable_acs_redir parameter. 878 */ 879 static void pci_disable_acs_redir(struct pci_dev *dev) 880 { 881 int ret = 0; 882 const char *p; 883 int pos; 884 u16 ctrl; 885 886 if (!disable_acs_redir_param) 887 return; 888 889 p = disable_acs_redir_param; 890 while (*p) { 891 ret = pci_dev_str_match(dev, p, &p); 892 if (ret < 0) { 893 pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n", 894 disable_acs_redir_param); 895 896 break; 897 } else if (ret == 1) { 898 /* Found a match */ 899 break; 900 } 901 902 if (*p != ';' && *p != ',') { 903 /* End of param or invalid format */ 904 break; 905 } 906 p++; 907 } 908 909 if (ret != 1) 910 return; 911 912 if (!pci_dev_specific_disable_acs_redir(dev)) 913 return; 914 915 pos = dev->acs_cap; 916 if (!pos) { 917 pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n"); 918 return; 919 } 920 921 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 922 923 /* P2P Request & Completion Redirect */ 924 ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC); 925 926 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 927 928 pci_info(dev, "disabled ACS redirect\n"); 929 } 930 931 /** 932 * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities 933 * @dev: the PCI device 934 */ 935 static void pci_std_enable_acs(struct pci_dev *dev) 936 { 937 int pos; 938 u16 cap; 939 u16 ctrl; 940 941 pos = dev->acs_cap; 942 if (!pos) 943 return; 944 945 pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap); 946 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 947 948 /* Source Validation */ 949 ctrl |= (cap & PCI_ACS_SV); 950 951 /* P2P Request Redirect */ 952 ctrl |= (cap & PCI_ACS_RR); 953 954 /* P2P Completion Redirect */ 955 ctrl |= (cap & PCI_ACS_CR); 956 957 /* Upstream Forwarding */ 958 ctrl |= (cap & PCI_ACS_UF); 959 960 /* Enable Translation Blocking for external devices and noats */ 961 if (pci_ats_disabled() || dev->external_facing || dev->untrusted) 962 ctrl |= (cap & PCI_ACS_TB); 963 964 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 965 } 966 967 /** 968 * pci_enable_acs - enable ACS if hardware support it 969 * @dev: the PCI device 970 */ 971 static void pci_enable_acs(struct pci_dev *dev) 972 { 973 if (!pci_acs_enable) 974 goto disable_acs_redir; 975 976 if (!pci_dev_specific_enable_acs(dev)) 977 goto disable_acs_redir; 978 979 pci_std_enable_acs(dev); 980 981 disable_acs_redir: 982 /* 983 * Note: pci_disable_acs_redir() must be called even if ACS was not 984 * enabled by the kernel because it may have been enabled by 985 * platform firmware. So if we are told to disable it, we should 986 * always disable it after setting the kernel's default 987 * preferences. 988 */ 989 pci_disable_acs_redir(dev); 990 } 991 992 /** 993 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up) 994 * @dev: PCI device to have its BARs restored 995 * 996 * Restore the BAR values for a given device, so as to make it 997 * accessible by its driver. 998 */ 999 static void pci_restore_bars(struct pci_dev *dev) 1000 { 1001 int i; 1002 1003 for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) 1004 pci_update_resource(dev, i); 1005 } 1006 1007 static inline bool platform_pci_power_manageable(struct pci_dev *dev) 1008 { 1009 if (pci_use_mid_pm()) 1010 return true; 1011 1012 return acpi_pci_power_manageable(dev); 1013 } 1014 1015 static inline int platform_pci_set_power_state(struct pci_dev *dev, 1016 pci_power_t t) 1017 { 1018 if (pci_use_mid_pm()) 1019 return mid_pci_set_power_state(dev, t); 1020 1021 return acpi_pci_set_power_state(dev, t); 1022 } 1023 1024 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev) 1025 { 1026 if (pci_use_mid_pm()) 1027 return mid_pci_get_power_state(dev); 1028 1029 return acpi_pci_get_power_state(dev); 1030 } 1031 1032 static inline void platform_pci_refresh_power_state(struct pci_dev *dev) 1033 { 1034 if (!pci_use_mid_pm()) 1035 acpi_pci_refresh_power_state(dev); 1036 } 1037 1038 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev) 1039 { 1040 if (pci_use_mid_pm()) 1041 return PCI_POWER_ERROR; 1042 1043 return acpi_pci_choose_state(dev); 1044 } 1045 1046 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable) 1047 { 1048 if (pci_use_mid_pm()) 1049 return PCI_POWER_ERROR; 1050 1051 return acpi_pci_wakeup(dev, enable); 1052 } 1053 1054 static inline bool platform_pci_need_resume(struct pci_dev *dev) 1055 { 1056 if (pci_use_mid_pm()) 1057 return false; 1058 1059 return acpi_pci_need_resume(dev); 1060 } 1061 1062 static inline bool platform_pci_bridge_d3(struct pci_dev *dev) 1063 { 1064 if (pci_use_mid_pm()) 1065 return false; 1066 1067 return acpi_pci_bridge_d3(dev); 1068 } 1069 1070 /** 1071 * pci_update_current_state - Read power state of given device and cache it 1072 * @dev: PCI device to handle. 1073 * @state: State to cache in case the device doesn't have the PM capability 1074 * 1075 * The power state is read from the PMCSR register, which however is 1076 * inaccessible in D3cold. The platform firmware is therefore queried first 1077 * to detect accessibility of the register. In case the platform firmware 1078 * reports an incorrect state or the device isn't power manageable by the 1079 * platform at all, we try to detect D3cold by testing accessibility of the 1080 * vendor ID in config space. 1081 */ 1082 void pci_update_current_state(struct pci_dev *dev, pci_power_t state) 1083 { 1084 if (platform_pci_get_power_state(dev) == PCI_D3cold) { 1085 dev->current_state = PCI_D3cold; 1086 } else if (dev->pm_cap) { 1087 u16 pmcsr; 1088 1089 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1090 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1091 dev->current_state = PCI_D3cold; 1092 return; 1093 } 1094 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1095 } else { 1096 dev->current_state = state; 1097 } 1098 } 1099 1100 /** 1101 * pci_refresh_power_state - Refresh the given device's power state data 1102 * @dev: Target PCI device. 1103 * 1104 * Ask the platform to refresh the devices power state information and invoke 1105 * pci_update_current_state() to update its current PCI power state. 1106 */ 1107 void pci_refresh_power_state(struct pci_dev *dev) 1108 { 1109 platform_pci_refresh_power_state(dev); 1110 pci_update_current_state(dev, dev->current_state); 1111 } 1112 1113 /** 1114 * pci_platform_power_transition - Use platform to change device power state 1115 * @dev: PCI device to handle. 1116 * @state: State to put the device into. 1117 */ 1118 int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state) 1119 { 1120 int error; 1121 1122 error = platform_pci_set_power_state(dev, state); 1123 if (!error) 1124 pci_update_current_state(dev, state); 1125 else if (!dev->pm_cap) /* Fall back to PCI_D0 */ 1126 dev->current_state = PCI_D0; 1127 1128 return error; 1129 } 1130 EXPORT_SYMBOL_GPL(pci_platform_power_transition); 1131 1132 static int pci_resume_one(struct pci_dev *pci_dev, void *ign) 1133 { 1134 pm_request_resume(&pci_dev->dev); 1135 return 0; 1136 } 1137 1138 /** 1139 * pci_resume_bus - Walk given bus and runtime resume devices on it 1140 * @bus: Top bus of the subtree to walk. 1141 */ 1142 void pci_resume_bus(struct pci_bus *bus) 1143 { 1144 if (bus) 1145 pci_walk_bus(bus, pci_resume_one, NULL); 1146 } 1147 1148 static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) 1149 { 1150 int delay = 1; 1151 u32 id; 1152 1153 /* 1154 * After reset, the device should not silently discard config 1155 * requests, but it may still indicate that it needs more time by 1156 * responding to them with CRS completions. The Root Port will 1157 * generally synthesize ~0 (PCI_ERROR_RESPONSE) data to complete 1158 * the read (except when CRS SV is enabled and the read was for the 1159 * Vendor ID; in that case it synthesizes 0x0001 data). 1160 * 1161 * Wait for the device to return a non-CRS completion. Read the 1162 * Command register instead of Vendor ID so we don't have to 1163 * contend with the CRS SV value. 1164 */ 1165 pci_read_config_dword(dev, PCI_COMMAND, &id); 1166 while (PCI_POSSIBLE_ERROR(id)) { 1167 if (delay > timeout) { 1168 pci_warn(dev, "not ready %dms after %s; giving up\n", 1169 delay - 1, reset_type); 1170 return -ENOTTY; 1171 } 1172 1173 if (delay > 1000) 1174 pci_info(dev, "not ready %dms after %s; waiting\n", 1175 delay - 1, reset_type); 1176 1177 msleep(delay); 1178 delay *= 2; 1179 pci_read_config_dword(dev, PCI_COMMAND, &id); 1180 } 1181 1182 if (delay > 1000) 1183 pci_info(dev, "ready %dms after %s\n", delay - 1, 1184 reset_type); 1185 1186 return 0; 1187 } 1188 1189 /** 1190 * pci_power_up - Put the given device into D0 1191 * @dev: PCI device to power up 1192 * 1193 * On success, return 0 or 1, depending on whether or not it is necessary to 1194 * restore the device's BARs subsequently (1 is returned in that case). 1195 */ 1196 int pci_power_up(struct pci_dev *dev) 1197 { 1198 bool need_restore; 1199 pci_power_t state; 1200 u16 pmcsr; 1201 1202 platform_pci_set_power_state(dev, PCI_D0); 1203 1204 if (!dev->pm_cap) { 1205 state = platform_pci_get_power_state(dev); 1206 if (state == PCI_UNKNOWN) 1207 dev->current_state = PCI_D0; 1208 else 1209 dev->current_state = state; 1210 1211 if (state == PCI_D0) 1212 return 0; 1213 1214 return -EIO; 1215 } 1216 1217 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1218 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1219 pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n", 1220 pci_power_name(dev->current_state)); 1221 dev->current_state = PCI_D3cold; 1222 return -EIO; 1223 } 1224 1225 state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1226 1227 need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) && 1228 !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 1229 1230 if (state == PCI_D0) 1231 goto end; 1232 1233 /* 1234 * Force the entire word to 0. This doesn't affect PME_Status, disables 1235 * PME_En, and sets PowerState to 0. 1236 */ 1237 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0); 1238 1239 /* Mandatory transition delays; see PCI PM 1.2. */ 1240 if (state == PCI_D3hot) 1241 pci_dev_d3_sleep(dev); 1242 else if (state == PCI_D2) 1243 udelay(PCI_PM_D2_DELAY); 1244 1245 end: 1246 dev->current_state = PCI_D0; 1247 if (need_restore) 1248 return 1; 1249 1250 return 0; 1251 } 1252 1253 /** 1254 * pci_set_full_power_state - Put a PCI device into D0 and update its state 1255 * @dev: PCI device to power up 1256 * 1257 * Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register 1258 * to confirm the state change, restore its BARs if they might be lost and 1259 * reconfigure ASPM in acordance with the new power state. 1260 * 1261 * If pci_restore_state() is going to be called right after a power state change 1262 * to D0, it is more efficient to use pci_power_up() directly instead of this 1263 * function. 1264 */ 1265 static int pci_set_full_power_state(struct pci_dev *dev) 1266 { 1267 u16 pmcsr; 1268 int ret; 1269 1270 ret = pci_power_up(dev); 1271 if (ret < 0) 1272 return ret; 1273 1274 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1275 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1276 if (dev->current_state != PCI_D0) { 1277 pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n", 1278 pci_power_name(dev->current_state)); 1279 } else if (ret > 0) { 1280 /* 1281 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT 1282 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning 1283 * from D3hot to D0 _may_ perform an internal reset, thereby 1284 * going to "D0 Uninitialized" rather than "D0 Initialized". 1285 * For example, at least some versions of the 3c905B and the 1286 * 3c556B exhibit this behaviour. 1287 * 1288 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave 1289 * devices in a D3hot state at boot. Consequently, we need to 1290 * restore at least the BARs so that the device will be 1291 * accessible to its driver. 1292 */ 1293 pci_restore_bars(dev); 1294 } 1295 1296 if (dev->bus->self) 1297 pcie_aspm_pm_state_change(dev->bus->self); 1298 1299 return 0; 1300 } 1301 1302 /** 1303 * __pci_dev_set_current_state - Set current state of a PCI device 1304 * @dev: Device to handle 1305 * @data: pointer to state to be set 1306 */ 1307 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data) 1308 { 1309 pci_power_t state = *(pci_power_t *)data; 1310 1311 dev->current_state = state; 1312 return 0; 1313 } 1314 1315 /** 1316 * pci_bus_set_current_state - Walk given bus and set current state of devices 1317 * @bus: Top bus of the subtree to walk. 1318 * @state: state to be set 1319 */ 1320 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state) 1321 { 1322 if (bus) 1323 pci_walk_bus(bus, __pci_dev_set_current_state, &state); 1324 } 1325 1326 /** 1327 * pci_set_low_power_state - Put a PCI device into a low-power state. 1328 * @dev: PCI device to handle. 1329 * @state: PCI power state (D1, D2, D3hot) to put the device into. 1330 * 1331 * Use the device's PCI_PM_CTRL register to put it into a low-power state. 1332 * 1333 * RETURN VALUE: 1334 * -EINVAL if the requested state is invalid. 1335 * -EIO if device does not support PCI PM or its PM capabilities register has a 1336 * wrong version, or device doesn't support the requested state. 1337 * 0 if device already is in the requested state. 1338 * 0 if device's power state has been successfully changed. 1339 */ 1340 static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state) 1341 { 1342 u16 pmcsr; 1343 1344 if (!dev->pm_cap) 1345 return -EIO; 1346 1347 /* 1348 * Validate transition: We can enter D0 from any state, but if 1349 * we're already in a low-power state, we can only go deeper. E.g., 1350 * we can go from D1 to D3, but we can't go directly from D3 to D1; 1351 * we'd have to go from D3 to D0, then to D1. 1352 */ 1353 if (dev->current_state <= PCI_D3cold && dev->current_state > state) { 1354 pci_dbg(dev, "Invalid power transition (from %s to %s)\n", 1355 pci_power_name(dev->current_state), 1356 pci_power_name(state)); 1357 return -EINVAL; 1358 } 1359 1360 /* Check if this device supports the desired state */ 1361 if ((state == PCI_D1 && !dev->d1_support) 1362 || (state == PCI_D2 && !dev->d2_support)) 1363 return -EIO; 1364 1365 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1366 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1367 pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n", 1368 pci_power_name(dev->current_state), 1369 pci_power_name(state)); 1370 dev->current_state = PCI_D3cold; 1371 return -EIO; 1372 } 1373 1374 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 1375 pmcsr |= state; 1376 1377 /* Enter specified state */ 1378 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 1379 1380 /* Mandatory power management transition delays; see PCI PM 1.2. */ 1381 if (state == PCI_D3hot) 1382 pci_dev_d3_sleep(dev); 1383 else if (state == PCI_D2) 1384 udelay(PCI_PM_D2_DELAY); 1385 1386 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1387 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1388 if (dev->current_state != state) 1389 pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n", 1390 pci_power_name(dev->current_state), 1391 pci_power_name(state)); 1392 1393 if (dev->bus->self) 1394 pcie_aspm_pm_state_change(dev->bus->self); 1395 1396 return 0; 1397 } 1398 1399 /** 1400 * pci_set_power_state - Set the power state of a PCI device 1401 * @dev: PCI device to handle. 1402 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. 1403 * 1404 * Transition a device to a new power state, using the platform firmware and/or 1405 * the device's PCI PM registers. 1406 * 1407 * RETURN VALUE: 1408 * -EINVAL if the requested state is invalid. 1409 * -EIO if device does not support PCI PM or its PM capabilities register has a 1410 * wrong version, or device doesn't support the requested state. 1411 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported. 1412 * 0 if device already is in the requested state. 1413 * 0 if the transition is to D3 but D3 is not supported. 1414 * 0 if device's power state has been successfully changed. 1415 */ 1416 int pci_set_power_state(struct pci_dev *dev, pci_power_t state) 1417 { 1418 int error; 1419 1420 /* Bound the state we're entering */ 1421 if (state > PCI_D3cold) 1422 state = PCI_D3cold; 1423 else if (state < PCI_D0) 1424 state = PCI_D0; 1425 else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev)) 1426 1427 /* 1428 * If the device or the parent bridge do not support PCI 1429 * PM, ignore the request if we're doing anything other 1430 * than putting it into D0 (which would only happen on 1431 * boot). 1432 */ 1433 return 0; 1434 1435 /* Check if we're already there */ 1436 if (dev->current_state == state) 1437 return 0; 1438 1439 if (state == PCI_D0) 1440 return pci_set_full_power_state(dev); 1441 1442 /* 1443 * This device is quirked not to be put into D3, so don't put it in 1444 * D3 1445 */ 1446 if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3)) 1447 return 0; 1448 1449 if (state == PCI_D3cold) { 1450 /* 1451 * To put the device in D3cold, put it into D3hot in the native 1452 * way, then put it into D3cold using platform ops. 1453 */ 1454 error = pci_set_low_power_state(dev, PCI_D3hot); 1455 1456 if (pci_platform_power_transition(dev, PCI_D3cold)) 1457 return error; 1458 1459 /* Powering off a bridge may power off the whole hierarchy */ 1460 if (dev->current_state == PCI_D3cold) 1461 pci_bus_set_current_state(dev->subordinate, PCI_D3cold); 1462 } else { 1463 error = pci_set_low_power_state(dev, state); 1464 1465 if (pci_platform_power_transition(dev, state)) 1466 return error; 1467 } 1468 1469 return 0; 1470 } 1471 EXPORT_SYMBOL(pci_set_power_state); 1472 1473 #define PCI_EXP_SAVE_REGS 7 1474 1475 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev, 1476 u16 cap, bool extended) 1477 { 1478 struct pci_cap_saved_state *tmp; 1479 1480 hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) { 1481 if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap) 1482 return tmp; 1483 } 1484 return NULL; 1485 } 1486 1487 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap) 1488 { 1489 return _pci_find_saved_cap(dev, cap, false); 1490 } 1491 1492 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap) 1493 { 1494 return _pci_find_saved_cap(dev, cap, true); 1495 } 1496 1497 static int pci_save_pcie_state(struct pci_dev *dev) 1498 { 1499 int i = 0; 1500 struct pci_cap_saved_state *save_state; 1501 u16 *cap; 1502 1503 if (!pci_is_pcie(dev)) 1504 return 0; 1505 1506 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1507 if (!save_state) { 1508 pci_err(dev, "buffer not found in %s\n", __func__); 1509 return -ENOMEM; 1510 } 1511 1512 cap = (u16 *)&save_state->cap.data[0]; 1513 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]); 1514 pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]); 1515 pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]); 1516 pcie_capability_read_word(dev, PCI_EXP_RTCTL, &cap[i++]); 1517 pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]); 1518 pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]); 1519 pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]); 1520 1521 return 0; 1522 } 1523 1524 void pci_bridge_reconfigure_ltr(struct pci_dev *dev) 1525 { 1526 #ifdef CONFIG_PCIEASPM 1527 struct pci_dev *bridge; 1528 u32 ctl; 1529 1530 bridge = pci_upstream_bridge(dev); 1531 if (bridge && bridge->ltr_path) { 1532 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, &ctl); 1533 if (!(ctl & PCI_EXP_DEVCTL2_LTR_EN)) { 1534 pci_dbg(bridge, "re-enabling LTR\n"); 1535 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, 1536 PCI_EXP_DEVCTL2_LTR_EN); 1537 } 1538 } 1539 #endif 1540 } 1541 1542 static void pci_restore_pcie_state(struct pci_dev *dev) 1543 { 1544 int i = 0; 1545 struct pci_cap_saved_state *save_state; 1546 u16 *cap; 1547 1548 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1549 if (!save_state) 1550 return; 1551 1552 /* 1553 * Downstream ports reset the LTR enable bit when link goes down. 1554 * Check and re-configure the bit here before restoring device. 1555 * PCIe r5.0, sec 7.5.3.16. 1556 */ 1557 pci_bridge_reconfigure_ltr(dev); 1558 1559 cap = (u16 *)&save_state->cap.data[0]; 1560 pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]); 1561 pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]); 1562 pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]); 1563 pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]); 1564 pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]); 1565 pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]); 1566 pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]); 1567 } 1568 1569 static int pci_save_pcix_state(struct pci_dev *dev) 1570 { 1571 int pos; 1572 struct pci_cap_saved_state *save_state; 1573 1574 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1575 if (!pos) 1576 return 0; 1577 1578 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1579 if (!save_state) { 1580 pci_err(dev, "buffer not found in %s\n", __func__); 1581 return -ENOMEM; 1582 } 1583 1584 pci_read_config_word(dev, pos + PCI_X_CMD, 1585 (u16 *)save_state->cap.data); 1586 1587 return 0; 1588 } 1589 1590 static void pci_restore_pcix_state(struct pci_dev *dev) 1591 { 1592 int i = 0, pos; 1593 struct pci_cap_saved_state *save_state; 1594 u16 *cap; 1595 1596 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1597 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1598 if (!save_state || !pos) 1599 return; 1600 cap = (u16 *)&save_state->cap.data[0]; 1601 1602 pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]); 1603 } 1604 1605 static void pci_save_ltr_state(struct pci_dev *dev) 1606 { 1607 int ltr; 1608 struct pci_cap_saved_state *save_state; 1609 u32 *cap; 1610 1611 if (!pci_is_pcie(dev)) 1612 return; 1613 1614 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR); 1615 if (!ltr) 1616 return; 1617 1618 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR); 1619 if (!save_state) { 1620 pci_err(dev, "no suspend buffer for LTR; ASPM issues possible after resume\n"); 1621 return; 1622 } 1623 1624 /* Some broken devices only support dword access to LTR */ 1625 cap = &save_state->cap.data[0]; 1626 pci_read_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap); 1627 } 1628 1629 static void pci_restore_ltr_state(struct pci_dev *dev) 1630 { 1631 struct pci_cap_saved_state *save_state; 1632 int ltr; 1633 u32 *cap; 1634 1635 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR); 1636 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR); 1637 if (!save_state || !ltr) 1638 return; 1639 1640 /* Some broken devices only support dword access to LTR */ 1641 cap = &save_state->cap.data[0]; 1642 pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap); 1643 } 1644 1645 /** 1646 * pci_save_state - save the PCI configuration space of a device before 1647 * suspending 1648 * @dev: PCI device that we're dealing with 1649 */ 1650 int pci_save_state(struct pci_dev *dev) 1651 { 1652 int i; 1653 /* XXX: 100% dword access ok here? */ 1654 for (i = 0; i < 16; i++) { 1655 pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]); 1656 pci_dbg(dev, "saving config space at offset %#x (reading %#x)\n", 1657 i * 4, dev->saved_config_space[i]); 1658 } 1659 dev->state_saved = true; 1660 1661 i = pci_save_pcie_state(dev); 1662 if (i != 0) 1663 return i; 1664 1665 i = pci_save_pcix_state(dev); 1666 if (i != 0) 1667 return i; 1668 1669 pci_save_ltr_state(dev); 1670 pci_save_dpc_state(dev); 1671 pci_save_aer_state(dev); 1672 pci_save_ptm_state(dev); 1673 return pci_save_vc_state(dev); 1674 } 1675 EXPORT_SYMBOL(pci_save_state); 1676 1677 static void pci_restore_config_dword(struct pci_dev *pdev, int offset, 1678 u32 saved_val, int retry, bool force) 1679 { 1680 u32 val; 1681 1682 pci_read_config_dword(pdev, offset, &val); 1683 if (!force && val == saved_val) 1684 return; 1685 1686 for (;;) { 1687 pci_dbg(pdev, "restoring config space at offset %#x (was %#x, writing %#x)\n", 1688 offset, val, saved_val); 1689 pci_write_config_dword(pdev, offset, saved_val); 1690 if (retry-- <= 0) 1691 return; 1692 1693 pci_read_config_dword(pdev, offset, &val); 1694 if (val == saved_val) 1695 return; 1696 1697 mdelay(1); 1698 } 1699 } 1700 1701 static void pci_restore_config_space_range(struct pci_dev *pdev, 1702 int start, int end, int retry, 1703 bool force) 1704 { 1705 int index; 1706 1707 for (index = end; index >= start; index--) 1708 pci_restore_config_dword(pdev, 4 * index, 1709 pdev->saved_config_space[index], 1710 retry, force); 1711 } 1712 1713 static void pci_restore_config_space(struct pci_dev *pdev) 1714 { 1715 if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) { 1716 pci_restore_config_space_range(pdev, 10, 15, 0, false); 1717 /* Restore BARs before the command register. */ 1718 pci_restore_config_space_range(pdev, 4, 9, 10, false); 1719 pci_restore_config_space_range(pdev, 0, 3, 0, false); 1720 } else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 1721 pci_restore_config_space_range(pdev, 12, 15, 0, false); 1722 1723 /* 1724 * Force rewriting of prefetch registers to avoid S3 resume 1725 * issues on Intel PCI bridges that occur when these 1726 * registers are not explicitly written. 1727 */ 1728 pci_restore_config_space_range(pdev, 9, 11, 0, true); 1729 pci_restore_config_space_range(pdev, 0, 8, 0, false); 1730 } else { 1731 pci_restore_config_space_range(pdev, 0, 15, 0, false); 1732 } 1733 } 1734 1735 static void pci_restore_rebar_state(struct pci_dev *pdev) 1736 { 1737 unsigned int pos, nbars, i; 1738 u32 ctrl; 1739 1740 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 1741 if (!pos) 1742 return; 1743 1744 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1745 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 1746 PCI_REBAR_CTRL_NBAR_SHIFT; 1747 1748 for (i = 0; i < nbars; i++, pos += 8) { 1749 struct resource *res; 1750 int bar_idx, size; 1751 1752 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1753 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 1754 res = pdev->resource + bar_idx; 1755 size = pci_rebar_bytes_to_size(resource_size(res)); 1756 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 1757 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 1758 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 1759 } 1760 } 1761 1762 /** 1763 * pci_restore_state - Restore the saved state of a PCI device 1764 * @dev: PCI device that we're dealing with 1765 */ 1766 void pci_restore_state(struct pci_dev *dev) 1767 { 1768 if (!dev->state_saved) 1769 return; 1770 1771 /* 1772 * Restore max latencies (in the LTR capability) before enabling 1773 * LTR itself (in the PCIe capability). 1774 */ 1775 pci_restore_ltr_state(dev); 1776 1777 pci_restore_pcie_state(dev); 1778 pci_restore_pasid_state(dev); 1779 pci_restore_pri_state(dev); 1780 pci_restore_ats_state(dev); 1781 pci_restore_vc_state(dev); 1782 pci_restore_rebar_state(dev); 1783 pci_restore_dpc_state(dev); 1784 pci_restore_ptm_state(dev); 1785 1786 pci_aer_clear_status(dev); 1787 pci_restore_aer_state(dev); 1788 1789 pci_restore_config_space(dev); 1790 1791 pci_restore_pcix_state(dev); 1792 pci_restore_msi_state(dev); 1793 1794 /* Restore ACS and IOV configuration state */ 1795 pci_enable_acs(dev); 1796 pci_restore_iov_state(dev); 1797 1798 dev->state_saved = false; 1799 } 1800 EXPORT_SYMBOL(pci_restore_state); 1801 1802 struct pci_saved_state { 1803 u32 config_space[16]; 1804 struct pci_cap_saved_data cap[]; 1805 }; 1806 1807 /** 1808 * pci_store_saved_state - Allocate and return an opaque struct containing 1809 * the device saved state. 1810 * @dev: PCI device that we're dealing with 1811 * 1812 * Return NULL if no state or error. 1813 */ 1814 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev) 1815 { 1816 struct pci_saved_state *state; 1817 struct pci_cap_saved_state *tmp; 1818 struct pci_cap_saved_data *cap; 1819 size_t size; 1820 1821 if (!dev->state_saved) 1822 return NULL; 1823 1824 size = sizeof(*state) + sizeof(struct pci_cap_saved_data); 1825 1826 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) 1827 size += sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1828 1829 state = kzalloc(size, GFP_KERNEL); 1830 if (!state) 1831 return NULL; 1832 1833 memcpy(state->config_space, dev->saved_config_space, 1834 sizeof(state->config_space)); 1835 1836 cap = state->cap; 1837 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) { 1838 size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1839 memcpy(cap, &tmp->cap, len); 1840 cap = (struct pci_cap_saved_data *)((u8 *)cap + len); 1841 } 1842 /* Empty cap_save terminates list */ 1843 1844 return state; 1845 } 1846 EXPORT_SYMBOL_GPL(pci_store_saved_state); 1847 1848 /** 1849 * pci_load_saved_state - Reload the provided save state into struct pci_dev. 1850 * @dev: PCI device that we're dealing with 1851 * @state: Saved state returned from pci_store_saved_state() 1852 */ 1853 int pci_load_saved_state(struct pci_dev *dev, 1854 struct pci_saved_state *state) 1855 { 1856 struct pci_cap_saved_data *cap; 1857 1858 dev->state_saved = false; 1859 1860 if (!state) 1861 return 0; 1862 1863 memcpy(dev->saved_config_space, state->config_space, 1864 sizeof(state->config_space)); 1865 1866 cap = state->cap; 1867 while (cap->size) { 1868 struct pci_cap_saved_state *tmp; 1869 1870 tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended); 1871 if (!tmp || tmp->cap.size != cap->size) 1872 return -EINVAL; 1873 1874 memcpy(tmp->cap.data, cap->data, tmp->cap.size); 1875 cap = (struct pci_cap_saved_data *)((u8 *)cap + 1876 sizeof(struct pci_cap_saved_data) + cap->size); 1877 } 1878 1879 dev->state_saved = true; 1880 return 0; 1881 } 1882 EXPORT_SYMBOL_GPL(pci_load_saved_state); 1883 1884 /** 1885 * pci_load_and_free_saved_state - Reload the save state pointed to by state, 1886 * and free the memory allocated for it. 1887 * @dev: PCI device that we're dealing with 1888 * @state: Pointer to saved state returned from pci_store_saved_state() 1889 */ 1890 int pci_load_and_free_saved_state(struct pci_dev *dev, 1891 struct pci_saved_state **state) 1892 { 1893 int ret = pci_load_saved_state(dev, *state); 1894 kfree(*state); 1895 *state = NULL; 1896 return ret; 1897 } 1898 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state); 1899 1900 int __weak pcibios_enable_device(struct pci_dev *dev, int bars) 1901 { 1902 return pci_enable_resources(dev, bars); 1903 } 1904 1905 static int do_pci_enable_device(struct pci_dev *dev, int bars) 1906 { 1907 int err; 1908 struct pci_dev *bridge; 1909 u16 cmd; 1910 u8 pin; 1911 1912 err = pci_set_power_state(dev, PCI_D0); 1913 if (err < 0 && err != -EIO) 1914 return err; 1915 1916 bridge = pci_upstream_bridge(dev); 1917 if (bridge) 1918 pcie_aspm_powersave_config_link(bridge); 1919 1920 err = pcibios_enable_device(dev, bars); 1921 if (err < 0) 1922 return err; 1923 pci_fixup_device(pci_fixup_enable, dev); 1924 1925 if (dev->msi_enabled || dev->msix_enabled) 1926 return 0; 1927 1928 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin); 1929 if (pin) { 1930 pci_read_config_word(dev, PCI_COMMAND, &cmd); 1931 if (cmd & PCI_COMMAND_INTX_DISABLE) 1932 pci_write_config_word(dev, PCI_COMMAND, 1933 cmd & ~PCI_COMMAND_INTX_DISABLE); 1934 } 1935 1936 return 0; 1937 } 1938 1939 /** 1940 * pci_reenable_device - Resume abandoned device 1941 * @dev: PCI device to be resumed 1942 * 1943 * NOTE: This function is a backend of pci_default_resume() and is not supposed 1944 * to be called by normal code, write proper resume handler and use it instead. 1945 */ 1946 int pci_reenable_device(struct pci_dev *dev) 1947 { 1948 if (pci_is_enabled(dev)) 1949 return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1); 1950 return 0; 1951 } 1952 EXPORT_SYMBOL(pci_reenable_device); 1953 1954 static void pci_enable_bridge(struct pci_dev *dev) 1955 { 1956 struct pci_dev *bridge; 1957 int retval; 1958 1959 bridge = pci_upstream_bridge(dev); 1960 if (bridge) 1961 pci_enable_bridge(bridge); 1962 1963 if (pci_is_enabled(dev)) { 1964 if (!dev->is_busmaster) 1965 pci_set_master(dev); 1966 return; 1967 } 1968 1969 retval = pci_enable_device(dev); 1970 if (retval) 1971 pci_err(dev, "Error enabling bridge (%d), continuing\n", 1972 retval); 1973 pci_set_master(dev); 1974 } 1975 1976 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) 1977 { 1978 struct pci_dev *bridge; 1979 int err; 1980 int i, bars = 0; 1981 1982 /* 1983 * Power state could be unknown at this point, either due to a fresh 1984 * boot or a device removal call. So get the current power state 1985 * so that things like MSI message writing will behave as expected 1986 * (e.g. if the device really is in D0 at enable time). 1987 */ 1988 pci_update_current_state(dev, dev->current_state); 1989 1990 if (atomic_inc_return(&dev->enable_cnt) > 1) 1991 return 0; /* already enabled */ 1992 1993 bridge = pci_upstream_bridge(dev); 1994 if (bridge) 1995 pci_enable_bridge(bridge); 1996 1997 /* only skip sriov related */ 1998 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 1999 if (dev->resource[i].flags & flags) 2000 bars |= (1 << i); 2001 for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++) 2002 if (dev->resource[i].flags & flags) 2003 bars |= (1 << i); 2004 2005 err = do_pci_enable_device(dev, bars); 2006 if (err < 0) 2007 atomic_dec(&dev->enable_cnt); 2008 return err; 2009 } 2010 2011 /** 2012 * pci_enable_device_io - Initialize a device for use with IO space 2013 * @dev: PCI device to be initialized 2014 * 2015 * Initialize device before it's used by a driver. Ask low-level code 2016 * to enable I/O resources. Wake up the device if it was suspended. 2017 * Beware, this function can fail. 2018 */ 2019 int pci_enable_device_io(struct pci_dev *dev) 2020 { 2021 return pci_enable_device_flags(dev, IORESOURCE_IO); 2022 } 2023 EXPORT_SYMBOL(pci_enable_device_io); 2024 2025 /** 2026 * pci_enable_device_mem - Initialize a device for use with Memory space 2027 * @dev: PCI device to be initialized 2028 * 2029 * Initialize device before it's used by a driver. Ask low-level code 2030 * to enable Memory resources. Wake up the device if it was suspended. 2031 * Beware, this function can fail. 2032 */ 2033 int pci_enable_device_mem(struct pci_dev *dev) 2034 { 2035 return pci_enable_device_flags(dev, IORESOURCE_MEM); 2036 } 2037 EXPORT_SYMBOL(pci_enable_device_mem); 2038 2039 /** 2040 * pci_enable_device - Initialize device before it's used by a driver. 2041 * @dev: PCI device to be initialized 2042 * 2043 * Initialize device before it's used by a driver. Ask low-level code 2044 * to enable I/O and memory. Wake up the device if it was suspended. 2045 * Beware, this function can fail. 2046 * 2047 * Note we don't actually enable the device many times if we call 2048 * this function repeatedly (we just increment the count). 2049 */ 2050 int pci_enable_device(struct pci_dev *dev) 2051 { 2052 return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO); 2053 } 2054 EXPORT_SYMBOL(pci_enable_device); 2055 2056 /* 2057 * Managed PCI resources. This manages device on/off, INTx/MSI/MSI-X 2058 * on/off and BAR regions. pci_dev itself records MSI/MSI-X status, so 2059 * there's no need to track it separately. pci_devres is initialized 2060 * when a device is enabled using managed PCI device enable interface. 2061 */ 2062 struct pci_devres { 2063 unsigned int enabled:1; 2064 unsigned int pinned:1; 2065 unsigned int orig_intx:1; 2066 unsigned int restore_intx:1; 2067 unsigned int mwi:1; 2068 u32 region_mask; 2069 }; 2070 2071 static void pcim_release(struct device *gendev, void *res) 2072 { 2073 struct pci_dev *dev = to_pci_dev(gendev); 2074 struct pci_devres *this = res; 2075 int i; 2076 2077 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) 2078 if (this->region_mask & (1 << i)) 2079 pci_release_region(dev, i); 2080 2081 if (this->mwi) 2082 pci_clear_mwi(dev); 2083 2084 if (this->restore_intx) 2085 pci_intx(dev, this->orig_intx); 2086 2087 if (this->enabled && !this->pinned) 2088 pci_disable_device(dev); 2089 } 2090 2091 static struct pci_devres *get_pci_dr(struct pci_dev *pdev) 2092 { 2093 struct pci_devres *dr, *new_dr; 2094 2095 dr = devres_find(&pdev->dev, pcim_release, NULL, NULL); 2096 if (dr) 2097 return dr; 2098 2099 new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL); 2100 if (!new_dr) 2101 return NULL; 2102 return devres_get(&pdev->dev, new_dr, NULL, NULL); 2103 } 2104 2105 static struct pci_devres *find_pci_dr(struct pci_dev *pdev) 2106 { 2107 if (pci_is_managed(pdev)) 2108 return devres_find(&pdev->dev, pcim_release, NULL, NULL); 2109 return NULL; 2110 } 2111 2112 /** 2113 * pcim_enable_device - Managed pci_enable_device() 2114 * @pdev: PCI device to be initialized 2115 * 2116 * Managed pci_enable_device(). 2117 */ 2118 int pcim_enable_device(struct pci_dev *pdev) 2119 { 2120 struct pci_devres *dr; 2121 int rc; 2122 2123 dr = get_pci_dr(pdev); 2124 if (unlikely(!dr)) 2125 return -ENOMEM; 2126 if (dr->enabled) 2127 return 0; 2128 2129 rc = pci_enable_device(pdev); 2130 if (!rc) { 2131 pdev->is_managed = 1; 2132 dr->enabled = 1; 2133 } 2134 return rc; 2135 } 2136 EXPORT_SYMBOL(pcim_enable_device); 2137 2138 /** 2139 * pcim_pin_device - Pin managed PCI device 2140 * @pdev: PCI device to pin 2141 * 2142 * Pin managed PCI device @pdev. Pinned device won't be disabled on 2143 * driver detach. @pdev must have been enabled with 2144 * pcim_enable_device(). 2145 */ 2146 void pcim_pin_device(struct pci_dev *pdev) 2147 { 2148 struct pci_devres *dr; 2149 2150 dr = find_pci_dr(pdev); 2151 WARN_ON(!dr || !dr->enabled); 2152 if (dr) 2153 dr->pinned = 1; 2154 } 2155 EXPORT_SYMBOL(pcim_pin_device); 2156 2157 /* 2158 * pcibios_device_add - provide arch specific hooks when adding device dev 2159 * @dev: the PCI device being added 2160 * 2161 * Permits the platform to provide architecture specific functionality when 2162 * devices are added. This is the default implementation. Architecture 2163 * implementations can override this. 2164 */ 2165 int __weak pcibios_device_add(struct pci_dev *dev) 2166 { 2167 return 0; 2168 } 2169 2170 /** 2171 * pcibios_release_device - provide arch specific hooks when releasing 2172 * device dev 2173 * @dev: the PCI device being released 2174 * 2175 * Permits the platform to provide architecture specific functionality when 2176 * devices are released. This is the default implementation. Architecture 2177 * implementations can override this. 2178 */ 2179 void __weak pcibios_release_device(struct pci_dev *dev) {} 2180 2181 /** 2182 * pcibios_disable_device - disable arch specific PCI resources for device dev 2183 * @dev: the PCI device to disable 2184 * 2185 * Disables architecture specific PCI resources for the device. This 2186 * is the default implementation. Architecture implementations can 2187 * override this. 2188 */ 2189 void __weak pcibios_disable_device(struct pci_dev *dev) {} 2190 2191 /** 2192 * pcibios_penalize_isa_irq - penalize an ISA IRQ 2193 * @irq: ISA IRQ to penalize 2194 * @active: IRQ active or not 2195 * 2196 * Permits the platform to provide architecture-specific functionality when 2197 * penalizing ISA IRQs. This is the default implementation. Architecture 2198 * implementations can override this. 2199 */ 2200 void __weak pcibios_penalize_isa_irq(int irq, int active) {} 2201 2202 static void do_pci_disable_device(struct pci_dev *dev) 2203 { 2204 u16 pci_command; 2205 2206 pci_read_config_word(dev, PCI_COMMAND, &pci_command); 2207 if (pci_command & PCI_COMMAND_MASTER) { 2208 pci_command &= ~PCI_COMMAND_MASTER; 2209 pci_write_config_word(dev, PCI_COMMAND, pci_command); 2210 } 2211 2212 pcibios_disable_device(dev); 2213 } 2214 2215 /** 2216 * pci_disable_enabled_device - Disable device without updating enable_cnt 2217 * @dev: PCI device to disable 2218 * 2219 * NOTE: This function is a backend of PCI power management routines and is 2220 * not supposed to be called drivers. 2221 */ 2222 void pci_disable_enabled_device(struct pci_dev *dev) 2223 { 2224 if (pci_is_enabled(dev)) 2225 do_pci_disable_device(dev); 2226 } 2227 2228 /** 2229 * pci_disable_device - Disable PCI device after use 2230 * @dev: PCI device to be disabled 2231 * 2232 * Signal to the system that the PCI device is not in use by the system 2233 * anymore. This only involves disabling PCI bus-mastering, if active. 2234 * 2235 * Note we don't actually disable the device until all callers of 2236 * pci_enable_device() have called pci_disable_device(). 2237 */ 2238 void pci_disable_device(struct pci_dev *dev) 2239 { 2240 struct pci_devres *dr; 2241 2242 dr = find_pci_dr(dev); 2243 if (dr) 2244 dr->enabled = 0; 2245 2246 dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0, 2247 "disabling already-disabled device"); 2248 2249 if (atomic_dec_return(&dev->enable_cnt) != 0) 2250 return; 2251 2252 do_pci_disable_device(dev); 2253 2254 dev->is_busmaster = 0; 2255 } 2256 EXPORT_SYMBOL(pci_disable_device); 2257 2258 /** 2259 * pcibios_set_pcie_reset_state - set reset state for device dev 2260 * @dev: the PCIe device reset 2261 * @state: Reset state to enter into 2262 * 2263 * Set the PCIe reset state for the device. This is the default 2264 * implementation. Architecture implementations can override this. 2265 */ 2266 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev, 2267 enum pcie_reset_state state) 2268 { 2269 return -EINVAL; 2270 } 2271 2272 /** 2273 * pci_set_pcie_reset_state - set reset state for device dev 2274 * @dev: the PCIe device reset 2275 * @state: Reset state to enter into 2276 * 2277 * Sets the PCI reset state for the device. 2278 */ 2279 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 2280 { 2281 return pcibios_set_pcie_reset_state(dev, state); 2282 } 2283 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state); 2284 2285 #ifdef CONFIG_PCIEAER 2286 void pcie_clear_device_status(struct pci_dev *dev) 2287 { 2288 u16 sta; 2289 2290 pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta); 2291 pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta); 2292 } 2293 #endif 2294 2295 /** 2296 * pcie_clear_root_pme_status - Clear root port PME interrupt status. 2297 * @dev: PCIe root port or event collector. 2298 */ 2299 void pcie_clear_root_pme_status(struct pci_dev *dev) 2300 { 2301 pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME); 2302 } 2303 2304 /** 2305 * pci_check_pme_status - Check if given device has generated PME. 2306 * @dev: Device to check. 2307 * 2308 * Check the PME status of the device and if set, clear it and clear PME enable 2309 * (if set). Return 'true' if PME status and PME enable were both set or 2310 * 'false' otherwise. 2311 */ 2312 bool pci_check_pme_status(struct pci_dev *dev) 2313 { 2314 int pmcsr_pos; 2315 u16 pmcsr; 2316 bool ret = false; 2317 2318 if (!dev->pm_cap) 2319 return false; 2320 2321 pmcsr_pos = dev->pm_cap + PCI_PM_CTRL; 2322 pci_read_config_word(dev, pmcsr_pos, &pmcsr); 2323 if (!(pmcsr & PCI_PM_CTRL_PME_STATUS)) 2324 return false; 2325 2326 /* Clear PME status. */ 2327 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2328 if (pmcsr & PCI_PM_CTRL_PME_ENABLE) { 2329 /* Disable PME to avoid interrupt flood. */ 2330 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2331 ret = true; 2332 } 2333 2334 pci_write_config_word(dev, pmcsr_pos, pmcsr); 2335 2336 return ret; 2337 } 2338 2339 /** 2340 * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set. 2341 * @dev: Device to handle. 2342 * @pme_poll_reset: Whether or not to reset the device's pme_poll flag. 2343 * 2344 * Check if @dev has generated PME and queue a resume request for it in that 2345 * case. 2346 */ 2347 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset) 2348 { 2349 if (pme_poll_reset && dev->pme_poll) 2350 dev->pme_poll = false; 2351 2352 if (pci_check_pme_status(dev)) { 2353 pci_wakeup_event(dev); 2354 pm_request_resume(&dev->dev); 2355 } 2356 return 0; 2357 } 2358 2359 /** 2360 * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary. 2361 * @bus: Top bus of the subtree to walk. 2362 */ 2363 void pci_pme_wakeup_bus(struct pci_bus *bus) 2364 { 2365 if (bus) 2366 pci_walk_bus(bus, pci_pme_wakeup, (void *)true); 2367 } 2368 2369 2370 /** 2371 * pci_pme_capable - check the capability of PCI device to generate PME# 2372 * @dev: PCI device to handle. 2373 * @state: PCI state from which device will issue PME#. 2374 */ 2375 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state) 2376 { 2377 if (!dev->pm_cap) 2378 return false; 2379 2380 return !!(dev->pme_support & (1 << state)); 2381 } 2382 EXPORT_SYMBOL(pci_pme_capable); 2383 2384 static void pci_pme_list_scan(struct work_struct *work) 2385 { 2386 struct pci_pme_device *pme_dev, *n; 2387 2388 mutex_lock(&pci_pme_list_mutex); 2389 list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 2390 if (pme_dev->dev->pme_poll) { 2391 struct pci_dev *bridge; 2392 2393 bridge = pme_dev->dev->bus->self; 2394 /* 2395 * If bridge is in low power state, the 2396 * configuration space of subordinate devices 2397 * may be not accessible 2398 */ 2399 if (bridge && bridge->current_state != PCI_D0) 2400 continue; 2401 /* 2402 * If the device is in D3cold it should not be 2403 * polled either. 2404 */ 2405 if (pme_dev->dev->current_state == PCI_D3cold) 2406 continue; 2407 2408 pci_pme_wakeup(pme_dev->dev, NULL); 2409 } else { 2410 list_del(&pme_dev->list); 2411 kfree(pme_dev); 2412 } 2413 } 2414 if (!list_empty(&pci_pme_list)) 2415 queue_delayed_work(system_freezable_wq, &pci_pme_work, 2416 msecs_to_jiffies(PME_TIMEOUT)); 2417 mutex_unlock(&pci_pme_list_mutex); 2418 } 2419 2420 static void __pci_pme_active(struct pci_dev *dev, bool enable) 2421 { 2422 u16 pmcsr; 2423 2424 if (!dev->pme_support) 2425 return; 2426 2427 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2428 /* Clear PME_Status by writing 1 to it and enable PME# */ 2429 pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE; 2430 if (!enable) 2431 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2432 2433 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2434 } 2435 2436 /** 2437 * pci_pme_restore - Restore PME configuration after config space restore. 2438 * @dev: PCI device to update. 2439 */ 2440 void pci_pme_restore(struct pci_dev *dev) 2441 { 2442 u16 pmcsr; 2443 2444 if (!dev->pme_support) 2445 return; 2446 2447 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2448 if (dev->wakeup_prepared) { 2449 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 2450 pmcsr &= ~PCI_PM_CTRL_PME_STATUS; 2451 } else { 2452 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2453 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2454 } 2455 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2456 } 2457 2458 /** 2459 * pci_pme_active - enable or disable PCI device's PME# function 2460 * @dev: PCI device to handle. 2461 * @enable: 'true' to enable PME# generation; 'false' to disable it. 2462 * 2463 * The caller must verify that the device is capable of generating PME# before 2464 * calling this function with @enable equal to 'true'. 2465 */ 2466 void pci_pme_active(struct pci_dev *dev, bool enable) 2467 { 2468 __pci_pme_active(dev, enable); 2469 2470 /* 2471 * PCI (as opposed to PCIe) PME requires that the device have 2472 * its PME# line hooked up correctly. Not all hardware vendors 2473 * do this, so the PME never gets delivered and the device 2474 * remains asleep. The easiest way around this is to 2475 * periodically walk the list of suspended devices and check 2476 * whether any have their PME flag set. The assumption is that 2477 * we'll wake up often enough anyway that this won't be a huge 2478 * hit, and the power savings from the devices will still be a 2479 * win. 2480 * 2481 * Although PCIe uses in-band PME message instead of PME# line 2482 * to report PME, PME does not work for some PCIe devices in 2483 * reality. For example, there are devices that set their PME 2484 * status bits, but don't really bother to send a PME message; 2485 * there are PCI Express Root Ports that don't bother to 2486 * trigger interrupts when they receive PME messages from the 2487 * devices below. So PME poll is used for PCIe devices too. 2488 */ 2489 2490 if (dev->pme_poll) { 2491 struct pci_pme_device *pme_dev; 2492 if (enable) { 2493 pme_dev = kmalloc(sizeof(struct pci_pme_device), 2494 GFP_KERNEL); 2495 if (!pme_dev) { 2496 pci_warn(dev, "can't enable PME#\n"); 2497 return; 2498 } 2499 pme_dev->dev = dev; 2500 mutex_lock(&pci_pme_list_mutex); 2501 list_add(&pme_dev->list, &pci_pme_list); 2502 if (list_is_singular(&pci_pme_list)) 2503 queue_delayed_work(system_freezable_wq, 2504 &pci_pme_work, 2505 msecs_to_jiffies(PME_TIMEOUT)); 2506 mutex_unlock(&pci_pme_list_mutex); 2507 } else { 2508 mutex_lock(&pci_pme_list_mutex); 2509 list_for_each_entry(pme_dev, &pci_pme_list, list) { 2510 if (pme_dev->dev == dev) { 2511 list_del(&pme_dev->list); 2512 kfree(pme_dev); 2513 break; 2514 } 2515 } 2516 mutex_unlock(&pci_pme_list_mutex); 2517 } 2518 } 2519 2520 pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled"); 2521 } 2522 EXPORT_SYMBOL(pci_pme_active); 2523 2524 /** 2525 * __pci_enable_wake - enable PCI device as wakeup event source 2526 * @dev: PCI device affected 2527 * @state: PCI state from which device will issue wakeup events 2528 * @enable: True to enable event generation; false to disable 2529 * 2530 * This enables the device as a wakeup event source, or disables it. 2531 * When such events involves platform-specific hooks, those hooks are 2532 * called automatically by this routine. 2533 * 2534 * Devices with legacy power management (no standard PCI PM capabilities) 2535 * always require such platform hooks. 2536 * 2537 * RETURN VALUE: 2538 * 0 is returned on success 2539 * -EINVAL is returned if device is not supposed to wake up the system 2540 * Error code depending on the platform is returned if both the platform and 2541 * the native mechanism fail to enable the generation of wake-up events 2542 */ 2543 static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable) 2544 { 2545 int ret = 0; 2546 2547 /* 2548 * Bridges that are not power-manageable directly only signal 2549 * wakeup on behalf of subordinate devices which is set up 2550 * elsewhere, so skip them. However, bridges that are 2551 * power-manageable may signal wakeup for themselves (for example, 2552 * on a hotplug event) and they need to be covered here. 2553 */ 2554 if (!pci_power_manageable(dev)) 2555 return 0; 2556 2557 /* Don't do the same thing twice in a row for one device. */ 2558 if (!!enable == !!dev->wakeup_prepared) 2559 return 0; 2560 2561 /* 2562 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don 2563 * Anderson we should be doing PME# wake enable followed by ACPI wake 2564 * enable. To disable wake-up we call the platform first, for symmetry. 2565 */ 2566 2567 if (enable) { 2568 int error; 2569 2570 /* 2571 * Enable PME signaling if the device can signal PME from 2572 * D3cold regardless of whether or not it can signal PME from 2573 * the current target state, because that will allow it to 2574 * signal PME when the hierarchy above it goes into D3cold and 2575 * the device itself ends up in D3cold as a result of that. 2576 */ 2577 if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold)) 2578 pci_pme_active(dev, true); 2579 else 2580 ret = 1; 2581 error = platform_pci_set_wakeup(dev, true); 2582 if (ret) 2583 ret = error; 2584 if (!ret) 2585 dev->wakeup_prepared = true; 2586 } else { 2587 platform_pci_set_wakeup(dev, false); 2588 pci_pme_active(dev, false); 2589 dev->wakeup_prepared = false; 2590 } 2591 2592 return ret; 2593 } 2594 2595 /** 2596 * pci_enable_wake - change wakeup settings for a PCI device 2597 * @pci_dev: Target device 2598 * @state: PCI state from which device will issue wakeup events 2599 * @enable: Whether or not to enable event generation 2600 * 2601 * If @enable is set, check device_may_wakeup() for the device before calling 2602 * __pci_enable_wake() for it. 2603 */ 2604 int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable) 2605 { 2606 if (enable && !device_may_wakeup(&pci_dev->dev)) 2607 return -EINVAL; 2608 2609 return __pci_enable_wake(pci_dev, state, enable); 2610 } 2611 EXPORT_SYMBOL(pci_enable_wake); 2612 2613 /** 2614 * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold 2615 * @dev: PCI device to prepare 2616 * @enable: True to enable wake-up event generation; false to disable 2617 * 2618 * Many drivers want the device to wake up the system from D3_hot or D3_cold 2619 * and this function allows them to set that up cleanly - pci_enable_wake() 2620 * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI 2621 * ordering constraints. 2622 * 2623 * This function only returns error code if the device is not allowed to wake 2624 * up the system from sleep or it is not capable of generating PME# from both 2625 * D3_hot and D3_cold and the platform is unable to enable wake-up power for it. 2626 */ 2627 int pci_wake_from_d3(struct pci_dev *dev, bool enable) 2628 { 2629 return pci_pme_capable(dev, PCI_D3cold) ? 2630 pci_enable_wake(dev, PCI_D3cold, enable) : 2631 pci_enable_wake(dev, PCI_D3hot, enable); 2632 } 2633 EXPORT_SYMBOL(pci_wake_from_d3); 2634 2635 /** 2636 * pci_target_state - find an appropriate low power state for a given PCI dev 2637 * @dev: PCI device 2638 * @wakeup: Whether or not wakeup functionality will be enabled for the device. 2639 * 2640 * Use underlying platform code to find a supported low power state for @dev. 2641 * If the platform can't manage @dev, return the deepest state from which it 2642 * can generate wake events, based on any available PME info. 2643 */ 2644 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup) 2645 { 2646 if (platform_pci_power_manageable(dev)) { 2647 /* 2648 * Call the platform to find the target state for the device. 2649 */ 2650 pci_power_t state = platform_pci_choose_state(dev); 2651 2652 switch (state) { 2653 case PCI_POWER_ERROR: 2654 case PCI_UNKNOWN: 2655 return PCI_D3hot; 2656 2657 case PCI_D1: 2658 case PCI_D2: 2659 if (pci_no_d1d2(dev)) 2660 return PCI_D3hot; 2661 } 2662 2663 return state; 2664 } 2665 2666 /* 2667 * If the device is in D3cold even though it's not power-manageable by 2668 * the platform, it may have been powered down by non-standard means. 2669 * Best to let it slumber. 2670 */ 2671 if (dev->current_state == PCI_D3cold) 2672 return PCI_D3cold; 2673 else if (!dev->pm_cap) 2674 return PCI_D0; 2675 2676 if (wakeup && dev->pme_support) { 2677 pci_power_t state = PCI_D3hot; 2678 2679 /* 2680 * Find the deepest state from which the device can generate 2681 * PME#. 2682 */ 2683 while (state && !(dev->pme_support & (1 << state))) 2684 state--; 2685 2686 if (state) 2687 return state; 2688 else if (dev->pme_support & 1) 2689 return PCI_D0; 2690 } 2691 2692 return PCI_D3hot; 2693 } 2694 2695 /** 2696 * pci_prepare_to_sleep - prepare PCI device for system-wide transition 2697 * into a sleep state 2698 * @dev: Device to handle. 2699 * 2700 * Choose the power state appropriate for the device depending on whether 2701 * it can wake up the system and/or is power manageable by the platform 2702 * (PCI_D3hot is the default) and put the device into that state. 2703 */ 2704 int pci_prepare_to_sleep(struct pci_dev *dev) 2705 { 2706 bool wakeup = device_may_wakeup(&dev->dev); 2707 pci_power_t target_state = pci_target_state(dev, wakeup); 2708 int error; 2709 2710 if (target_state == PCI_POWER_ERROR) 2711 return -EIO; 2712 2713 /* 2714 * There are systems (for example, Intel mobile chips since Coffee 2715 * Lake) where the power drawn while suspended can be significantly 2716 * reduced by disabling PTM on PCIe root ports as this allows the 2717 * port to enter a lower-power PM state and the SoC to reach a 2718 * lower-power idle state as a whole. 2719 */ 2720 if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) 2721 pci_disable_ptm(dev); 2722 2723 pci_enable_wake(dev, target_state, wakeup); 2724 2725 error = pci_set_power_state(dev, target_state); 2726 2727 if (error) { 2728 pci_enable_wake(dev, target_state, false); 2729 pci_restore_ptm_state(dev); 2730 } 2731 2732 return error; 2733 } 2734 EXPORT_SYMBOL(pci_prepare_to_sleep); 2735 2736 /** 2737 * pci_back_from_sleep - turn PCI device on during system-wide transition 2738 * into working state 2739 * @dev: Device to handle. 2740 * 2741 * Disable device's system wake-up capability and put it into D0. 2742 */ 2743 int pci_back_from_sleep(struct pci_dev *dev) 2744 { 2745 int ret = pci_set_power_state(dev, PCI_D0); 2746 2747 if (ret) 2748 return ret; 2749 2750 pci_enable_wake(dev, PCI_D0, false); 2751 return 0; 2752 } 2753 EXPORT_SYMBOL(pci_back_from_sleep); 2754 2755 /** 2756 * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend. 2757 * @dev: PCI device being suspended. 2758 * 2759 * Prepare @dev to generate wake-up events at run time and put it into a low 2760 * power state. 2761 */ 2762 int pci_finish_runtime_suspend(struct pci_dev *dev) 2763 { 2764 pci_power_t target_state; 2765 int error; 2766 2767 target_state = pci_target_state(dev, device_can_wakeup(&dev->dev)); 2768 if (target_state == PCI_POWER_ERROR) 2769 return -EIO; 2770 2771 /* 2772 * There are systems (for example, Intel mobile chips since Coffee 2773 * Lake) where the power drawn while suspended can be significantly 2774 * reduced by disabling PTM on PCIe root ports as this allows the 2775 * port to enter a lower-power PM state and the SoC to reach a 2776 * lower-power idle state as a whole. 2777 */ 2778 if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) 2779 pci_disable_ptm(dev); 2780 2781 __pci_enable_wake(dev, target_state, pci_dev_run_wake(dev)); 2782 2783 error = pci_set_power_state(dev, target_state); 2784 2785 if (error) { 2786 pci_enable_wake(dev, target_state, false); 2787 pci_restore_ptm_state(dev); 2788 } 2789 2790 return error; 2791 } 2792 2793 /** 2794 * pci_dev_run_wake - Check if device can generate run-time wake-up events. 2795 * @dev: Device to check. 2796 * 2797 * Return true if the device itself is capable of generating wake-up events 2798 * (through the platform or using the native PCIe PME) or if the device supports 2799 * PME and one of its upstream bridges can generate wake-up events. 2800 */ 2801 bool pci_dev_run_wake(struct pci_dev *dev) 2802 { 2803 struct pci_bus *bus = dev->bus; 2804 2805 if (!dev->pme_support) 2806 return false; 2807 2808 /* PME-capable in principle, but not from the target power state */ 2809 if (!pci_pme_capable(dev, pci_target_state(dev, true))) 2810 return false; 2811 2812 if (device_can_wakeup(&dev->dev)) 2813 return true; 2814 2815 while (bus->parent) { 2816 struct pci_dev *bridge = bus->self; 2817 2818 if (device_can_wakeup(&bridge->dev)) 2819 return true; 2820 2821 bus = bus->parent; 2822 } 2823 2824 /* We have reached the root bus. */ 2825 if (bus->bridge) 2826 return device_can_wakeup(bus->bridge); 2827 2828 return false; 2829 } 2830 EXPORT_SYMBOL_GPL(pci_dev_run_wake); 2831 2832 /** 2833 * pci_dev_need_resume - Check if it is necessary to resume the device. 2834 * @pci_dev: Device to check. 2835 * 2836 * Return 'true' if the device is not runtime-suspended or it has to be 2837 * reconfigured due to wakeup settings difference between system and runtime 2838 * suspend, or the current power state of it is not suitable for the upcoming 2839 * (system-wide) transition. 2840 */ 2841 bool pci_dev_need_resume(struct pci_dev *pci_dev) 2842 { 2843 struct device *dev = &pci_dev->dev; 2844 pci_power_t target_state; 2845 2846 if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev)) 2847 return true; 2848 2849 target_state = pci_target_state(pci_dev, device_may_wakeup(dev)); 2850 2851 /* 2852 * If the earlier platform check has not triggered, D3cold is just power 2853 * removal on top of D3hot, so no need to resume the device in that 2854 * case. 2855 */ 2856 return target_state != pci_dev->current_state && 2857 target_state != PCI_D3cold && 2858 pci_dev->current_state != PCI_D3hot; 2859 } 2860 2861 /** 2862 * pci_dev_adjust_pme - Adjust PME setting for a suspended device. 2863 * @pci_dev: Device to check. 2864 * 2865 * If the device is suspended and it is not configured for system wakeup, 2866 * disable PME for it to prevent it from waking up the system unnecessarily. 2867 * 2868 * Note that if the device's power state is D3cold and the platform check in 2869 * pci_dev_need_resume() has not triggered, the device's configuration need not 2870 * be changed. 2871 */ 2872 void pci_dev_adjust_pme(struct pci_dev *pci_dev) 2873 { 2874 struct device *dev = &pci_dev->dev; 2875 2876 spin_lock_irq(&dev->power.lock); 2877 2878 if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) && 2879 pci_dev->current_state < PCI_D3cold) 2880 __pci_pme_active(pci_dev, false); 2881 2882 spin_unlock_irq(&dev->power.lock); 2883 } 2884 2885 /** 2886 * pci_dev_complete_resume - Finalize resume from system sleep for a device. 2887 * @pci_dev: Device to handle. 2888 * 2889 * If the device is runtime suspended and wakeup-capable, enable PME for it as 2890 * it might have been disabled during the prepare phase of system suspend if 2891 * the device was not configured for system wakeup. 2892 */ 2893 void pci_dev_complete_resume(struct pci_dev *pci_dev) 2894 { 2895 struct device *dev = &pci_dev->dev; 2896 2897 if (!pci_dev_run_wake(pci_dev)) 2898 return; 2899 2900 spin_lock_irq(&dev->power.lock); 2901 2902 if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold) 2903 __pci_pme_active(pci_dev, true); 2904 2905 spin_unlock_irq(&dev->power.lock); 2906 } 2907 2908 /** 2909 * pci_choose_state - Choose the power state of a PCI device. 2910 * @dev: Target PCI device. 2911 * @state: Target state for the whole system. 2912 * 2913 * Returns PCI power state suitable for @dev and @state. 2914 */ 2915 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) 2916 { 2917 if (state.event == PM_EVENT_ON) 2918 return PCI_D0; 2919 2920 return pci_target_state(dev, false); 2921 } 2922 EXPORT_SYMBOL(pci_choose_state); 2923 2924 void pci_config_pm_runtime_get(struct pci_dev *pdev) 2925 { 2926 struct device *dev = &pdev->dev; 2927 struct device *parent = dev->parent; 2928 2929 if (parent) 2930 pm_runtime_get_sync(parent); 2931 pm_runtime_get_noresume(dev); 2932 /* 2933 * pdev->current_state is set to PCI_D3cold during suspending, 2934 * so wait until suspending completes 2935 */ 2936 pm_runtime_barrier(dev); 2937 /* 2938 * Only need to resume devices in D3cold, because config 2939 * registers are still accessible for devices suspended but 2940 * not in D3cold. 2941 */ 2942 if (pdev->current_state == PCI_D3cold) 2943 pm_runtime_resume(dev); 2944 } 2945 2946 void pci_config_pm_runtime_put(struct pci_dev *pdev) 2947 { 2948 struct device *dev = &pdev->dev; 2949 struct device *parent = dev->parent; 2950 2951 pm_runtime_put(dev); 2952 if (parent) 2953 pm_runtime_put_sync(parent); 2954 } 2955 2956 static const struct dmi_system_id bridge_d3_blacklist[] = { 2957 #ifdef CONFIG_X86 2958 { 2959 /* 2960 * Gigabyte X299 root port is not marked as hotplug capable 2961 * which allows Linux to power manage it. However, this 2962 * confuses the BIOS SMI handler so don't power manage root 2963 * ports on that system. 2964 */ 2965 .ident = "X299 DESIGNARE EX-CF", 2966 .matches = { 2967 DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."), 2968 DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"), 2969 }, 2970 }, 2971 { 2972 /* 2973 * Downstream device is not accessible after putting a root port 2974 * into D3cold and back into D0 on Elo i2. 2975 */ 2976 .ident = "Elo i2", 2977 .matches = { 2978 DMI_MATCH(DMI_SYS_VENDOR, "Elo Touch Solutions"), 2979 DMI_MATCH(DMI_PRODUCT_NAME, "Elo i2"), 2980 DMI_MATCH(DMI_PRODUCT_VERSION, "RevB"), 2981 }, 2982 }, 2983 #endif 2984 { } 2985 }; 2986 2987 /** 2988 * pci_bridge_d3_possible - Is it possible to put the bridge into D3 2989 * @bridge: Bridge to check 2990 * 2991 * This function checks if it is possible to move the bridge to D3. 2992 * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt. 2993 */ 2994 bool pci_bridge_d3_possible(struct pci_dev *bridge) 2995 { 2996 if (!pci_is_pcie(bridge)) 2997 return false; 2998 2999 switch (pci_pcie_type(bridge)) { 3000 case PCI_EXP_TYPE_ROOT_PORT: 3001 case PCI_EXP_TYPE_UPSTREAM: 3002 case PCI_EXP_TYPE_DOWNSTREAM: 3003 if (pci_bridge_d3_disable) 3004 return false; 3005 3006 /* 3007 * Hotplug ports handled by firmware in System Management Mode 3008 * may not be put into D3 by the OS (Thunderbolt on non-Macs). 3009 */ 3010 if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge)) 3011 return false; 3012 3013 if (pci_bridge_d3_force) 3014 return true; 3015 3016 /* Even the oldest 2010 Thunderbolt controller supports D3. */ 3017 if (bridge->is_thunderbolt) 3018 return true; 3019 3020 /* Platform might know better if the bridge supports D3 */ 3021 if (platform_pci_bridge_d3(bridge)) 3022 return true; 3023 3024 /* 3025 * Hotplug ports handled natively by the OS were not validated 3026 * by vendors for runtime D3 at least until 2018 because there 3027 * was no OS support. 3028 */ 3029 if (bridge->is_hotplug_bridge) 3030 return false; 3031 3032 if (dmi_check_system(bridge_d3_blacklist)) 3033 return false; 3034 3035 /* 3036 * It should be safe to put PCIe ports from 2015 or newer 3037 * to D3. 3038 */ 3039 if (dmi_get_bios_year() >= 2015) 3040 return true; 3041 break; 3042 } 3043 3044 return false; 3045 } 3046 3047 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) 3048 { 3049 bool *d3cold_ok = data; 3050 3051 if (/* The device needs to be allowed to go D3cold ... */ 3052 dev->no_d3cold || !dev->d3cold_allowed || 3053 3054 /* ... and if it is wakeup capable to do so from D3cold. */ 3055 (device_may_wakeup(&dev->dev) && 3056 !pci_pme_capable(dev, PCI_D3cold)) || 3057 3058 /* If it is a bridge it must be allowed to go to D3. */ 3059 !pci_power_manageable(dev)) 3060 3061 *d3cold_ok = false; 3062 3063 return !*d3cold_ok; 3064 } 3065 3066 /* 3067 * pci_bridge_d3_update - Update bridge D3 capabilities 3068 * @dev: PCI device which is changed 3069 * 3070 * Update upstream bridge PM capabilities accordingly depending on if the 3071 * device PM configuration was changed or the device is being removed. The 3072 * change is also propagated upstream. 3073 */ 3074 void pci_bridge_d3_update(struct pci_dev *dev) 3075 { 3076 bool remove = !device_is_registered(&dev->dev); 3077 struct pci_dev *bridge; 3078 bool d3cold_ok = true; 3079 3080 bridge = pci_upstream_bridge(dev); 3081 if (!bridge || !pci_bridge_d3_possible(bridge)) 3082 return; 3083 3084 /* 3085 * If D3 is currently allowed for the bridge, removing one of its 3086 * children won't change that. 3087 */ 3088 if (remove && bridge->bridge_d3) 3089 return; 3090 3091 /* 3092 * If D3 is currently allowed for the bridge and a child is added or 3093 * changed, disallowance of D3 can only be caused by that child, so 3094 * we only need to check that single device, not any of its siblings. 3095 * 3096 * If D3 is currently not allowed for the bridge, checking the device 3097 * first may allow us to skip checking its siblings. 3098 */ 3099 if (!remove) 3100 pci_dev_check_d3cold(dev, &d3cold_ok); 3101 3102 /* 3103 * If D3 is currently not allowed for the bridge, this may be caused 3104 * either by the device being changed/removed or any of its siblings, 3105 * so we need to go through all children to find out if one of them 3106 * continues to block D3. 3107 */ 3108 if (d3cold_ok && !bridge->bridge_d3) 3109 pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold, 3110 &d3cold_ok); 3111 3112 if (bridge->bridge_d3 != d3cold_ok) { 3113 bridge->bridge_d3 = d3cold_ok; 3114 /* Propagate change to upstream bridges */ 3115 pci_bridge_d3_update(bridge); 3116 } 3117 } 3118 3119 /** 3120 * pci_d3cold_enable - Enable D3cold for device 3121 * @dev: PCI device to handle 3122 * 3123 * This function can be used in drivers to enable D3cold from the device 3124 * they handle. It also updates upstream PCI bridge PM capabilities 3125 * accordingly. 3126 */ 3127 void pci_d3cold_enable(struct pci_dev *dev) 3128 { 3129 if (dev->no_d3cold) { 3130 dev->no_d3cold = false; 3131 pci_bridge_d3_update(dev); 3132 } 3133 } 3134 EXPORT_SYMBOL_GPL(pci_d3cold_enable); 3135 3136 /** 3137 * pci_d3cold_disable - Disable D3cold for device 3138 * @dev: PCI device to handle 3139 * 3140 * This function can be used in drivers to disable D3cold from the device 3141 * they handle. It also updates upstream PCI bridge PM capabilities 3142 * accordingly. 3143 */ 3144 void pci_d3cold_disable(struct pci_dev *dev) 3145 { 3146 if (!dev->no_d3cold) { 3147 dev->no_d3cold = true; 3148 pci_bridge_d3_update(dev); 3149 } 3150 } 3151 EXPORT_SYMBOL_GPL(pci_d3cold_disable); 3152 3153 /** 3154 * pci_pm_init - Initialize PM functions of given PCI device 3155 * @dev: PCI device to handle. 3156 */ 3157 void pci_pm_init(struct pci_dev *dev) 3158 { 3159 int pm; 3160 u16 status; 3161 u16 pmc; 3162 3163 pm_runtime_forbid(&dev->dev); 3164 pm_runtime_set_active(&dev->dev); 3165 pm_runtime_enable(&dev->dev); 3166 device_enable_async_suspend(&dev->dev); 3167 dev->wakeup_prepared = false; 3168 3169 dev->pm_cap = 0; 3170 dev->pme_support = 0; 3171 3172 /* find PCI PM capability in list */ 3173 pm = pci_find_capability(dev, PCI_CAP_ID_PM); 3174 if (!pm) 3175 return; 3176 /* Check device's ability to generate PME# */ 3177 pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc); 3178 3179 if ((pmc & PCI_PM_CAP_VER_MASK) > 3) { 3180 pci_err(dev, "unsupported PM cap regs version (%u)\n", 3181 pmc & PCI_PM_CAP_VER_MASK); 3182 return; 3183 } 3184 3185 dev->pm_cap = pm; 3186 dev->d3hot_delay = PCI_PM_D3HOT_WAIT; 3187 dev->d3cold_delay = PCI_PM_D3COLD_WAIT; 3188 dev->bridge_d3 = pci_bridge_d3_possible(dev); 3189 dev->d3cold_allowed = true; 3190 3191 dev->d1_support = false; 3192 dev->d2_support = false; 3193 if (!pci_no_d1d2(dev)) { 3194 if (pmc & PCI_PM_CAP_D1) 3195 dev->d1_support = true; 3196 if (pmc & PCI_PM_CAP_D2) 3197 dev->d2_support = true; 3198 3199 if (dev->d1_support || dev->d2_support) 3200 pci_info(dev, "supports%s%s\n", 3201 dev->d1_support ? " D1" : "", 3202 dev->d2_support ? " D2" : ""); 3203 } 3204 3205 pmc &= PCI_PM_CAP_PME_MASK; 3206 if (pmc) { 3207 pci_info(dev, "PME# supported from%s%s%s%s%s\n", 3208 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "", 3209 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "", 3210 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "", 3211 (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "", 3212 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : ""); 3213 dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT; 3214 dev->pme_poll = true; 3215 /* 3216 * Make device's PM flags reflect the wake-up capability, but 3217 * let the user space enable it to wake up the system as needed. 3218 */ 3219 device_set_wakeup_capable(&dev->dev, true); 3220 /* Disable the PME# generation functionality */ 3221 pci_pme_active(dev, false); 3222 } 3223 3224 pci_read_config_word(dev, PCI_STATUS, &status); 3225 if (status & PCI_STATUS_IMM_READY) 3226 dev->imm_ready = 1; 3227 } 3228 3229 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) 3230 { 3231 unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI; 3232 3233 switch (prop) { 3234 case PCI_EA_P_MEM: 3235 case PCI_EA_P_VF_MEM: 3236 flags |= IORESOURCE_MEM; 3237 break; 3238 case PCI_EA_P_MEM_PREFETCH: 3239 case PCI_EA_P_VF_MEM_PREFETCH: 3240 flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH; 3241 break; 3242 case PCI_EA_P_IO: 3243 flags |= IORESOURCE_IO; 3244 break; 3245 default: 3246 return 0; 3247 } 3248 3249 return flags; 3250 } 3251 3252 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei, 3253 u8 prop) 3254 { 3255 if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO) 3256 return &dev->resource[bei]; 3257 #ifdef CONFIG_PCI_IOV 3258 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 && 3259 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH)) 3260 return &dev->resource[PCI_IOV_RESOURCES + 3261 bei - PCI_EA_BEI_VF_BAR0]; 3262 #endif 3263 else if (bei == PCI_EA_BEI_ROM) 3264 return &dev->resource[PCI_ROM_RESOURCE]; 3265 else 3266 return NULL; 3267 } 3268 3269 /* Read an Enhanced Allocation (EA) entry */ 3270 static int pci_ea_read(struct pci_dev *dev, int offset) 3271 { 3272 struct resource *res; 3273 int ent_size, ent_offset = offset; 3274 resource_size_t start, end; 3275 unsigned long flags; 3276 u32 dw0, bei, base, max_offset; 3277 u8 prop; 3278 bool support_64 = (sizeof(resource_size_t) >= 8); 3279 3280 pci_read_config_dword(dev, ent_offset, &dw0); 3281 ent_offset += 4; 3282 3283 /* Entry size field indicates DWORDs after 1st */ 3284 ent_size = ((dw0 & PCI_EA_ES) + 1) << 2; 3285 3286 if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */ 3287 goto out; 3288 3289 bei = (dw0 & PCI_EA_BEI) >> 4; 3290 prop = (dw0 & PCI_EA_PP) >> 8; 3291 3292 /* 3293 * If the Property is in the reserved range, try the Secondary 3294 * Property instead. 3295 */ 3296 if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED) 3297 prop = (dw0 & PCI_EA_SP) >> 16; 3298 if (prop > PCI_EA_P_BRIDGE_IO) 3299 goto out; 3300 3301 res = pci_ea_get_resource(dev, bei, prop); 3302 if (!res) { 3303 pci_err(dev, "Unsupported EA entry BEI: %u\n", bei); 3304 goto out; 3305 } 3306 3307 flags = pci_ea_flags(dev, prop); 3308 if (!flags) { 3309 pci_err(dev, "Unsupported EA properties: %#x\n", prop); 3310 goto out; 3311 } 3312 3313 /* Read Base */ 3314 pci_read_config_dword(dev, ent_offset, &base); 3315 start = (base & PCI_EA_FIELD_MASK); 3316 ent_offset += 4; 3317 3318 /* Read MaxOffset */ 3319 pci_read_config_dword(dev, ent_offset, &max_offset); 3320 ent_offset += 4; 3321 3322 /* Read Base MSBs (if 64-bit entry) */ 3323 if (base & PCI_EA_IS_64) { 3324 u32 base_upper; 3325 3326 pci_read_config_dword(dev, ent_offset, &base_upper); 3327 ent_offset += 4; 3328 3329 flags |= IORESOURCE_MEM_64; 3330 3331 /* entry starts above 32-bit boundary, can't use */ 3332 if (!support_64 && base_upper) 3333 goto out; 3334 3335 if (support_64) 3336 start |= ((u64)base_upper << 32); 3337 } 3338 3339 end = start + (max_offset | 0x03); 3340 3341 /* Read MaxOffset MSBs (if 64-bit entry) */ 3342 if (max_offset & PCI_EA_IS_64) { 3343 u32 max_offset_upper; 3344 3345 pci_read_config_dword(dev, ent_offset, &max_offset_upper); 3346 ent_offset += 4; 3347 3348 flags |= IORESOURCE_MEM_64; 3349 3350 /* entry too big, can't use */ 3351 if (!support_64 && max_offset_upper) 3352 goto out; 3353 3354 if (support_64) 3355 end += ((u64)max_offset_upper << 32); 3356 } 3357 3358 if (end < start) { 3359 pci_err(dev, "EA Entry crosses address boundary\n"); 3360 goto out; 3361 } 3362 3363 if (ent_size != ent_offset - offset) { 3364 pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n", 3365 ent_size, ent_offset - offset); 3366 goto out; 3367 } 3368 3369 res->name = pci_name(dev); 3370 res->start = start; 3371 res->end = end; 3372 res->flags = flags; 3373 3374 if (bei <= PCI_EA_BEI_BAR5) 3375 pci_info(dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 3376 bei, res, prop); 3377 else if (bei == PCI_EA_BEI_ROM) 3378 pci_info(dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n", 3379 res, prop); 3380 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5) 3381 pci_info(dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 3382 bei - PCI_EA_BEI_VF_BAR0, res, prop); 3383 else 3384 pci_info(dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n", 3385 bei, res, prop); 3386 3387 out: 3388 return offset + ent_size; 3389 } 3390 3391 /* Enhanced Allocation Initialization */ 3392 void pci_ea_init(struct pci_dev *dev) 3393 { 3394 int ea; 3395 u8 num_ent; 3396 int offset; 3397 int i; 3398 3399 /* find PCI EA capability in list */ 3400 ea = pci_find_capability(dev, PCI_CAP_ID_EA); 3401 if (!ea) 3402 return; 3403 3404 /* determine the number of entries */ 3405 pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT, 3406 &num_ent); 3407 num_ent &= PCI_EA_NUM_ENT_MASK; 3408 3409 offset = ea + PCI_EA_FIRST_ENT; 3410 3411 /* Skip DWORD 2 for type 1 functions */ 3412 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 3413 offset += 4; 3414 3415 /* parse each EA entry */ 3416 for (i = 0; i < num_ent; ++i) 3417 offset = pci_ea_read(dev, offset); 3418 } 3419 3420 static void pci_add_saved_cap(struct pci_dev *pci_dev, 3421 struct pci_cap_saved_state *new_cap) 3422 { 3423 hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space); 3424 } 3425 3426 /** 3427 * _pci_add_cap_save_buffer - allocate buffer for saving given 3428 * capability registers 3429 * @dev: the PCI device 3430 * @cap: the capability to allocate the buffer for 3431 * @extended: Standard or Extended capability ID 3432 * @size: requested size of the buffer 3433 */ 3434 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap, 3435 bool extended, unsigned int size) 3436 { 3437 int pos; 3438 struct pci_cap_saved_state *save_state; 3439 3440 if (extended) 3441 pos = pci_find_ext_capability(dev, cap); 3442 else 3443 pos = pci_find_capability(dev, cap); 3444 3445 if (!pos) 3446 return 0; 3447 3448 save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL); 3449 if (!save_state) 3450 return -ENOMEM; 3451 3452 save_state->cap.cap_nr = cap; 3453 save_state->cap.cap_extended = extended; 3454 save_state->cap.size = size; 3455 pci_add_saved_cap(dev, save_state); 3456 3457 return 0; 3458 } 3459 3460 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size) 3461 { 3462 return _pci_add_cap_save_buffer(dev, cap, false, size); 3463 } 3464 3465 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size) 3466 { 3467 return _pci_add_cap_save_buffer(dev, cap, true, size); 3468 } 3469 3470 /** 3471 * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities 3472 * @dev: the PCI device 3473 */ 3474 void pci_allocate_cap_save_buffers(struct pci_dev *dev) 3475 { 3476 int error; 3477 3478 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP, 3479 PCI_EXP_SAVE_REGS * sizeof(u16)); 3480 if (error) 3481 pci_err(dev, "unable to preallocate PCI Express save buffer\n"); 3482 3483 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16)); 3484 if (error) 3485 pci_err(dev, "unable to preallocate PCI-X save buffer\n"); 3486 3487 error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR, 3488 2 * sizeof(u16)); 3489 if (error) 3490 pci_err(dev, "unable to allocate suspend buffer for LTR\n"); 3491 3492 pci_allocate_vc_save_buffers(dev); 3493 } 3494 3495 void pci_free_cap_save_buffers(struct pci_dev *dev) 3496 { 3497 struct pci_cap_saved_state *tmp; 3498 struct hlist_node *n; 3499 3500 hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next) 3501 kfree(tmp); 3502 } 3503 3504 /** 3505 * pci_configure_ari - enable or disable ARI forwarding 3506 * @dev: the PCI device 3507 * 3508 * If @dev and its upstream bridge both support ARI, enable ARI in the 3509 * bridge. Otherwise, disable ARI in the bridge. 3510 */ 3511 void pci_configure_ari(struct pci_dev *dev) 3512 { 3513 u32 cap; 3514 struct pci_dev *bridge; 3515 3516 if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn) 3517 return; 3518 3519 bridge = dev->bus->self; 3520 if (!bridge) 3521 return; 3522 3523 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3524 if (!(cap & PCI_EXP_DEVCAP2_ARI)) 3525 return; 3526 3527 if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) { 3528 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, 3529 PCI_EXP_DEVCTL2_ARI); 3530 bridge->ari_enabled = 1; 3531 } else { 3532 pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2, 3533 PCI_EXP_DEVCTL2_ARI); 3534 bridge->ari_enabled = 0; 3535 } 3536 } 3537 3538 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags) 3539 { 3540 int pos; 3541 u16 cap, ctrl; 3542 3543 pos = pdev->acs_cap; 3544 if (!pos) 3545 return false; 3546 3547 /* 3548 * Except for egress control, capabilities are either required 3549 * or only required if controllable. Features missing from the 3550 * capability field can therefore be assumed as hard-wired enabled. 3551 */ 3552 pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap); 3553 acs_flags &= (cap | PCI_ACS_EC); 3554 3555 pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl); 3556 return (ctrl & acs_flags) == acs_flags; 3557 } 3558 3559 /** 3560 * pci_acs_enabled - test ACS against required flags for a given device 3561 * @pdev: device to test 3562 * @acs_flags: required PCI ACS flags 3563 * 3564 * Return true if the device supports the provided flags. Automatically 3565 * filters out flags that are not implemented on multifunction devices. 3566 * 3567 * Note that this interface checks the effective ACS capabilities of the 3568 * device rather than the actual capabilities. For instance, most single 3569 * function endpoints are not required to support ACS because they have no 3570 * opportunity for peer-to-peer access. We therefore return 'true' 3571 * regardless of whether the device exposes an ACS capability. This makes 3572 * it much easier for callers of this function to ignore the actual type 3573 * or topology of the device when testing ACS support. 3574 */ 3575 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags) 3576 { 3577 int ret; 3578 3579 ret = pci_dev_specific_acs_enabled(pdev, acs_flags); 3580 if (ret >= 0) 3581 return ret > 0; 3582 3583 /* 3584 * Conventional PCI and PCI-X devices never support ACS, either 3585 * effectively or actually. The shared bus topology implies that 3586 * any device on the bus can receive or snoop DMA. 3587 */ 3588 if (!pci_is_pcie(pdev)) 3589 return false; 3590 3591 switch (pci_pcie_type(pdev)) { 3592 /* 3593 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec, 3594 * but since their primary interface is PCI/X, we conservatively 3595 * handle them as we would a non-PCIe device. 3596 */ 3597 case PCI_EXP_TYPE_PCIE_BRIDGE: 3598 /* 3599 * PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never 3600 * applicable... must never implement an ACS Extended Capability...". 3601 * This seems arbitrary, but we take a conservative interpretation 3602 * of this statement. 3603 */ 3604 case PCI_EXP_TYPE_PCI_BRIDGE: 3605 case PCI_EXP_TYPE_RC_EC: 3606 return false; 3607 /* 3608 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should 3609 * implement ACS in order to indicate their peer-to-peer capabilities, 3610 * regardless of whether they are single- or multi-function devices. 3611 */ 3612 case PCI_EXP_TYPE_DOWNSTREAM: 3613 case PCI_EXP_TYPE_ROOT_PORT: 3614 return pci_acs_flags_enabled(pdev, acs_flags); 3615 /* 3616 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be 3617 * implemented by the remaining PCIe types to indicate peer-to-peer 3618 * capabilities, but only when they are part of a multifunction 3619 * device. The footnote for section 6.12 indicates the specific 3620 * PCIe types included here. 3621 */ 3622 case PCI_EXP_TYPE_ENDPOINT: 3623 case PCI_EXP_TYPE_UPSTREAM: 3624 case PCI_EXP_TYPE_LEG_END: 3625 case PCI_EXP_TYPE_RC_END: 3626 if (!pdev->multifunction) 3627 break; 3628 3629 return pci_acs_flags_enabled(pdev, acs_flags); 3630 } 3631 3632 /* 3633 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable 3634 * to single function devices with the exception of downstream ports. 3635 */ 3636 return true; 3637 } 3638 3639 /** 3640 * pci_acs_path_enabled - test ACS flags from start to end in a hierarchy 3641 * @start: starting downstream device 3642 * @end: ending upstream device or NULL to search to the root bus 3643 * @acs_flags: required flags 3644 * 3645 * Walk up a device tree from start to end testing PCI ACS support. If 3646 * any step along the way does not support the required flags, return false. 3647 */ 3648 bool pci_acs_path_enabled(struct pci_dev *start, 3649 struct pci_dev *end, u16 acs_flags) 3650 { 3651 struct pci_dev *pdev, *parent = start; 3652 3653 do { 3654 pdev = parent; 3655 3656 if (!pci_acs_enabled(pdev, acs_flags)) 3657 return false; 3658 3659 if (pci_is_root_bus(pdev->bus)) 3660 return (end == NULL); 3661 3662 parent = pdev->bus->self; 3663 } while (pdev != end); 3664 3665 return true; 3666 } 3667 3668 /** 3669 * pci_acs_init - Initialize ACS if hardware supports it 3670 * @dev: the PCI device 3671 */ 3672 void pci_acs_init(struct pci_dev *dev) 3673 { 3674 dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); 3675 3676 /* 3677 * Attempt to enable ACS regardless of capability because some Root 3678 * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have 3679 * the standard ACS capability but still support ACS via those 3680 * quirks. 3681 */ 3682 pci_enable_acs(dev); 3683 } 3684 3685 /** 3686 * pci_rebar_find_pos - find position of resize ctrl reg for BAR 3687 * @pdev: PCI device 3688 * @bar: BAR to find 3689 * 3690 * Helper to find the position of the ctrl register for a BAR. 3691 * Returns -ENOTSUPP if resizable BARs are not supported at all. 3692 * Returns -ENOENT if no ctrl register for the BAR could be found. 3693 */ 3694 static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) 3695 { 3696 unsigned int pos, nbars, i; 3697 u32 ctrl; 3698 3699 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 3700 if (!pos) 3701 return -ENOTSUPP; 3702 3703 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3704 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 3705 PCI_REBAR_CTRL_NBAR_SHIFT; 3706 3707 for (i = 0; i < nbars; i++, pos += 8) { 3708 int bar_idx; 3709 3710 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3711 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 3712 if (bar_idx == bar) 3713 return pos; 3714 } 3715 3716 return -ENOENT; 3717 } 3718 3719 /** 3720 * pci_rebar_get_possible_sizes - get possible sizes for BAR 3721 * @pdev: PCI device 3722 * @bar: BAR to query 3723 * 3724 * Get the possible sizes of a resizable BAR as bitmask defined in the spec 3725 * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable. 3726 */ 3727 u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar) 3728 { 3729 int pos; 3730 u32 cap; 3731 3732 pos = pci_rebar_find_pos(pdev, bar); 3733 if (pos < 0) 3734 return 0; 3735 3736 pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap); 3737 cap &= PCI_REBAR_CAP_SIZES; 3738 3739 /* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */ 3740 if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f && 3741 bar == 0 && cap == 0x7000) 3742 cap = 0x3f000; 3743 3744 return cap >> 4; 3745 } 3746 EXPORT_SYMBOL(pci_rebar_get_possible_sizes); 3747 3748 /** 3749 * pci_rebar_get_current_size - get the current size of a BAR 3750 * @pdev: PCI device 3751 * @bar: BAR to set size to 3752 * 3753 * Read the size of a BAR from the resizable BAR config. 3754 * Returns size if found or negative error code. 3755 */ 3756 int pci_rebar_get_current_size(struct pci_dev *pdev, int bar) 3757 { 3758 int pos; 3759 u32 ctrl; 3760 3761 pos = pci_rebar_find_pos(pdev, bar); 3762 if (pos < 0) 3763 return pos; 3764 3765 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3766 return (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT; 3767 } 3768 3769 /** 3770 * pci_rebar_set_size - set a new size for a BAR 3771 * @pdev: PCI device 3772 * @bar: BAR to set size to 3773 * @size: new size as defined in the spec (0=1MB, 19=512GB) 3774 * 3775 * Set the new size of a BAR as defined in the spec. 3776 * Returns zero if resizing was successful, error code otherwise. 3777 */ 3778 int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size) 3779 { 3780 int pos; 3781 u32 ctrl; 3782 3783 pos = pci_rebar_find_pos(pdev, bar); 3784 if (pos < 0) 3785 return pos; 3786 3787 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3788 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 3789 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 3790 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 3791 return 0; 3792 } 3793 3794 /** 3795 * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port 3796 * @dev: the PCI device 3797 * @cap_mask: mask of desired AtomicOp sizes, including one or more of: 3798 * PCI_EXP_DEVCAP2_ATOMIC_COMP32 3799 * PCI_EXP_DEVCAP2_ATOMIC_COMP64 3800 * PCI_EXP_DEVCAP2_ATOMIC_COMP128 3801 * 3802 * Return 0 if all upstream bridges support AtomicOp routing, egress 3803 * blocking is disabled on all upstream ports, and the root port supports 3804 * the requested completion capabilities (32-bit, 64-bit and/or 128-bit 3805 * AtomicOp completion), or negative otherwise. 3806 */ 3807 int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) 3808 { 3809 struct pci_bus *bus = dev->bus; 3810 struct pci_dev *bridge; 3811 u32 cap, ctl2; 3812 3813 /* 3814 * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit 3815 * in Device Control 2 is reserved in VFs and the PF value applies 3816 * to all associated VFs. 3817 */ 3818 if (dev->is_virtfn) 3819 return -EINVAL; 3820 3821 if (!pci_is_pcie(dev)) 3822 return -EINVAL; 3823 3824 /* 3825 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be 3826 * AtomicOp requesters. For now, we only support endpoints as 3827 * requesters and root ports as completers. No endpoints as 3828 * completers, and no peer-to-peer. 3829 */ 3830 3831 switch (pci_pcie_type(dev)) { 3832 case PCI_EXP_TYPE_ENDPOINT: 3833 case PCI_EXP_TYPE_LEG_END: 3834 case PCI_EXP_TYPE_RC_END: 3835 break; 3836 default: 3837 return -EINVAL; 3838 } 3839 3840 while (bus->parent) { 3841 bridge = bus->self; 3842 3843 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3844 3845 switch (pci_pcie_type(bridge)) { 3846 /* Ensure switch ports support AtomicOp routing */ 3847 case PCI_EXP_TYPE_UPSTREAM: 3848 case PCI_EXP_TYPE_DOWNSTREAM: 3849 if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) 3850 return -EINVAL; 3851 break; 3852 3853 /* Ensure root port supports all the sizes we care about */ 3854 case PCI_EXP_TYPE_ROOT_PORT: 3855 if ((cap & cap_mask) != cap_mask) 3856 return -EINVAL; 3857 break; 3858 } 3859 3860 /* Ensure upstream ports don't block AtomicOps on egress */ 3861 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) { 3862 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, 3863 &ctl2); 3864 if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK) 3865 return -EINVAL; 3866 } 3867 3868 bus = bus->parent; 3869 } 3870 3871 pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, 3872 PCI_EXP_DEVCTL2_ATOMIC_REQ); 3873 return 0; 3874 } 3875 EXPORT_SYMBOL(pci_enable_atomic_ops_to_root); 3876 3877 /** 3878 * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge 3879 * @dev: the PCI device 3880 * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) 3881 * 3882 * Perform INTx swizzling for a device behind one level of bridge. This is 3883 * required by section 9.1 of the PCI-to-PCI bridge specification for devices 3884 * behind bridges on add-in cards. For devices with ARI enabled, the slot 3885 * number is always 0 (see the Implementation Note in section 2.2.8.1 of 3886 * the PCI Express Base Specification, Revision 2.1) 3887 */ 3888 u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin) 3889 { 3890 int slot; 3891 3892 if (pci_ari_enabled(dev->bus)) 3893 slot = 0; 3894 else 3895 slot = PCI_SLOT(dev->devfn); 3896 3897 return (((pin - 1) + slot) % 4) + 1; 3898 } 3899 3900 int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge) 3901 { 3902 u8 pin; 3903 3904 pin = dev->pin; 3905 if (!pin) 3906 return -1; 3907 3908 while (!pci_is_root_bus(dev->bus)) { 3909 pin = pci_swizzle_interrupt_pin(dev, pin); 3910 dev = dev->bus->self; 3911 } 3912 *bridge = dev; 3913 return pin; 3914 } 3915 3916 /** 3917 * pci_common_swizzle - swizzle INTx all the way to root bridge 3918 * @dev: the PCI device 3919 * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD) 3920 * 3921 * Perform INTx swizzling for a device. This traverses through all PCI-to-PCI 3922 * bridges all the way up to a PCI root bus. 3923 */ 3924 u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp) 3925 { 3926 u8 pin = *pinp; 3927 3928 while (!pci_is_root_bus(dev->bus)) { 3929 pin = pci_swizzle_interrupt_pin(dev, pin); 3930 dev = dev->bus->self; 3931 } 3932 *pinp = pin; 3933 return PCI_SLOT(dev->devfn); 3934 } 3935 EXPORT_SYMBOL_GPL(pci_common_swizzle); 3936 3937 /** 3938 * pci_release_region - Release a PCI bar 3939 * @pdev: PCI device whose resources were previously reserved by 3940 * pci_request_region() 3941 * @bar: BAR to release 3942 * 3943 * Releases the PCI I/O and memory resources previously reserved by a 3944 * successful call to pci_request_region(). Call this function only 3945 * after all use of the PCI regions has ceased. 3946 */ 3947 void pci_release_region(struct pci_dev *pdev, int bar) 3948 { 3949 struct pci_devres *dr; 3950 3951 if (pci_resource_len(pdev, bar) == 0) 3952 return; 3953 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) 3954 release_region(pci_resource_start(pdev, bar), 3955 pci_resource_len(pdev, bar)); 3956 else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) 3957 release_mem_region(pci_resource_start(pdev, bar), 3958 pci_resource_len(pdev, bar)); 3959 3960 dr = find_pci_dr(pdev); 3961 if (dr) 3962 dr->region_mask &= ~(1 << bar); 3963 } 3964 EXPORT_SYMBOL(pci_release_region); 3965 3966 /** 3967 * __pci_request_region - Reserved PCI I/O and memory resource 3968 * @pdev: PCI device whose resources are to be reserved 3969 * @bar: BAR to be reserved 3970 * @res_name: Name to be associated with resource. 3971 * @exclusive: whether the region access is exclusive or not 3972 * 3973 * Mark the PCI region associated with PCI device @pdev BAR @bar as 3974 * being reserved by owner @res_name. Do not access any 3975 * address inside the PCI regions unless this call returns 3976 * successfully. 3977 * 3978 * If @exclusive is set, then the region is marked so that userspace 3979 * is explicitly not allowed to map the resource via /dev/mem or 3980 * sysfs MMIO access. 3981 * 3982 * Returns 0 on success, or %EBUSY on error. A warning 3983 * message is also printed on failure. 3984 */ 3985 static int __pci_request_region(struct pci_dev *pdev, int bar, 3986 const char *res_name, int exclusive) 3987 { 3988 struct pci_devres *dr; 3989 3990 if (pci_resource_len(pdev, bar) == 0) 3991 return 0; 3992 3993 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) { 3994 if (!request_region(pci_resource_start(pdev, bar), 3995 pci_resource_len(pdev, bar), res_name)) 3996 goto err_out; 3997 } else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) { 3998 if (!__request_mem_region(pci_resource_start(pdev, bar), 3999 pci_resource_len(pdev, bar), res_name, 4000 exclusive)) 4001 goto err_out; 4002 } 4003 4004 dr = find_pci_dr(pdev); 4005 if (dr) 4006 dr->region_mask |= 1 << bar; 4007 4008 return 0; 4009 4010 err_out: 4011 pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar, 4012 &pdev->resource[bar]); 4013 return -EBUSY; 4014 } 4015 4016 /** 4017 * pci_request_region - Reserve PCI I/O and memory resource 4018 * @pdev: PCI device whose resources are to be reserved 4019 * @bar: BAR to be reserved 4020 * @res_name: Name to be associated with resource 4021 * 4022 * Mark the PCI region associated with PCI device @pdev BAR @bar as 4023 * being reserved by owner @res_name. Do not access any 4024 * address inside the PCI regions unless this call returns 4025 * successfully. 4026 * 4027 * Returns 0 on success, or %EBUSY on error. A warning 4028 * message is also printed on failure. 4029 */ 4030 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 4031 { 4032 return __pci_request_region(pdev, bar, res_name, 0); 4033 } 4034 EXPORT_SYMBOL(pci_request_region); 4035 4036 /** 4037 * pci_release_selected_regions - Release selected PCI I/O and memory resources 4038 * @pdev: PCI device whose resources were previously reserved 4039 * @bars: Bitmask of BARs to be released 4040 * 4041 * Release selected PCI I/O and memory resources previously reserved. 4042 * Call this function only after all use of the PCI regions has ceased. 4043 */ 4044 void pci_release_selected_regions(struct pci_dev *pdev, int bars) 4045 { 4046 int i; 4047 4048 for (i = 0; i < PCI_STD_NUM_BARS; i++) 4049 if (bars & (1 << i)) 4050 pci_release_region(pdev, i); 4051 } 4052 EXPORT_SYMBOL(pci_release_selected_regions); 4053 4054 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars, 4055 const char *res_name, int excl) 4056 { 4057 int i; 4058 4059 for (i = 0; i < PCI_STD_NUM_BARS; i++) 4060 if (bars & (1 << i)) 4061 if (__pci_request_region(pdev, i, res_name, excl)) 4062 goto err_out; 4063 return 0; 4064 4065 err_out: 4066 while (--i >= 0) 4067 if (bars & (1 << i)) 4068 pci_release_region(pdev, i); 4069 4070 return -EBUSY; 4071 } 4072 4073 4074 /** 4075 * pci_request_selected_regions - Reserve selected PCI I/O and memory resources 4076 * @pdev: PCI device whose resources are to be reserved 4077 * @bars: Bitmask of BARs to be requested 4078 * @res_name: Name to be associated with resource 4079 */ 4080 int pci_request_selected_regions(struct pci_dev *pdev, int bars, 4081 const char *res_name) 4082 { 4083 return __pci_request_selected_regions(pdev, bars, res_name, 0); 4084 } 4085 EXPORT_SYMBOL(pci_request_selected_regions); 4086 4087 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars, 4088 const char *res_name) 4089 { 4090 return __pci_request_selected_regions(pdev, bars, res_name, 4091 IORESOURCE_EXCLUSIVE); 4092 } 4093 EXPORT_SYMBOL(pci_request_selected_regions_exclusive); 4094 4095 /** 4096 * pci_release_regions - Release reserved PCI I/O and memory resources 4097 * @pdev: PCI device whose resources were previously reserved by 4098 * pci_request_regions() 4099 * 4100 * Releases all PCI I/O and memory resources previously reserved by a 4101 * successful call to pci_request_regions(). Call this function only 4102 * after all use of the PCI regions has ceased. 4103 */ 4104 4105 void pci_release_regions(struct pci_dev *pdev) 4106 { 4107 pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1); 4108 } 4109 EXPORT_SYMBOL(pci_release_regions); 4110 4111 /** 4112 * pci_request_regions - Reserve PCI I/O and memory resources 4113 * @pdev: PCI device whose resources are to be reserved 4114 * @res_name: Name to be associated with resource. 4115 * 4116 * Mark all PCI regions associated with PCI device @pdev as 4117 * being reserved by owner @res_name. Do not access any 4118 * address inside the PCI regions unless this call returns 4119 * successfully. 4120 * 4121 * Returns 0 on success, or %EBUSY on error. A warning 4122 * message is also printed on failure. 4123 */ 4124 int pci_request_regions(struct pci_dev *pdev, const char *res_name) 4125 { 4126 return pci_request_selected_regions(pdev, 4127 ((1 << PCI_STD_NUM_BARS) - 1), res_name); 4128 } 4129 EXPORT_SYMBOL(pci_request_regions); 4130 4131 /** 4132 * pci_request_regions_exclusive - Reserve PCI I/O and memory resources 4133 * @pdev: PCI device whose resources are to be reserved 4134 * @res_name: Name to be associated with resource. 4135 * 4136 * Mark all PCI regions associated with PCI device @pdev as being reserved 4137 * by owner @res_name. Do not access any address inside the PCI regions 4138 * unless this call returns successfully. 4139 * 4140 * pci_request_regions_exclusive() will mark the region so that /dev/mem 4141 * and the sysfs MMIO access will not be allowed. 4142 * 4143 * Returns 0 on success, or %EBUSY on error. A warning message is also 4144 * printed on failure. 4145 */ 4146 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name) 4147 { 4148 return pci_request_selected_regions_exclusive(pdev, 4149 ((1 << PCI_STD_NUM_BARS) - 1), res_name); 4150 } 4151 EXPORT_SYMBOL(pci_request_regions_exclusive); 4152 4153 /* 4154 * Record the PCI IO range (expressed as CPU physical address + size). 4155 * Return a negative value if an error has occurred, zero otherwise 4156 */ 4157 int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr, 4158 resource_size_t size) 4159 { 4160 int ret = 0; 4161 #ifdef PCI_IOBASE 4162 struct logic_pio_hwaddr *range; 4163 4164 if (!size || addr + size < addr) 4165 return -EINVAL; 4166 4167 range = kzalloc(sizeof(*range), GFP_ATOMIC); 4168 if (!range) 4169 return -ENOMEM; 4170 4171 range->fwnode = fwnode; 4172 range->size = size; 4173 range->hw_start = addr; 4174 range->flags = LOGIC_PIO_CPU_MMIO; 4175 4176 ret = logic_pio_register_range(range); 4177 if (ret) 4178 kfree(range); 4179 4180 /* Ignore duplicates due to deferred probing */ 4181 if (ret == -EEXIST) 4182 ret = 0; 4183 #endif 4184 4185 return ret; 4186 } 4187 4188 phys_addr_t pci_pio_to_address(unsigned long pio) 4189 { 4190 phys_addr_t address = (phys_addr_t)OF_BAD_ADDR; 4191 4192 #ifdef PCI_IOBASE 4193 if (pio >= MMIO_UPPER_LIMIT) 4194 return address; 4195 4196 address = logic_pio_to_hwaddr(pio); 4197 #endif 4198 4199 return address; 4200 } 4201 EXPORT_SYMBOL_GPL(pci_pio_to_address); 4202 4203 unsigned long __weak pci_address_to_pio(phys_addr_t address) 4204 { 4205 #ifdef PCI_IOBASE 4206 return logic_pio_trans_cpuaddr(address); 4207 #else 4208 if (address > IO_SPACE_LIMIT) 4209 return (unsigned long)-1; 4210 4211 return (unsigned long) address; 4212 #endif 4213 } 4214 4215 /** 4216 * pci_remap_iospace - Remap the memory mapped I/O space 4217 * @res: Resource describing the I/O space 4218 * @phys_addr: physical address of range to be mapped 4219 * 4220 * Remap the memory mapped I/O space described by the @res and the CPU 4221 * physical address @phys_addr into virtual address space. Only 4222 * architectures that have memory mapped IO functions defined (and the 4223 * PCI_IOBASE value defined) should call this function. 4224 */ 4225 #ifndef pci_remap_iospace 4226 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) 4227 { 4228 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 4229 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4230 4231 if (!(res->flags & IORESOURCE_IO)) 4232 return -EINVAL; 4233 4234 if (res->end > IO_SPACE_LIMIT) 4235 return -EINVAL; 4236 4237 return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr, 4238 pgprot_device(PAGE_KERNEL)); 4239 #else 4240 /* 4241 * This architecture does not have memory mapped I/O space, 4242 * so this function should never be called 4243 */ 4244 WARN_ONCE(1, "This architecture does not support memory mapped I/O\n"); 4245 return -ENODEV; 4246 #endif 4247 } 4248 EXPORT_SYMBOL(pci_remap_iospace); 4249 #endif 4250 4251 /** 4252 * pci_unmap_iospace - Unmap the memory mapped I/O space 4253 * @res: resource to be unmapped 4254 * 4255 * Unmap the CPU virtual address @res from virtual address space. Only 4256 * architectures that have memory mapped IO functions defined (and the 4257 * PCI_IOBASE value defined) should call this function. 4258 */ 4259 void pci_unmap_iospace(struct resource *res) 4260 { 4261 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 4262 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4263 4264 vunmap_range(vaddr, vaddr + resource_size(res)); 4265 #endif 4266 } 4267 EXPORT_SYMBOL(pci_unmap_iospace); 4268 4269 static void devm_pci_unmap_iospace(struct device *dev, void *ptr) 4270 { 4271 struct resource **res = ptr; 4272 4273 pci_unmap_iospace(*res); 4274 } 4275 4276 /** 4277 * devm_pci_remap_iospace - Managed pci_remap_iospace() 4278 * @dev: Generic device to remap IO address for 4279 * @res: Resource describing the I/O space 4280 * @phys_addr: physical address of range to be mapped 4281 * 4282 * Managed pci_remap_iospace(). Map is automatically unmapped on driver 4283 * detach. 4284 */ 4285 int devm_pci_remap_iospace(struct device *dev, const struct resource *res, 4286 phys_addr_t phys_addr) 4287 { 4288 const struct resource **ptr; 4289 int error; 4290 4291 ptr = devres_alloc(devm_pci_unmap_iospace, sizeof(*ptr), GFP_KERNEL); 4292 if (!ptr) 4293 return -ENOMEM; 4294 4295 error = pci_remap_iospace(res, phys_addr); 4296 if (error) { 4297 devres_free(ptr); 4298 } else { 4299 *ptr = res; 4300 devres_add(dev, ptr); 4301 } 4302 4303 return error; 4304 } 4305 EXPORT_SYMBOL(devm_pci_remap_iospace); 4306 4307 /** 4308 * devm_pci_remap_cfgspace - Managed pci_remap_cfgspace() 4309 * @dev: Generic device to remap IO address for 4310 * @offset: Resource address to map 4311 * @size: Size of map 4312 * 4313 * Managed pci_remap_cfgspace(). Map is automatically unmapped on driver 4314 * detach. 4315 */ 4316 void __iomem *devm_pci_remap_cfgspace(struct device *dev, 4317 resource_size_t offset, 4318 resource_size_t size) 4319 { 4320 void __iomem **ptr, *addr; 4321 4322 ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL); 4323 if (!ptr) 4324 return NULL; 4325 4326 addr = pci_remap_cfgspace(offset, size); 4327 if (addr) { 4328 *ptr = addr; 4329 devres_add(dev, ptr); 4330 } else 4331 devres_free(ptr); 4332 4333 return addr; 4334 } 4335 EXPORT_SYMBOL(devm_pci_remap_cfgspace); 4336 4337 /** 4338 * devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource 4339 * @dev: generic device to handle the resource for 4340 * @res: configuration space resource to be handled 4341 * 4342 * Checks that a resource is a valid memory region, requests the memory 4343 * region and ioremaps with pci_remap_cfgspace() API that ensures the 4344 * proper PCI configuration space memory attributes are guaranteed. 4345 * 4346 * All operations are managed and will be undone on driver detach. 4347 * 4348 * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code 4349 * on failure. Usage example:: 4350 * 4351 * res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4352 * base = devm_pci_remap_cfg_resource(&pdev->dev, res); 4353 * if (IS_ERR(base)) 4354 * return PTR_ERR(base); 4355 */ 4356 void __iomem *devm_pci_remap_cfg_resource(struct device *dev, 4357 struct resource *res) 4358 { 4359 resource_size_t size; 4360 const char *name; 4361 void __iomem *dest_ptr; 4362 4363 BUG_ON(!dev); 4364 4365 if (!res || resource_type(res) != IORESOURCE_MEM) { 4366 dev_err(dev, "invalid resource\n"); 4367 return IOMEM_ERR_PTR(-EINVAL); 4368 } 4369 4370 size = resource_size(res); 4371 4372 if (res->name) 4373 name = devm_kasprintf(dev, GFP_KERNEL, "%s %s", dev_name(dev), 4374 res->name); 4375 else 4376 name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL); 4377 if (!name) 4378 return IOMEM_ERR_PTR(-ENOMEM); 4379 4380 if (!devm_request_mem_region(dev, res->start, size, name)) { 4381 dev_err(dev, "can't request region for resource %pR\n", res); 4382 return IOMEM_ERR_PTR(-EBUSY); 4383 } 4384 4385 dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size); 4386 if (!dest_ptr) { 4387 dev_err(dev, "ioremap failed for resource %pR\n", res); 4388 devm_release_mem_region(dev, res->start, size); 4389 dest_ptr = IOMEM_ERR_PTR(-ENOMEM); 4390 } 4391 4392 return dest_ptr; 4393 } 4394 EXPORT_SYMBOL(devm_pci_remap_cfg_resource); 4395 4396 static void __pci_set_master(struct pci_dev *dev, bool enable) 4397 { 4398 u16 old_cmd, cmd; 4399 4400 pci_read_config_word(dev, PCI_COMMAND, &old_cmd); 4401 if (enable) 4402 cmd = old_cmd | PCI_COMMAND_MASTER; 4403 else 4404 cmd = old_cmd & ~PCI_COMMAND_MASTER; 4405 if (cmd != old_cmd) { 4406 pci_dbg(dev, "%s bus mastering\n", 4407 enable ? "enabling" : "disabling"); 4408 pci_write_config_word(dev, PCI_COMMAND, cmd); 4409 } 4410 dev->is_busmaster = enable; 4411 } 4412 4413 /** 4414 * pcibios_setup - process "pci=" kernel boot arguments 4415 * @str: string used to pass in "pci=" kernel boot arguments 4416 * 4417 * Process kernel boot arguments. This is the default implementation. 4418 * Architecture specific implementations can override this as necessary. 4419 */ 4420 char * __weak __init pcibios_setup(char *str) 4421 { 4422 return str; 4423 } 4424 4425 /** 4426 * pcibios_set_master - enable PCI bus-mastering for device dev 4427 * @dev: the PCI device to enable 4428 * 4429 * Enables PCI bus-mastering for the device. This is the default 4430 * implementation. Architecture specific implementations can override 4431 * this if necessary. 4432 */ 4433 void __weak pcibios_set_master(struct pci_dev *dev) 4434 { 4435 u8 lat; 4436 4437 /* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */ 4438 if (pci_is_pcie(dev)) 4439 return; 4440 4441 pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat); 4442 if (lat < 16) 4443 lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency; 4444 else if (lat > pcibios_max_latency) 4445 lat = pcibios_max_latency; 4446 else 4447 return; 4448 4449 pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat); 4450 } 4451 4452 /** 4453 * pci_set_master - enables bus-mastering for device dev 4454 * @dev: the PCI device to enable 4455 * 4456 * Enables bus-mastering on the device and calls pcibios_set_master() 4457 * to do the needed arch specific settings. 4458 */ 4459 void pci_set_master(struct pci_dev *dev) 4460 { 4461 __pci_set_master(dev, true); 4462 pcibios_set_master(dev); 4463 } 4464 EXPORT_SYMBOL(pci_set_master); 4465 4466 /** 4467 * pci_clear_master - disables bus-mastering for device dev 4468 * @dev: the PCI device to disable 4469 */ 4470 void pci_clear_master(struct pci_dev *dev) 4471 { 4472 __pci_set_master(dev, false); 4473 } 4474 EXPORT_SYMBOL(pci_clear_master); 4475 4476 /** 4477 * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed 4478 * @dev: the PCI device for which MWI is to be enabled 4479 * 4480 * Helper function for pci_set_mwi. 4481 * Originally copied from drivers/net/acenic.c. 4482 * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>. 4483 * 4484 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4485 */ 4486 int pci_set_cacheline_size(struct pci_dev *dev) 4487 { 4488 u8 cacheline_size; 4489 4490 if (!pci_cache_line_size) 4491 return -EINVAL; 4492 4493 /* Validate current setting: the PCI_CACHE_LINE_SIZE must be 4494 equal to or multiple of the right value. */ 4495 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4496 if (cacheline_size >= pci_cache_line_size && 4497 (cacheline_size % pci_cache_line_size) == 0) 4498 return 0; 4499 4500 /* Write the correct value. */ 4501 pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size); 4502 /* Read it back. */ 4503 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4504 if (cacheline_size == pci_cache_line_size) 4505 return 0; 4506 4507 pci_dbg(dev, "cache line size of %d is not supported\n", 4508 pci_cache_line_size << 2); 4509 4510 return -EINVAL; 4511 } 4512 EXPORT_SYMBOL_GPL(pci_set_cacheline_size); 4513 4514 /** 4515 * pci_set_mwi - enables memory-write-invalidate PCI transaction 4516 * @dev: the PCI device for which MWI is enabled 4517 * 4518 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4519 * 4520 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4521 */ 4522 int pci_set_mwi(struct pci_dev *dev) 4523 { 4524 #ifdef PCI_DISABLE_MWI 4525 return 0; 4526 #else 4527 int rc; 4528 u16 cmd; 4529 4530 rc = pci_set_cacheline_size(dev); 4531 if (rc) 4532 return rc; 4533 4534 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4535 if (!(cmd & PCI_COMMAND_INVALIDATE)) { 4536 pci_dbg(dev, "enabling Mem-Wr-Inval\n"); 4537 cmd |= PCI_COMMAND_INVALIDATE; 4538 pci_write_config_word(dev, PCI_COMMAND, cmd); 4539 } 4540 return 0; 4541 #endif 4542 } 4543 EXPORT_SYMBOL(pci_set_mwi); 4544 4545 /** 4546 * pcim_set_mwi - a device-managed pci_set_mwi() 4547 * @dev: the PCI device for which MWI is enabled 4548 * 4549 * Managed pci_set_mwi(). 4550 * 4551 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4552 */ 4553 int pcim_set_mwi(struct pci_dev *dev) 4554 { 4555 struct pci_devres *dr; 4556 4557 dr = find_pci_dr(dev); 4558 if (!dr) 4559 return -ENOMEM; 4560 4561 dr->mwi = 1; 4562 return pci_set_mwi(dev); 4563 } 4564 EXPORT_SYMBOL(pcim_set_mwi); 4565 4566 /** 4567 * pci_try_set_mwi - enables memory-write-invalidate PCI transaction 4568 * @dev: the PCI device for which MWI is enabled 4569 * 4570 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4571 * Callers are not required to check the return value. 4572 * 4573 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4574 */ 4575 int pci_try_set_mwi(struct pci_dev *dev) 4576 { 4577 #ifdef PCI_DISABLE_MWI 4578 return 0; 4579 #else 4580 return pci_set_mwi(dev); 4581 #endif 4582 } 4583 EXPORT_SYMBOL(pci_try_set_mwi); 4584 4585 /** 4586 * pci_clear_mwi - disables Memory-Write-Invalidate for device dev 4587 * @dev: the PCI device to disable 4588 * 4589 * Disables PCI Memory-Write-Invalidate transaction on the device 4590 */ 4591 void pci_clear_mwi(struct pci_dev *dev) 4592 { 4593 #ifndef PCI_DISABLE_MWI 4594 u16 cmd; 4595 4596 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4597 if (cmd & PCI_COMMAND_INVALIDATE) { 4598 cmd &= ~PCI_COMMAND_INVALIDATE; 4599 pci_write_config_word(dev, PCI_COMMAND, cmd); 4600 } 4601 #endif 4602 } 4603 EXPORT_SYMBOL(pci_clear_mwi); 4604 4605 /** 4606 * pci_disable_parity - disable parity checking for device 4607 * @dev: the PCI device to operate on 4608 * 4609 * Disable parity checking for device @dev 4610 */ 4611 void pci_disable_parity(struct pci_dev *dev) 4612 { 4613 u16 cmd; 4614 4615 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4616 if (cmd & PCI_COMMAND_PARITY) { 4617 cmd &= ~PCI_COMMAND_PARITY; 4618 pci_write_config_word(dev, PCI_COMMAND, cmd); 4619 } 4620 } 4621 4622 /** 4623 * pci_intx - enables/disables PCI INTx for device dev 4624 * @pdev: the PCI device to operate on 4625 * @enable: boolean: whether to enable or disable PCI INTx 4626 * 4627 * Enables/disables PCI INTx for device @pdev 4628 */ 4629 void pci_intx(struct pci_dev *pdev, int enable) 4630 { 4631 u16 pci_command, new; 4632 4633 pci_read_config_word(pdev, PCI_COMMAND, &pci_command); 4634 4635 if (enable) 4636 new = pci_command & ~PCI_COMMAND_INTX_DISABLE; 4637 else 4638 new = pci_command | PCI_COMMAND_INTX_DISABLE; 4639 4640 if (new != pci_command) { 4641 struct pci_devres *dr; 4642 4643 pci_write_config_word(pdev, PCI_COMMAND, new); 4644 4645 dr = find_pci_dr(pdev); 4646 if (dr && !dr->restore_intx) { 4647 dr->restore_intx = 1; 4648 dr->orig_intx = !enable; 4649 } 4650 } 4651 } 4652 EXPORT_SYMBOL_GPL(pci_intx); 4653 4654 static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask) 4655 { 4656 struct pci_bus *bus = dev->bus; 4657 bool mask_updated = true; 4658 u32 cmd_status_dword; 4659 u16 origcmd, newcmd; 4660 unsigned long flags; 4661 bool irq_pending; 4662 4663 /* 4664 * We do a single dword read to retrieve both command and status. 4665 * Document assumptions that make this possible. 4666 */ 4667 BUILD_BUG_ON(PCI_COMMAND % 4); 4668 BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS); 4669 4670 raw_spin_lock_irqsave(&pci_lock, flags); 4671 4672 bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword); 4673 4674 irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT; 4675 4676 /* 4677 * Check interrupt status register to see whether our device 4678 * triggered the interrupt (when masking) or the next IRQ is 4679 * already pending (when unmasking). 4680 */ 4681 if (mask != irq_pending) { 4682 mask_updated = false; 4683 goto done; 4684 } 4685 4686 origcmd = cmd_status_dword; 4687 newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE; 4688 if (mask) 4689 newcmd |= PCI_COMMAND_INTX_DISABLE; 4690 if (newcmd != origcmd) 4691 bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd); 4692 4693 done: 4694 raw_spin_unlock_irqrestore(&pci_lock, flags); 4695 4696 return mask_updated; 4697 } 4698 4699 /** 4700 * pci_check_and_mask_intx - mask INTx on pending interrupt 4701 * @dev: the PCI device to operate on 4702 * 4703 * Check if the device dev has its INTx line asserted, mask it and return 4704 * true in that case. False is returned if no interrupt was pending. 4705 */ 4706 bool pci_check_and_mask_intx(struct pci_dev *dev) 4707 { 4708 return pci_check_and_set_intx_mask(dev, true); 4709 } 4710 EXPORT_SYMBOL_GPL(pci_check_and_mask_intx); 4711 4712 /** 4713 * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending 4714 * @dev: the PCI device to operate on 4715 * 4716 * Check if the device dev has its INTx line asserted, unmask it if not and 4717 * return true. False is returned and the mask remains active if there was 4718 * still an interrupt pending. 4719 */ 4720 bool pci_check_and_unmask_intx(struct pci_dev *dev) 4721 { 4722 return pci_check_and_set_intx_mask(dev, false); 4723 } 4724 EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx); 4725 4726 /** 4727 * pci_wait_for_pending_transaction - wait for pending transaction 4728 * @dev: the PCI device to operate on 4729 * 4730 * Return 0 if transaction is pending 1 otherwise. 4731 */ 4732 int pci_wait_for_pending_transaction(struct pci_dev *dev) 4733 { 4734 if (!pci_is_pcie(dev)) 4735 return 1; 4736 4737 return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA, 4738 PCI_EXP_DEVSTA_TRPND); 4739 } 4740 EXPORT_SYMBOL(pci_wait_for_pending_transaction); 4741 4742 /** 4743 * pcie_flr - initiate a PCIe function level reset 4744 * @dev: device to reset 4745 * 4746 * Initiate a function level reset unconditionally on @dev without 4747 * checking any flags and DEVCAP 4748 */ 4749 int pcie_flr(struct pci_dev *dev) 4750 { 4751 if (!pci_wait_for_pending_transaction(dev)) 4752 pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n"); 4753 4754 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); 4755 4756 if (dev->imm_ready) 4757 return 0; 4758 4759 /* 4760 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within 4761 * 100ms, but may silently discard requests while the FLR is in 4762 * progress. Wait 100ms before trying to access the device. 4763 */ 4764 msleep(100); 4765 4766 return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS); 4767 } 4768 EXPORT_SYMBOL_GPL(pcie_flr); 4769 4770 /** 4771 * pcie_reset_flr - initiate a PCIe function level reset 4772 * @dev: device to reset 4773 * @probe: if true, return 0 if device can be reset this way 4774 * 4775 * Initiate a function level reset on @dev. 4776 */ 4777 int pcie_reset_flr(struct pci_dev *dev, bool probe) 4778 { 4779 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4780 return -ENOTTY; 4781 4782 if (!(dev->devcap & PCI_EXP_DEVCAP_FLR)) 4783 return -ENOTTY; 4784 4785 if (probe) 4786 return 0; 4787 4788 return pcie_flr(dev); 4789 } 4790 EXPORT_SYMBOL_GPL(pcie_reset_flr); 4791 4792 static int pci_af_flr(struct pci_dev *dev, bool probe) 4793 { 4794 int pos; 4795 u8 cap; 4796 4797 pos = pci_find_capability(dev, PCI_CAP_ID_AF); 4798 if (!pos) 4799 return -ENOTTY; 4800 4801 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4802 return -ENOTTY; 4803 4804 pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); 4805 if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR)) 4806 return -ENOTTY; 4807 4808 if (probe) 4809 return 0; 4810 4811 /* 4812 * Wait for Transaction Pending bit to clear. A word-aligned test 4813 * is used, so we use the control offset rather than status and shift 4814 * the test bit to match. 4815 */ 4816 if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL, 4817 PCI_AF_STATUS_TP << 8)) 4818 pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n"); 4819 4820 pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR); 4821 4822 if (dev->imm_ready) 4823 return 0; 4824 4825 /* 4826 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006, 4827 * updated 27 July 2006; a device must complete an FLR within 4828 * 100ms, but may silently discard requests while the FLR is in 4829 * progress. Wait 100ms before trying to access the device. 4830 */ 4831 msleep(100); 4832 4833 return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS); 4834 } 4835 4836 /** 4837 * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0. 4838 * @dev: Device to reset. 4839 * @probe: if true, return 0 if the device can be reset this way. 4840 * 4841 * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is 4842 * unset, it will be reinitialized internally when going from PCI_D3hot to 4843 * PCI_D0. If that's the case and the device is not in a low-power state 4844 * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset. 4845 * 4846 * NOTE: This causes the caller to sleep for twice the device power transition 4847 * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms 4848 * by default (i.e. unless the @dev's d3hot_delay field has a different value). 4849 * Moreover, only devices in D0 can be reset by this function. 4850 */ 4851 static int pci_pm_reset(struct pci_dev *dev, bool probe) 4852 { 4853 u16 csr; 4854 4855 if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET) 4856 return -ENOTTY; 4857 4858 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr); 4859 if (csr & PCI_PM_CTRL_NO_SOFT_RESET) 4860 return -ENOTTY; 4861 4862 if (probe) 4863 return 0; 4864 4865 if (dev->current_state != PCI_D0) 4866 return -EINVAL; 4867 4868 csr &= ~PCI_PM_CTRL_STATE_MASK; 4869 csr |= PCI_D3hot; 4870 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4871 pci_dev_d3_sleep(dev); 4872 4873 csr &= ~PCI_PM_CTRL_STATE_MASK; 4874 csr |= PCI_D0; 4875 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4876 pci_dev_d3_sleep(dev); 4877 4878 return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS); 4879 } 4880 4881 /** 4882 * pcie_wait_for_link_delay - Wait until link is active or inactive 4883 * @pdev: Bridge device 4884 * @active: waiting for active or inactive? 4885 * @delay: Delay to wait after link has become active (in ms) 4886 * 4887 * Use this to wait till link becomes active or inactive. 4888 */ 4889 static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active, 4890 int delay) 4891 { 4892 int timeout = 1000; 4893 bool ret; 4894 u16 lnk_status; 4895 4896 /* 4897 * Some controllers might not implement link active reporting. In this 4898 * case, we wait for 1000 ms + any delay requested by the caller. 4899 */ 4900 if (!pdev->link_active_reporting) { 4901 msleep(timeout + delay); 4902 return true; 4903 } 4904 4905 /* 4906 * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms, 4907 * after which we should expect an link active if the reset was 4908 * successful. If so, software must wait a minimum 100ms before sending 4909 * configuration requests to devices downstream this port. 4910 * 4911 * If the link fails to activate, either the device was physically 4912 * removed or the link is permanently failed. 4913 */ 4914 if (active) 4915 msleep(20); 4916 for (;;) { 4917 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status); 4918 ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA); 4919 if (ret == active) 4920 break; 4921 if (timeout <= 0) 4922 break; 4923 msleep(10); 4924 timeout -= 10; 4925 } 4926 if (active && ret) 4927 msleep(delay); 4928 4929 return ret == active; 4930 } 4931 4932 /** 4933 * pcie_wait_for_link - Wait until link is active or inactive 4934 * @pdev: Bridge device 4935 * @active: waiting for active or inactive? 4936 * 4937 * Use this to wait till link becomes active or inactive. 4938 */ 4939 bool pcie_wait_for_link(struct pci_dev *pdev, bool active) 4940 { 4941 return pcie_wait_for_link_delay(pdev, active, 100); 4942 } 4943 4944 /* 4945 * Find maximum D3cold delay required by all the devices on the bus. The 4946 * spec says 100 ms, but firmware can lower it and we allow drivers to 4947 * increase it as well. 4948 * 4949 * Called with @pci_bus_sem locked for reading. 4950 */ 4951 static int pci_bus_max_d3cold_delay(const struct pci_bus *bus) 4952 { 4953 const struct pci_dev *pdev; 4954 int min_delay = 100; 4955 int max_delay = 0; 4956 4957 list_for_each_entry(pdev, &bus->devices, bus_list) { 4958 if (pdev->d3cold_delay < min_delay) 4959 min_delay = pdev->d3cold_delay; 4960 if (pdev->d3cold_delay > max_delay) 4961 max_delay = pdev->d3cold_delay; 4962 } 4963 4964 return max(min_delay, max_delay); 4965 } 4966 4967 /** 4968 * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible 4969 * @dev: PCI bridge 4970 * 4971 * Handle necessary delays before access to the devices on the secondary 4972 * side of the bridge are permitted after D3cold to D0 transition. 4973 * 4974 * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For 4975 * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section 4976 * 4.3.2. 4977 */ 4978 void pci_bridge_wait_for_secondary_bus(struct pci_dev *dev) 4979 { 4980 struct pci_dev *child; 4981 int delay; 4982 4983 if (pci_dev_is_disconnected(dev)) 4984 return; 4985 4986 if (!pci_is_bridge(dev) || !dev->bridge_d3) 4987 return; 4988 4989 down_read(&pci_bus_sem); 4990 4991 /* 4992 * We only deal with devices that are present currently on the bus. 4993 * For any hot-added devices the access delay is handled in pciehp 4994 * board_added(). In case of ACPI hotplug the firmware is expected 4995 * to configure the devices before OS is notified. 4996 */ 4997 if (!dev->subordinate || list_empty(&dev->subordinate->devices)) { 4998 up_read(&pci_bus_sem); 4999 return; 5000 } 5001 5002 /* Take d3cold_delay requirements into account */ 5003 delay = pci_bus_max_d3cold_delay(dev->subordinate); 5004 if (!delay) { 5005 up_read(&pci_bus_sem); 5006 return; 5007 } 5008 5009 child = list_first_entry(&dev->subordinate->devices, struct pci_dev, 5010 bus_list); 5011 up_read(&pci_bus_sem); 5012 5013 /* 5014 * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before 5015 * accessing the device after reset (that is 1000 ms + 100 ms). In 5016 * practice this should not be needed because we don't do power 5017 * management for them (see pci_bridge_d3_possible()). 5018 */ 5019 if (!pci_is_pcie(dev)) { 5020 pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay); 5021 msleep(1000 + delay); 5022 return; 5023 } 5024 5025 /* 5026 * For PCIe downstream and root ports that do not support speeds 5027 * greater than 5 GT/s need to wait minimum 100 ms. For higher 5028 * speeds (gen3) we need to wait first for the data link layer to 5029 * become active. 5030 * 5031 * However, 100 ms is the minimum and the PCIe spec says the 5032 * software must allow at least 1s before it can determine that the 5033 * device that did not respond is a broken device. There is 5034 * evidence that 100 ms is not always enough, for example certain 5035 * Titan Ridge xHCI controller does not always respond to 5036 * configuration requests if we only wait for 100 ms (see 5037 * https://bugzilla.kernel.org/show_bug.cgi?id=203885). 5038 * 5039 * Therefore we wait for 100 ms and check for the device presence. 5040 * If it is still not present give it an additional 100 ms. 5041 */ 5042 if (!pcie_downstream_port(dev)) 5043 return; 5044 5045 if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) { 5046 pci_dbg(dev, "waiting %d ms for downstream link\n", delay); 5047 msleep(delay); 5048 } else { 5049 pci_dbg(dev, "waiting %d ms for downstream link, after activation\n", 5050 delay); 5051 if (!pcie_wait_for_link_delay(dev, true, delay)) { 5052 /* Did not train, no need to wait any further */ 5053 pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n"); 5054 return; 5055 } 5056 } 5057 5058 if (!pci_device_is_present(child)) { 5059 pci_dbg(child, "waiting additional %d ms to become accessible\n", delay); 5060 msleep(delay); 5061 } 5062 } 5063 5064 void pci_reset_secondary_bus(struct pci_dev *dev) 5065 { 5066 u16 ctrl; 5067 5068 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl); 5069 ctrl |= PCI_BRIDGE_CTL_BUS_RESET; 5070 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 5071 5072 /* 5073 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double 5074 * this to 2ms to ensure that we meet the minimum requirement. 5075 */ 5076 msleep(2); 5077 5078 ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; 5079 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 5080 5081 /* 5082 * Trhfa for conventional PCI is 2^25 clock cycles. 5083 * Assuming a minimum 33MHz clock this results in a 1s 5084 * delay before we can consider subordinate devices to 5085 * be re-initialized. PCIe has some ways to shorten this, 5086 * but we don't make use of them yet. 5087 */ 5088 ssleep(1); 5089 } 5090 5091 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) 5092 { 5093 pci_reset_secondary_bus(dev); 5094 } 5095 5096 /** 5097 * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge. 5098 * @dev: Bridge device 5099 * 5100 * Use the bridge control register to assert reset on the secondary bus. 5101 * Devices on the secondary bus are left in power-on state. 5102 */ 5103 int pci_bridge_secondary_bus_reset(struct pci_dev *dev) 5104 { 5105 pcibios_reset_secondary_bus(dev); 5106 5107 return pci_dev_wait(dev, "bus reset", PCIE_RESET_READY_POLL_MS); 5108 } 5109 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset); 5110 5111 static int pci_parent_bus_reset(struct pci_dev *dev, bool probe) 5112 { 5113 struct pci_dev *pdev; 5114 5115 if (pci_is_root_bus(dev->bus) || dev->subordinate || 5116 !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 5117 return -ENOTTY; 5118 5119 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 5120 if (pdev != dev) 5121 return -ENOTTY; 5122 5123 if (probe) 5124 return 0; 5125 5126 return pci_bridge_secondary_bus_reset(dev->bus->self); 5127 } 5128 5129 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe) 5130 { 5131 int rc = -ENOTTY; 5132 5133 if (!hotplug || !try_module_get(hotplug->owner)) 5134 return rc; 5135 5136 if (hotplug->ops->reset_slot) 5137 rc = hotplug->ops->reset_slot(hotplug, probe); 5138 5139 module_put(hotplug->owner); 5140 5141 return rc; 5142 } 5143 5144 static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe) 5145 { 5146 if (dev->multifunction || dev->subordinate || !dev->slot || 5147 dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 5148 return -ENOTTY; 5149 5150 return pci_reset_hotplug_slot(dev->slot->hotplug, probe); 5151 } 5152 5153 static int pci_reset_bus_function(struct pci_dev *dev, bool probe) 5154 { 5155 int rc; 5156 5157 rc = pci_dev_reset_slot_function(dev, probe); 5158 if (rc != -ENOTTY) 5159 return rc; 5160 return pci_parent_bus_reset(dev, probe); 5161 } 5162 5163 void pci_dev_lock(struct pci_dev *dev) 5164 { 5165 /* block PM suspend, driver probe, etc. */ 5166 device_lock(&dev->dev); 5167 pci_cfg_access_lock(dev); 5168 } 5169 EXPORT_SYMBOL_GPL(pci_dev_lock); 5170 5171 /* Return 1 on successful lock, 0 on contention */ 5172 int pci_dev_trylock(struct pci_dev *dev) 5173 { 5174 if (device_trylock(&dev->dev)) { 5175 if (pci_cfg_access_trylock(dev)) 5176 return 1; 5177 device_unlock(&dev->dev); 5178 } 5179 5180 return 0; 5181 } 5182 EXPORT_SYMBOL_GPL(pci_dev_trylock); 5183 5184 void pci_dev_unlock(struct pci_dev *dev) 5185 { 5186 pci_cfg_access_unlock(dev); 5187 device_unlock(&dev->dev); 5188 } 5189 EXPORT_SYMBOL_GPL(pci_dev_unlock); 5190 5191 static void pci_dev_save_and_disable(struct pci_dev *dev) 5192 { 5193 const struct pci_error_handlers *err_handler = 5194 dev->driver ? dev->driver->err_handler : NULL; 5195 5196 /* 5197 * dev->driver->err_handler->reset_prepare() is protected against 5198 * races with ->remove() by the device lock, which must be held by 5199 * the caller. 5200 */ 5201 if (err_handler && err_handler->reset_prepare) 5202 err_handler->reset_prepare(dev); 5203 5204 /* 5205 * Wake-up device prior to save. PM registers default to D0 after 5206 * reset and a simple register restore doesn't reliably return 5207 * to a non-D0 state anyway. 5208 */ 5209 pci_set_power_state(dev, PCI_D0); 5210 5211 pci_save_state(dev); 5212 /* 5213 * Disable the device by clearing the Command register, except for 5214 * INTx-disable which is set. This not only disables MMIO and I/O port 5215 * BARs, but also prevents the device from being Bus Master, preventing 5216 * DMA from the device including MSI/MSI-X interrupts. For PCI 2.3 5217 * compliant devices, INTx-disable prevents legacy interrupts. 5218 */ 5219 pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 5220 } 5221 5222 static void pci_dev_restore(struct pci_dev *dev) 5223 { 5224 const struct pci_error_handlers *err_handler = 5225 dev->driver ? dev->driver->err_handler : NULL; 5226 5227 pci_restore_state(dev); 5228 5229 /* 5230 * dev->driver->err_handler->reset_done() is protected against 5231 * races with ->remove() by the device lock, which must be held by 5232 * the caller. 5233 */ 5234 if (err_handler && err_handler->reset_done) 5235 err_handler->reset_done(dev); 5236 } 5237 5238 /* dev->reset_methods[] is a 0-terminated list of indices into this array */ 5239 static const struct pci_reset_fn_method pci_reset_fn_methods[] = { 5240 { }, 5241 { pci_dev_specific_reset, .name = "device_specific" }, 5242 { pci_dev_acpi_reset, .name = "acpi" }, 5243 { pcie_reset_flr, .name = "flr" }, 5244 { pci_af_flr, .name = "af_flr" }, 5245 { pci_pm_reset, .name = "pm" }, 5246 { pci_reset_bus_function, .name = "bus" }, 5247 }; 5248 5249 static ssize_t reset_method_show(struct device *dev, 5250 struct device_attribute *attr, char *buf) 5251 { 5252 struct pci_dev *pdev = to_pci_dev(dev); 5253 ssize_t len = 0; 5254 int i, m; 5255 5256 for (i = 0; i < PCI_NUM_RESET_METHODS; i++) { 5257 m = pdev->reset_methods[i]; 5258 if (!m) 5259 break; 5260 5261 len += sysfs_emit_at(buf, len, "%s%s", len ? " " : "", 5262 pci_reset_fn_methods[m].name); 5263 } 5264 5265 if (len) 5266 len += sysfs_emit_at(buf, len, "\n"); 5267 5268 return len; 5269 } 5270 5271 static int reset_method_lookup(const char *name) 5272 { 5273 int m; 5274 5275 for (m = 1; m < PCI_NUM_RESET_METHODS; m++) { 5276 if (sysfs_streq(name, pci_reset_fn_methods[m].name)) 5277 return m; 5278 } 5279 5280 return 0; /* not found */ 5281 } 5282 5283 static ssize_t reset_method_store(struct device *dev, 5284 struct device_attribute *attr, 5285 const char *buf, size_t count) 5286 { 5287 struct pci_dev *pdev = to_pci_dev(dev); 5288 char *options, *name; 5289 int m, n; 5290 u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 }; 5291 5292 if (sysfs_streq(buf, "")) { 5293 pdev->reset_methods[0] = 0; 5294 pci_warn(pdev, "All device reset methods disabled by user"); 5295 return count; 5296 } 5297 5298 if (sysfs_streq(buf, "default")) { 5299 pci_init_reset_methods(pdev); 5300 return count; 5301 } 5302 5303 options = kstrndup(buf, count, GFP_KERNEL); 5304 if (!options) 5305 return -ENOMEM; 5306 5307 n = 0; 5308 while ((name = strsep(&options, " ")) != NULL) { 5309 if (sysfs_streq(name, "")) 5310 continue; 5311 5312 name = strim(name); 5313 5314 m = reset_method_lookup(name); 5315 if (!m) { 5316 pci_err(pdev, "Invalid reset method '%s'", name); 5317 goto error; 5318 } 5319 5320 if (pci_reset_fn_methods[m].reset_fn(pdev, PCI_RESET_PROBE)) { 5321 pci_err(pdev, "Unsupported reset method '%s'", name); 5322 goto error; 5323 } 5324 5325 if (n == PCI_NUM_RESET_METHODS - 1) { 5326 pci_err(pdev, "Too many reset methods\n"); 5327 goto error; 5328 } 5329 5330 reset_methods[n++] = m; 5331 } 5332 5333 reset_methods[n] = 0; 5334 5335 /* Warn if dev-specific supported but not highest priority */ 5336 if (pci_reset_fn_methods[1].reset_fn(pdev, PCI_RESET_PROBE) == 0 && 5337 reset_methods[0] != 1) 5338 pci_warn(pdev, "Device-specific reset disabled/de-prioritized by user"); 5339 memcpy(pdev->reset_methods, reset_methods, sizeof(pdev->reset_methods)); 5340 kfree(options); 5341 return count; 5342 5343 error: 5344 /* Leave previous methods unchanged */ 5345 kfree(options); 5346 return -EINVAL; 5347 } 5348 static DEVICE_ATTR_RW(reset_method); 5349 5350 static struct attribute *pci_dev_reset_method_attrs[] = { 5351 &dev_attr_reset_method.attr, 5352 NULL, 5353 }; 5354 5355 static umode_t pci_dev_reset_method_attr_is_visible(struct kobject *kobj, 5356 struct attribute *a, int n) 5357 { 5358 struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); 5359 5360 if (!pci_reset_supported(pdev)) 5361 return 0; 5362 5363 return a->mode; 5364 } 5365 5366 const struct attribute_group pci_dev_reset_method_attr_group = { 5367 .attrs = pci_dev_reset_method_attrs, 5368 .is_visible = pci_dev_reset_method_attr_is_visible, 5369 }; 5370 5371 /** 5372 * __pci_reset_function_locked - reset a PCI device function while holding 5373 * the @dev mutex lock. 5374 * @dev: PCI device to reset 5375 * 5376 * Some devices allow an individual function to be reset without affecting 5377 * other functions in the same device. The PCI device must be responsive 5378 * to PCI config space in order to use this function. 5379 * 5380 * The device function is presumed to be unused and the caller is holding 5381 * the device mutex lock when this function is called. 5382 * 5383 * Resetting the device will make the contents of PCI configuration space 5384 * random, so any caller of this must be prepared to reinitialise the 5385 * device including MSI, bus mastering, BARs, decoding IO and memory spaces, 5386 * etc. 5387 * 5388 * Returns 0 if the device function was successfully reset or negative if the 5389 * device doesn't support resetting a single function. 5390 */ 5391 int __pci_reset_function_locked(struct pci_dev *dev) 5392 { 5393 int i, m, rc; 5394 5395 might_sleep(); 5396 5397 /* 5398 * A reset method returns -ENOTTY if it doesn't support this device and 5399 * we should try the next method. 5400 * 5401 * If it returns 0 (success), we're finished. If it returns any other 5402 * error, we're also finished: this indicates that further reset 5403 * mechanisms might be broken on the device. 5404 */ 5405 for (i = 0; i < PCI_NUM_RESET_METHODS; i++) { 5406 m = dev->reset_methods[i]; 5407 if (!m) 5408 return -ENOTTY; 5409 5410 rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET); 5411 if (!rc) 5412 return 0; 5413 if (rc != -ENOTTY) 5414 return rc; 5415 } 5416 5417 return -ENOTTY; 5418 } 5419 EXPORT_SYMBOL_GPL(__pci_reset_function_locked); 5420 5421 /** 5422 * pci_init_reset_methods - check whether device can be safely reset 5423 * and store supported reset mechanisms. 5424 * @dev: PCI device to check for reset mechanisms 5425 * 5426 * Some devices allow an individual function to be reset without affecting 5427 * other functions in the same device. The PCI device must be in D0-D3hot 5428 * state. 5429 * 5430 * Stores reset mechanisms supported by device in reset_methods byte array 5431 * which is a member of struct pci_dev. 5432 */ 5433 void pci_init_reset_methods(struct pci_dev *dev) 5434 { 5435 int m, i, rc; 5436 5437 BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS); 5438 5439 might_sleep(); 5440 5441 i = 0; 5442 for (m = 1; m < PCI_NUM_RESET_METHODS; m++) { 5443 rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE); 5444 if (!rc) 5445 dev->reset_methods[i++] = m; 5446 else if (rc != -ENOTTY) 5447 break; 5448 } 5449 5450 dev->reset_methods[i] = 0; 5451 } 5452 5453 /** 5454 * pci_reset_function - quiesce and reset a PCI device function 5455 * @dev: PCI device to reset 5456 * 5457 * Some devices allow an individual function to be reset without affecting 5458 * other functions in the same device. The PCI device must be responsive 5459 * to PCI config space in order to use this function. 5460 * 5461 * This function does not just reset the PCI portion of a device, but 5462 * clears all the state associated with the device. This function differs 5463 * from __pci_reset_function_locked() in that it saves and restores device state 5464 * over the reset and takes the PCI device lock. 5465 * 5466 * Returns 0 if the device function was successfully reset or negative if the 5467 * device doesn't support resetting a single function. 5468 */ 5469 int pci_reset_function(struct pci_dev *dev) 5470 { 5471 int rc; 5472 5473 if (!pci_reset_supported(dev)) 5474 return -ENOTTY; 5475 5476 pci_dev_lock(dev); 5477 pci_dev_save_and_disable(dev); 5478 5479 rc = __pci_reset_function_locked(dev); 5480 5481 pci_dev_restore(dev); 5482 pci_dev_unlock(dev); 5483 5484 return rc; 5485 } 5486 EXPORT_SYMBOL_GPL(pci_reset_function); 5487 5488 /** 5489 * pci_reset_function_locked - quiesce and reset a PCI device function 5490 * @dev: PCI device to reset 5491 * 5492 * Some devices allow an individual function to be reset without affecting 5493 * other functions in the same device. The PCI device must be responsive 5494 * to PCI config space in order to use this function. 5495 * 5496 * This function does not just reset the PCI portion of a device, but 5497 * clears all the state associated with the device. This function differs 5498 * from __pci_reset_function_locked() in that it saves and restores device state 5499 * over the reset. It also differs from pci_reset_function() in that it 5500 * requires the PCI device lock to be held. 5501 * 5502 * Returns 0 if the device function was successfully reset or negative if the 5503 * device doesn't support resetting a single function. 5504 */ 5505 int pci_reset_function_locked(struct pci_dev *dev) 5506 { 5507 int rc; 5508 5509 if (!pci_reset_supported(dev)) 5510 return -ENOTTY; 5511 5512 pci_dev_save_and_disable(dev); 5513 5514 rc = __pci_reset_function_locked(dev); 5515 5516 pci_dev_restore(dev); 5517 5518 return rc; 5519 } 5520 EXPORT_SYMBOL_GPL(pci_reset_function_locked); 5521 5522 /** 5523 * pci_try_reset_function - quiesce and reset a PCI device function 5524 * @dev: PCI device to reset 5525 * 5526 * Same as above, except return -EAGAIN if unable to lock device. 5527 */ 5528 int pci_try_reset_function(struct pci_dev *dev) 5529 { 5530 int rc; 5531 5532 if (!pci_reset_supported(dev)) 5533 return -ENOTTY; 5534 5535 if (!pci_dev_trylock(dev)) 5536 return -EAGAIN; 5537 5538 pci_dev_save_and_disable(dev); 5539 rc = __pci_reset_function_locked(dev); 5540 pci_dev_restore(dev); 5541 pci_dev_unlock(dev); 5542 5543 return rc; 5544 } 5545 EXPORT_SYMBOL_GPL(pci_try_reset_function); 5546 5547 /* Do any devices on or below this bus prevent a bus reset? */ 5548 static bool pci_bus_resetable(struct pci_bus *bus) 5549 { 5550 struct pci_dev *dev; 5551 5552 5553 if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5554 return false; 5555 5556 list_for_each_entry(dev, &bus->devices, bus_list) { 5557 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5558 (dev->subordinate && !pci_bus_resetable(dev->subordinate))) 5559 return false; 5560 } 5561 5562 return true; 5563 } 5564 5565 /* Lock devices from the top of the tree down */ 5566 static void pci_bus_lock(struct pci_bus *bus) 5567 { 5568 struct pci_dev *dev; 5569 5570 list_for_each_entry(dev, &bus->devices, bus_list) { 5571 pci_dev_lock(dev); 5572 if (dev->subordinate) 5573 pci_bus_lock(dev->subordinate); 5574 } 5575 } 5576 5577 /* Unlock devices from the bottom of the tree up */ 5578 static void pci_bus_unlock(struct pci_bus *bus) 5579 { 5580 struct pci_dev *dev; 5581 5582 list_for_each_entry(dev, &bus->devices, bus_list) { 5583 if (dev->subordinate) 5584 pci_bus_unlock(dev->subordinate); 5585 pci_dev_unlock(dev); 5586 } 5587 } 5588 5589 /* Return 1 on successful lock, 0 on contention */ 5590 static int pci_bus_trylock(struct pci_bus *bus) 5591 { 5592 struct pci_dev *dev; 5593 5594 list_for_each_entry(dev, &bus->devices, bus_list) { 5595 if (!pci_dev_trylock(dev)) 5596 goto unlock; 5597 if (dev->subordinate) { 5598 if (!pci_bus_trylock(dev->subordinate)) { 5599 pci_dev_unlock(dev); 5600 goto unlock; 5601 } 5602 } 5603 } 5604 return 1; 5605 5606 unlock: 5607 list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) { 5608 if (dev->subordinate) 5609 pci_bus_unlock(dev->subordinate); 5610 pci_dev_unlock(dev); 5611 } 5612 return 0; 5613 } 5614 5615 /* Do any devices on or below this slot prevent a bus reset? */ 5616 static bool pci_slot_resetable(struct pci_slot *slot) 5617 { 5618 struct pci_dev *dev; 5619 5620 if (slot->bus->self && 5621 (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5622 return false; 5623 5624 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5625 if (!dev->slot || dev->slot != slot) 5626 continue; 5627 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5628 (dev->subordinate && !pci_bus_resetable(dev->subordinate))) 5629 return false; 5630 } 5631 5632 return true; 5633 } 5634 5635 /* Lock devices from the top of the tree down */ 5636 static void pci_slot_lock(struct pci_slot *slot) 5637 { 5638 struct pci_dev *dev; 5639 5640 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5641 if (!dev->slot || dev->slot != slot) 5642 continue; 5643 pci_dev_lock(dev); 5644 if (dev->subordinate) 5645 pci_bus_lock(dev->subordinate); 5646 } 5647 } 5648 5649 /* Unlock devices from the bottom of the tree up */ 5650 static void pci_slot_unlock(struct pci_slot *slot) 5651 { 5652 struct pci_dev *dev; 5653 5654 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5655 if (!dev->slot || dev->slot != slot) 5656 continue; 5657 if (dev->subordinate) 5658 pci_bus_unlock(dev->subordinate); 5659 pci_dev_unlock(dev); 5660 } 5661 } 5662 5663 /* Return 1 on successful lock, 0 on contention */ 5664 static int pci_slot_trylock(struct pci_slot *slot) 5665 { 5666 struct pci_dev *dev; 5667 5668 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5669 if (!dev->slot || dev->slot != slot) 5670 continue; 5671 if (!pci_dev_trylock(dev)) 5672 goto unlock; 5673 if (dev->subordinate) { 5674 if (!pci_bus_trylock(dev->subordinate)) { 5675 pci_dev_unlock(dev); 5676 goto unlock; 5677 } 5678 } 5679 } 5680 return 1; 5681 5682 unlock: 5683 list_for_each_entry_continue_reverse(dev, 5684 &slot->bus->devices, bus_list) { 5685 if (!dev->slot || dev->slot != slot) 5686 continue; 5687 if (dev->subordinate) 5688 pci_bus_unlock(dev->subordinate); 5689 pci_dev_unlock(dev); 5690 } 5691 return 0; 5692 } 5693 5694 /* 5695 * Save and disable devices from the top of the tree down while holding 5696 * the @dev mutex lock for the entire tree. 5697 */ 5698 static void pci_bus_save_and_disable_locked(struct pci_bus *bus) 5699 { 5700 struct pci_dev *dev; 5701 5702 list_for_each_entry(dev, &bus->devices, bus_list) { 5703 pci_dev_save_and_disable(dev); 5704 if (dev->subordinate) 5705 pci_bus_save_and_disable_locked(dev->subordinate); 5706 } 5707 } 5708 5709 /* 5710 * Restore devices from top of the tree down while holding @dev mutex lock 5711 * for the entire tree. Parent bridges need to be restored before we can 5712 * get to subordinate devices. 5713 */ 5714 static void pci_bus_restore_locked(struct pci_bus *bus) 5715 { 5716 struct pci_dev *dev; 5717 5718 list_for_each_entry(dev, &bus->devices, bus_list) { 5719 pci_dev_restore(dev); 5720 if (dev->subordinate) 5721 pci_bus_restore_locked(dev->subordinate); 5722 } 5723 } 5724 5725 /* 5726 * Save and disable devices from the top of the tree down while holding 5727 * the @dev mutex lock for the entire tree. 5728 */ 5729 static void pci_slot_save_and_disable_locked(struct pci_slot *slot) 5730 { 5731 struct pci_dev *dev; 5732 5733 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5734 if (!dev->slot || dev->slot != slot) 5735 continue; 5736 pci_dev_save_and_disable(dev); 5737 if (dev->subordinate) 5738 pci_bus_save_and_disable_locked(dev->subordinate); 5739 } 5740 } 5741 5742 /* 5743 * Restore devices from top of the tree down while holding @dev mutex lock 5744 * for the entire tree. Parent bridges need to be restored before we can 5745 * get to subordinate devices. 5746 */ 5747 static void pci_slot_restore_locked(struct pci_slot *slot) 5748 { 5749 struct pci_dev *dev; 5750 5751 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5752 if (!dev->slot || dev->slot != slot) 5753 continue; 5754 pci_dev_restore(dev); 5755 if (dev->subordinate) 5756 pci_bus_restore_locked(dev->subordinate); 5757 } 5758 } 5759 5760 static int pci_slot_reset(struct pci_slot *slot, bool probe) 5761 { 5762 int rc; 5763 5764 if (!slot || !pci_slot_resetable(slot)) 5765 return -ENOTTY; 5766 5767 if (!probe) 5768 pci_slot_lock(slot); 5769 5770 might_sleep(); 5771 5772 rc = pci_reset_hotplug_slot(slot->hotplug, probe); 5773 5774 if (!probe) 5775 pci_slot_unlock(slot); 5776 5777 return rc; 5778 } 5779 5780 /** 5781 * pci_probe_reset_slot - probe whether a PCI slot can be reset 5782 * @slot: PCI slot to probe 5783 * 5784 * Return 0 if slot can be reset, negative if a slot reset is not supported. 5785 */ 5786 int pci_probe_reset_slot(struct pci_slot *slot) 5787 { 5788 return pci_slot_reset(slot, PCI_RESET_PROBE); 5789 } 5790 EXPORT_SYMBOL_GPL(pci_probe_reset_slot); 5791 5792 /** 5793 * __pci_reset_slot - Try to reset a PCI slot 5794 * @slot: PCI slot to reset 5795 * 5796 * A PCI bus may host multiple slots, each slot may support a reset mechanism 5797 * independent of other slots. For instance, some slots may support slot power 5798 * control. In the case of a 1:1 bus to slot architecture, this function may 5799 * wrap the bus reset to avoid spurious slot related events such as hotplug. 5800 * Generally a slot reset should be attempted before a bus reset. All of the 5801 * function of the slot and any subordinate buses behind the slot are reset 5802 * through this function. PCI config space of all devices in the slot and 5803 * behind the slot is saved before and restored after reset. 5804 * 5805 * Same as above except return -EAGAIN if the slot cannot be locked 5806 */ 5807 static int __pci_reset_slot(struct pci_slot *slot) 5808 { 5809 int rc; 5810 5811 rc = pci_slot_reset(slot, PCI_RESET_PROBE); 5812 if (rc) 5813 return rc; 5814 5815 if (pci_slot_trylock(slot)) { 5816 pci_slot_save_and_disable_locked(slot); 5817 might_sleep(); 5818 rc = pci_reset_hotplug_slot(slot->hotplug, PCI_RESET_DO_RESET); 5819 pci_slot_restore_locked(slot); 5820 pci_slot_unlock(slot); 5821 } else 5822 rc = -EAGAIN; 5823 5824 return rc; 5825 } 5826 5827 static int pci_bus_reset(struct pci_bus *bus, bool probe) 5828 { 5829 int ret; 5830 5831 if (!bus->self || !pci_bus_resetable(bus)) 5832 return -ENOTTY; 5833 5834 if (probe) 5835 return 0; 5836 5837 pci_bus_lock(bus); 5838 5839 might_sleep(); 5840 5841 ret = pci_bridge_secondary_bus_reset(bus->self); 5842 5843 pci_bus_unlock(bus); 5844 5845 return ret; 5846 } 5847 5848 /** 5849 * pci_bus_error_reset - reset the bridge's subordinate bus 5850 * @bridge: The parent device that connects to the bus to reset 5851 * 5852 * This function will first try to reset the slots on this bus if the method is 5853 * available. If slot reset fails or is not available, this will fall back to a 5854 * secondary bus reset. 5855 */ 5856 int pci_bus_error_reset(struct pci_dev *bridge) 5857 { 5858 struct pci_bus *bus = bridge->subordinate; 5859 struct pci_slot *slot; 5860 5861 if (!bus) 5862 return -ENOTTY; 5863 5864 mutex_lock(&pci_slot_mutex); 5865 if (list_empty(&bus->slots)) 5866 goto bus_reset; 5867 5868 list_for_each_entry(slot, &bus->slots, list) 5869 if (pci_probe_reset_slot(slot)) 5870 goto bus_reset; 5871 5872 list_for_each_entry(slot, &bus->slots, list) 5873 if (pci_slot_reset(slot, PCI_RESET_DO_RESET)) 5874 goto bus_reset; 5875 5876 mutex_unlock(&pci_slot_mutex); 5877 return 0; 5878 bus_reset: 5879 mutex_unlock(&pci_slot_mutex); 5880 return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET); 5881 } 5882 5883 /** 5884 * pci_probe_reset_bus - probe whether a PCI bus can be reset 5885 * @bus: PCI bus to probe 5886 * 5887 * Return 0 if bus can be reset, negative if a bus reset is not supported. 5888 */ 5889 int pci_probe_reset_bus(struct pci_bus *bus) 5890 { 5891 return pci_bus_reset(bus, PCI_RESET_PROBE); 5892 } 5893 EXPORT_SYMBOL_GPL(pci_probe_reset_bus); 5894 5895 /** 5896 * __pci_reset_bus - Try to reset a PCI bus 5897 * @bus: top level PCI bus to reset 5898 * 5899 * Same as above except return -EAGAIN if the bus cannot be locked 5900 */ 5901 static int __pci_reset_bus(struct pci_bus *bus) 5902 { 5903 int rc; 5904 5905 rc = pci_bus_reset(bus, PCI_RESET_PROBE); 5906 if (rc) 5907 return rc; 5908 5909 if (pci_bus_trylock(bus)) { 5910 pci_bus_save_and_disable_locked(bus); 5911 might_sleep(); 5912 rc = pci_bridge_secondary_bus_reset(bus->self); 5913 pci_bus_restore_locked(bus); 5914 pci_bus_unlock(bus); 5915 } else 5916 rc = -EAGAIN; 5917 5918 return rc; 5919 } 5920 5921 /** 5922 * pci_reset_bus - Try to reset a PCI bus 5923 * @pdev: top level PCI device to reset via slot/bus 5924 * 5925 * Same as above except return -EAGAIN if the bus cannot be locked 5926 */ 5927 int pci_reset_bus(struct pci_dev *pdev) 5928 { 5929 return (!pci_probe_reset_slot(pdev->slot)) ? 5930 __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus); 5931 } 5932 EXPORT_SYMBOL_GPL(pci_reset_bus); 5933 5934 /** 5935 * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count 5936 * @dev: PCI device to query 5937 * 5938 * Returns mmrbc: maximum designed memory read count in bytes or 5939 * appropriate error value. 5940 */ 5941 int pcix_get_max_mmrbc(struct pci_dev *dev) 5942 { 5943 int cap; 5944 u32 stat; 5945 5946 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5947 if (!cap) 5948 return -EINVAL; 5949 5950 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 5951 return -EINVAL; 5952 5953 return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21); 5954 } 5955 EXPORT_SYMBOL(pcix_get_max_mmrbc); 5956 5957 /** 5958 * pcix_get_mmrbc - get PCI-X maximum memory read byte count 5959 * @dev: PCI device to query 5960 * 5961 * Returns mmrbc: maximum memory read count in bytes or appropriate error 5962 * value. 5963 */ 5964 int pcix_get_mmrbc(struct pci_dev *dev) 5965 { 5966 int cap; 5967 u16 cmd; 5968 5969 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5970 if (!cap) 5971 return -EINVAL; 5972 5973 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 5974 return -EINVAL; 5975 5976 return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2); 5977 } 5978 EXPORT_SYMBOL(pcix_get_mmrbc); 5979 5980 /** 5981 * pcix_set_mmrbc - set PCI-X maximum memory read byte count 5982 * @dev: PCI device to query 5983 * @mmrbc: maximum memory read count in bytes 5984 * valid values are 512, 1024, 2048, 4096 5985 * 5986 * If possible sets maximum memory read byte count, some bridges have errata 5987 * that prevent this. 5988 */ 5989 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc) 5990 { 5991 int cap; 5992 u32 stat, v, o; 5993 u16 cmd; 5994 5995 if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc)) 5996 return -EINVAL; 5997 5998 v = ffs(mmrbc) - 10; 5999 6000 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 6001 if (!cap) 6002 return -EINVAL; 6003 6004 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 6005 return -EINVAL; 6006 6007 if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21) 6008 return -E2BIG; 6009 6010 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 6011 return -EINVAL; 6012 6013 o = (cmd & PCI_X_CMD_MAX_READ) >> 2; 6014 if (o != v) { 6015 if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC)) 6016 return -EIO; 6017 6018 cmd &= ~PCI_X_CMD_MAX_READ; 6019 cmd |= v << 2; 6020 if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd)) 6021 return -EIO; 6022 } 6023 return 0; 6024 } 6025 EXPORT_SYMBOL(pcix_set_mmrbc); 6026 6027 /** 6028 * pcie_get_readrq - get PCI Express read request size 6029 * @dev: PCI device to query 6030 * 6031 * Returns maximum memory read request in bytes or appropriate error value. 6032 */ 6033 int pcie_get_readrq(struct pci_dev *dev) 6034 { 6035 u16 ctl; 6036 6037 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 6038 6039 return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12); 6040 } 6041 EXPORT_SYMBOL(pcie_get_readrq); 6042 6043 /** 6044 * pcie_set_readrq - set PCI Express maximum memory read request 6045 * @dev: PCI device to query 6046 * @rq: maximum memory read count in bytes 6047 * valid values are 128, 256, 512, 1024, 2048, 4096 6048 * 6049 * If possible sets maximum memory read request in bytes 6050 */ 6051 int pcie_set_readrq(struct pci_dev *dev, int rq) 6052 { 6053 u16 v; 6054 int ret; 6055 6056 if (rq < 128 || rq > 4096 || !is_power_of_2(rq)) 6057 return -EINVAL; 6058 6059 /* 6060 * If using the "performance" PCIe config, we clamp the read rq 6061 * size to the max packet size to keep the host bridge from 6062 * generating requests larger than we can cope with. 6063 */ 6064 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) { 6065 int mps = pcie_get_mps(dev); 6066 6067 if (mps < rq) 6068 rq = mps; 6069 } 6070 6071 v = (ffs(rq) - 8) << 12; 6072 6073 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 6074 PCI_EXP_DEVCTL_READRQ, v); 6075 6076 return pcibios_err_to_errno(ret); 6077 } 6078 EXPORT_SYMBOL(pcie_set_readrq); 6079 6080 /** 6081 * pcie_get_mps - get PCI Express maximum payload size 6082 * @dev: PCI device to query 6083 * 6084 * Returns maximum payload size in bytes 6085 */ 6086 int pcie_get_mps(struct pci_dev *dev) 6087 { 6088 u16 ctl; 6089 6090 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 6091 6092 return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5); 6093 } 6094 EXPORT_SYMBOL(pcie_get_mps); 6095 6096 /** 6097 * pcie_set_mps - set PCI Express maximum payload size 6098 * @dev: PCI device to query 6099 * @mps: maximum payload size in bytes 6100 * valid values are 128, 256, 512, 1024, 2048, 4096 6101 * 6102 * If possible sets maximum payload size 6103 */ 6104 int pcie_set_mps(struct pci_dev *dev, int mps) 6105 { 6106 u16 v; 6107 int ret; 6108 6109 if (mps < 128 || mps > 4096 || !is_power_of_2(mps)) 6110 return -EINVAL; 6111 6112 v = ffs(mps) - 8; 6113 if (v > dev->pcie_mpss) 6114 return -EINVAL; 6115 v <<= 5; 6116 6117 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 6118 PCI_EXP_DEVCTL_PAYLOAD, v); 6119 6120 return pcibios_err_to_errno(ret); 6121 } 6122 EXPORT_SYMBOL(pcie_set_mps); 6123 6124 /** 6125 * pcie_bandwidth_available - determine minimum link settings of a PCIe 6126 * device and its bandwidth limitation 6127 * @dev: PCI device to query 6128 * @limiting_dev: storage for device causing the bandwidth limitation 6129 * @speed: storage for speed of limiting device 6130 * @width: storage for width of limiting device 6131 * 6132 * Walk up the PCI device chain and find the point where the minimum 6133 * bandwidth is available. Return the bandwidth available there and (if 6134 * limiting_dev, speed, and width pointers are supplied) information about 6135 * that point. The bandwidth returned is in Mb/s, i.e., megabits/second of 6136 * raw bandwidth. 6137 */ 6138 u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, 6139 enum pci_bus_speed *speed, 6140 enum pcie_link_width *width) 6141 { 6142 u16 lnksta; 6143 enum pci_bus_speed next_speed; 6144 enum pcie_link_width next_width; 6145 u32 bw, next_bw; 6146 6147 if (speed) 6148 *speed = PCI_SPEED_UNKNOWN; 6149 if (width) 6150 *width = PCIE_LNK_WIDTH_UNKNOWN; 6151 6152 bw = 0; 6153 6154 while (dev) { 6155 pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); 6156 6157 next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS]; 6158 next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >> 6159 PCI_EXP_LNKSTA_NLW_SHIFT; 6160 6161 next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed); 6162 6163 /* Check if current device limits the total bandwidth */ 6164 if (!bw || next_bw <= bw) { 6165 bw = next_bw; 6166 6167 if (limiting_dev) 6168 *limiting_dev = dev; 6169 if (speed) 6170 *speed = next_speed; 6171 if (width) 6172 *width = next_width; 6173 } 6174 6175 dev = pci_upstream_bridge(dev); 6176 } 6177 6178 return bw; 6179 } 6180 EXPORT_SYMBOL(pcie_bandwidth_available); 6181 6182 /** 6183 * pcie_get_speed_cap - query for the PCI device's link speed capability 6184 * @dev: PCI device to query 6185 * 6186 * Query the PCI device speed capability. Return the maximum link speed 6187 * supported by the device. 6188 */ 6189 enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev) 6190 { 6191 u32 lnkcap2, lnkcap; 6192 6193 /* 6194 * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. The 6195 * implementation note there recommends using the Supported Link 6196 * Speeds Vector in Link Capabilities 2 when supported. 6197 * 6198 * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software 6199 * should use the Supported Link Speeds field in Link Capabilities, 6200 * where only 2.5 GT/s and 5.0 GT/s speeds were defined. 6201 */ 6202 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2); 6203 6204 /* PCIe r3.0-compliant */ 6205 if (lnkcap2) 6206 return PCIE_LNKCAP2_SLS2SPEED(lnkcap2); 6207 6208 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 6209 if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB) 6210 return PCIE_SPEED_5_0GT; 6211 else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB) 6212 return PCIE_SPEED_2_5GT; 6213 6214 return PCI_SPEED_UNKNOWN; 6215 } 6216 EXPORT_SYMBOL(pcie_get_speed_cap); 6217 6218 /** 6219 * pcie_get_width_cap - query for the PCI device's link width capability 6220 * @dev: PCI device to query 6221 * 6222 * Query the PCI device width capability. Return the maximum link width 6223 * supported by the device. 6224 */ 6225 enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev) 6226 { 6227 u32 lnkcap; 6228 6229 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 6230 if (lnkcap) 6231 return (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4; 6232 6233 return PCIE_LNK_WIDTH_UNKNOWN; 6234 } 6235 EXPORT_SYMBOL(pcie_get_width_cap); 6236 6237 /** 6238 * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability 6239 * @dev: PCI device 6240 * @speed: storage for link speed 6241 * @width: storage for link width 6242 * 6243 * Calculate a PCI device's link bandwidth by querying for its link speed 6244 * and width, multiplying them, and applying encoding overhead. The result 6245 * is in Mb/s, i.e., megabits/second of raw bandwidth. 6246 */ 6247 u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed, 6248 enum pcie_link_width *width) 6249 { 6250 *speed = pcie_get_speed_cap(dev); 6251 *width = pcie_get_width_cap(dev); 6252 6253 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 6254 return 0; 6255 6256 return *width * PCIE_SPEED2MBS_ENC(*speed); 6257 } 6258 6259 /** 6260 * __pcie_print_link_status - Report the PCI device's link speed and width 6261 * @dev: PCI device to query 6262 * @verbose: Print info even when enough bandwidth is available 6263 * 6264 * If the available bandwidth at the device is less than the device is 6265 * capable of, report the device's maximum possible bandwidth and the 6266 * upstream link that limits its performance. If @verbose, always print 6267 * the available bandwidth, even if the device isn't constrained. 6268 */ 6269 void __pcie_print_link_status(struct pci_dev *dev, bool verbose) 6270 { 6271 enum pcie_link_width width, width_cap; 6272 enum pci_bus_speed speed, speed_cap; 6273 struct pci_dev *limiting_dev = NULL; 6274 u32 bw_avail, bw_cap; 6275 6276 bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap); 6277 bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width); 6278 6279 if (bw_avail >= bw_cap && verbose) 6280 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n", 6281 bw_cap / 1000, bw_cap % 1000, 6282 pci_speed_string(speed_cap), width_cap); 6283 else if (bw_avail < bw_cap) 6284 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n", 6285 bw_avail / 1000, bw_avail % 1000, 6286 pci_speed_string(speed), width, 6287 limiting_dev ? pci_name(limiting_dev) : "<unknown>", 6288 bw_cap / 1000, bw_cap % 1000, 6289 pci_speed_string(speed_cap), width_cap); 6290 } 6291 6292 /** 6293 * pcie_print_link_status - Report the PCI device's link speed and width 6294 * @dev: PCI device to query 6295 * 6296 * Report the available bandwidth at the device. 6297 */ 6298 void pcie_print_link_status(struct pci_dev *dev) 6299 { 6300 __pcie_print_link_status(dev, true); 6301 } 6302 EXPORT_SYMBOL(pcie_print_link_status); 6303 6304 /** 6305 * pci_select_bars - Make BAR mask from the type of resource 6306 * @dev: the PCI device for which BAR mask is made 6307 * @flags: resource type mask to be selected 6308 * 6309 * This helper routine makes bar mask from the type of resource. 6310 */ 6311 int pci_select_bars(struct pci_dev *dev, unsigned long flags) 6312 { 6313 int i, bars = 0; 6314 for (i = 0; i < PCI_NUM_RESOURCES; i++) 6315 if (pci_resource_flags(dev, i) & flags) 6316 bars |= (1 << i); 6317 return bars; 6318 } 6319 EXPORT_SYMBOL(pci_select_bars); 6320 6321 /* Some architectures require additional programming to enable VGA */ 6322 static arch_set_vga_state_t arch_set_vga_state; 6323 6324 void __init pci_register_set_vga_state(arch_set_vga_state_t func) 6325 { 6326 arch_set_vga_state = func; /* NULL disables */ 6327 } 6328 6329 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode, 6330 unsigned int command_bits, u32 flags) 6331 { 6332 if (arch_set_vga_state) 6333 return arch_set_vga_state(dev, decode, command_bits, 6334 flags); 6335 return 0; 6336 } 6337 6338 /** 6339 * pci_set_vga_state - set VGA decode state on device and parents if requested 6340 * @dev: the PCI device 6341 * @decode: true = enable decoding, false = disable decoding 6342 * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY 6343 * @flags: traverse ancestors and change bridges 6344 * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE 6345 */ 6346 int pci_set_vga_state(struct pci_dev *dev, bool decode, 6347 unsigned int command_bits, u32 flags) 6348 { 6349 struct pci_bus *bus; 6350 struct pci_dev *bridge; 6351 u16 cmd; 6352 int rc; 6353 6354 WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY))); 6355 6356 /* ARCH specific VGA enables */ 6357 rc = pci_set_vga_state_arch(dev, decode, command_bits, flags); 6358 if (rc) 6359 return rc; 6360 6361 if (flags & PCI_VGA_STATE_CHANGE_DECODES) { 6362 pci_read_config_word(dev, PCI_COMMAND, &cmd); 6363 if (decode) 6364 cmd |= command_bits; 6365 else 6366 cmd &= ~command_bits; 6367 pci_write_config_word(dev, PCI_COMMAND, cmd); 6368 } 6369 6370 if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE)) 6371 return 0; 6372 6373 bus = dev->bus; 6374 while (bus) { 6375 bridge = bus->self; 6376 if (bridge) { 6377 pci_read_config_word(bridge, PCI_BRIDGE_CONTROL, 6378 &cmd); 6379 if (decode) 6380 cmd |= PCI_BRIDGE_CTL_VGA; 6381 else 6382 cmd &= ~PCI_BRIDGE_CTL_VGA; 6383 pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, 6384 cmd); 6385 } 6386 bus = bus->parent; 6387 } 6388 return 0; 6389 } 6390 6391 #ifdef CONFIG_ACPI 6392 bool pci_pr3_present(struct pci_dev *pdev) 6393 { 6394 struct acpi_device *adev; 6395 6396 if (acpi_disabled) 6397 return false; 6398 6399 adev = ACPI_COMPANION(&pdev->dev); 6400 if (!adev) 6401 return false; 6402 6403 return adev->power.flags.power_resources && 6404 acpi_has_method(adev->handle, "_PR3"); 6405 } 6406 EXPORT_SYMBOL_GPL(pci_pr3_present); 6407 #endif 6408 6409 /** 6410 * pci_add_dma_alias - Add a DMA devfn alias for a device 6411 * @dev: the PCI device for which alias is added 6412 * @devfn_from: alias slot and function 6413 * @nr_devfns: number of subsequent devfns to alias 6414 * 6415 * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask 6416 * which is used to program permissible bus-devfn source addresses for DMA 6417 * requests in an IOMMU. These aliases factor into IOMMU group creation 6418 * and are useful for devices generating DMA requests beyond or different 6419 * from their logical bus-devfn. Examples include device quirks where the 6420 * device simply uses the wrong devfn, as well as non-transparent bridges 6421 * where the alias may be a proxy for devices in another domain. 6422 * 6423 * IOMMU group creation is performed during device discovery or addition, 6424 * prior to any potential DMA mapping and therefore prior to driver probing 6425 * (especially for userspace assigned devices where IOMMU group definition 6426 * cannot be left as a userspace activity). DMA aliases should therefore 6427 * be configured via quirks, such as the PCI fixup header quirk. 6428 */ 6429 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, 6430 unsigned int nr_devfns) 6431 { 6432 int devfn_to; 6433 6434 nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from); 6435 devfn_to = devfn_from + nr_devfns - 1; 6436 6437 if (!dev->dma_alias_mask) 6438 dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL); 6439 if (!dev->dma_alias_mask) { 6440 pci_warn(dev, "Unable to allocate DMA alias mask\n"); 6441 return; 6442 } 6443 6444 bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns); 6445 6446 if (nr_devfns == 1) 6447 pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n", 6448 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from)); 6449 else if (nr_devfns > 1) 6450 pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n", 6451 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from), 6452 PCI_SLOT(devfn_to), PCI_FUNC(devfn_to)); 6453 } 6454 6455 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) 6456 { 6457 return (dev1->dma_alias_mask && 6458 test_bit(dev2->devfn, dev1->dma_alias_mask)) || 6459 (dev2->dma_alias_mask && 6460 test_bit(dev1->devfn, dev2->dma_alias_mask)) || 6461 pci_real_dma_dev(dev1) == dev2 || 6462 pci_real_dma_dev(dev2) == dev1; 6463 } 6464 6465 bool pci_device_is_present(struct pci_dev *pdev) 6466 { 6467 u32 v; 6468 6469 if (pci_dev_is_disconnected(pdev)) 6470 return false; 6471 return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0); 6472 } 6473 EXPORT_SYMBOL_GPL(pci_device_is_present); 6474 6475 void pci_ignore_hotplug(struct pci_dev *dev) 6476 { 6477 struct pci_dev *bridge = dev->bus->self; 6478 6479 dev->ignore_hotplug = 1; 6480 /* Propagate the "ignore hotplug" setting to the parent bridge. */ 6481 if (bridge) 6482 bridge->ignore_hotplug = 1; 6483 } 6484 EXPORT_SYMBOL_GPL(pci_ignore_hotplug); 6485 6486 /** 6487 * pci_real_dma_dev - Get PCI DMA device for PCI device 6488 * @dev: the PCI device that may have a PCI DMA alias 6489 * 6490 * Permits the platform to provide architecture-specific functionality to 6491 * devices needing to alias DMA to another PCI device on another PCI bus. If 6492 * the PCI device is on the same bus, it is recommended to use 6493 * pci_add_dma_alias(). This is the default implementation. Architecture 6494 * implementations can override this. 6495 */ 6496 struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev) 6497 { 6498 return dev; 6499 } 6500 6501 resource_size_t __weak pcibios_default_alignment(void) 6502 { 6503 return 0; 6504 } 6505 6506 /* 6507 * Arches that don't want to expose struct resource to userland as-is in 6508 * sysfs and /proc can implement their own pci_resource_to_user(). 6509 */ 6510 void __weak pci_resource_to_user(const struct pci_dev *dev, int bar, 6511 const struct resource *rsrc, 6512 resource_size_t *start, resource_size_t *end) 6513 { 6514 *start = rsrc->start; 6515 *end = rsrc->end; 6516 } 6517 6518 static char *resource_alignment_param; 6519 static DEFINE_SPINLOCK(resource_alignment_lock); 6520 6521 /** 6522 * pci_specified_resource_alignment - get resource alignment specified by user. 6523 * @dev: the PCI device to get 6524 * @resize: whether or not to change resources' size when reassigning alignment 6525 * 6526 * RETURNS: Resource alignment if it is specified. 6527 * Zero if it is not specified. 6528 */ 6529 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev, 6530 bool *resize) 6531 { 6532 int align_order, count; 6533 resource_size_t align = pcibios_default_alignment(); 6534 const char *p; 6535 int ret; 6536 6537 spin_lock(&resource_alignment_lock); 6538 p = resource_alignment_param; 6539 if (!p || !*p) 6540 goto out; 6541 if (pci_has_flag(PCI_PROBE_ONLY)) { 6542 align = 0; 6543 pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n"); 6544 goto out; 6545 } 6546 6547 while (*p) { 6548 count = 0; 6549 if (sscanf(p, "%d%n", &align_order, &count) == 1 && 6550 p[count] == '@') { 6551 p += count + 1; 6552 if (align_order > 63) { 6553 pr_err("PCI: Invalid requested alignment (order %d)\n", 6554 align_order); 6555 align_order = PAGE_SHIFT; 6556 } 6557 } else { 6558 align_order = PAGE_SHIFT; 6559 } 6560 6561 ret = pci_dev_str_match(dev, p, &p); 6562 if (ret == 1) { 6563 *resize = true; 6564 align = 1ULL << align_order; 6565 break; 6566 } else if (ret < 0) { 6567 pr_err("PCI: Can't parse resource_alignment parameter: %s\n", 6568 p); 6569 break; 6570 } 6571 6572 if (*p != ';' && *p != ',') { 6573 /* End of param or invalid format */ 6574 break; 6575 } 6576 p++; 6577 } 6578 out: 6579 spin_unlock(&resource_alignment_lock); 6580 return align; 6581 } 6582 6583 static void pci_request_resource_alignment(struct pci_dev *dev, int bar, 6584 resource_size_t align, bool resize) 6585 { 6586 struct resource *r = &dev->resource[bar]; 6587 resource_size_t size; 6588 6589 if (!(r->flags & IORESOURCE_MEM)) 6590 return; 6591 6592 if (r->flags & IORESOURCE_PCI_FIXED) { 6593 pci_info(dev, "BAR%d %pR: ignoring requested alignment %#llx\n", 6594 bar, r, (unsigned long long)align); 6595 return; 6596 } 6597 6598 size = resource_size(r); 6599 if (size >= align) 6600 return; 6601 6602 /* 6603 * Increase the alignment of the resource. There are two ways we 6604 * can do this: 6605 * 6606 * 1) Increase the size of the resource. BARs are aligned on their 6607 * size, so when we reallocate space for this resource, we'll 6608 * allocate it with the larger alignment. This also prevents 6609 * assignment of any other BARs inside the alignment region, so 6610 * if we're requesting page alignment, this means no other BARs 6611 * will share the page. 6612 * 6613 * The disadvantage is that this makes the resource larger than 6614 * the hardware BAR, which may break drivers that compute things 6615 * based on the resource size, e.g., to find registers at a 6616 * fixed offset before the end of the BAR. 6617 * 6618 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and 6619 * set r->start to the desired alignment. By itself this 6620 * doesn't prevent other BARs being put inside the alignment 6621 * region, but if we realign *every* resource of every device in 6622 * the system, none of them will share an alignment region. 6623 * 6624 * When the user has requested alignment for only some devices via 6625 * the "pci=resource_alignment" argument, "resize" is true and we 6626 * use the first method. Otherwise we assume we're aligning all 6627 * devices and we use the second. 6628 */ 6629 6630 pci_info(dev, "BAR%d %pR: requesting alignment to %#llx\n", 6631 bar, r, (unsigned long long)align); 6632 6633 if (resize) { 6634 r->start = 0; 6635 r->end = align - 1; 6636 } else { 6637 r->flags &= ~IORESOURCE_SIZEALIGN; 6638 r->flags |= IORESOURCE_STARTALIGN; 6639 r->start = align; 6640 r->end = r->start + size - 1; 6641 } 6642 r->flags |= IORESOURCE_UNSET; 6643 } 6644 6645 /* 6646 * This function disables memory decoding and releases memory resources 6647 * of the device specified by kernel's boot parameter 'pci=resource_alignment='. 6648 * It also rounds up size to specified alignment. 6649 * Later on, the kernel will assign page-aligned memory resource back 6650 * to the device. 6651 */ 6652 void pci_reassigndev_resource_alignment(struct pci_dev *dev) 6653 { 6654 int i; 6655 struct resource *r; 6656 resource_size_t align; 6657 u16 command; 6658 bool resize = false; 6659 6660 /* 6661 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec 6662 * 3.4.1.11. Their resources are allocated from the space 6663 * described by the VF BARx register in the PF's SR-IOV capability. 6664 * We can't influence their alignment here. 6665 */ 6666 if (dev->is_virtfn) 6667 return; 6668 6669 /* check if specified PCI is target device to reassign */ 6670 align = pci_specified_resource_alignment(dev, &resize); 6671 if (!align) 6672 return; 6673 6674 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL && 6675 (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) { 6676 pci_warn(dev, "Can't reassign resources to host bridge\n"); 6677 return; 6678 } 6679 6680 pci_read_config_word(dev, PCI_COMMAND, &command); 6681 command &= ~PCI_COMMAND_MEMORY; 6682 pci_write_config_word(dev, PCI_COMMAND, command); 6683 6684 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 6685 pci_request_resource_alignment(dev, i, align, resize); 6686 6687 /* 6688 * Need to disable bridge's resource window, 6689 * to enable the kernel to reassign new resource 6690 * window later on. 6691 */ 6692 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 6693 for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) { 6694 r = &dev->resource[i]; 6695 if (!(r->flags & IORESOURCE_MEM)) 6696 continue; 6697 r->flags |= IORESOURCE_UNSET; 6698 r->end = resource_size(r) - 1; 6699 r->start = 0; 6700 } 6701 pci_disable_bridge_window(dev); 6702 } 6703 } 6704 6705 static ssize_t resource_alignment_show(struct bus_type *bus, char *buf) 6706 { 6707 size_t count = 0; 6708 6709 spin_lock(&resource_alignment_lock); 6710 if (resource_alignment_param) 6711 count = sysfs_emit(buf, "%s\n", resource_alignment_param); 6712 spin_unlock(&resource_alignment_lock); 6713 6714 return count; 6715 } 6716 6717 static ssize_t resource_alignment_store(struct bus_type *bus, 6718 const char *buf, size_t count) 6719 { 6720 char *param, *old, *end; 6721 6722 if (count >= (PAGE_SIZE - 1)) 6723 return -EINVAL; 6724 6725 param = kstrndup(buf, count, GFP_KERNEL); 6726 if (!param) 6727 return -ENOMEM; 6728 6729 end = strchr(param, '\n'); 6730 if (end) 6731 *end = '\0'; 6732 6733 spin_lock(&resource_alignment_lock); 6734 old = resource_alignment_param; 6735 if (strlen(param)) { 6736 resource_alignment_param = param; 6737 } else { 6738 kfree(param); 6739 resource_alignment_param = NULL; 6740 } 6741 spin_unlock(&resource_alignment_lock); 6742 6743 kfree(old); 6744 6745 return count; 6746 } 6747 6748 static BUS_ATTR_RW(resource_alignment); 6749 6750 static int __init pci_resource_alignment_sysfs_init(void) 6751 { 6752 return bus_create_file(&pci_bus_type, 6753 &bus_attr_resource_alignment); 6754 } 6755 late_initcall(pci_resource_alignment_sysfs_init); 6756 6757 static void pci_no_domains(void) 6758 { 6759 #ifdef CONFIG_PCI_DOMAINS 6760 pci_domains_supported = 0; 6761 #endif 6762 } 6763 6764 #ifdef CONFIG_PCI_DOMAINS_GENERIC 6765 static atomic_t __domain_nr = ATOMIC_INIT(-1); 6766 6767 static int pci_get_new_domain_nr(void) 6768 { 6769 return atomic_inc_return(&__domain_nr); 6770 } 6771 6772 static int of_pci_bus_find_domain_nr(struct device *parent) 6773 { 6774 static int use_dt_domains = -1; 6775 int domain = -1; 6776 6777 if (parent) 6778 domain = of_get_pci_domain_nr(parent->of_node); 6779 6780 /* 6781 * Check DT domain and use_dt_domains values. 6782 * 6783 * If DT domain property is valid (domain >= 0) and 6784 * use_dt_domains != 0, the DT assignment is valid since this means 6785 * we have not previously allocated a domain number by using 6786 * pci_get_new_domain_nr(); we should also update use_dt_domains to 6787 * 1, to indicate that we have just assigned a domain number from 6788 * DT. 6789 * 6790 * If DT domain property value is not valid (ie domain < 0), and we 6791 * have not previously assigned a domain number from DT 6792 * (use_dt_domains != 1) we should assign a domain number by 6793 * using the: 6794 * 6795 * pci_get_new_domain_nr() 6796 * 6797 * API and update the use_dt_domains value to keep track of method we 6798 * are using to assign domain numbers (use_dt_domains = 0). 6799 * 6800 * All other combinations imply we have a platform that is trying 6801 * to mix domain numbers obtained from DT and pci_get_new_domain_nr(), 6802 * which is a recipe for domain mishandling and it is prevented by 6803 * invalidating the domain value (domain = -1) and printing a 6804 * corresponding error. 6805 */ 6806 if (domain >= 0 && use_dt_domains) { 6807 use_dt_domains = 1; 6808 } else if (domain < 0 && use_dt_domains != 1) { 6809 use_dt_domains = 0; 6810 domain = pci_get_new_domain_nr(); 6811 } else { 6812 if (parent) 6813 pr_err("Node %pOF has ", parent->of_node); 6814 pr_err("Inconsistent \"linux,pci-domain\" property in DT\n"); 6815 domain = -1; 6816 } 6817 6818 return domain; 6819 } 6820 6821 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent) 6822 { 6823 return acpi_disabled ? of_pci_bus_find_domain_nr(parent) : 6824 acpi_pci_bus_find_domain_nr(bus); 6825 } 6826 #endif 6827 6828 /** 6829 * pci_ext_cfg_avail - can we access extended PCI config space? 6830 * 6831 * Returns 1 if we can access PCI extended config space (offsets 6832 * greater than 0xff). This is the default implementation. Architecture 6833 * implementations can override this. 6834 */ 6835 int __weak pci_ext_cfg_avail(void) 6836 { 6837 return 1; 6838 } 6839 6840 void __weak pci_fixup_cardbus(struct pci_bus *bus) 6841 { 6842 } 6843 EXPORT_SYMBOL(pci_fixup_cardbus); 6844 6845 static int __init pci_setup(char *str) 6846 { 6847 while (str) { 6848 char *k = strchr(str, ','); 6849 if (k) 6850 *k++ = 0; 6851 if (*str && (str = pcibios_setup(str)) && *str) { 6852 if (!strcmp(str, "nomsi")) { 6853 pci_no_msi(); 6854 } else if (!strncmp(str, "noats", 5)) { 6855 pr_info("PCIe: ATS is disabled\n"); 6856 pcie_ats_disabled = true; 6857 } else if (!strcmp(str, "noaer")) { 6858 pci_no_aer(); 6859 } else if (!strcmp(str, "earlydump")) { 6860 pci_early_dump = true; 6861 } else if (!strncmp(str, "realloc=", 8)) { 6862 pci_realloc_get_opt(str + 8); 6863 } else if (!strncmp(str, "realloc", 7)) { 6864 pci_realloc_get_opt("on"); 6865 } else if (!strcmp(str, "nodomains")) { 6866 pci_no_domains(); 6867 } else if (!strncmp(str, "noari", 5)) { 6868 pcie_ari_disabled = true; 6869 } else if (!strncmp(str, "cbiosize=", 9)) { 6870 pci_cardbus_io_size = memparse(str + 9, &str); 6871 } else if (!strncmp(str, "cbmemsize=", 10)) { 6872 pci_cardbus_mem_size = memparse(str + 10, &str); 6873 } else if (!strncmp(str, "resource_alignment=", 19)) { 6874 resource_alignment_param = str + 19; 6875 } else if (!strncmp(str, "ecrc=", 5)) { 6876 pcie_ecrc_get_policy(str + 5); 6877 } else if (!strncmp(str, "hpiosize=", 9)) { 6878 pci_hotplug_io_size = memparse(str + 9, &str); 6879 } else if (!strncmp(str, "hpmmiosize=", 11)) { 6880 pci_hotplug_mmio_size = memparse(str + 11, &str); 6881 } else if (!strncmp(str, "hpmmioprefsize=", 15)) { 6882 pci_hotplug_mmio_pref_size = memparse(str + 15, &str); 6883 } else if (!strncmp(str, "hpmemsize=", 10)) { 6884 pci_hotplug_mmio_size = memparse(str + 10, &str); 6885 pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size; 6886 } else if (!strncmp(str, "hpbussize=", 10)) { 6887 pci_hotplug_bus_size = 6888 simple_strtoul(str + 10, &str, 0); 6889 if (pci_hotplug_bus_size > 0xff) 6890 pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 6891 } else if (!strncmp(str, "pcie_bus_tune_off", 17)) { 6892 pcie_bus_config = PCIE_BUS_TUNE_OFF; 6893 } else if (!strncmp(str, "pcie_bus_safe", 13)) { 6894 pcie_bus_config = PCIE_BUS_SAFE; 6895 } else if (!strncmp(str, "pcie_bus_perf", 13)) { 6896 pcie_bus_config = PCIE_BUS_PERFORMANCE; 6897 } else if (!strncmp(str, "pcie_bus_peer2peer", 18)) { 6898 pcie_bus_config = PCIE_BUS_PEER2PEER; 6899 } else if (!strncmp(str, "pcie_scan_all", 13)) { 6900 pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS); 6901 } else if (!strncmp(str, "disable_acs_redir=", 18)) { 6902 disable_acs_redir_param = str + 18; 6903 } else { 6904 pr_err("PCI: Unknown option `%s'\n", str); 6905 } 6906 } 6907 str = k; 6908 } 6909 return 0; 6910 } 6911 early_param("pci", pci_setup); 6912 6913 /* 6914 * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized 6915 * in pci_setup(), above, to point to data in the __initdata section which 6916 * will be freed after the init sequence is complete. We can't allocate memory 6917 * in pci_setup() because some architectures do not have any memory allocation 6918 * service available during an early_param() call. So we allocate memory and 6919 * copy the variable here before the init section is freed. 6920 * 6921 */ 6922 static int __init pci_realloc_setup_params(void) 6923 { 6924 resource_alignment_param = kstrdup(resource_alignment_param, 6925 GFP_KERNEL); 6926 disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL); 6927 6928 return 0; 6929 } 6930 pure_initcall(pci_realloc_setup_params); 6931