xref: /linux/drivers/pci/pci.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI Bus Services, see include/linux/pci.h for further explanation.
4  *
5  * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
6  * David Mosberger-Tang
7  *
8  * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
9  */
10 
11 #include <linux/acpi.h>
12 #include <linux/kernel.h>
13 #include <linux/delay.h>
14 #include <linux/dmi.h>
15 #include <linux/init.h>
16 #include <linux/msi.h>
17 #include <linux/of.h>
18 #include <linux/pci.h>
19 #include <linux/pm.h>
20 #include <linux/slab.h>
21 #include <linux/module.h>
22 #include <linux/spinlock.h>
23 #include <linux/string.h>
24 #include <linux/log2.h>
25 #include <linux/logic_pio.h>
26 #include <linux/device.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/pci_hotplug.h>
29 #include <linux/vmalloc.h>
30 #include <asm/dma.h>
31 #include <linux/aer.h>
32 #include <linux/bitfield.h>
33 #include "pci.h"
34 
35 DEFINE_MUTEX(pci_slot_mutex);
36 
37 const char *pci_power_names[] = {
38 	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
39 };
40 EXPORT_SYMBOL_GPL(pci_power_names);
41 
42 #ifdef CONFIG_X86_32
43 int isa_dma_bridge_buggy;
44 EXPORT_SYMBOL(isa_dma_bridge_buggy);
45 #endif
46 
47 int pci_pci_problems;
48 EXPORT_SYMBOL(pci_pci_problems);
49 
50 unsigned int pci_pm_d3hot_delay;
51 
52 static void pci_pme_list_scan(struct work_struct *work);
53 
54 static LIST_HEAD(pci_pme_list);
55 static DEFINE_MUTEX(pci_pme_list_mutex);
56 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
57 
58 struct pci_pme_device {
59 	struct list_head list;
60 	struct pci_dev *dev;
61 };
62 
63 #define PME_TIMEOUT 1000 /* How long between PME checks */
64 
65 /*
66  * Following exit from Conventional Reset, devices must be ready within 1 sec
67  * (PCIe r6.0 sec 6.6.1).  A D3cold to D0 transition implies a Conventional
68  * Reset (PCIe r6.0 sec 5.8).
69  */
70 #define PCI_RESET_WAIT 1000 /* msec */
71 
72 /*
73  * Devices may extend the 1 sec period through Request Retry Status
74  * completions (PCIe r6.0 sec 2.3.1).  The spec does not provide an upper
75  * limit, but 60 sec ought to be enough for any device to become
76  * responsive.
77  */
78 #define PCIE_RESET_READY_POLL_MS 60000 /* msec */
79 
pci_dev_d3_sleep(struct pci_dev * dev)80 static void pci_dev_d3_sleep(struct pci_dev *dev)
81 {
82 	unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay);
83 	unsigned int upper;
84 
85 	if (delay_ms) {
86 		/* Use a 20% upper bound, 1ms minimum */
87 		upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U);
88 		usleep_range(delay_ms * USEC_PER_MSEC,
89 			     (delay_ms + upper) * USEC_PER_MSEC);
90 	}
91 }
92 
pci_reset_supported(struct pci_dev * dev)93 bool pci_reset_supported(struct pci_dev *dev)
94 {
95 	return dev->reset_methods[0] != 0;
96 }
97 
98 #ifdef CONFIG_PCI_DOMAINS
99 int pci_domains_supported = 1;
100 #endif
101 
102 #define DEFAULT_CARDBUS_IO_SIZE		(256)
103 #define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
104 /* pci=cbmemsize=nnM,cbiosize=nn can override this */
105 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
106 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
107 
108 #define DEFAULT_HOTPLUG_IO_SIZE		(256)
109 #define DEFAULT_HOTPLUG_MMIO_SIZE	(2*1024*1024)
110 #define DEFAULT_HOTPLUG_MMIO_PREF_SIZE	(2*1024*1024)
111 /* hpiosize=nn can override this */
112 unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
113 /*
114  * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size,
115  * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size;
116  * pci=hpmemsize=nnM overrides both
117  */
118 unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE;
119 unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE;
120 
121 #define DEFAULT_HOTPLUG_BUS_SIZE	1
122 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
123 
124 
125 /* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */
126 #ifdef CONFIG_PCIE_BUS_TUNE_OFF
127 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
128 #elif defined CONFIG_PCIE_BUS_SAFE
129 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE;
130 #elif defined CONFIG_PCIE_BUS_PERFORMANCE
131 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE;
132 #elif defined CONFIG_PCIE_BUS_PEER2PEER
133 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER;
134 #else
135 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
136 #endif
137 
138 /*
139  * The default CLS is used if arch didn't set CLS explicitly and not
140  * all pci devices agree on the same value.  Arch can override either
141  * the dfl or actual value as it sees fit.  Don't forget this is
142  * measured in 32-bit words, not bytes.
143  */
144 u8 pci_dfl_cache_line_size __ro_after_init = L1_CACHE_BYTES >> 2;
145 u8 pci_cache_line_size __ro_after_init ;
146 
147 /*
148  * If we set up a device for bus mastering, we need to check the latency
149  * timer as certain BIOSes forget to set it properly.
150  */
151 unsigned int pcibios_max_latency = 255;
152 
153 /* If set, the PCIe ARI capability will not be used. */
154 static bool pcie_ari_disabled;
155 
156 /* If set, the PCIe ATS capability will not be used. */
157 static bool pcie_ats_disabled;
158 
159 /* If set, the PCI config space of each device is printed during boot. */
160 bool pci_early_dump;
161 
pci_ats_disabled(void)162 bool pci_ats_disabled(void)
163 {
164 	return pcie_ats_disabled;
165 }
166 EXPORT_SYMBOL_GPL(pci_ats_disabled);
167 
168 /* Disable bridge_d3 for all PCIe ports */
169 static bool pci_bridge_d3_disable;
170 /* Force bridge_d3 for all PCIe ports */
171 static bool pci_bridge_d3_force;
172 
pcie_port_pm_setup(char * str)173 static int __init pcie_port_pm_setup(char *str)
174 {
175 	if (!strcmp(str, "off"))
176 		pci_bridge_d3_disable = true;
177 	else if (!strcmp(str, "force"))
178 		pci_bridge_d3_force = true;
179 	return 1;
180 }
181 __setup("pcie_port_pm=", pcie_port_pm_setup);
182 
183 /**
184  * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
185  * @bus: pointer to PCI bus structure to search
186  *
187  * Given a PCI bus, returns the highest PCI bus number present in the set
188  * including the given PCI bus and its list of child PCI buses.
189  */
pci_bus_max_busnr(struct pci_bus * bus)190 unsigned char pci_bus_max_busnr(struct pci_bus *bus)
191 {
192 	struct pci_bus *tmp;
193 	unsigned char max, n;
194 
195 	max = bus->busn_res.end;
196 	list_for_each_entry(tmp, &bus->children, node) {
197 		n = pci_bus_max_busnr(tmp);
198 		if (n > max)
199 			max = n;
200 	}
201 	return max;
202 }
203 EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
204 
205 /**
206  * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS
207  * @pdev: the PCI device
208  *
209  * Returns error bits set in PCI_STATUS and clears them.
210  */
pci_status_get_and_clear_errors(struct pci_dev * pdev)211 int pci_status_get_and_clear_errors(struct pci_dev *pdev)
212 {
213 	u16 status;
214 	int ret;
215 
216 	ret = pci_read_config_word(pdev, PCI_STATUS, &status);
217 	if (ret != PCIBIOS_SUCCESSFUL)
218 		return -EIO;
219 
220 	status &= PCI_STATUS_ERROR_BITS;
221 	if (status)
222 		pci_write_config_word(pdev, PCI_STATUS, status);
223 
224 	return status;
225 }
226 EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors);
227 
228 #ifdef CONFIG_HAS_IOMEM
__pci_ioremap_resource(struct pci_dev * pdev,int bar,bool write_combine)229 static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar,
230 					    bool write_combine)
231 {
232 	struct resource *res = &pdev->resource[bar];
233 	resource_size_t start = res->start;
234 	resource_size_t size = resource_size(res);
235 
236 	/*
237 	 * Make sure the BAR is actually a memory resource, not an IO resource
238 	 */
239 	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
240 		pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res);
241 		return NULL;
242 	}
243 
244 	if (write_combine)
245 		return ioremap_wc(start, size);
246 
247 	return ioremap(start, size);
248 }
249 
pci_ioremap_bar(struct pci_dev * pdev,int bar)250 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
251 {
252 	return __pci_ioremap_resource(pdev, bar, false);
253 }
254 EXPORT_SYMBOL_GPL(pci_ioremap_bar);
255 
pci_ioremap_wc_bar(struct pci_dev * pdev,int bar)256 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
257 {
258 	return __pci_ioremap_resource(pdev, bar, true);
259 }
260 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
261 #endif
262 
263 /**
264  * pci_dev_str_match_path - test if a path string matches a device
265  * @dev: the PCI device to test
266  * @path: string to match the device against
267  * @endptr: pointer to the string after the match
268  *
269  * Test if a string (typically from a kernel parameter) formatted as a
270  * path of device/function addresses matches a PCI device. The string must
271  * be of the form:
272  *
273  *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
274  *
275  * A path for a device can be obtained using 'lspci -t'.  Using a path
276  * is more robust against bus renumbering than using only a single bus,
277  * device and function address.
278  *
279  * Returns 1 if the string matches the device, 0 if it does not and
280  * a negative error code if it fails to parse the string.
281  */
pci_dev_str_match_path(struct pci_dev * dev,const char * path,const char ** endptr)282 static int pci_dev_str_match_path(struct pci_dev *dev, const char *path,
283 				  const char **endptr)
284 {
285 	int ret;
286 	unsigned int seg, bus, slot, func;
287 	char *wpath, *p;
288 	char end;
289 
290 	*endptr = strchrnul(path, ';');
291 
292 	wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC);
293 	if (!wpath)
294 		return -ENOMEM;
295 
296 	while (1) {
297 		p = strrchr(wpath, '/');
298 		if (!p)
299 			break;
300 		ret = sscanf(p, "/%x.%x%c", &slot, &func, &end);
301 		if (ret != 2) {
302 			ret = -EINVAL;
303 			goto free_and_exit;
304 		}
305 
306 		if (dev->devfn != PCI_DEVFN(slot, func)) {
307 			ret = 0;
308 			goto free_and_exit;
309 		}
310 
311 		/*
312 		 * Note: we don't need to get a reference to the upstream
313 		 * bridge because we hold a reference to the top level
314 		 * device which should hold a reference to the bridge,
315 		 * and so on.
316 		 */
317 		dev = pci_upstream_bridge(dev);
318 		if (!dev) {
319 			ret = 0;
320 			goto free_and_exit;
321 		}
322 
323 		*p = 0;
324 	}
325 
326 	ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot,
327 		     &func, &end);
328 	if (ret != 4) {
329 		seg = 0;
330 		ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end);
331 		if (ret != 3) {
332 			ret = -EINVAL;
333 			goto free_and_exit;
334 		}
335 	}
336 
337 	ret = (seg == pci_domain_nr(dev->bus) &&
338 	       bus == dev->bus->number &&
339 	       dev->devfn == PCI_DEVFN(slot, func));
340 
341 free_and_exit:
342 	kfree(wpath);
343 	return ret;
344 }
345 
346 /**
347  * pci_dev_str_match - test if a string matches a device
348  * @dev: the PCI device to test
349  * @p: string to match the device against
350  * @endptr: pointer to the string after the match
351  *
352  * Test if a string (typically from a kernel parameter) matches a specified
353  * PCI device. The string may be of one of the following formats:
354  *
355  *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
356  *   pci:<vendor>:<device>[:<subvendor>:<subdevice>]
357  *
358  * The first format specifies a PCI bus/device/function address which
359  * may change if new hardware is inserted, if motherboard firmware changes,
360  * or due to changes caused in kernel parameters. If the domain is
361  * left unspecified, it is taken to be 0.  In order to be robust against
362  * bus renumbering issues, a path of PCI device/function numbers may be used
363  * to address the specific device.  The path for a device can be determined
364  * through the use of 'lspci -t'.
365  *
366  * The second format matches devices using IDs in the configuration
367  * space which may match multiple devices in the system. A value of 0
368  * for any field will match all devices. (Note: this differs from
369  * in-kernel code that uses PCI_ANY_ID which is ~0; this is for
370  * legacy reasons and convenience so users don't have to specify
371  * FFFFFFFFs on the command line.)
372  *
373  * Returns 1 if the string matches the device, 0 if it does not and
374  * a negative error code if the string cannot be parsed.
375  */
pci_dev_str_match(struct pci_dev * dev,const char * p,const char ** endptr)376 static int pci_dev_str_match(struct pci_dev *dev, const char *p,
377 			     const char **endptr)
378 {
379 	int ret;
380 	int count;
381 	unsigned short vendor, device, subsystem_vendor, subsystem_device;
382 
383 	if (strncmp(p, "pci:", 4) == 0) {
384 		/* PCI vendor/device (subvendor/subdevice) IDs are specified */
385 		p += 4;
386 		ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device,
387 			     &subsystem_vendor, &subsystem_device, &count);
388 		if (ret != 4) {
389 			ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count);
390 			if (ret != 2)
391 				return -EINVAL;
392 
393 			subsystem_vendor = 0;
394 			subsystem_device = 0;
395 		}
396 
397 		p += count;
398 
399 		if ((!vendor || vendor == dev->vendor) &&
400 		    (!device || device == dev->device) &&
401 		    (!subsystem_vendor ||
402 			    subsystem_vendor == dev->subsystem_vendor) &&
403 		    (!subsystem_device ||
404 			    subsystem_device == dev->subsystem_device))
405 			goto found;
406 	} else {
407 		/*
408 		 * PCI Bus, Device, Function IDs are specified
409 		 * (optionally, may include a path of devfns following it)
410 		 */
411 		ret = pci_dev_str_match_path(dev, p, &p);
412 		if (ret < 0)
413 			return ret;
414 		else if (ret)
415 			goto found;
416 	}
417 
418 	*endptr = p;
419 	return 0;
420 
421 found:
422 	*endptr = p;
423 	return 1;
424 }
425 
__pci_find_next_cap_ttl(struct pci_bus * bus,unsigned int devfn,u8 pos,int cap,int * ttl)426 static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
427 				  u8 pos, int cap, int *ttl)
428 {
429 	u8 id;
430 	u16 ent;
431 
432 	pci_bus_read_config_byte(bus, devfn, pos, &pos);
433 
434 	while ((*ttl)--) {
435 		if (pos < 0x40)
436 			break;
437 		pos &= ~3;
438 		pci_bus_read_config_word(bus, devfn, pos, &ent);
439 
440 		id = ent & 0xff;
441 		if (id == 0xff)
442 			break;
443 		if (id == cap)
444 			return pos;
445 		pos = (ent >> 8);
446 	}
447 	return 0;
448 }
449 
__pci_find_next_cap(struct pci_bus * bus,unsigned int devfn,u8 pos,int cap)450 static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
451 			      u8 pos, int cap)
452 {
453 	int ttl = PCI_FIND_CAP_TTL;
454 
455 	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
456 }
457 
pci_find_next_capability(struct pci_dev * dev,u8 pos,int cap)458 u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
459 {
460 	return __pci_find_next_cap(dev->bus, dev->devfn,
461 				   pos + PCI_CAP_LIST_NEXT, cap);
462 }
463 EXPORT_SYMBOL_GPL(pci_find_next_capability);
464 
__pci_bus_find_cap_start(struct pci_bus * bus,unsigned int devfn,u8 hdr_type)465 static u8 __pci_bus_find_cap_start(struct pci_bus *bus,
466 				    unsigned int devfn, u8 hdr_type)
467 {
468 	u16 status;
469 
470 	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
471 	if (!(status & PCI_STATUS_CAP_LIST))
472 		return 0;
473 
474 	switch (hdr_type) {
475 	case PCI_HEADER_TYPE_NORMAL:
476 	case PCI_HEADER_TYPE_BRIDGE:
477 		return PCI_CAPABILITY_LIST;
478 	case PCI_HEADER_TYPE_CARDBUS:
479 		return PCI_CB_CAPABILITY_LIST;
480 	}
481 
482 	return 0;
483 }
484 
485 /**
486  * pci_find_capability - query for devices' capabilities
487  * @dev: PCI device to query
488  * @cap: capability code
489  *
490  * Tell if a device supports a given PCI capability.
491  * Returns the address of the requested capability structure within the
492  * device's PCI configuration space or 0 in case the device does not
493  * support it.  Possible values for @cap include:
494  *
495  *  %PCI_CAP_ID_PM           Power Management
496  *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
497  *  %PCI_CAP_ID_VPD          Vital Product Data
498  *  %PCI_CAP_ID_SLOTID       Slot Identification
499  *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
500  *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
501  *  %PCI_CAP_ID_PCIX         PCI-X
502  *  %PCI_CAP_ID_EXP          PCI Express
503  */
pci_find_capability(struct pci_dev * dev,int cap)504 u8 pci_find_capability(struct pci_dev *dev, int cap)
505 {
506 	u8 pos;
507 
508 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
509 	if (pos)
510 		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);
511 
512 	return pos;
513 }
514 EXPORT_SYMBOL(pci_find_capability);
515 
516 /**
517  * pci_bus_find_capability - query for devices' capabilities
518  * @bus: the PCI bus to query
519  * @devfn: PCI device to query
520  * @cap: capability code
521  *
522  * Like pci_find_capability() but works for PCI devices that do not have a
523  * pci_dev structure set up yet.
524  *
525  * Returns the address of the requested capability structure within the
526  * device's PCI configuration space or 0 in case the device does not
527  * support it.
528  */
pci_bus_find_capability(struct pci_bus * bus,unsigned int devfn,int cap)529 u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
530 {
531 	u8 hdr_type, pos;
532 
533 	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);
534 
535 	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & PCI_HEADER_TYPE_MASK);
536 	if (pos)
537 		pos = __pci_find_next_cap(bus, devfn, pos, cap);
538 
539 	return pos;
540 }
541 EXPORT_SYMBOL(pci_bus_find_capability);
542 
543 /**
544  * pci_find_next_ext_capability - Find an extended capability
545  * @dev: PCI device to query
546  * @start: address at which to start looking (0 to start at beginning of list)
547  * @cap: capability code
548  *
549  * Returns the address of the next matching extended capability structure
550  * within the device's PCI configuration space or 0 if the device does
551  * not support it.  Some capabilities can occur several times, e.g., the
552  * vendor-specific capability, and this provides a way to find them all.
553  */
pci_find_next_ext_capability(struct pci_dev * dev,u16 start,int cap)554 u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap)
555 {
556 	u32 header;
557 	int ttl;
558 	u16 pos = PCI_CFG_SPACE_SIZE;
559 
560 	/* minimum 8 bytes per capability */
561 	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
562 
563 	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
564 		return 0;
565 
566 	if (start)
567 		pos = start;
568 
569 	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
570 		return 0;
571 
572 	/*
573 	 * If we have no capabilities, this is indicated by cap ID,
574 	 * cap version and next pointer all being 0.
575 	 */
576 	if (header == 0)
577 		return 0;
578 
579 	while (ttl-- > 0) {
580 		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
581 			return pos;
582 
583 		pos = PCI_EXT_CAP_NEXT(header);
584 		if (pos < PCI_CFG_SPACE_SIZE)
585 			break;
586 
587 		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
588 			break;
589 	}
590 
591 	return 0;
592 }
593 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
594 
595 /**
596  * pci_find_ext_capability - Find an extended capability
597  * @dev: PCI device to query
598  * @cap: capability code
599  *
600  * Returns the address of the requested extended capability structure
601  * within the device's PCI configuration space or 0 if the device does
602  * not support it.  Possible values for @cap include:
603  *
604  *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
605  *  %PCI_EXT_CAP_ID_VC		Virtual Channel
606  *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
607  *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
608  */
pci_find_ext_capability(struct pci_dev * dev,int cap)609 u16 pci_find_ext_capability(struct pci_dev *dev, int cap)
610 {
611 	return pci_find_next_ext_capability(dev, 0, cap);
612 }
613 EXPORT_SYMBOL_GPL(pci_find_ext_capability);
614 
615 /**
616  * pci_get_dsn - Read and return the 8-byte Device Serial Number
617  * @dev: PCI device to query
618  *
619  * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial
620  * Number.
621  *
622  * Returns the DSN, or zero if the capability does not exist.
623  */
pci_get_dsn(struct pci_dev * dev)624 u64 pci_get_dsn(struct pci_dev *dev)
625 {
626 	u32 dword;
627 	u64 dsn;
628 	int pos;
629 
630 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
631 	if (!pos)
632 		return 0;
633 
634 	/*
635 	 * The Device Serial Number is two dwords offset 4 bytes from the
636 	 * capability position. The specification says that the first dword is
637 	 * the lower half, and the second dword is the upper half.
638 	 */
639 	pos += 4;
640 	pci_read_config_dword(dev, pos, &dword);
641 	dsn = (u64)dword;
642 	pci_read_config_dword(dev, pos + 4, &dword);
643 	dsn |= ((u64)dword) << 32;
644 
645 	return dsn;
646 }
647 EXPORT_SYMBOL_GPL(pci_get_dsn);
648 
__pci_find_next_ht_cap(struct pci_dev * dev,u8 pos,int ht_cap)649 static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap)
650 {
651 	int rc, ttl = PCI_FIND_CAP_TTL;
652 	u8 cap, mask;
653 
654 	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
655 		mask = HT_3BIT_CAP_MASK;
656 	else
657 		mask = HT_5BIT_CAP_MASK;
658 
659 	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
660 				      PCI_CAP_ID_HT, &ttl);
661 	while (pos) {
662 		rc = pci_read_config_byte(dev, pos + 3, &cap);
663 		if (rc != PCIBIOS_SUCCESSFUL)
664 			return 0;
665 
666 		if ((cap & mask) == ht_cap)
667 			return pos;
668 
669 		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
670 					      pos + PCI_CAP_LIST_NEXT,
671 					      PCI_CAP_ID_HT, &ttl);
672 	}
673 
674 	return 0;
675 }
676 
677 /**
678  * pci_find_next_ht_capability - query a device's HyperTransport capabilities
679  * @dev: PCI device to query
680  * @pos: Position from which to continue searching
681  * @ht_cap: HyperTransport capability code
682  *
683  * To be used in conjunction with pci_find_ht_capability() to search for
684  * all capabilities matching @ht_cap. @pos should always be a value returned
685  * from pci_find_ht_capability().
686  *
687  * NB. To be 100% safe against broken PCI devices, the caller should take
688  * steps to avoid an infinite loop.
689  */
pci_find_next_ht_capability(struct pci_dev * dev,u8 pos,int ht_cap)690 u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap)
691 {
692 	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
693 }
694 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
695 
696 /**
697  * pci_find_ht_capability - query a device's HyperTransport capabilities
698  * @dev: PCI device to query
699  * @ht_cap: HyperTransport capability code
700  *
701  * Tell if a device supports a given HyperTransport capability.
702  * Returns an address within the device's PCI configuration space
703  * or 0 in case the device does not support the request capability.
704  * The address points to the PCI capability, of type PCI_CAP_ID_HT,
705  * which has a HyperTransport capability matching @ht_cap.
706  */
pci_find_ht_capability(struct pci_dev * dev,int ht_cap)707 u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
708 {
709 	u8 pos;
710 
711 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
712 	if (pos)
713 		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
714 
715 	return pos;
716 }
717 EXPORT_SYMBOL_GPL(pci_find_ht_capability);
718 
719 /**
720  * pci_find_vsec_capability - Find a vendor-specific extended capability
721  * @dev: PCI device to query
722  * @vendor: Vendor ID for which capability is defined
723  * @cap: Vendor-specific capability ID
724  *
725  * If @dev has Vendor ID @vendor, search for a VSEC capability with
726  * VSEC ID @cap. If found, return the capability offset in
727  * config space; otherwise return 0.
728  */
pci_find_vsec_capability(struct pci_dev * dev,u16 vendor,int cap)729 u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap)
730 {
731 	u16 vsec = 0;
732 	u32 header;
733 	int ret;
734 
735 	if (vendor != dev->vendor)
736 		return 0;
737 
738 	while ((vsec = pci_find_next_ext_capability(dev, vsec,
739 						     PCI_EXT_CAP_ID_VNDR))) {
740 		ret = pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, &header);
741 		if (ret != PCIBIOS_SUCCESSFUL)
742 			continue;
743 
744 		if (PCI_VNDR_HEADER_ID(header) == cap)
745 			return vsec;
746 	}
747 
748 	return 0;
749 }
750 EXPORT_SYMBOL_GPL(pci_find_vsec_capability);
751 
752 /**
753  * pci_find_dvsec_capability - Find DVSEC for vendor
754  * @dev: PCI device to query
755  * @vendor: Vendor ID to match for the DVSEC
756  * @dvsec: Designated Vendor-specific capability ID
757  *
758  * If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability
759  * offset in config space; otherwise return 0.
760  */
pci_find_dvsec_capability(struct pci_dev * dev,u16 vendor,u16 dvsec)761 u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec)
762 {
763 	int pos;
764 
765 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC);
766 	if (!pos)
767 		return 0;
768 
769 	while (pos) {
770 		u16 v, id;
771 
772 		pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v);
773 		pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id);
774 		if (vendor == v && dvsec == id)
775 			return pos;
776 
777 		pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC);
778 	}
779 
780 	return 0;
781 }
782 EXPORT_SYMBOL_GPL(pci_find_dvsec_capability);
783 
784 /**
785  * pci_find_parent_resource - return resource region of parent bus of given
786  *			      region
787  * @dev: PCI device structure contains resources to be searched
788  * @res: child resource record for which parent is sought
789  *
790  * For given resource region of given device, return the resource region of
791  * parent bus the given region is contained in.
792  */
pci_find_parent_resource(const struct pci_dev * dev,struct resource * res)793 struct resource *pci_find_parent_resource(const struct pci_dev *dev,
794 					  struct resource *res)
795 {
796 	const struct pci_bus *bus = dev->bus;
797 	struct resource *r;
798 
799 	pci_bus_for_each_resource(bus, r) {
800 		if (!r)
801 			continue;
802 		if (resource_contains(r, res)) {
803 
804 			/*
805 			 * If the window is prefetchable but the BAR is
806 			 * not, the allocator made a mistake.
807 			 */
808 			if (r->flags & IORESOURCE_PREFETCH &&
809 			    !(res->flags & IORESOURCE_PREFETCH))
810 				return NULL;
811 
812 			/*
813 			 * If we're below a transparent bridge, there may
814 			 * be both a positively-decoded aperture and a
815 			 * subtractively-decoded region that contain the BAR.
816 			 * We want the positively-decoded one, so this depends
817 			 * on pci_bus_for_each_resource() giving us those
818 			 * first.
819 			 */
820 			return r;
821 		}
822 	}
823 	return NULL;
824 }
825 EXPORT_SYMBOL(pci_find_parent_resource);
826 
827 /**
828  * pci_find_resource - Return matching PCI device resource
829  * @dev: PCI device to query
830  * @res: Resource to look for
831  *
832  * Goes over standard PCI resources (BARs) and checks if the given resource
833  * is partially or fully contained in any of them. In that case the
834  * matching resource is returned, %NULL otherwise.
835  */
pci_find_resource(struct pci_dev * dev,struct resource * res)836 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
837 {
838 	int i;
839 
840 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
841 		struct resource *r = &dev->resource[i];
842 
843 		if (r->start && resource_contains(r, res))
844 			return r;
845 	}
846 
847 	return NULL;
848 }
849 EXPORT_SYMBOL(pci_find_resource);
850 
851 /**
852  * pci_resource_name - Return the name of the PCI resource
853  * @dev: PCI device to query
854  * @i: index of the resource
855  *
856  * Return the standard PCI resource (BAR) name according to their index.
857  */
pci_resource_name(struct pci_dev * dev,unsigned int i)858 const char *pci_resource_name(struct pci_dev *dev, unsigned int i)
859 {
860 	static const char * const bar_name[] = {
861 		"BAR 0",
862 		"BAR 1",
863 		"BAR 2",
864 		"BAR 3",
865 		"BAR 4",
866 		"BAR 5",
867 		"ROM",
868 #ifdef CONFIG_PCI_IOV
869 		"VF BAR 0",
870 		"VF BAR 1",
871 		"VF BAR 2",
872 		"VF BAR 3",
873 		"VF BAR 4",
874 		"VF BAR 5",
875 #endif
876 		"bridge window",	/* "io" included in %pR */
877 		"bridge window",	/* "mem" included in %pR */
878 		"bridge window",	/* "mem pref" included in %pR */
879 	};
880 	static const char * const cardbus_name[] = {
881 		"BAR 1",
882 		"unknown",
883 		"unknown",
884 		"unknown",
885 		"unknown",
886 		"unknown",
887 #ifdef CONFIG_PCI_IOV
888 		"unknown",
889 		"unknown",
890 		"unknown",
891 		"unknown",
892 		"unknown",
893 		"unknown",
894 #endif
895 		"CardBus bridge window 0",	/* I/O */
896 		"CardBus bridge window 1",	/* I/O */
897 		"CardBus bridge window 0",	/* mem */
898 		"CardBus bridge window 1",	/* mem */
899 	};
900 
901 	if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS &&
902 	    i < ARRAY_SIZE(cardbus_name))
903 		return cardbus_name[i];
904 
905 	if (i < ARRAY_SIZE(bar_name))
906 		return bar_name[i];
907 
908 	return "unknown";
909 }
910 
911 /**
912  * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
913  * @dev: the PCI device to operate on
914  * @pos: config space offset of status word
915  * @mask: mask of bit(s) to care about in status word
916  *
917  * Return 1 when mask bit(s) in status word clear, 0 otherwise.
918  */
pci_wait_for_pending(struct pci_dev * dev,int pos,u16 mask)919 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
920 {
921 	int i;
922 
923 	/* Wait for Transaction Pending bit clean */
924 	for (i = 0; i < 4; i++) {
925 		u16 status;
926 		if (i)
927 			msleep((1 << (i - 1)) * 100);
928 
929 		pci_read_config_word(dev, pos, &status);
930 		if (!(status & mask))
931 			return 1;
932 	}
933 
934 	return 0;
935 }
936 
937 static int pci_acs_enable;
938 
939 /**
940  * pci_request_acs - ask for ACS to be enabled if supported
941  */
pci_request_acs(void)942 void pci_request_acs(void)
943 {
944 	pci_acs_enable = 1;
945 }
946 
947 static const char *disable_acs_redir_param;
948 static const char *config_acs_param;
949 
950 struct pci_acs {
951 	u16 cap;
952 	u16 ctrl;
953 	u16 fw_ctrl;
954 };
955 
__pci_config_acs(struct pci_dev * dev,struct pci_acs * caps,const char * p,u16 mask,u16 flags)956 static void __pci_config_acs(struct pci_dev *dev, struct pci_acs *caps,
957 			     const char *p, u16 mask, u16 flags)
958 {
959 	char *delimit;
960 	int ret = 0;
961 
962 	if (!p)
963 		return;
964 
965 	while (*p) {
966 		if (!mask) {
967 			/* Check for ACS flags */
968 			delimit = strstr(p, "@");
969 			if (delimit) {
970 				int end;
971 				u32 shift = 0;
972 
973 				end = delimit - p - 1;
974 
975 				while (end > -1) {
976 					if (*(p + end) == '0') {
977 						mask |= 1 << shift;
978 						shift++;
979 						end--;
980 					} else if (*(p + end) == '1') {
981 						mask |= 1 << shift;
982 						flags |= 1 << shift;
983 						shift++;
984 						end--;
985 					} else if ((*(p + end) == 'x') || (*(p + end) == 'X')) {
986 						shift++;
987 						end--;
988 					} else {
989 						pci_err(dev, "Invalid ACS flags... Ignoring\n");
990 						return;
991 					}
992 				}
993 				p = delimit + 1;
994 			} else {
995 				pci_err(dev, "ACS Flags missing\n");
996 				return;
997 			}
998 		}
999 
1000 		if (mask & ~(PCI_ACS_SV | PCI_ACS_TB | PCI_ACS_RR | PCI_ACS_CR |
1001 			    PCI_ACS_UF | PCI_ACS_EC | PCI_ACS_DT)) {
1002 			pci_err(dev, "Invalid ACS flags specified\n");
1003 			return;
1004 		}
1005 
1006 		ret = pci_dev_str_match(dev, p, &p);
1007 		if (ret < 0) {
1008 			pr_info_once("PCI: Can't parse ACS command line parameter\n");
1009 			break;
1010 		} else if (ret == 1) {
1011 			/* Found a match */
1012 			break;
1013 		}
1014 
1015 		if (*p != ';' && *p != ',') {
1016 			/* End of param or invalid format */
1017 			break;
1018 		}
1019 		p++;
1020 	}
1021 
1022 	if (ret != 1)
1023 		return;
1024 
1025 	if (!pci_dev_specific_disable_acs_redir(dev))
1026 		return;
1027 
1028 	pci_dbg(dev, "ACS mask  = %#06x\n", mask);
1029 	pci_dbg(dev, "ACS flags = %#06x\n", flags);
1030 
1031 	/* If mask is 0 then we copy the bit from the firmware setting. */
1032 	caps->ctrl = (caps->ctrl & ~mask) | (caps->fw_ctrl & mask);
1033 	caps->ctrl |= flags;
1034 
1035 	pci_info(dev, "Configured ACS to %#06x\n", caps->ctrl);
1036 }
1037 
1038 /**
1039  * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities
1040  * @dev: the PCI device
1041  * @caps: default ACS controls
1042  */
pci_std_enable_acs(struct pci_dev * dev,struct pci_acs * caps)1043 static void pci_std_enable_acs(struct pci_dev *dev, struct pci_acs *caps)
1044 {
1045 	/* Source Validation */
1046 	caps->ctrl |= (caps->cap & PCI_ACS_SV);
1047 
1048 	/* P2P Request Redirect */
1049 	caps->ctrl |= (caps->cap & PCI_ACS_RR);
1050 
1051 	/* P2P Completion Redirect */
1052 	caps->ctrl |= (caps->cap & PCI_ACS_CR);
1053 
1054 	/* Upstream Forwarding */
1055 	caps->ctrl |= (caps->cap & PCI_ACS_UF);
1056 
1057 	/* Enable Translation Blocking for external devices and noats */
1058 	if (pci_ats_disabled() || dev->external_facing || dev->untrusted)
1059 		caps->ctrl |= (caps->cap & PCI_ACS_TB);
1060 }
1061 
1062 /**
1063  * pci_enable_acs - enable ACS if hardware support it
1064  * @dev: the PCI device
1065  */
pci_enable_acs(struct pci_dev * dev)1066 static void pci_enable_acs(struct pci_dev *dev)
1067 {
1068 	struct pci_acs caps;
1069 	bool enable_acs = false;
1070 	int pos;
1071 
1072 	/* If an iommu is present we start with kernel default caps */
1073 	if (pci_acs_enable) {
1074 		if (pci_dev_specific_enable_acs(dev))
1075 			enable_acs = true;
1076 	}
1077 
1078 	pos = dev->acs_cap;
1079 	if (!pos)
1080 		return;
1081 
1082 	pci_read_config_word(dev, pos + PCI_ACS_CAP, &caps.cap);
1083 	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &caps.ctrl);
1084 	caps.fw_ctrl = caps.ctrl;
1085 
1086 	if (enable_acs)
1087 		pci_std_enable_acs(dev, &caps);
1088 
1089 	/*
1090 	 * Always apply caps from the command line, even if there is no iommu.
1091 	 * Trust that the admin has a reason to change the ACS settings.
1092 	 */
1093 	__pci_config_acs(dev, &caps, disable_acs_redir_param,
1094 			 PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC,
1095 			 ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC));
1096 	__pci_config_acs(dev, &caps, config_acs_param, 0, 0);
1097 
1098 	pci_write_config_word(dev, pos + PCI_ACS_CTRL, caps.ctrl);
1099 }
1100 
1101 /**
1102  * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
1103  * @dev: PCI device to have its BARs restored
1104  *
1105  * Restore the BAR values for a given device, so as to make it
1106  * accessible by its driver.
1107  */
pci_restore_bars(struct pci_dev * dev)1108 static void pci_restore_bars(struct pci_dev *dev)
1109 {
1110 	int i;
1111 
1112 	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
1113 		pci_update_resource(dev, i);
1114 }
1115 
platform_pci_power_manageable(struct pci_dev * dev)1116 static inline bool platform_pci_power_manageable(struct pci_dev *dev)
1117 {
1118 	if (pci_use_mid_pm())
1119 		return true;
1120 
1121 	return acpi_pci_power_manageable(dev);
1122 }
1123 
platform_pci_set_power_state(struct pci_dev * dev,pci_power_t t)1124 static inline int platform_pci_set_power_state(struct pci_dev *dev,
1125 					       pci_power_t t)
1126 {
1127 	if (pci_use_mid_pm())
1128 		return mid_pci_set_power_state(dev, t);
1129 
1130 	return acpi_pci_set_power_state(dev, t);
1131 }
1132 
platform_pci_get_power_state(struct pci_dev * dev)1133 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
1134 {
1135 	if (pci_use_mid_pm())
1136 		return mid_pci_get_power_state(dev);
1137 
1138 	return acpi_pci_get_power_state(dev);
1139 }
1140 
platform_pci_refresh_power_state(struct pci_dev * dev)1141 static inline void platform_pci_refresh_power_state(struct pci_dev *dev)
1142 {
1143 	if (!pci_use_mid_pm())
1144 		acpi_pci_refresh_power_state(dev);
1145 }
1146 
platform_pci_choose_state(struct pci_dev * dev)1147 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
1148 {
1149 	if (pci_use_mid_pm())
1150 		return PCI_POWER_ERROR;
1151 
1152 	return acpi_pci_choose_state(dev);
1153 }
1154 
platform_pci_set_wakeup(struct pci_dev * dev,bool enable)1155 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
1156 {
1157 	if (pci_use_mid_pm())
1158 		return PCI_POWER_ERROR;
1159 
1160 	return acpi_pci_wakeup(dev, enable);
1161 }
1162 
platform_pci_need_resume(struct pci_dev * dev)1163 static inline bool platform_pci_need_resume(struct pci_dev *dev)
1164 {
1165 	if (pci_use_mid_pm())
1166 		return false;
1167 
1168 	return acpi_pci_need_resume(dev);
1169 }
1170 
platform_pci_bridge_d3(struct pci_dev * dev)1171 static inline bool platform_pci_bridge_d3(struct pci_dev *dev)
1172 {
1173 	if (pci_use_mid_pm())
1174 		return false;
1175 
1176 	return acpi_pci_bridge_d3(dev);
1177 }
1178 
1179 /**
1180  * pci_update_current_state - Read power state of given device and cache it
1181  * @dev: PCI device to handle.
1182  * @state: State to cache in case the device doesn't have the PM capability
1183  *
1184  * The power state is read from the PMCSR register, which however is
1185  * inaccessible in D3cold.  The platform firmware is therefore queried first
1186  * to detect accessibility of the register.  In case the platform firmware
1187  * reports an incorrect state or the device isn't power manageable by the
1188  * platform at all, we try to detect D3cold by testing accessibility of the
1189  * vendor ID in config space.
1190  */
pci_update_current_state(struct pci_dev * dev,pci_power_t state)1191 void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
1192 {
1193 	if (platform_pci_get_power_state(dev) == PCI_D3cold) {
1194 		dev->current_state = PCI_D3cold;
1195 	} else if (dev->pm_cap) {
1196 		u16 pmcsr;
1197 
1198 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1199 		if (PCI_POSSIBLE_ERROR(pmcsr)) {
1200 			dev->current_state = PCI_D3cold;
1201 			return;
1202 		}
1203 		dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1204 	} else {
1205 		dev->current_state = state;
1206 	}
1207 }
1208 
1209 /**
1210  * pci_refresh_power_state - Refresh the given device's power state data
1211  * @dev: Target PCI device.
1212  *
1213  * Ask the platform to refresh the devices power state information and invoke
1214  * pci_update_current_state() to update its current PCI power state.
1215  */
pci_refresh_power_state(struct pci_dev * dev)1216 void pci_refresh_power_state(struct pci_dev *dev)
1217 {
1218 	platform_pci_refresh_power_state(dev);
1219 	pci_update_current_state(dev, dev->current_state);
1220 }
1221 
1222 /**
1223  * pci_platform_power_transition - Use platform to change device power state
1224  * @dev: PCI device to handle.
1225  * @state: State to put the device into.
1226  */
pci_platform_power_transition(struct pci_dev * dev,pci_power_t state)1227 int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
1228 {
1229 	int error;
1230 
1231 	error = platform_pci_set_power_state(dev, state);
1232 	if (!error)
1233 		pci_update_current_state(dev, state);
1234 	else if (!dev->pm_cap) /* Fall back to PCI_D0 */
1235 		dev->current_state = PCI_D0;
1236 
1237 	return error;
1238 }
1239 EXPORT_SYMBOL_GPL(pci_platform_power_transition);
1240 
pci_resume_one(struct pci_dev * pci_dev,void * ign)1241 static int pci_resume_one(struct pci_dev *pci_dev, void *ign)
1242 {
1243 	pm_request_resume(&pci_dev->dev);
1244 	return 0;
1245 }
1246 
1247 /**
1248  * pci_resume_bus - Walk given bus and runtime resume devices on it
1249  * @bus: Top bus of the subtree to walk.
1250  */
pci_resume_bus(struct pci_bus * bus)1251 void pci_resume_bus(struct pci_bus *bus)
1252 {
1253 	if (bus)
1254 		pci_walk_bus(bus, pci_resume_one, NULL);
1255 }
1256 
pci_dev_wait(struct pci_dev * dev,char * reset_type,int timeout)1257 static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
1258 {
1259 	int delay = 1;
1260 	bool retrain = false;
1261 	struct pci_dev *root, *bridge;
1262 
1263 	root = pcie_find_root_port(dev);
1264 
1265 	if (pci_is_pcie(dev)) {
1266 		bridge = pci_upstream_bridge(dev);
1267 		if (bridge)
1268 			retrain = true;
1269 	}
1270 
1271 	/*
1272 	 * The caller has already waited long enough after a reset that the
1273 	 * device should respond to config requests, but it may respond
1274 	 * with Request Retry Status (RRS) if it needs more time to
1275 	 * initialize.
1276 	 *
1277 	 * If the device is below a Root Port with Configuration RRS
1278 	 * Software Visibility enabled, reading the Vendor ID returns a
1279 	 * special data value if the device responded with RRS.  Read the
1280 	 * Vendor ID until we get non-RRS status.
1281 	 *
1282 	 * If there's no Root Port or Configuration RRS Software Visibility
1283 	 * is not enabled, the device may still respond with RRS, but
1284 	 * hardware may retry the config request.  If no retries receive
1285 	 * Successful Completion, hardware generally synthesizes ~0
1286 	 * (PCI_ERROR_RESPONSE) data to complete the read.  Reading Vendor
1287 	 * ID for VFs and non-existent devices also returns ~0, so read the
1288 	 * Command register until it returns something other than ~0.
1289 	 */
1290 	for (;;) {
1291 		u32 id;
1292 
1293 		if (pci_dev_is_disconnected(dev)) {
1294 			pci_dbg(dev, "disconnected; not waiting\n");
1295 			return -ENOTTY;
1296 		}
1297 
1298 		if (root && root->config_rrs_sv) {
1299 			pci_read_config_dword(dev, PCI_VENDOR_ID, &id);
1300 			if (!pci_bus_rrs_vendor_id(id))
1301 				break;
1302 		} else {
1303 			pci_read_config_dword(dev, PCI_COMMAND, &id);
1304 			if (!PCI_POSSIBLE_ERROR(id))
1305 				break;
1306 		}
1307 
1308 		if (delay > timeout) {
1309 			pci_warn(dev, "not ready %dms after %s; giving up\n",
1310 				 delay - 1, reset_type);
1311 			return -ENOTTY;
1312 		}
1313 
1314 		if (delay > PCI_RESET_WAIT) {
1315 			if (retrain) {
1316 				retrain = false;
1317 				if (pcie_failed_link_retrain(bridge) == 0) {
1318 					delay = 1;
1319 					continue;
1320 				}
1321 			}
1322 			pci_info(dev, "not ready %dms after %s; waiting\n",
1323 				 delay - 1, reset_type);
1324 		}
1325 
1326 		msleep(delay);
1327 		delay *= 2;
1328 	}
1329 
1330 	if (delay > PCI_RESET_WAIT)
1331 		pci_info(dev, "ready %dms after %s\n", delay - 1,
1332 			 reset_type);
1333 	else
1334 		pci_dbg(dev, "ready %dms after %s\n", delay - 1,
1335 			reset_type);
1336 
1337 	return 0;
1338 }
1339 
1340 /**
1341  * pci_power_up - Put the given device into D0
1342  * @dev: PCI device to power up
1343  *
1344  * On success, return 0 or 1, depending on whether or not it is necessary to
1345  * restore the device's BARs subsequently (1 is returned in that case).
1346  *
1347  * On failure, return a negative error code.  Always return failure if @dev
1348  * lacks a Power Management Capability, even if the platform was able to
1349  * put the device in D0 via non-PCI means.
1350  */
pci_power_up(struct pci_dev * dev)1351 int pci_power_up(struct pci_dev *dev)
1352 {
1353 	bool need_restore;
1354 	pci_power_t state;
1355 	u16 pmcsr;
1356 
1357 	platform_pci_set_power_state(dev, PCI_D0);
1358 
1359 	if (!dev->pm_cap) {
1360 		state = platform_pci_get_power_state(dev);
1361 		if (state == PCI_UNKNOWN)
1362 			dev->current_state = PCI_D0;
1363 		else
1364 			dev->current_state = state;
1365 
1366 		return -EIO;
1367 	}
1368 
1369 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1370 	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1371 		pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n",
1372 			pci_power_name(dev->current_state));
1373 		dev->current_state = PCI_D3cold;
1374 		return -EIO;
1375 	}
1376 
1377 	state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1378 
1379 	need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) &&
1380 			!(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
1381 
1382 	if (state == PCI_D0)
1383 		goto end;
1384 
1385 	/*
1386 	 * Force the entire word to 0. This doesn't affect PME_Status, disables
1387 	 * PME_En, and sets PowerState to 0.
1388 	 */
1389 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0);
1390 
1391 	/* Mandatory transition delays; see PCI PM 1.2. */
1392 	if (state == PCI_D3hot)
1393 		pci_dev_d3_sleep(dev);
1394 	else if (state == PCI_D2)
1395 		udelay(PCI_PM_D2_DELAY);
1396 
1397 end:
1398 	dev->current_state = PCI_D0;
1399 	if (need_restore)
1400 		return 1;
1401 
1402 	return 0;
1403 }
1404 
1405 /**
1406  * pci_set_full_power_state - Put a PCI device into D0 and update its state
1407  * @dev: PCI device to power up
1408  * @locked: whether pci_bus_sem is held
1409  *
1410  * Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register
1411  * to confirm the state change, restore its BARs if they might be lost and
1412  * reconfigure ASPM in accordance with the new power state.
1413  *
1414  * If pci_restore_state() is going to be called right after a power state change
1415  * to D0, it is more efficient to use pci_power_up() directly instead of this
1416  * function.
1417  */
pci_set_full_power_state(struct pci_dev * dev,bool locked)1418 static int pci_set_full_power_state(struct pci_dev *dev, bool locked)
1419 {
1420 	u16 pmcsr;
1421 	int ret;
1422 
1423 	ret = pci_power_up(dev);
1424 	if (ret < 0) {
1425 		if (dev->current_state == PCI_D0)
1426 			return 0;
1427 
1428 		return ret;
1429 	}
1430 
1431 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1432 	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1433 	if (dev->current_state != PCI_D0) {
1434 		pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n",
1435 				     pci_power_name(dev->current_state));
1436 	} else if (ret > 0) {
1437 		/*
1438 		 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
1439 		 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
1440 		 * from D3hot to D0 _may_ perform an internal reset, thereby
1441 		 * going to "D0 Uninitialized" rather than "D0 Initialized".
1442 		 * For example, at least some versions of the 3c905B and the
1443 		 * 3c556B exhibit this behaviour.
1444 		 *
1445 		 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
1446 		 * devices in a D3hot state at boot.  Consequently, we need to
1447 		 * restore at least the BARs so that the device will be
1448 		 * accessible to its driver.
1449 		 */
1450 		pci_restore_bars(dev);
1451 	}
1452 
1453 	if (dev->bus->self)
1454 		pcie_aspm_pm_state_change(dev->bus->self, locked);
1455 
1456 	return 0;
1457 }
1458 
1459 /**
1460  * __pci_dev_set_current_state - Set current state of a PCI device
1461  * @dev: Device to handle
1462  * @data: pointer to state to be set
1463  */
__pci_dev_set_current_state(struct pci_dev * dev,void * data)1464 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
1465 {
1466 	pci_power_t state = *(pci_power_t *)data;
1467 
1468 	dev->current_state = state;
1469 	return 0;
1470 }
1471 
1472 /**
1473  * pci_bus_set_current_state - Walk given bus and set current state of devices
1474  * @bus: Top bus of the subtree to walk.
1475  * @state: state to be set
1476  */
pci_bus_set_current_state(struct pci_bus * bus,pci_power_t state)1477 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
1478 {
1479 	if (bus)
1480 		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
1481 }
1482 
__pci_bus_set_current_state(struct pci_bus * bus,pci_power_t state,bool locked)1483 static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state, bool locked)
1484 {
1485 	if (!bus)
1486 		return;
1487 
1488 	if (locked)
1489 		pci_walk_bus_locked(bus, __pci_dev_set_current_state, &state);
1490 	else
1491 		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
1492 }
1493 
1494 /**
1495  * pci_set_low_power_state - Put a PCI device into a low-power state.
1496  * @dev: PCI device to handle.
1497  * @state: PCI power state (D1, D2, D3hot) to put the device into.
1498  * @locked: whether pci_bus_sem is held
1499  *
1500  * Use the device's PCI_PM_CTRL register to put it into a low-power state.
1501  *
1502  * RETURN VALUE:
1503  * -EINVAL if the requested state is invalid.
1504  * -EIO if device does not support PCI PM or its PM capabilities register has a
1505  * wrong version, or device doesn't support the requested state.
1506  * 0 if device already is in the requested state.
1507  * 0 if device's power state has been successfully changed.
1508  */
pci_set_low_power_state(struct pci_dev * dev,pci_power_t state,bool locked)1509 static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1510 {
1511 	u16 pmcsr;
1512 
1513 	if (!dev->pm_cap)
1514 		return -EIO;
1515 
1516 	/*
1517 	 * Validate transition: We can enter D0 from any state, but if
1518 	 * we're already in a low-power state, we can only go deeper.  E.g.,
1519 	 * we can go from D1 to D3, but we can't go directly from D3 to D1;
1520 	 * we'd have to go from D3 to D0, then to D1.
1521 	 */
1522 	if (dev->current_state <= PCI_D3cold && dev->current_state > state) {
1523 		pci_dbg(dev, "Invalid power transition (from %s to %s)\n",
1524 			pci_power_name(dev->current_state),
1525 			pci_power_name(state));
1526 		return -EINVAL;
1527 	}
1528 
1529 	/* Check if this device supports the desired state */
1530 	if ((state == PCI_D1 && !dev->d1_support)
1531 	   || (state == PCI_D2 && !dev->d2_support))
1532 		return -EIO;
1533 
1534 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1535 	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1536 		pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n",
1537 			pci_power_name(dev->current_state),
1538 			pci_power_name(state));
1539 		dev->current_state = PCI_D3cold;
1540 		return -EIO;
1541 	}
1542 
1543 	pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1544 	pmcsr |= state;
1545 
1546 	/* Enter specified state */
1547 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
1548 
1549 	/* Mandatory power management transition delays; see PCI PM 1.2. */
1550 	if (state == PCI_D3hot)
1551 		pci_dev_d3_sleep(dev);
1552 	else if (state == PCI_D2)
1553 		udelay(PCI_PM_D2_DELAY);
1554 
1555 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1556 	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1557 	if (dev->current_state != state)
1558 		pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n",
1559 				     pci_power_name(dev->current_state),
1560 				     pci_power_name(state));
1561 
1562 	if (dev->bus->self)
1563 		pcie_aspm_pm_state_change(dev->bus->self, locked);
1564 
1565 	return 0;
1566 }
1567 
__pci_set_power_state(struct pci_dev * dev,pci_power_t state,bool locked)1568 static int __pci_set_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1569 {
1570 	int error;
1571 
1572 	/* Bound the state we're entering */
1573 	if (state > PCI_D3cold)
1574 		state = PCI_D3cold;
1575 	else if (state < PCI_D0)
1576 		state = PCI_D0;
1577 	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
1578 
1579 		/*
1580 		 * If the device or the parent bridge do not support PCI
1581 		 * PM, ignore the request if we're doing anything other
1582 		 * than putting it into D0 (which would only happen on
1583 		 * boot).
1584 		 */
1585 		return 0;
1586 
1587 	/* Check if we're already there */
1588 	if (dev->current_state == state)
1589 		return 0;
1590 
1591 	if (state == PCI_D0)
1592 		return pci_set_full_power_state(dev, locked);
1593 
1594 	/*
1595 	 * This device is quirked not to be put into D3, so don't put it in
1596 	 * D3
1597 	 */
1598 	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
1599 		return 0;
1600 
1601 	if (state == PCI_D3cold) {
1602 		/*
1603 		 * To put the device in D3cold, put it into D3hot in the native
1604 		 * way, then put it into D3cold using platform ops.
1605 		 */
1606 		error = pci_set_low_power_state(dev, PCI_D3hot, locked);
1607 
1608 		if (pci_platform_power_transition(dev, PCI_D3cold))
1609 			return error;
1610 
1611 		/* Powering off a bridge may power off the whole hierarchy */
1612 		if (dev->current_state == PCI_D3cold)
1613 			__pci_bus_set_current_state(dev->subordinate, PCI_D3cold, locked);
1614 	} else {
1615 		error = pci_set_low_power_state(dev, state, locked);
1616 
1617 		if (pci_platform_power_transition(dev, state))
1618 			return error;
1619 	}
1620 
1621 	return 0;
1622 }
1623 
1624 /**
1625  * pci_set_power_state - Set the power state of a PCI device
1626  * @dev: PCI device to handle.
1627  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
1628  *
1629  * Transition a device to a new power state, using the platform firmware and/or
1630  * the device's PCI PM registers.
1631  *
1632  * RETURN VALUE:
1633  * -EINVAL if the requested state is invalid.
1634  * -EIO if device does not support PCI PM or its PM capabilities register has a
1635  * wrong version, or device doesn't support the requested state.
1636  * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
1637  * 0 if device already is in the requested state.
1638  * 0 if the transition is to D3 but D3 is not supported.
1639  * 0 if device's power state has been successfully changed.
1640  */
pci_set_power_state(struct pci_dev * dev,pci_power_t state)1641 int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
1642 {
1643 	return __pci_set_power_state(dev, state, false);
1644 }
1645 EXPORT_SYMBOL(pci_set_power_state);
1646 
pci_set_power_state_locked(struct pci_dev * dev,pci_power_t state)1647 int pci_set_power_state_locked(struct pci_dev *dev, pci_power_t state)
1648 {
1649 	lockdep_assert_held(&pci_bus_sem);
1650 
1651 	return __pci_set_power_state(dev, state, true);
1652 }
1653 EXPORT_SYMBOL(pci_set_power_state_locked);
1654 
1655 #define PCI_EXP_SAVE_REGS	7
1656 
_pci_find_saved_cap(struct pci_dev * pci_dev,u16 cap,bool extended)1657 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
1658 						       u16 cap, bool extended)
1659 {
1660 	struct pci_cap_saved_state *tmp;
1661 
1662 	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
1663 		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
1664 			return tmp;
1665 	}
1666 	return NULL;
1667 }
1668 
pci_find_saved_cap(struct pci_dev * dev,char cap)1669 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
1670 {
1671 	return _pci_find_saved_cap(dev, cap, false);
1672 }
1673 
pci_find_saved_ext_cap(struct pci_dev * dev,u16 cap)1674 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
1675 {
1676 	return _pci_find_saved_cap(dev, cap, true);
1677 }
1678 
pci_save_pcie_state(struct pci_dev * dev)1679 static int pci_save_pcie_state(struct pci_dev *dev)
1680 {
1681 	int i = 0;
1682 	struct pci_cap_saved_state *save_state;
1683 	u16 *cap;
1684 
1685 	if (!pci_is_pcie(dev))
1686 		return 0;
1687 
1688 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1689 	if (!save_state) {
1690 		pci_err(dev, "buffer not found in %s\n", __func__);
1691 		return -ENOMEM;
1692 	}
1693 
1694 	cap = (u16 *)&save_state->cap.data[0];
1695 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
1696 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
1697 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
1698 	pcie_capability_read_word(dev, PCI_EXP_RTCTL,  &cap[i++]);
1699 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
1700 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
1701 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
1702 
1703 	pci_save_aspm_l1ss_state(dev);
1704 	pci_save_ltr_state(dev);
1705 
1706 	return 0;
1707 }
1708 
pci_restore_pcie_state(struct pci_dev * dev)1709 static void pci_restore_pcie_state(struct pci_dev *dev)
1710 {
1711 	int i = 0;
1712 	struct pci_cap_saved_state *save_state;
1713 	u16 *cap;
1714 
1715 	/*
1716 	 * Restore max latencies (in the LTR capability) before enabling
1717 	 * LTR itself in PCI_EXP_DEVCTL2.
1718 	 */
1719 	pci_restore_ltr_state(dev);
1720 	pci_restore_aspm_l1ss_state(dev);
1721 
1722 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1723 	if (!save_state)
1724 		return;
1725 
1726 	/*
1727 	 * Downstream ports reset the LTR enable bit when link goes down.
1728 	 * Check and re-configure the bit here before restoring device.
1729 	 * PCIe r5.0, sec 7.5.3.16.
1730 	 */
1731 	pci_bridge_reconfigure_ltr(dev);
1732 
1733 	cap = (u16 *)&save_state->cap.data[0];
1734 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
1735 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
1736 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
1737 	pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
1738 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
1739 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
1740 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
1741 }
1742 
pci_save_pcix_state(struct pci_dev * dev)1743 static int pci_save_pcix_state(struct pci_dev *dev)
1744 {
1745 	int pos;
1746 	struct pci_cap_saved_state *save_state;
1747 
1748 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1749 	if (!pos)
1750 		return 0;
1751 
1752 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1753 	if (!save_state) {
1754 		pci_err(dev, "buffer not found in %s\n", __func__);
1755 		return -ENOMEM;
1756 	}
1757 
1758 	pci_read_config_word(dev, pos + PCI_X_CMD,
1759 			     (u16 *)save_state->cap.data);
1760 
1761 	return 0;
1762 }
1763 
pci_restore_pcix_state(struct pci_dev * dev)1764 static void pci_restore_pcix_state(struct pci_dev *dev)
1765 {
1766 	int i = 0, pos;
1767 	struct pci_cap_saved_state *save_state;
1768 	u16 *cap;
1769 
1770 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1771 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1772 	if (!save_state || !pos)
1773 		return;
1774 	cap = (u16 *)&save_state->cap.data[0];
1775 
1776 	pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
1777 }
1778 
1779 /**
1780  * pci_save_state - save the PCI configuration space of a device before
1781  *		    suspending
1782  * @dev: PCI device that we're dealing with
1783  */
pci_save_state(struct pci_dev * dev)1784 int pci_save_state(struct pci_dev *dev)
1785 {
1786 	int i;
1787 	/* XXX: 100% dword access ok here? */
1788 	for (i = 0; i < 16; i++) {
1789 		pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
1790 		pci_dbg(dev, "save config %#04x: %#010x\n",
1791 			i * 4, dev->saved_config_space[i]);
1792 	}
1793 	dev->state_saved = true;
1794 
1795 	i = pci_save_pcie_state(dev);
1796 	if (i != 0)
1797 		return i;
1798 
1799 	i = pci_save_pcix_state(dev);
1800 	if (i != 0)
1801 		return i;
1802 
1803 	pci_save_dpc_state(dev);
1804 	pci_save_aer_state(dev);
1805 	pci_save_ptm_state(dev);
1806 	pci_save_tph_state(dev);
1807 	return pci_save_vc_state(dev);
1808 }
1809 EXPORT_SYMBOL(pci_save_state);
1810 
pci_restore_config_dword(struct pci_dev * pdev,int offset,u32 saved_val,int retry,bool force)1811 static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1812 				     u32 saved_val, int retry, bool force)
1813 {
1814 	u32 val;
1815 
1816 	pci_read_config_dword(pdev, offset, &val);
1817 	if (!force && val == saved_val)
1818 		return;
1819 
1820 	for (;;) {
1821 		pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n",
1822 			offset, val, saved_val);
1823 		pci_write_config_dword(pdev, offset, saved_val);
1824 		if (retry-- <= 0)
1825 			return;
1826 
1827 		pci_read_config_dword(pdev, offset, &val);
1828 		if (val == saved_val)
1829 			return;
1830 
1831 		mdelay(1);
1832 	}
1833 }
1834 
pci_restore_config_space_range(struct pci_dev * pdev,int start,int end,int retry,bool force)1835 static void pci_restore_config_space_range(struct pci_dev *pdev,
1836 					   int start, int end, int retry,
1837 					   bool force)
1838 {
1839 	int index;
1840 
1841 	for (index = end; index >= start; index--)
1842 		pci_restore_config_dword(pdev, 4 * index,
1843 					 pdev->saved_config_space[index],
1844 					 retry, force);
1845 }
1846 
pci_restore_config_space(struct pci_dev * pdev)1847 static void pci_restore_config_space(struct pci_dev *pdev)
1848 {
1849 	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1850 		pci_restore_config_space_range(pdev, 10, 15, 0, false);
1851 		/* Restore BARs before the command register. */
1852 		pci_restore_config_space_range(pdev, 4, 9, 10, false);
1853 		pci_restore_config_space_range(pdev, 0, 3, 0, false);
1854 	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1855 		pci_restore_config_space_range(pdev, 12, 15, 0, false);
1856 
1857 		/*
1858 		 * Force rewriting of prefetch registers to avoid S3 resume
1859 		 * issues on Intel PCI bridges that occur when these
1860 		 * registers are not explicitly written.
1861 		 */
1862 		pci_restore_config_space_range(pdev, 9, 11, 0, true);
1863 		pci_restore_config_space_range(pdev, 0, 8, 0, false);
1864 	} else {
1865 		pci_restore_config_space_range(pdev, 0, 15, 0, false);
1866 	}
1867 }
1868 
pci_restore_rebar_state(struct pci_dev * pdev)1869 static void pci_restore_rebar_state(struct pci_dev *pdev)
1870 {
1871 	unsigned int pos, nbars, i;
1872 	u32 ctrl;
1873 
1874 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
1875 	if (!pos)
1876 		return;
1877 
1878 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1879 	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
1880 
1881 	for (i = 0; i < nbars; i++, pos += 8) {
1882 		struct resource *res;
1883 		int bar_idx, size;
1884 
1885 		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1886 		bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
1887 		res = pdev->resource + bar_idx;
1888 		size = pci_rebar_bytes_to_size(resource_size(res));
1889 		ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
1890 		ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
1891 		pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
1892 	}
1893 }
1894 
1895 /**
1896  * pci_restore_state - Restore the saved state of a PCI device
1897  * @dev: PCI device that we're dealing with
1898  */
pci_restore_state(struct pci_dev * dev)1899 void pci_restore_state(struct pci_dev *dev)
1900 {
1901 	if (!dev->state_saved)
1902 		return;
1903 
1904 	pci_restore_pcie_state(dev);
1905 	pci_restore_pasid_state(dev);
1906 	pci_restore_pri_state(dev);
1907 	pci_restore_ats_state(dev);
1908 	pci_restore_vc_state(dev);
1909 	pci_restore_rebar_state(dev);
1910 	pci_restore_dpc_state(dev);
1911 	pci_restore_ptm_state(dev);
1912 	pci_restore_tph_state(dev);
1913 
1914 	pci_aer_clear_status(dev);
1915 	pci_restore_aer_state(dev);
1916 
1917 	pci_restore_config_space(dev);
1918 
1919 	pci_restore_pcix_state(dev);
1920 	pci_restore_msi_state(dev);
1921 
1922 	/* Restore ACS and IOV configuration state */
1923 	pci_enable_acs(dev);
1924 	pci_restore_iov_state(dev);
1925 
1926 	dev->state_saved = false;
1927 }
1928 EXPORT_SYMBOL(pci_restore_state);
1929 
1930 struct pci_saved_state {
1931 	u32 config_space[16];
1932 	struct pci_cap_saved_data cap[];
1933 };
1934 
1935 /**
1936  * pci_store_saved_state - Allocate and return an opaque struct containing
1937  *			   the device saved state.
1938  * @dev: PCI device that we're dealing with
1939  *
1940  * Return NULL if no state or error.
1941  */
pci_store_saved_state(struct pci_dev * dev)1942 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1943 {
1944 	struct pci_saved_state *state;
1945 	struct pci_cap_saved_state *tmp;
1946 	struct pci_cap_saved_data *cap;
1947 	size_t size;
1948 
1949 	if (!dev->state_saved)
1950 		return NULL;
1951 
1952 	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1953 
1954 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1955 		size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1956 
1957 	state = kzalloc(size, GFP_KERNEL);
1958 	if (!state)
1959 		return NULL;
1960 
1961 	memcpy(state->config_space, dev->saved_config_space,
1962 	       sizeof(state->config_space));
1963 
1964 	cap = state->cap;
1965 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1966 		size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1967 		memcpy(cap, &tmp->cap, len);
1968 		cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1969 	}
1970 	/* Empty cap_save terminates list */
1971 
1972 	return state;
1973 }
1974 EXPORT_SYMBOL_GPL(pci_store_saved_state);
1975 
1976 /**
1977  * pci_load_saved_state - Reload the provided save state into struct pci_dev.
1978  * @dev: PCI device that we're dealing with
1979  * @state: Saved state returned from pci_store_saved_state()
1980  */
pci_load_saved_state(struct pci_dev * dev,struct pci_saved_state * state)1981 int pci_load_saved_state(struct pci_dev *dev,
1982 			 struct pci_saved_state *state)
1983 {
1984 	struct pci_cap_saved_data *cap;
1985 
1986 	dev->state_saved = false;
1987 
1988 	if (!state)
1989 		return 0;
1990 
1991 	memcpy(dev->saved_config_space, state->config_space,
1992 	       sizeof(state->config_space));
1993 
1994 	cap = state->cap;
1995 	while (cap->size) {
1996 		struct pci_cap_saved_state *tmp;
1997 
1998 		tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
1999 		if (!tmp || tmp->cap.size != cap->size)
2000 			return -EINVAL;
2001 
2002 		memcpy(tmp->cap.data, cap->data, tmp->cap.size);
2003 		cap = (struct pci_cap_saved_data *)((u8 *)cap +
2004 		       sizeof(struct pci_cap_saved_data) + cap->size);
2005 	}
2006 
2007 	dev->state_saved = true;
2008 	return 0;
2009 }
2010 EXPORT_SYMBOL_GPL(pci_load_saved_state);
2011 
2012 /**
2013  * pci_load_and_free_saved_state - Reload the save state pointed to by state,
2014  *				   and free the memory allocated for it.
2015  * @dev: PCI device that we're dealing with
2016  * @state: Pointer to saved state returned from pci_store_saved_state()
2017  */
pci_load_and_free_saved_state(struct pci_dev * dev,struct pci_saved_state ** state)2018 int pci_load_and_free_saved_state(struct pci_dev *dev,
2019 				  struct pci_saved_state **state)
2020 {
2021 	int ret = pci_load_saved_state(dev, *state);
2022 	kfree(*state);
2023 	*state = NULL;
2024 	return ret;
2025 }
2026 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
2027 
pcibios_enable_device(struct pci_dev * dev,int bars)2028 int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
2029 {
2030 	return pci_enable_resources(dev, bars);
2031 }
2032 
pci_host_bridge_enable_device(struct pci_dev * dev)2033 static int pci_host_bridge_enable_device(struct pci_dev *dev)
2034 {
2035 	struct pci_host_bridge *host_bridge = pci_find_host_bridge(dev->bus);
2036 	int err;
2037 
2038 	if (host_bridge && host_bridge->enable_device) {
2039 		err = host_bridge->enable_device(host_bridge, dev);
2040 		if (err)
2041 			return err;
2042 	}
2043 
2044 	return 0;
2045 }
2046 
pci_host_bridge_disable_device(struct pci_dev * dev)2047 static void pci_host_bridge_disable_device(struct pci_dev *dev)
2048 {
2049 	struct pci_host_bridge *host_bridge = pci_find_host_bridge(dev->bus);
2050 
2051 	if (host_bridge && host_bridge->disable_device)
2052 		host_bridge->disable_device(host_bridge, dev);
2053 }
2054 
do_pci_enable_device(struct pci_dev * dev,int bars)2055 static int do_pci_enable_device(struct pci_dev *dev, int bars)
2056 {
2057 	int err;
2058 	struct pci_dev *bridge;
2059 	u16 cmd;
2060 	u8 pin;
2061 
2062 	err = pci_set_power_state(dev, PCI_D0);
2063 	if (err < 0 && err != -EIO)
2064 		return err;
2065 
2066 	bridge = pci_upstream_bridge(dev);
2067 	if (bridge)
2068 		pcie_aspm_powersave_config_link(bridge);
2069 
2070 	err = pci_host_bridge_enable_device(dev);
2071 	if (err)
2072 		return err;
2073 
2074 	err = pcibios_enable_device(dev, bars);
2075 	if (err < 0)
2076 		goto err_enable;
2077 	pci_fixup_device(pci_fixup_enable, dev);
2078 
2079 	if (dev->msi_enabled || dev->msix_enabled)
2080 		return 0;
2081 
2082 	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
2083 	if (pin) {
2084 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
2085 		if (cmd & PCI_COMMAND_INTX_DISABLE)
2086 			pci_write_config_word(dev, PCI_COMMAND,
2087 					      cmd & ~PCI_COMMAND_INTX_DISABLE);
2088 	}
2089 
2090 	return 0;
2091 
2092 err_enable:
2093 	pci_host_bridge_disable_device(dev);
2094 
2095 	return err;
2096 
2097 }
2098 
2099 /**
2100  * pci_reenable_device - Resume abandoned device
2101  * @dev: PCI device to be resumed
2102  *
2103  * NOTE: This function is a backend of pci_default_resume() and is not supposed
2104  * to be called by normal code, write proper resume handler and use it instead.
2105  */
pci_reenable_device(struct pci_dev * dev)2106 int pci_reenable_device(struct pci_dev *dev)
2107 {
2108 	if (pci_is_enabled(dev))
2109 		return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
2110 	return 0;
2111 }
2112 EXPORT_SYMBOL(pci_reenable_device);
2113 
pci_enable_bridge(struct pci_dev * dev)2114 static void pci_enable_bridge(struct pci_dev *dev)
2115 {
2116 	struct pci_dev *bridge;
2117 	int retval;
2118 
2119 	bridge = pci_upstream_bridge(dev);
2120 	if (bridge)
2121 		pci_enable_bridge(bridge);
2122 
2123 	if (pci_is_enabled(dev)) {
2124 		if (!dev->is_busmaster)
2125 			pci_set_master(dev);
2126 		return;
2127 	}
2128 
2129 	retval = pci_enable_device(dev);
2130 	if (retval)
2131 		pci_err(dev, "Error enabling bridge (%d), continuing\n",
2132 			retval);
2133 	pci_set_master(dev);
2134 }
2135 
pci_enable_device_flags(struct pci_dev * dev,unsigned long flags)2136 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
2137 {
2138 	struct pci_dev *bridge;
2139 	int err;
2140 	int i, bars = 0;
2141 
2142 	/*
2143 	 * Power state could be unknown at this point, either due to a fresh
2144 	 * boot or a device removal call.  So get the current power state
2145 	 * so that things like MSI message writing will behave as expected
2146 	 * (e.g. if the device really is in D0 at enable time).
2147 	 */
2148 	pci_update_current_state(dev, dev->current_state);
2149 
2150 	if (atomic_inc_return(&dev->enable_cnt) > 1)
2151 		return 0;		/* already enabled */
2152 
2153 	bridge = pci_upstream_bridge(dev);
2154 	if (bridge)
2155 		pci_enable_bridge(bridge);
2156 
2157 	/* only skip sriov related */
2158 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
2159 		if (dev->resource[i].flags & flags)
2160 			bars |= (1 << i);
2161 	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
2162 		if (dev->resource[i].flags & flags)
2163 			bars |= (1 << i);
2164 
2165 	err = do_pci_enable_device(dev, bars);
2166 	if (err < 0)
2167 		atomic_dec(&dev->enable_cnt);
2168 	return err;
2169 }
2170 
2171 /**
2172  * pci_enable_device_mem - Initialize a device for use with Memory space
2173  * @dev: PCI device to be initialized
2174  *
2175  * Initialize device before it's used by a driver. Ask low-level code
2176  * to enable Memory resources. Wake up the device if it was suspended.
2177  * Beware, this function can fail.
2178  */
pci_enable_device_mem(struct pci_dev * dev)2179 int pci_enable_device_mem(struct pci_dev *dev)
2180 {
2181 	return pci_enable_device_flags(dev, IORESOURCE_MEM);
2182 }
2183 EXPORT_SYMBOL(pci_enable_device_mem);
2184 
2185 /**
2186  * pci_enable_device - Initialize device before it's used by a driver.
2187  * @dev: PCI device to be initialized
2188  *
2189  * Initialize device before it's used by a driver. Ask low-level code
2190  * to enable I/O and memory. Wake up the device if it was suspended.
2191  * Beware, this function can fail.
2192  *
2193  * Note we don't actually enable the device many times if we call
2194  * this function repeatedly (we just increment the count).
2195  */
pci_enable_device(struct pci_dev * dev)2196 int pci_enable_device(struct pci_dev *dev)
2197 {
2198 	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
2199 }
2200 EXPORT_SYMBOL(pci_enable_device);
2201 
2202 /*
2203  * pcibios_device_add - provide arch specific hooks when adding device dev
2204  * @dev: the PCI device being added
2205  *
2206  * Permits the platform to provide architecture specific functionality when
2207  * devices are added. This is the default implementation. Architecture
2208  * implementations can override this.
2209  */
pcibios_device_add(struct pci_dev * dev)2210 int __weak pcibios_device_add(struct pci_dev *dev)
2211 {
2212 	return 0;
2213 }
2214 
2215 /**
2216  * pcibios_release_device - provide arch specific hooks when releasing
2217  *			    device dev
2218  * @dev: the PCI device being released
2219  *
2220  * Permits the platform to provide architecture specific functionality when
2221  * devices are released. This is the default implementation. Architecture
2222  * implementations can override this.
2223  */
pcibios_release_device(struct pci_dev * dev)2224 void __weak pcibios_release_device(struct pci_dev *dev) {}
2225 
2226 /**
2227  * pcibios_disable_device - disable arch specific PCI resources for device dev
2228  * @dev: the PCI device to disable
2229  *
2230  * Disables architecture specific PCI resources for the device. This
2231  * is the default implementation. Architecture implementations can
2232  * override this.
2233  */
pcibios_disable_device(struct pci_dev * dev)2234 void __weak pcibios_disable_device(struct pci_dev *dev) {}
2235 
do_pci_disable_device(struct pci_dev * dev)2236 static void do_pci_disable_device(struct pci_dev *dev)
2237 {
2238 	u16 pci_command;
2239 
2240 	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
2241 	if (pci_command & PCI_COMMAND_MASTER) {
2242 		pci_command &= ~PCI_COMMAND_MASTER;
2243 		pci_write_config_word(dev, PCI_COMMAND, pci_command);
2244 	}
2245 
2246 	pcibios_disable_device(dev);
2247 }
2248 
2249 /**
2250  * pci_disable_enabled_device - Disable device without updating enable_cnt
2251  * @dev: PCI device to disable
2252  *
2253  * NOTE: This function is a backend of PCI power management routines and is
2254  * not supposed to be called drivers.
2255  */
pci_disable_enabled_device(struct pci_dev * dev)2256 void pci_disable_enabled_device(struct pci_dev *dev)
2257 {
2258 	if (pci_is_enabled(dev))
2259 		do_pci_disable_device(dev);
2260 }
2261 
2262 /**
2263  * pci_disable_device - Disable PCI device after use
2264  * @dev: PCI device to be disabled
2265  *
2266  * Signal to the system that the PCI device is not in use by the system
2267  * anymore.  This only involves disabling PCI bus-mastering, if active.
2268  *
2269  * Note we don't actually disable the device until all callers of
2270  * pci_enable_device() have called pci_disable_device().
2271  */
pci_disable_device(struct pci_dev * dev)2272 void pci_disable_device(struct pci_dev *dev)
2273 {
2274 	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
2275 		      "disabling already-disabled device");
2276 
2277 	if (atomic_dec_return(&dev->enable_cnt) != 0)
2278 		return;
2279 
2280 	pci_host_bridge_disable_device(dev);
2281 
2282 	do_pci_disable_device(dev);
2283 
2284 	dev->is_busmaster = 0;
2285 }
2286 EXPORT_SYMBOL(pci_disable_device);
2287 
2288 /**
2289  * pcibios_set_pcie_reset_state - set reset state for device dev
2290  * @dev: the PCIe device reset
2291  * @state: Reset state to enter into
2292  *
2293  * Set the PCIe reset state for the device. This is the default
2294  * implementation. Architecture implementations can override this.
2295  */
pcibios_set_pcie_reset_state(struct pci_dev * dev,enum pcie_reset_state state)2296 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
2297 					enum pcie_reset_state state)
2298 {
2299 	return -EINVAL;
2300 }
2301 
2302 /**
2303  * pci_set_pcie_reset_state - set reset state for device dev
2304  * @dev: the PCIe device reset
2305  * @state: Reset state to enter into
2306  *
2307  * Sets the PCI reset state for the device.
2308  */
pci_set_pcie_reset_state(struct pci_dev * dev,enum pcie_reset_state state)2309 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
2310 {
2311 	return pcibios_set_pcie_reset_state(dev, state);
2312 }
2313 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
2314 
2315 #ifdef CONFIG_PCIEAER
pcie_clear_device_status(struct pci_dev * dev)2316 void pcie_clear_device_status(struct pci_dev *dev)
2317 {
2318 	u16 sta;
2319 
2320 	pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta);
2321 	pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta);
2322 }
2323 #endif
2324 
2325 /**
2326  * pcie_clear_root_pme_status - Clear root port PME interrupt status.
2327  * @dev: PCIe root port or event collector.
2328  */
pcie_clear_root_pme_status(struct pci_dev * dev)2329 void pcie_clear_root_pme_status(struct pci_dev *dev)
2330 {
2331 	pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME);
2332 }
2333 
2334 /**
2335  * pci_check_pme_status - Check if given device has generated PME.
2336  * @dev: Device to check.
2337  *
2338  * Check the PME status of the device and if set, clear it and clear PME enable
2339  * (if set).  Return 'true' if PME status and PME enable were both set or
2340  * 'false' otherwise.
2341  */
pci_check_pme_status(struct pci_dev * dev)2342 bool pci_check_pme_status(struct pci_dev *dev)
2343 {
2344 	int pmcsr_pos;
2345 	u16 pmcsr;
2346 	bool ret = false;
2347 
2348 	if (!dev->pm_cap)
2349 		return false;
2350 
2351 	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
2352 	pci_read_config_word(dev, pmcsr_pos, &pmcsr);
2353 	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
2354 		return false;
2355 
2356 	/* Clear PME status. */
2357 	pmcsr |= PCI_PM_CTRL_PME_STATUS;
2358 	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
2359 		/* Disable PME to avoid interrupt flood. */
2360 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2361 		ret = true;
2362 	}
2363 
2364 	pci_write_config_word(dev, pmcsr_pos, pmcsr);
2365 
2366 	return ret;
2367 }
2368 
2369 /**
2370  * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
2371  * @dev: Device to handle.
2372  * @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
2373  *
2374  * Check if @dev has generated PME and queue a resume request for it in that
2375  * case.
2376  */
pci_pme_wakeup(struct pci_dev * dev,void * pme_poll_reset)2377 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
2378 {
2379 	if (pme_poll_reset && dev->pme_poll)
2380 		dev->pme_poll = false;
2381 
2382 	if (pci_check_pme_status(dev)) {
2383 		pci_wakeup_event(dev);
2384 		pm_request_resume(&dev->dev);
2385 	}
2386 	return 0;
2387 }
2388 
2389 /**
2390  * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
2391  * @bus: Top bus of the subtree to walk.
2392  */
pci_pme_wakeup_bus(struct pci_bus * bus)2393 void pci_pme_wakeup_bus(struct pci_bus *bus)
2394 {
2395 	if (bus)
2396 		pci_walk_bus(bus, pci_pme_wakeup, (void *)true);
2397 }
2398 
2399 
2400 /**
2401  * pci_pme_capable - check the capability of PCI device to generate PME#
2402  * @dev: PCI device to handle.
2403  * @state: PCI state from which device will issue PME#.
2404  */
pci_pme_capable(struct pci_dev * dev,pci_power_t state)2405 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
2406 {
2407 	if (!dev->pm_cap)
2408 		return false;
2409 
2410 	return !!(dev->pme_support & (1 << state));
2411 }
2412 EXPORT_SYMBOL(pci_pme_capable);
2413 
pci_pme_list_scan(struct work_struct * work)2414 static void pci_pme_list_scan(struct work_struct *work)
2415 {
2416 	struct pci_pme_device *pme_dev, *n;
2417 
2418 	mutex_lock(&pci_pme_list_mutex);
2419 	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
2420 		struct pci_dev *pdev = pme_dev->dev;
2421 
2422 		if (pdev->pme_poll) {
2423 			struct pci_dev *bridge = pdev->bus->self;
2424 			struct device *dev = &pdev->dev;
2425 			struct device *bdev = bridge ? &bridge->dev : NULL;
2426 			int bref = 0;
2427 
2428 			/*
2429 			 * If we have a bridge, it should be in an active/D0
2430 			 * state or the configuration space of subordinate
2431 			 * devices may not be accessible or stable over the
2432 			 * course of the call.
2433 			 */
2434 			if (bdev) {
2435 				bref = pm_runtime_get_if_active(bdev);
2436 				if (!bref)
2437 					continue;
2438 
2439 				if (bridge->current_state != PCI_D0)
2440 					goto put_bridge;
2441 			}
2442 
2443 			/*
2444 			 * The device itself should be suspended but config
2445 			 * space must be accessible, therefore it cannot be in
2446 			 * D3cold.
2447 			 */
2448 			if (pm_runtime_suspended(dev) &&
2449 			    pdev->current_state != PCI_D3cold)
2450 				pci_pme_wakeup(pdev, NULL);
2451 
2452 put_bridge:
2453 			if (bref > 0)
2454 				pm_runtime_put(bdev);
2455 		} else {
2456 			list_del(&pme_dev->list);
2457 			kfree(pme_dev);
2458 		}
2459 	}
2460 	if (!list_empty(&pci_pme_list))
2461 		queue_delayed_work(system_freezable_wq, &pci_pme_work,
2462 				   msecs_to_jiffies(PME_TIMEOUT));
2463 	mutex_unlock(&pci_pme_list_mutex);
2464 }
2465 
__pci_pme_active(struct pci_dev * dev,bool enable)2466 static void __pci_pme_active(struct pci_dev *dev, bool enable)
2467 {
2468 	u16 pmcsr;
2469 
2470 	if (!dev->pme_support)
2471 		return;
2472 
2473 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2474 	/* Clear PME_Status by writing 1 to it and enable PME# */
2475 	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
2476 	if (!enable)
2477 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2478 
2479 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2480 }
2481 
2482 /**
2483  * pci_pme_restore - Restore PME configuration after config space restore.
2484  * @dev: PCI device to update.
2485  */
pci_pme_restore(struct pci_dev * dev)2486 void pci_pme_restore(struct pci_dev *dev)
2487 {
2488 	u16 pmcsr;
2489 
2490 	if (!dev->pme_support)
2491 		return;
2492 
2493 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2494 	if (dev->wakeup_prepared) {
2495 		pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2496 		pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
2497 	} else {
2498 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2499 		pmcsr |= PCI_PM_CTRL_PME_STATUS;
2500 	}
2501 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2502 }
2503 
2504 /**
2505  * pci_pme_active - enable or disable PCI device's PME# function
2506  * @dev: PCI device to handle.
2507  * @enable: 'true' to enable PME# generation; 'false' to disable it.
2508  *
2509  * The caller must verify that the device is capable of generating PME# before
2510  * calling this function with @enable equal to 'true'.
2511  */
pci_pme_active(struct pci_dev * dev,bool enable)2512 void pci_pme_active(struct pci_dev *dev, bool enable)
2513 {
2514 	__pci_pme_active(dev, enable);
2515 
2516 	/*
2517 	 * PCI (as opposed to PCIe) PME requires that the device have
2518 	 * its PME# line hooked up correctly. Not all hardware vendors
2519 	 * do this, so the PME never gets delivered and the device
2520 	 * remains asleep. The easiest way around this is to
2521 	 * periodically walk the list of suspended devices and check
2522 	 * whether any have their PME flag set. The assumption is that
2523 	 * we'll wake up often enough anyway that this won't be a huge
2524 	 * hit, and the power savings from the devices will still be a
2525 	 * win.
2526 	 *
2527 	 * Although PCIe uses in-band PME message instead of PME# line
2528 	 * to report PME, PME does not work for some PCIe devices in
2529 	 * reality.  For example, there are devices that set their PME
2530 	 * status bits, but don't really bother to send a PME message;
2531 	 * there are PCI Express Root Ports that don't bother to
2532 	 * trigger interrupts when they receive PME messages from the
2533 	 * devices below.  So PME poll is used for PCIe devices too.
2534 	 */
2535 
2536 	if (dev->pme_poll) {
2537 		struct pci_pme_device *pme_dev;
2538 		if (enable) {
2539 			pme_dev = kmalloc(sizeof(struct pci_pme_device),
2540 					  GFP_KERNEL);
2541 			if (!pme_dev) {
2542 				pci_warn(dev, "can't enable PME#\n");
2543 				return;
2544 			}
2545 			pme_dev->dev = dev;
2546 			mutex_lock(&pci_pme_list_mutex);
2547 			list_add(&pme_dev->list, &pci_pme_list);
2548 			if (list_is_singular(&pci_pme_list))
2549 				queue_delayed_work(system_freezable_wq,
2550 						   &pci_pme_work,
2551 						   msecs_to_jiffies(PME_TIMEOUT));
2552 			mutex_unlock(&pci_pme_list_mutex);
2553 		} else {
2554 			mutex_lock(&pci_pme_list_mutex);
2555 			list_for_each_entry(pme_dev, &pci_pme_list, list) {
2556 				if (pme_dev->dev == dev) {
2557 					list_del(&pme_dev->list);
2558 					kfree(pme_dev);
2559 					break;
2560 				}
2561 			}
2562 			mutex_unlock(&pci_pme_list_mutex);
2563 		}
2564 	}
2565 
2566 	pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled");
2567 }
2568 EXPORT_SYMBOL(pci_pme_active);
2569 
2570 /**
2571  * __pci_enable_wake - enable PCI device as wakeup event source
2572  * @dev: PCI device affected
2573  * @state: PCI state from which device will issue wakeup events
2574  * @enable: True to enable event generation; false to disable
2575  *
2576  * This enables the device as a wakeup event source, or disables it.
2577  * When such events involves platform-specific hooks, those hooks are
2578  * called automatically by this routine.
2579  *
2580  * Devices with legacy power management (no standard PCI PM capabilities)
2581  * always require such platform hooks.
2582  *
2583  * RETURN VALUE:
2584  * 0 is returned on success
2585  * -EINVAL is returned if device is not supposed to wake up the system
2586  * Error code depending on the platform is returned if both the platform and
2587  * the native mechanism fail to enable the generation of wake-up events
2588  */
__pci_enable_wake(struct pci_dev * dev,pci_power_t state,bool enable)2589 static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
2590 {
2591 	int ret = 0;
2592 
2593 	/*
2594 	 * Bridges that are not power-manageable directly only signal
2595 	 * wakeup on behalf of subordinate devices which is set up
2596 	 * elsewhere, so skip them. However, bridges that are
2597 	 * power-manageable may signal wakeup for themselves (for example,
2598 	 * on a hotplug event) and they need to be covered here.
2599 	 */
2600 	if (!pci_power_manageable(dev))
2601 		return 0;
2602 
2603 	/* Don't do the same thing twice in a row for one device. */
2604 	if (!!enable == !!dev->wakeup_prepared)
2605 		return 0;
2606 
2607 	/*
2608 	 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
2609 	 * Anderson we should be doing PME# wake enable followed by ACPI wake
2610 	 * enable.  To disable wake-up we call the platform first, for symmetry.
2611 	 */
2612 
2613 	if (enable) {
2614 		int error;
2615 
2616 		/*
2617 		 * Enable PME signaling if the device can signal PME from
2618 		 * D3cold regardless of whether or not it can signal PME from
2619 		 * the current target state, because that will allow it to
2620 		 * signal PME when the hierarchy above it goes into D3cold and
2621 		 * the device itself ends up in D3cold as a result of that.
2622 		 */
2623 		if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold))
2624 			pci_pme_active(dev, true);
2625 		else
2626 			ret = 1;
2627 		error = platform_pci_set_wakeup(dev, true);
2628 		if (ret)
2629 			ret = error;
2630 		if (!ret)
2631 			dev->wakeup_prepared = true;
2632 	} else {
2633 		platform_pci_set_wakeup(dev, false);
2634 		pci_pme_active(dev, false);
2635 		dev->wakeup_prepared = false;
2636 	}
2637 
2638 	return ret;
2639 }
2640 
2641 /**
2642  * pci_enable_wake - change wakeup settings for a PCI device
2643  * @pci_dev: Target device
2644  * @state: PCI state from which device will issue wakeup events
2645  * @enable: Whether or not to enable event generation
2646  *
2647  * If @enable is set, check device_may_wakeup() for the device before calling
2648  * __pci_enable_wake() for it.
2649  */
pci_enable_wake(struct pci_dev * pci_dev,pci_power_t state,bool enable)2650 int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable)
2651 {
2652 	if (enable && !device_may_wakeup(&pci_dev->dev))
2653 		return -EINVAL;
2654 
2655 	return __pci_enable_wake(pci_dev, state, enable);
2656 }
2657 EXPORT_SYMBOL(pci_enable_wake);
2658 
2659 /**
2660  * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
2661  * @dev: PCI device to prepare
2662  * @enable: True to enable wake-up event generation; false to disable
2663  *
2664  * Many drivers want the device to wake up the system from D3_hot or D3_cold
2665  * and this function allows them to set that up cleanly - pci_enable_wake()
2666  * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
2667  * ordering constraints.
2668  *
2669  * This function only returns error code if the device is not allowed to wake
2670  * up the system from sleep or it is not capable of generating PME# from both
2671  * D3_hot and D3_cold and the platform is unable to enable wake-up power for it.
2672  */
pci_wake_from_d3(struct pci_dev * dev,bool enable)2673 int pci_wake_from_d3(struct pci_dev *dev, bool enable)
2674 {
2675 	return pci_pme_capable(dev, PCI_D3cold) ?
2676 			pci_enable_wake(dev, PCI_D3cold, enable) :
2677 			pci_enable_wake(dev, PCI_D3hot, enable);
2678 }
2679 EXPORT_SYMBOL(pci_wake_from_d3);
2680 
2681 /**
2682  * pci_target_state - find an appropriate low power state for a given PCI dev
2683  * @dev: PCI device
2684  * @wakeup: Whether or not wakeup functionality will be enabled for the device.
2685  *
2686  * Use underlying platform code to find a supported low power state for @dev.
2687  * If the platform can't manage @dev, return the deepest state from which it
2688  * can generate wake events, based on any available PME info.
2689  */
pci_target_state(struct pci_dev * dev,bool wakeup)2690 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
2691 {
2692 	if (platform_pci_power_manageable(dev)) {
2693 		/*
2694 		 * Call the platform to find the target state for the device.
2695 		 */
2696 		pci_power_t state = platform_pci_choose_state(dev);
2697 
2698 		switch (state) {
2699 		case PCI_POWER_ERROR:
2700 		case PCI_UNKNOWN:
2701 			return PCI_D3hot;
2702 
2703 		case PCI_D1:
2704 		case PCI_D2:
2705 			if (pci_no_d1d2(dev))
2706 				return PCI_D3hot;
2707 		}
2708 
2709 		return state;
2710 	}
2711 
2712 	/*
2713 	 * If the device is in D3cold even though it's not power-manageable by
2714 	 * the platform, it may have been powered down by non-standard means.
2715 	 * Best to let it slumber.
2716 	 */
2717 	if (dev->current_state == PCI_D3cold)
2718 		return PCI_D3cold;
2719 	else if (!dev->pm_cap)
2720 		return PCI_D0;
2721 
2722 	if (wakeup && dev->pme_support) {
2723 		pci_power_t state = PCI_D3hot;
2724 
2725 		/*
2726 		 * Find the deepest state from which the device can generate
2727 		 * PME#.
2728 		 */
2729 		while (state && !(dev->pme_support & (1 << state)))
2730 			state--;
2731 
2732 		if (state)
2733 			return state;
2734 		else if (dev->pme_support & 1)
2735 			return PCI_D0;
2736 	}
2737 
2738 	return PCI_D3hot;
2739 }
2740 
2741 /**
2742  * pci_prepare_to_sleep - prepare PCI device for system-wide transition
2743  *			  into a sleep state
2744  * @dev: Device to handle.
2745  *
2746  * Choose the power state appropriate for the device depending on whether
2747  * it can wake up the system and/or is power manageable by the platform
2748  * (PCI_D3hot is the default) and put the device into that state.
2749  */
pci_prepare_to_sleep(struct pci_dev * dev)2750 int pci_prepare_to_sleep(struct pci_dev *dev)
2751 {
2752 	bool wakeup = device_may_wakeup(&dev->dev);
2753 	pci_power_t target_state = pci_target_state(dev, wakeup);
2754 	int error;
2755 
2756 	if (target_state == PCI_POWER_ERROR)
2757 		return -EIO;
2758 
2759 	pci_enable_wake(dev, target_state, wakeup);
2760 
2761 	error = pci_set_power_state(dev, target_state);
2762 
2763 	if (error)
2764 		pci_enable_wake(dev, target_state, false);
2765 
2766 	return error;
2767 }
2768 EXPORT_SYMBOL(pci_prepare_to_sleep);
2769 
2770 /**
2771  * pci_back_from_sleep - turn PCI device on during system-wide transition
2772  *			 into working state
2773  * @dev: Device to handle.
2774  *
2775  * Disable device's system wake-up capability and put it into D0.
2776  */
pci_back_from_sleep(struct pci_dev * dev)2777 int pci_back_from_sleep(struct pci_dev *dev)
2778 {
2779 	int ret = pci_set_power_state(dev, PCI_D0);
2780 
2781 	if (ret)
2782 		return ret;
2783 
2784 	pci_enable_wake(dev, PCI_D0, false);
2785 	return 0;
2786 }
2787 EXPORT_SYMBOL(pci_back_from_sleep);
2788 
2789 /**
2790  * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
2791  * @dev: PCI device being suspended.
2792  *
2793  * Prepare @dev to generate wake-up events at run time and put it into a low
2794  * power state.
2795  */
pci_finish_runtime_suspend(struct pci_dev * dev)2796 int pci_finish_runtime_suspend(struct pci_dev *dev)
2797 {
2798 	pci_power_t target_state;
2799 	int error;
2800 
2801 	target_state = pci_target_state(dev, device_can_wakeup(&dev->dev));
2802 	if (target_state == PCI_POWER_ERROR)
2803 		return -EIO;
2804 
2805 	__pci_enable_wake(dev, target_state, pci_dev_run_wake(dev));
2806 
2807 	error = pci_set_power_state(dev, target_state);
2808 
2809 	if (error)
2810 		pci_enable_wake(dev, target_state, false);
2811 
2812 	return error;
2813 }
2814 
2815 /**
2816  * pci_dev_run_wake - Check if device can generate run-time wake-up events.
2817  * @dev: Device to check.
2818  *
2819  * Return true if the device itself is capable of generating wake-up events
2820  * (through the platform or using the native PCIe PME) or if the device supports
2821  * PME and one of its upstream bridges can generate wake-up events.
2822  */
pci_dev_run_wake(struct pci_dev * dev)2823 bool pci_dev_run_wake(struct pci_dev *dev)
2824 {
2825 	struct pci_bus *bus = dev->bus;
2826 
2827 	if (!dev->pme_support)
2828 		return false;
2829 
2830 	/* PME-capable in principle, but not from the target power state */
2831 	if (!pci_pme_capable(dev, pci_target_state(dev, true)))
2832 		return false;
2833 
2834 	if (device_can_wakeup(&dev->dev))
2835 		return true;
2836 
2837 	while (bus->parent) {
2838 		struct pci_dev *bridge = bus->self;
2839 
2840 		if (device_can_wakeup(&bridge->dev))
2841 			return true;
2842 
2843 		bus = bus->parent;
2844 	}
2845 
2846 	/* We have reached the root bus. */
2847 	if (bus->bridge)
2848 		return device_can_wakeup(bus->bridge);
2849 
2850 	return false;
2851 }
2852 EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2853 
2854 /**
2855  * pci_dev_need_resume - Check if it is necessary to resume the device.
2856  * @pci_dev: Device to check.
2857  *
2858  * Return 'true' if the device is not runtime-suspended or it has to be
2859  * reconfigured due to wakeup settings difference between system and runtime
2860  * suspend, or the current power state of it is not suitable for the upcoming
2861  * (system-wide) transition.
2862  */
pci_dev_need_resume(struct pci_dev * pci_dev)2863 bool pci_dev_need_resume(struct pci_dev *pci_dev)
2864 {
2865 	struct device *dev = &pci_dev->dev;
2866 	pci_power_t target_state;
2867 
2868 	if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev))
2869 		return true;
2870 
2871 	target_state = pci_target_state(pci_dev, device_may_wakeup(dev));
2872 
2873 	/*
2874 	 * If the earlier platform check has not triggered, D3cold is just power
2875 	 * removal on top of D3hot, so no need to resume the device in that
2876 	 * case.
2877 	 */
2878 	return target_state != pci_dev->current_state &&
2879 		target_state != PCI_D3cold &&
2880 		pci_dev->current_state != PCI_D3hot;
2881 }
2882 
2883 /**
2884  * pci_dev_adjust_pme - Adjust PME setting for a suspended device.
2885  * @pci_dev: Device to check.
2886  *
2887  * If the device is suspended and it is not configured for system wakeup,
2888  * disable PME for it to prevent it from waking up the system unnecessarily.
2889  *
2890  * Note that if the device's power state is D3cold and the platform check in
2891  * pci_dev_need_resume() has not triggered, the device's configuration need not
2892  * be changed.
2893  */
pci_dev_adjust_pme(struct pci_dev * pci_dev)2894 void pci_dev_adjust_pme(struct pci_dev *pci_dev)
2895 {
2896 	struct device *dev = &pci_dev->dev;
2897 
2898 	spin_lock_irq(&dev->power.lock);
2899 
2900 	if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) &&
2901 	    pci_dev->current_state < PCI_D3cold)
2902 		__pci_pme_active(pci_dev, false);
2903 
2904 	spin_unlock_irq(&dev->power.lock);
2905 }
2906 
2907 /**
2908  * pci_dev_complete_resume - Finalize resume from system sleep for a device.
2909  * @pci_dev: Device to handle.
2910  *
2911  * If the device is runtime suspended and wakeup-capable, enable PME for it as
2912  * it might have been disabled during the prepare phase of system suspend if
2913  * the device was not configured for system wakeup.
2914  */
pci_dev_complete_resume(struct pci_dev * pci_dev)2915 void pci_dev_complete_resume(struct pci_dev *pci_dev)
2916 {
2917 	struct device *dev = &pci_dev->dev;
2918 
2919 	if (!pci_dev_run_wake(pci_dev))
2920 		return;
2921 
2922 	spin_lock_irq(&dev->power.lock);
2923 
2924 	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
2925 		__pci_pme_active(pci_dev, true);
2926 
2927 	spin_unlock_irq(&dev->power.lock);
2928 }
2929 
2930 /**
2931  * pci_choose_state - Choose the power state of a PCI device.
2932  * @dev: Target PCI device.
2933  * @state: Target state for the whole system.
2934  *
2935  * Returns PCI power state suitable for @dev and @state.
2936  */
pci_choose_state(struct pci_dev * dev,pm_message_t state)2937 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
2938 {
2939 	if (state.event == PM_EVENT_ON)
2940 		return PCI_D0;
2941 
2942 	return pci_target_state(dev, false);
2943 }
2944 EXPORT_SYMBOL(pci_choose_state);
2945 
pci_config_pm_runtime_get(struct pci_dev * pdev)2946 void pci_config_pm_runtime_get(struct pci_dev *pdev)
2947 {
2948 	struct device *dev = &pdev->dev;
2949 	struct device *parent = dev->parent;
2950 
2951 	if (parent)
2952 		pm_runtime_get_sync(parent);
2953 	pm_runtime_get_noresume(dev);
2954 	/*
2955 	 * pdev->current_state is set to PCI_D3cold during suspending,
2956 	 * so wait until suspending completes
2957 	 */
2958 	pm_runtime_barrier(dev);
2959 	/*
2960 	 * Only need to resume devices in D3cold, because config
2961 	 * registers are still accessible for devices suspended but
2962 	 * not in D3cold.
2963 	 */
2964 	if (pdev->current_state == PCI_D3cold)
2965 		pm_runtime_resume(dev);
2966 }
2967 
pci_config_pm_runtime_put(struct pci_dev * pdev)2968 void pci_config_pm_runtime_put(struct pci_dev *pdev)
2969 {
2970 	struct device *dev = &pdev->dev;
2971 	struct device *parent = dev->parent;
2972 
2973 	pm_runtime_put(dev);
2974 	if (parent)
2975 		pm_runtime_put_sync(parent);
2976 }
2977 
2978 static const struct dmi_system_id bridge_d3_blacklist[] = {
2979 #ifdef CONFIG_X86
2980 	{
2981 		/*
2982 		 * Gigabyte X299 root port is not marked as hotplug capable
2983 		 * which allows Linux to power manage it.  However, this
2984 		 * confuses the BIOS SMI handler so don't power manage root
2985 		 * ports on that system.
2986 		 */
2987 		.ident = "X299 DESIGNARE EX-CF",
2988 		.matches = {
2989 			DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
2990 			DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
2991 		},
2992 	},
2993 	{
2994 		/*
2995 		 * Downstream device is not accessible after putting a root port
2996 		 * into D3cold and back into D0 on Elo Continental Z2 board
2997 		 */
2998 		.ident = "Elo Continental Z2",
2999 		.matches = {
3000 			DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"),
3001 			DMI_MATCH(DMI_BOARD_NAME, "Geminilake"),
3002 			DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"),
3003 		},
3004 	},
3005 	{
3006 		/*
3007 		 * Changing power state of root port dGPU is connected fails
3008 		 * https://gitlab.freedesktop.org/drm/amd/-/issues/3229
3009 		 */
3010 		.ident = "Hewlett-Packard HP Pavilion 17 Notebook PC/1972",
3011 		.matches = {
3012 			DMI_MATCH(DMI_BOARD_VENDOR, "Hewlett-Packard"),
3013 			DMI_MATCH(DMI_BOARD_NAME, "1972"),
3014 			DMI_MATCH(DMI_BOARD_VERSION, "95.33"),
3015 		},
3016 	},
3017 #endif
3018 	{ }
3019 };
3020 
3021 /**
3022  * pci_bridge_d3_possible - Is it possible to put the bridge into D3
3023  * @bridge: Bridge to check
3024  *
3025  * This function checks if it is possible to move the bridge to D3.
3026  * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
3027  */
pci_bridge_d3_possible(struct pci_dev * bridge)3028 bool pci_bridge_d3_possible(struct pci_dev *bridge)
3029 {
3030 	if (!pci_is_pcie(bridge))
3031 		return false;
3032 
3033 	switch (pci_pcie_type(bridge)) {
3034 	case PCI_EXP_TYPE_ROOT_PORT:
3035 	case PCI_EXP_TYPE_UPSTREAM:
3036 	case PCI_EXP_TYPE_DOWNSTREAM:
3037 		if (pci_bridge_d3_disable)
3038 			return false;
3039 
3040 		/*
3041 		 * Hotplug ports handled by firmware in System Management Mode
3042 		 * may not be put into D3 by the OS (Thunderbolt on non-Macs).
3043 		 */
3044 		if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
3045 			return false;
3046 
3047 		if (pci_bridge_d3_force)
3048 			return true;
3049 
3050 		/* Even the oldest 2010 Thunderbolt controller supports D3. */
3051 		if (bridge->is_thunderbolt)
3052 			return true;
3053 
3054 		/* Platform might know better if the bridge supports D3 */
3055 		if (platform_pci_bridge_d3(bridge))
3056 			return true;
3057 
3058 		/*
3059 		 * Hotplug ports handled natively by the OS were not validated
3060 		 * by vendors for runtime D3 at least until 2018 because there
3061 		 * was no OS support.
3062 		 */
3063 		if (bridge->is_hotplug_bridge)
3064 			return false;
3065 
3066 		if (dmi_check_system(bridge_d3_blacklist))
3067 			return false;
3068 
3069 		/*
3070 		 * It should be safe to put PCIe ports from 2015 or newer
3071 		 * to D3.
3072 		 */
3073 		if (dmi_get_bios_year() >= 2015)
3074 			return true;
3075 		break;
3076 	}
3077 
3078 	return false;
3079 }
3080 
pci_dev_check_d3cold(struct pci_dev * dev,void * data)3081 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
3082 {
3083 	bool *d3cold_ok = data;
3084 
3085 	if (/* The device needs to be allowed to go D3cold ... */
3086 	    dev->no_d3cold || !dev->d3cold_allowed ||
3087 
3088 	    /* ... and if it is wakeup capable to do so from D3cold. */
3089 	    (device_may_wakeup(&dev->dev) &&
3090 	     !pci_pme_capable(dev, PCI_D3cold)) ||
3091 
3092 	    /* If it is a bridge it must be allowed to go to D3. */
3093 	    !pci_power_manageable(dev))
3094 
3095 		*d3cold_ok = false;
3096 
3097 	return !*d3cold_ok;
3098 }
3099 
3100 /*
3101  * pci_bridge_d3_update - Update bridge D3 capabilities
3102  * @dev: PCI device which is changed
3103  *
3104  * Update upstream bridge PM capabilities accordingly depending on if the
3105  * device PM configuration was changed or the device is being removed.  The
3106  * change is also propagated upstream.
3107  */
pci_bridge_d3_update(struct pci_dev * dev)3108 void pci_bridge_d3_update(struct pci_dev *dev)
3109 {
3110 	bool remove = !device_is_registered(&dev->dev);
3111 	struct pci_dev *bridge;
3112 	bool d3cold_ok = true;
3113 
3114 	bridge = pci_upstream_bridge(dev);
3115 	if (!bridge || !pci_bridge_d3_possible(bridge))
3116 		return;
3117 
3118 	/*
3119 	 * If D3 is currently allowed for the bridge, removing one of its
3120 	 * children won't change that.
3121 	 */
3122 	if (remove && bridge->bridge_d3)
3123 		return;
3124 
3125 	/*
3126 	 * If D3 is currently allowed for the bridge and a child is added or
3127 	 * changed, disallowance of D3 can only be caused by that child, so
3128 	 * we only need to check that single device, not any of its siblings.
3129 	 *
3130 	 * If D3 is currently not allowed for the bridge, checking the device
3131 	 * first may allow us to skip checking its siblings.
3132 	 */
3133 	if (!remove)
3134 		pci_dev_check_d3cold(dev, &d3cold_ok);
3135 
3136 	/*
3137 	 * If D3 is currently not allowed for the bridge, this may be caused
3138 	 * either by the device being changed/removed or any of its siblings,
3139 	 * so we need to go through all children to find out if one of them
3140 	 * continues to block D3.
3141 	 */
3142 	if (d3cold_ok && !bridge->bridge_d3)
3143 		pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
3144 			     &d3cold_ok);
3145 
3146 	if (bridge->bridge_d3 != d3cold_ok) {
3147 		bridge->bridge_d3 = d3cold_ok;
3148 		/* Propagate change to upstream bridges */
3149 		pci_bridge_d3_update(bridge);
3150 	}
3151 }
3152 
3153 /**
3154  * pci_d3cold_enable - Enable D3cold for device
3155  * @dev: PCI device to handle
3156  *
3157  * This function can be used in drivers to enable D3cold from the device
3158  * they handle.  It also updates upstream PCI bridge PM capabilities
3159  * accordingly.
3160  */
pci_d3cold_enable(struct pci_dev * dev)3161 void pci_d3cold_enable(struct pci_dev *dev)
3162 {
3163 	if (dev->no_d3cold) {
3164 		dev->no_d3cold = false;
3165 		pci_bridge_d3_update(dev);
3166 	}
3167 }
3168 EXPORT_SYMBOL_GPL(pci_d3cold_enable);
3169 
3170 /**
3171  * pci_d3cold_disable - Disable D3cold for device
3172  * @dev: PCI device to handle
3173  *
3174  * This function can be used in drivers to disable D3cold from the device
3175  * they handle.  It also updates upstream PCI bridge PM capabilities
3176  * accordingly.
3177  */
pci_d3cold_disable(struct pci_dev * dev)3178 void pci_d3cold_disable(struct pci_dev *dev)
3179 {
3180 	if (!dev->no_d3cold) {
3181 		dev->no_d3cold = true;
3182 		pci_bridge_d3_update(dev);
3183 	}
3184 }
3185 EXPORT_SYMBOL_GPL(pci_d3cold_disable);
3186 
3187 /**
3188  * pci_pm_init - Initialize PM functions of given PCI device
3189  * @dev: PCI device to handle.
3190  */
pci_pm_init(struct pci_dev * dev)3191 void pci_pm_init(struct pci_dev *dev)
3192 {
3193 	int pm;
3194 	u16 status;
3195 	u16 pmc;
3196 
3197 	pm_runtime_forbid(&dev->dev);
3198 	pm_runtime_set_active(&dev->dev);
3199 	pm_runtime_enable(&dev->dev);
3200 	device_enable_async_suspend(&dev->dev);
3201 	dev->wakeup_prepared = false;
3202 
3203 	dev->pm_cap = 0;
3204 	dev->pme_support = 0;
3205 
3206 	/* find PCI PM capability in list */
3207 	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
3208 	if (!pm)
3209 		return;
3210 	/* Check device's ability to generate PME# */
3211 	pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc);
3212 
3213 	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
3214 		pci_err(dev, "unsupported PM cap regs version (%u)\n",
3215 			pmc & PCI_PM_CAP_VER_MASK);
3216 		return;
3217 	}
3218 
3219 	dev->pm_cap = pm;
3220 	dev->d3hot_delay = PCI_PM_D3HOT_WAIT;
3221 	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
3222 	dev->bridge_d3 = pci_bridge_d3_possible(dev);
3223 	dev->d3cold_allowed = true;
3224 
3225 	dev->d1_support = false;
3226 	dev->d2_support = false;
3227 	if (!pci_no_d1d2(dev)) {
3228 		if (pmc & PCI_PM_CAP_D1)
3229 			dev->d1_support = true;
3230 		if (pmc & PCI_PM_CAP_D2)
3231 			dev->d2_support = true;
3232 
3233 		if (dev->d1_support || dev->d2_support)
3234 			pci_info(dev, "supports%s%s\n",
3235 				   dev->d1_support ? " D1" : "",
3236 				   dev->d2_support ? " D2" : "");
3237 	}
3238 
3239 	pmc &= PCI_PM_CAP_PME_MASK;
3240 	if (pmc) {
3241 		pci_info(dev, "PME# supported from%s%s%s%s%s\n",
3242 			 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
3243 			 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
3244 			 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
3245 			 (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "",
3246 			 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
3247 		dev->pme_support = FIELD_GET(PCI_PM_CAP_PME_MASK, pmc);
3248 		dev->pme_poll = true;
3249 		/*
3250 		 * Make device's PM flags reflect the wake-up capability, but
3251 		 * let the user space enable it to wake up the system as needed.
3252 		 */
3253 		device_set_wakeup_capable(&dev->dev, true);
3254 		/* Disable the PME# generation functionality */
3255 		pci_pme_active(dev, false);
3256 	}
3257 
3258 	pci_read_config_word(dev, PCI_STATUS, &status);
3259 	if (status & PCI_STATUS_IMM_READY)
3260 		dev->imm_ready = 1;
3261 }
3262 
pci_ea_flags(struct pci_dev * dev,u8 prop)3263 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
3264 {
3265 	unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
3266 
3267 	switch (prop) {
3268 	case PCI_EA_P_MEM:
3269 	case PCI_EA_P_VF_MEM:
3270 		flags |= IORESOURCE_MEM;
3271 		break;
3272 	case PCI_EA_P_MEM_PREFETCH:
3273 	case PCI_EA_P_VF_MEM_PREFETCH:
3274 		flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
3275 		break;
3276 	case PCI_EA_P_IO:
3277 		flags |= IORESOURCE_IO;
3278 		break;
3279 	default:
3280 		return 0;
3281 	}
3282 
3283 	return flags;
3284 }
3285 
pci_ea_get_resource(struct pci_dev * dev,u8 bei,u8 prop)3286 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
3287 					    u8 prop)
3288 {
3289 	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
3290 		return &dev->resource[bei];
3291 #ifdef CONFIG_PCI_IOV
3292 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
3293 		 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
3294 		return &dev->resource[PCI_IOV_RESOURCES +
3295 				      bei - PCI_EA_BEI_VF_BAR0];
3296 #endif
3297 	else if (bei == PCI_EA_BEI_ROM)
3298 		return &dev->resource[PCI_ROM_RESOURCE];
3299 	else
3300 		return NULL;
3301 }
3302 
3303 /* Read an Enhanced Allocation (EA) entry */
pci_ea_read(struct pci_dev * dev,int offset)3304 static int pci_ea_read(struct pci_dev *dev, int offset)
3305 {
3306 	struct resource *res;
3307 	const char *res_name;
3308 	int ent_size, ent_offset = offset;
3309 	resource_size_t start, end;
3310 	unsigned long flags;
3311 	u32 dw0, bei, base, max_offset;
3312 	u8 prop;
3313 	bool support_64 = (sizeof(resource_size_t) >= 8);
3314 
3315 	pci_read_config_dword(dev, ent_offset, &dw0);
3316 	ent_offset += 4;
3317 
3318 	/* Entry size field indicates DWORDs after 1st */
3319 	ent_size = (FIELD_GET(PCI_EA_ES, dw0) + 1) << 2;
3320 
3321 	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
3322 		goto out;
3323 
3324 	bei = FIELD_GET(PCI_EA_BEI, dw0);
3325 	prop = FIELD_GET(PCI_EA_PP, dw0);
3326 
3327 	/*
3328 	 * If the Property is in the reserved range, try the Secondary
3329 	 * Property instead.
3330 	 */
3331 	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
3332 		prop = FIELD_GET(PCI_EA_SP, dw0);
3333 	if (prop > PCI_EA_P_BRIDGE_IO)
3334 		goto out;
3335 
3336 	res = pci_ea_get_resource(dev, bei, prop);
3337 	res_name = pci_resource_name(dev, bei);
3338 	if (!res) {
3339 		pci_err(dev, "Unsupported EA entry BEI: %u\n", bei);
3340 		goto out;
3341 	}
3342 
3343 	flags = pci_ea_flags(dev, prop);
3344 	if (!flags) {
3345 		pci_err(dev, "Unsupported EA properties: %#x\n", prop);
3346 		goto out;
3347 	}
3348 
3349 	/* Read Base */
3350 	pci_read_config_dword(dev, ent_offset, &base);
3351 	start = (base & PCI_EA_FIELD_MASK);
3352 	ent_offset += 4;
3353 
3354 	/* Read MaxOffset */
3355 	pci_read_config_dword(dev, ent_offset, &max_offset);
3356 	ent_offset += 4;
3357 
3358 	/* Read Base MSBs (if 64-bit entry) */
3359 	if (base & PCI_EA_IS_64) {
3360 		u32 base_upper;
3361 
3362 		pci_read_config_dword(dev, ent_offset, &base_upper);
3363 		ent_offset += 4;
3364 
3365 		flags |= IORESOURCE_MEM_64;
3366 
3367 		/* entry starts above 32-bit boundary, can't use */
3368 		if (!support_64 && base_upper)
3369 			goto out;
3370 
3371 		if (support_64)
3372 			start |= ((u64)base_upper << 32);
3373 	}
3374 
3375 	end = start + (max_offset | 0x03);
3376 
3377 	/* Read MaxOffset MSBs (if 64-bit entry) */
3378 	if (max_offset & PCI_EA_IS_64) {
3379 		u32 max_offset_upper;
3380 
3381 		pci_read_config_dword(dev, ent_offset, &max_offset_upper);
3382 		ent_offset += 4;
3383 
3384 		flags |= IORESOURCE_MEM_64;
3385 
3386 		/* entry too big, can't use */
3387 		if (!support_64 && max_offset_upper)
3388 			goto out;
3389 
3390 		if (support_64)
3391 			end += ((u64)max_offset_upper << 32);
3392 	}
3393 
3394 	if (end < start) {
3395 		pci_err(dev, "EA Entry crosses address boundary\n");
3396 		goto out;
3397 	}
3398 
3399 	if (ent_size != ent_offset - offset) {
3400 		pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n",
3401 			ent_size, ent_offset - offset);
3402 		goto out;
3403 	}
3404 
3405 	res->name = pci_name(dev);
3406 	res->start = start;
3407 	res->end = end;
3408 	res->flags = flags;
3409 
3410 	if (bei <= PCI_EA_BEI_BAR5)
3411 		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3412 			 res_name, res, prop);
3413 	else if (bei == PCI_EA_BEI_ROM)
3414 		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3415 			 res_name, res, prop);
3416 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
3417 		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3418 			 res_name, res, prop);
3419 	else
3420 		pci_info(dev, "BEI %d %pR: from Enhanced Allocation, properties %#02x\n",
3421 			   bei, res, prop);
3422 
3423 out:
3424 	return offset + ent_size;
3425 }
3426 
3427 /* Enhanced Allocation Initialization */
pci_ea_init(struct pci_dev * dev)3428 void pci_ea_init(struct pci_dev *dev)
3429 {
3430 	int ea;
3431 	u8 num_ent;
3432 	int offset;
3433 	int i;
3434 
3435 	/* find PCI EA capability in list */
3436 	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
3437 	if (!ea)
3438 		return;
3439 
3440 	/* determine the number of entries */
3441 	pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT,
3442 					&num_ent);
3443 	num_ent &= PCI_EA_NUM_ENT_MASK;
3444 
3445 	offset = ea + PCI_EA_FIRST_ENT;
3446 
3447 	/* Skip DWORD 2 for type 1 functions */
3448 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
3449 		offset += 4;
3450 
3451 	/* parse each EA entry */
3452 	for (i = 0; i < num_ent; ++i)
3453 		offset = pci_ea_read(dev, offset);
3454 }
3455 
pci_add_saved_cap(struct pci_dev * pci_dev,struct pci_cap_saved_state * new_cap)3456 static void pci_add_saved_cap(struct pci_dev *pci_dev,
3457 	struct pci_cap_saved_state *new_cap)
3458 {
3459 	hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space);
3460 }
3461 
3462 /**
3463  * _pci_add_cap_save_buffer - allocate buffer for saving given
3464  *			      capability registers
3465  * @dev: the PCI device
3466  * @cap: the capability to allocate the buffer for
3467  * @extended: Standard or Extended capability ID
3468  * @size: requested size of the buffer
3469  */
_pci_add_cap_save_buffer(struct pci_dev * dev,u16 cap,bool extended,unsigned int size)3470 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
3471 				    bool extended, unsigned int size)
3472 {
3473 	int pos;
3474 	struct pci_cap_saved_state *save_state;
3475 
3476 	if (extended)
3477 		pos = pci_find_ext_capability(dev, cap);
3478 	else
3479 		pos = pci_find_capability(dev, cap);
3480 
3481 	if (!pos)
3482 		return 0;
3483 
3484 	save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL);
3485 	if (!save_state)
3486 		return -ENOMEM;
3487 
3488 	save_state->cap.cap_nr = cap;
3489 	save_state->cap.cap_extended = extended;
3490 	save_state->cap.size = size;
3491 	pci_add_saved_cap(dev, save_state);
3492 
3493 	return 0;
3494 }
3495 
pci_add_cap_save_buffer(struct pci_dev * dev,char cap,unsigned int size)3496 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
3497 {
3498 	return _pci_add_cap_save_buffer(dev, cap, false, size);
3499 }
3500 
pci_add_ext_cap_save_buffer(struct pci_dev * dev,u16 cap,unsigned int size)3501 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
3502 {
3503 	return _pci_add_cap_save_buffer(dev, cap, true, size);
3504 }
3505 
3506 /**
3507  * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
3508  * @dev: the PCI device
3509  */
pci_allocate_cap_save_buffers(struct pci_dev * dev)3510 void pci_allocate_cap_save_buffers(struct pci_dev *dev)
3511 {
3512 	int error;
3513 
3514 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
3515 					PCI_EXP_SAVE_REGS * sizeof(u16));
3516 	if (error)
3517 		pci_err(dev, "unable to preallocate PCI Express save buffer\n");
3518 
3519 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16));
3520 	if (error)
3521 		pci_err(dev, "unable to preallocate PCI-X save buffer\n");
3522 
3523 	error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR,
3524 					    2 * sizeof(u16));
3525 	if (error)
3526 		pci_err(dev, "unable to allocate suspend buffer for LTR\n");
3527 
3528 	pci_allocate_vc_save_buffers(dev);
3529 }
3530 
pci_free_cap_save_buffers(struct pci_dev * dev)3531 void pci_free_cap_save_buffers(struct pci_dev *dev)
3532 {
3533 	struct pci_cap_saved_state *tmp;
3534 	struct hlist_node *n;
3535 
3536 	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
3537 		kfree(tmp);
3538 }
3539 
3540 /**
3541  * pci_configure_ari - enable or disable ARI forwarding
3542  * @dev: the PCI device
3543  *
3544  * If @dev and its upstream bridge both support ARI, enable ARI in the
3545  * bridge.  Otherwise, disable ARI in the bridge.
3546  */
pci_configure_ari(struct pci_dev * dev)3547 void pci_configure_ari(struct pci_dev *dev)
3548 {
3549 	u32 cap;
3550 	struct pci_dev *bridge;
3551 
3552 	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
3553 		return;
3554 
3555 	bridge = dev->bus->self;
3556 	if (!bridge)
3557 		return;
3558 
3559 	pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3560 	if (!(cap & PCI_EXP_DEVCAP2_ARI))
3561 		return;
3562 
3563 	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
3564 		pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
3565 					 PCI_EXP_DEVCTL2_ARI);
3566 		bridge->ari_enabled = 1;
3567 	} else {
3568 		pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2,
3569 					   PCI_EXP_DEVCTL2_ARI);
3570 		bridge->ari_enabled = 0;
3571 	}
3572 }
3573 
pci_acs_flags_enabled(struct pci_dev * pdev,u16 acs_flags)3574 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
3575 {
3576 	int pos;
3577 	u16 cap, ctrl;
3578 
3579 	pos = pdev->acs_cap;
3580 	if (!pos)
3581 		return false;
3582 
3583 	/*
3584 	 * Except for egress control, capabilities are either required
3585 	 * or only required if controllable.  Features missing from the
3586 	 * capability field can therefore be assumed as hard-wired enabled.
3587 	 */
3588 	pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap);
3589 	acs_flags &= (cap | PCI_ACS_EC);
3590 
3591 	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
3592 	return (ctrl & acs_flags) == acs_flags;
3593 }
3594 
3595 /**
3596  * pci_acs_enabled - test ACS against required flags for a given device
3597  * @pdev: device to test
3598  * @acs_flags: required PCI ACS flags
3599  *
3600  * Return true if the device supports the provided flags.  Automatically
3601  * filters out flags that are not implemented on multifunction devices.
3602  *
3603  * Note that this interface checks the effective ACS capabilities of the
3604  * device rather than the actual capabilities.  For instance, most single
3605  * function endpoints are not required to support ACS because they have no
3606  * opportunity for peer-to-peer access.  We therefore return 'true'
3607  * regardless of whether the device exposes an ACS capability.  This makes
3608  * it much easier for callers of this function to ignore the actual type
3609  * or topology of the device when testing ACS support.
3610  */
pci_acs_enabled(struct pci_dev * pdev,u16 acs_flags)3611 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
3612 {
3613 	int ret;
3614 
3615 	ret = pci_dev_specific_acs_enabled(pdev, acs_flags);
3616 	if (ret >= 0)
3617 		return ret > 0;
3618 
3619 	/*
3620 	 * Conventional PCI and PCI-X devices never support ACS, either
3621 	 * effectively or actually.  The shared bus topology implies that
3622 	 * any device on the bus can receive or snoop DMA.
3623 	 */
3624 	if (!pci_is_pcie(pdev))
3625 		return false;
3626 
3627 	switch (pci_pcie_type(pdev)) {
3628 	/*
3629 	 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
3630 	 * but since their primary interface is PCI/X, we conservatively
3631 	 * handle them as we would a non-PCIe device.
3632 	 */
3633 	case PCI_EXP_TYPE_PCIE_BRIDGE:
3634 	/*
3635 	 * PCIe 3.0, 6.12.1 excludes ACS on these devices.  "ACS is never
3636 	 * applicable... must never implement an ACS Extended Capability...".
3637 	 * This seems arbitrary, but we take a conservative interpretation
3638 	 * of this statement.
3639 	 */
3640 	case PCI_EXP_TYPE_PCI_BRIDGE:
3641 	case PCI_EXP_TYPE_RC_EC:
3642 		return false;
3643 	/*
3644 	 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
3645 	 * implement ACS in order to indicate their peer-to-peer capabilities,
3646 	 * regardless of whether they are single- or multi-function devices.
3647 	 */
3648 	case PCI_EXP_TYPE_DOWNSTREAM:
3649 	case PCI_EXP_TYPE_ROOT_PORT:
3650 		return pci_acs_flags_enabled(pdev, acs_flags);
3651 	/*
3652 	 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
3653 	 * implemented by the remaining PCIe types to indicate peer-to-peer
3654 	 * capabilities, but only when they are part of a multifunction
3655 	 * device.  The footnote for section 6.12 indicates the specific
3656 	 * PCIe types included here.
3657 	 */
3658 	case PCI_EXP_TYPE_ENDPOINT:
3659 	case PCI_EXP_TYPE_UPSTREAM:
3660 	case PCI_EXP_TYPE_LEG_END:
3661 	case PCI_EXP_TYPE_RC_END:
3662 		if (!pdev->multifunction)
3663 			break;
3664 
3665 		return pci_acs_flags_enabled(pdev, acs_flags);
3666 	}
3667 
3668 	/*
3669 	 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
3670 	 * to single function devices with the exception of downstream ports.
3671 	 */
3672 	return true;
3673 }
3674 
3675 /**
3676  * pci_acs_path_enabled - test ACS flags from start to end in a hierarchy
3677  * @start: starting downstream device
3678  * @end: ending upstream device or NULL to search to the root bus
3679  * @acs_flags: required flags
3680  *
3681  * Walk up a device tree from start to end testing PCI ACS support.  If
3682  * any step along the way does not support the required flags, return false.
3683  */
pci_acs_path_enabled(struct pci_dev * start,struct pci_dev * end,u16 acs_flags)3684 bool pci_acs_path_enabled(struct pci_dev *start,
3685 			  struct pci_dev *end, u16 acs_flags)
3686 {
3687 	struct pci_dev *pdev, *parent = start;
3688 
3689 	do {
3690 		pdev = parent;
3691 
3692 		if (!pci_acs_enabled(pdev, acs_flags))
3693 			return false;
3694 
3695 		if (pci_is_root_bus(pdev->bus))
3696 			return (end == NULL);
3697 
3698 		parent = pdev->bus->self;
3699 	} while (pdev != end);
3700 
3701 	return true;
3702 }
3703 
3704 /**
3705  * pci_acs_init - Initialize ACS if hardware supports it
3706  * @dev: the PCI device
3707  */
pci_acs_init(struct pci_dev * dev)3708 void pci_acs_init(struct pci_dev *dev)
3709 {
3710 	dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
3711 
3712 	/*
3713 	 * Attempt to enable ACS regardless of capability because some Root
3714 	 * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have
3715 	 * the standard ACS capability but still support ACS via those
3716 	 * quirks.
3717 	 */
3718 	pci_enable_acs(dev);
3719 }
3720 
3721 /**
3722  * pci_rebar_find_pos - find position of resize ctrl reg for BAR
3723  * @pdev: PCI device
3724  * @bar: BAR to find
3725  *
3726  * Helper to find the position of the ctrl register for a BAR.
3727  * Returns -ENOTSUPP if resizable BARs are not supported at all.
3728  * Returns -ENOENT if no ctrl register for the BAR could be found.
3729  */
pci_rebar_find_pos(struct pci_dev * pdev,int bar)3730 static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
3731 {
3732 	unsigned int pos, nbars, i;
3733 	u32 ctrl;
3734 
3735 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
3736 	if (!pos)
3737 		return -ENOTSUPP;
3738 
3739 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3740 	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
3741 
3742 	for (i = 0; i < nbars; i++, pos += 8) {
3743 		int bar_idx;
3744 
3745 		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3746 		bar_idx = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, ctrl);
3747 		if (bar_idx == bar)
3748 			return pos;
3749 	}
3750 
3751 	return -ENOENT;
3752 }
3753 
3754 /**
3755  * pci_rebar_get_possible_sizes - get possible sizes for BAR
3756  * @pdev: PCI device
3757  * @bar: BAR to query
3758  *
3759  * Get the possible sizes of a resizable BAR as bitmask defined in the spec
3760  * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable.
3761  */
pci_rebar_get_possible_sizes(struct pci_dev * pdev,int bar)3762 u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar)
3763 {
3764 	int pos;
3765 	u32 cap;
3766 
3767 	pos = pci_rebar_find_pos(pdev, bar);
3768 	if (pos < 0)
3769 		return 0;
3770 
3771 	pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap);
3772 	cap = FIELD_GET(PCI_REBAR_CAP_SIZES, cap);
3773 
3774 	/* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */
3775 	if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f &&
3776 	    bar == 0 && cap == 0x700)
3777 		return 0x3f00;
3778 
3779 	return cap;
3780 }
3781 EXPORT_SYMBOL(pci_rebar_get_possible_sizes);
3782 
3783 /**
3784  * pci_rebar_get_current_size - get the current size of a BAR
3785  * @pdev: PCI device
3786  * @bar: BAR to set size to
3787  *
3788  * Read the size of a BAR from the resizable BAR config.
3789  * Returns size if found or negative error code.
3790  */
pci_rebar_get_current_size(struct pci_dev * pdev,int bar)3791 int pci_rebar_get_current_size(struct pci_dev *pdev, int bar)
3792 {
3793 	int pos;
3794 	u32 ctrl;
3795 
3796 	pos = pci_rebar_find_pos(pdev, bar);
3797 	if (pos < 0)
3798 		return pos;
3799 
3800 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3801 	return FIELD_GET(PCI_REBAR_CTRL_BAR_SIZE, ctrl);
3802 }
3803 
3804 /**
3805  * pci_rebar_set_size - set a new size for a BAR
3806  * @pdev: PCI device
3807  * @bar: BAR to set size to
3808  * @size: new size as defined in the spec (0=1MB, 19=512GB)
3809  *
3810  * Set the new size of a BAR as defined in the spec.
3811  * Returns zero if resizing was successful, error code otherwise.
3812  */
pci_rebar_set_size(struct pci_dev * pdev,int bar,int size)3813 int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
3814 {
3815 	int pos;
3816 	u32 ctrl;
3817 
3818 	pos = pci_rebar_find_pos(pdev, bar);
3819 	if (pos < 0)
3820 		return pos;
3821 
3822 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3823 	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
3824 	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
3825 	pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
3826 	return 0;
3827 }
3828 
3829 /**
3830  * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
3831  * @dev: the PCI device
3832  * @cap_mask: mask of desired AtomicOp sizes, including one or more of:
3833  *	PCI_EXP_DEVCAP2_ATOMIC_COMP32
3834  *	PCI_EXP_DEVCAP2_ATOMIC_COMP64
3835  *	PCI_EXP_DEVCAP2_ATOMIC_COMP128
3836  *
3837  * Return 0 if all upstream bridges support AtomicOp routing, egress
3838  * blocking is disabled on all upstream ports, and the root port supports
3839  * the requested completion capabilities (32-bit, 64-bit and/or 128-bit
3840  * AtomicOp completion), or negative otherwise.
3841  */
pci_enable_atomic_ops_to_root(struct pci_dev * dev,u32 cap_mask)3842 int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
3843 {
3844 	struct pci_bus *bus = dev->bus;
3845 	struct pci_dev *bridge;
3846 	u32 cap, ctl2;
3847 
3848 	/*
3849 	 * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit
3850 	 * in Device Control 2 is reserved in VFs and the PF value applies
3851 	 * to all associated VFs.
3852 	 */
3853 	if (dev->is_virtfn)
3854 		return -EINVAL;
3855 
3856 	if (!pci_is_pcie(dev))
3857 		return -EINVAL;
3858 
3859 	/*
3860 	 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be
3861 	 * AtomicOp requesters.  For now, we only support endpoints as
3862 	 * requesters and root ports as completers.  No endpoints as
3863 	 * completers, and no peer-to-peer.
3864 	 */
3865 
3866 	switch (pci_pcie_type(dev)) {
3867 	case PCI_EXP_TYPE_ENDPOINT:
3868 	case PCI_EXP_TYPE_LEG_END:
3869 	case PCI_EXP_TYPE_RC_END:
3870 		break;
3871 	default:
3872 		return -EINVAL;
3873 	}
3874 
3875 	while (bus->parent) {
3876 		bridge = bus->self;
3877 
3878 		pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3879 
3880 		switch (pci_pcie_type(bridge)) {
3881 		/* Ensure switch ports support AtomicOp routing */
3882 		case PCI_EXP_TYPE_UPSTREAM:
3883 		case PCI_EXP_TYPE_DOWNSTREAM:
3884 			if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
3885 				return -EINVAL;
3886 			break;
3887 
3888 		/* Ensure root port supports all the sizes we care about */
3889 		case PCI_EXP_TYPE_ROOT_PORT:
3890 			if ((cap & cap_mask) != cap_mask)
3891 				return -EINVAL;
3892 			break;
3893 		}
3894 
3895 		/* Ensure upstream ports don't block AtomicOps on egress */
3896 		if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) {
3897 			pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2,
3898 						   &ctl2);
3899 			if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK)
3900 				return -EINVAL;
3901 		}
3902 
3903 		bus = bus->parent;
3904 	}
3905 
3906 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
3907 				 PCI_EXP_DEVCTL2_ATOMIC_REQ);
3908 	return 0;
3909 }
3910 EXPORT_SYMBOL(pci_enable_atomic_ops_to_root);
3911 
3912 /**
3913  * pci_release_region - Release a PCI bar
3914  * @pdev: PCI device whose resources were previously reserved by
3915  *	  pci_request_region()
3916  * @bar: BAR to release
3917  *
3918  * Releases the PCI I/O and memory resources previously reserved by a
3919  * successful call to pci_request_region().  Call this function only
3920  * after all use of the PCI regions has ceased.
3921  */
pci_release_region(struct pci_dev * pdev,int bar)3922 void pci_release_region(struct pci_dev *pdev, int bar)
3923 {
3924 	/*
3925 	 * This is done for backwards compatibility, because the old PCI devres
3926 	 * API had a mode in which the function became managed if it had been
3927 	 * enabled with pcim_enable_device() instead of pci_enable_device().
3928 	 */
3929 	if (pci_is_managed(pdev)) {
3930 		pcim_release_region(pdev, bar);
3931 		return;
3932 	}
3933 
3934 	if (pci_resource_len(pdev, bar) == 0)
3935 		return;
3936 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
3937 		release_region(pci_resource_start(pdev, bar),
3938 				pci_resource_len(pdev, bar));
3939 	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
3940 		release_mem_region(pci_resource_start(pdev, bar),
3941 				pci_resource_len(pdev, bar));
3942 }
3943 EXPORT_SYMBOL(pci_release_region);
3944 
3945 /**
3946  * __pci_request_region - Reserved PCI I/O and memory resource
3947  * @pdev: PCI device whose resources are to be reserved
3948  * @bar: BAR to be reserved
3949  * @name: name of the driver requesting the resource
3950  * @exclusive: whether the region access is exclusive or not
3951  *
3952  * Returns: 0 on success, negative error code on failure.
3953  *
3954  * Mark the PCI region associated with PCI device @pdev BAR @bar as being
3955  * reserved by owner @name. Do not access any address inside the PCI regions
3956  * unless this call returns successfully.
3957  *
3958  * If @exclusive is set, then the region is marked so that userspace
3959  * is explicitly not allowed to map the resource via /dev/mem or
3960  * sysfs MMIO access.
3961  *
3962  * Returns 0 on success, or %EBUSY on error.  A warning
3963  * message is also printed on failure.
3964  */
__pci_request_region(struct pci_dev * pdev,int bar,const char * name,int exclusive)3965 static int __pci_request_region(struct pci_dev *pdev, int bar,
3966 				const char *name, int exclusive)
3967 {
3968 	if (pci_is_managed(pdev)) {
3969 		if (exclusive == IORESOURCE_EXCLUSIVE)
3970 			return pcim_request_region_exclusive(pdev, bar, name);
3971 
3972 		return pcim_request_region(pdev, bar, name);
3973 	}
3974 
3975 	if (pci_resource_len(pdev, bar) == 0)
3976 		return 0;
3977 
3978 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
3979 		if (!request_region(pci_resource_start(pdev, bar),
3980 			    pci_resource_len(pdev, bar), name))
3981 			goto err_out;
3982 	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
3983 		if (!__request_mem_region(pci_resource_start(pdev, bar),
3984 					pci_resource_len(pdev, bar), name,
3985 					exclusive))
3986 			goto err_out;
3987 	}
3988 
3989 	return 0;
3990 
3991 err_out:
3992 	pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar,
3993 		 &pdev->resource[bar]);
3994 	return -EBUSY;
3995 }
3996 
3997 /**
3998  * pci_request_region - Reserve PCI I/O and memory resource
3999  * @pdev: PCI device whose resources are to be reserved
4000  * @bar: BAR to be reserved
4001  * @name: name of the driver requesting the resource
4002  *
4003  * Returns: 0 on success, negative error code on failure.
4004  *
4005  * Mark the PCI region associated with PCI device @pdev BAR @bar as being
4006  * reserved by owner @name. Do not access any address inside the PCI regions
4007  * unless this call returns successfully.
4008  *
4009  * Returns 0 on success, or %EBUSY on error.  A warning
4010  * message is also printed on failure.
4011  *
4012  * NOTE:
4013  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4014  * when pcim_enable_device() has been called in advance. This hybrid feature is
4015  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4016  */
pci_request_region(struct pci_dev * pdev,int bar,const char * name)4017 int pci_request_region(struct pci_dev *pdev, int bar, const char *name)
4018 {
4019 	return __pci_request_region(pdev, bar, name, 0);
4020 }
4021 EXPORT_SYMBOL(pci_request_region);
4022 
4023 /**
4024  * pci_release_selected_regions - Release selected PCI I/O and memory resources
4025  * @pdev: PCI device whose resources were previously reserved
4026  * @bars: Bitmask of BARs to be released
4027  *
4028  * Release selected PCI I/O and memory resources previously reserved.
4029  * Call this function only after all use of the PCI regions has ceased.
4030  */
pci_release_selected_regions(struct pci_dev * pdev,int bars)4031 void pci_release_selected_regions(struct pci_dev *pdev, int bars)
4032 {
4033 	int i;
4034 
4035 	for (i = 0; i < PCI_STD_NUM_BARS; i++)
4036 		if (bars & (1 << i))
4037 			pci_release_region(pdev, i);
4038 }
4039 EXPORT_SYMBOL(pci_release_selected_regions);
4040 
__pci_request_selected_regions(struct pci_dev * pdev,int bars,const char * name,int excl)4041 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
4042 					  const char *name, int excl)
4043 {
4044 	int i;
4045 
4046 	for (i = 0; i < PCI_STD_NUM_BARS; i++)
4047 		if (bars & (1 << i))
4048 			if (__pci_request_region(pdev, i, name, excl))
4049 				goto err_out;
4050 	return 0;
4051 
4052 err_out:
4053 	while (--i >= 0)
4054 		if (bars & (1 << i))
4055 			pci_release_region(pdev, i);
4056 
4057 	return -EBUSY;
4058 }
4059 
4060 
4061 /**
4062  * pci_request_selected_regions - Reserve selected PCI I/O and memory resources
4063  * @pdev: PCI device whose resources are to be reserved
4064  * @bars: Bitmask of BARs to be requested
4065  * @name: Name of the driver requesting the resources
4066  *
4067  * Returns: 0 on success, negative error code on failure.
4068  *
4069  * NOTE:
4070  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4071  * when pcim_enable_device() has been called in advance. This hybrid feature is
4072  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4073  */
pci_request_selected_regions(struct pci_dev * pdev,int bars,const char * name)4074 int pci_request_selected_regions(struct pci_dev *pdev, int bars,
4075 				 const char *name)
4076 {
4077 	return __pci_request_selected_regions(pdev, bars, name, 0);
4078 }
4079 EXPORT_SYMBOL(pci_request_selected_regions);
4080 
4081 /**
4082  * pci_request_selected_regions_exclusive - Request regions exclusively
4083  * @pdev: PCI device to request regions from
4084  * @bars: bit mask of BARs to request
4085  * @name: name of the driver requesting the resources
4086  *
4087  * Returns: 0 on success, negative error code on failure.
4088  *
4089  * NOTE:
4090  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4091  * when pcim_enable_device() has been called in advance. This hybrid feature is
4092  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4093  */
pci_request_selected_regions_exclusive(struct pci_dev * pdev,int bars,const char * name)4094 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
4095 					   const char *name)
4096 {
4097 	return __pci_request_selected_regions(pdev, bars, name,
4098 			IORESOURCE_EXCLUSIVE);
4099 }
4100 EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
4101 
4102 /**
4103  * pci_release_regions - Release reserved PCI I/O and memory resources
4104  * @pdev: PCI device whose resources were previously reserved by
4105  *	  pci_request_regions()
4106  *
4107  * Releases all PCI I/O and memory resources previously reserved by a
4108  * successful call to pci_request_regions().  Call this function only
4109  * after all use of the PCI regions has ceased.
4110  */
pci_release_regions(struct pci_dev * pdev)4111 void pci_release_regions(struct pci_dev *pdev)
4112 {
4113 	pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1);
4114 }
4115 EXPORT_SYMBOL(pci_release_regions);
4116 
4117 /**
4118  * pci_request_regions - Reserve PCI I/O and memory resources
4119  * @pdev: PCI device whose resources are to be reserved
4120  * @name: name of the driver requesting the resources
4121  *
4122  * Mark all PCI regions associated with PCI device @pdev as being reserved by
4123  * owner @name. Do not access any address inside the PCI regions unless this
4124  * call returns successfully.
4125  *
4126  * Returns 0 on success, or %EBUSY on error.  A warning
4127  * message is also printed on failure.
4128  *
4129  * NOTE:
4130  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4131  * when pcim_enable_device() has been called in advance. This hybrid feature is
4132  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4133  */
pci_request_regions(struct pci_dev * pdev,const char * name)4134 int pci_request_regions(struct pci_dev *pdev, const char *name)
4135 {
4136 	return pci_request_selected_regions(pdev,
4137 			((1 << PCI_STD_NUM_BARS) - 1), name);
4138 }
4139 EXPORT_SYMBOL(pci_request_regions);
4140 
4141 /**
4142  * pci_request_regions_exclusive - Reserve PCI I/O and memory resources
4143  * @pdev: PCI device whose resources are to be reserved
4144  * @name: name of the driver requesting the resources
4145  *
4146  * Returns: 0 on success, negative error code on failure.
4147  *
4148  * Mark all PCI regions associated with PCI device @pdev as being reserved
4149  * by owner @name. Do not access any address inside the PCI regions
4150  * unless this call returns successfully.
4151  *
4152  * pci_request_regions_exclusive() will mark the region so that /dev/mem
4153  * and the sysfs MMIO access will not be allowed.
4154  *
4155  * Returns 0 on success, or %EBUSY on error.  A warning message is also
4156  * printed on failure.
4157  *
4158  * NOTE:
4159  * This is a "hybrid" function: It's normally unmanaged, but becomes managed
4160  * when pcim_enable_device() has been called in advance. This hybrid feature is
4161  * DEPRECATED! If you want managed cleanup, use the pcim_* functions instead.
4162  */
pci_request_regions_exclusive(struct pci_dev * pdev,const char * name)4163 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *name)
4164 {
4165 	return pci_request_selected_regions_exclusive(pdev,
4166 				((1 << PCI_STD_NUM_BARS) - 1), name);
4167 }
4168 EXPORT_SYMBOL(pci_request_regions_exclusive);
4169 
4170 /*
4171  * Record the PCI IO range (expressed as CPU physical address + size).
4172  * Return a negative value if an error has occurred, zero otherwise
4173  */
pci_register_io_range(const struct fwnode_handle * fwnode,phys_addr_t addr,resource_size_t size)4174 int pci_register_io_range(const struct fwnode_handle *fwnode, phys_addr_t addr,
4175 			resource_size_t	size)
4176 {
4177 	int ret = 0;
4178 #ifdef PCI_IOBASE
4179 	struct logic_pio_hwaddr *range;
4180 
4181 	if (!size || addr + size < addr)
4182 		return -EINVAL;
4183 
4184 	range = kzalloc(sizeof(*range), GFP_ATOMIC);
4185 	if (!range)
4186 		return -ENOMEM;
4187 
4188 	range->fwnode = fwnode;
4189 	range->size = size;
4190 	range->hw_start = addr;
4191 	range->flags = LOGIC_PIO_CPU_MMIO;
4192 
4193 	ret = logic_pio_register_range(range);
4194 	if (ret)
4195 		kfree(range);
4196 
4197 	/* Ignore duplicates due to deferred probing */
4198 	if (ret == -EEXIST)
4199 		ret = 0;
4200 #endif
4201 
4202 	return ret;
4203 }
4204 
pci_pio_to_address(unsigned long pio)4205 phys_addr_t pci_pio_to_address(unsigned long pio)
4206 {
4207 #ifdef PCI_IOBASE
4208 	if (pio < MMIO_UPPER_LIMIT)
4209 		return logic_pio_to_hwaddr(pio);
4210 #endif
4211 
4212 	return (phys_addr_t) OF_BAD_ADDR;
4213 }
4214 EXPORT_SYMBOL_GPL(pci_pio_to_address);
4215 
pci_address_to_pio(phys_addr_t address)4216 unsigned long __weak pci_address_to_pio(phys_addr_t address)
4217 {
4218 #ifdef PCI_IOBASE
4219 	return logic_pio_trans_cpuaddr(address);
4220 #else
4221 	if (address > IO_SPACE_LIMIT)
4222 		return (unsigned long)-1;
4223 
4224 	return (unsigned long) address;
4225 #endif
4226 }
4227 
4228 /**
4229  * pci_remap_iospace - Remap the memory mapped I/O space
4230  * @res: Resource describing the I/O space
4231  * @phys_addr: physical address of range to be mapped
4232  *
4233  * Remap the memory mapped I/O space described by the @res and the CPU
4234  * physical address @phys_addr into virtual address space.  Only
4235  * architectures that have memory mapped IO functions defined (and the
4236  * PCI_IOBASE value defined) should call this function.
4237  */
4238 #ifndef pci_remap_iospace
pci_remap_iospace(const struct resource * res,phys_addr_t phys_addr)4239 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
4240 {
4241 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4242 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4243 
4244 	if (!(res->flags & IORESOURCE_IO))
4245 		return -EINVAL;
4246 
4247 	if (res->end > IO_SPACE_LIMIT)
4248 		return -EINVAL;
4249 
4250 	return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
4251 			       pgprot_device(PAGE_KERNEL));
4252 #else
4253 	/*
4254 	 * This architecture does not have memory mapped I/O space,
4255 	 * so this function should never be called
4256 	 */
4257 	WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
4258 	return -ENODEV;
4259 #endif
4260 }
4261 EXPORT_SYMBOL(pci_remap_iospace);
4262 #endif
4263 
4264 /**
4265  * pci_unmap_iospace - Unmap the memory mapped I/O space
4266  * @res: resource to be unmapped
4267  *
4268  * Unmap the CPU virtual address @res from virtual address space.  Only
4269  * architectures that have memory mapped IO functions defined (and the
4270  * PCI_IOBASE value defined) should call this function.
4271  */
pci_unmap_iospace(struct resource * res)4272 void pci_unmap_iospace(struct resource *res)
4273 {
4274 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4275 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4276 
4277 	vunmap_range(vaddr, vaddr + resource_size(res));
4278 #endif
4279 }
4280 EXPORT_SYMBOL(pci_unmap_iospace);
4281 
__pci_set_master(struct pci_dev * dev,bool enable)4282 static void __pci_set_master(struct pci_dev *dev, bool enable)
4283 {
4284 	u16 old_cmd, cmd;
4285 
4286 	pci_read_config_word(dev, PCI_COMMAND, &old_cmd);
4287 	if (enable)
4288 		cmd = old_cmd | PCI_COMMAND_MASTER;
4289 	else
4290 		cmd = old_cmd & ~PCI_COMMAND_MASTER;
4291 	if (cmd != old_cmd) {
4292 		pci_dbg(dev, "%s bus mastering\n",
4293 			enable ? "enabling" : "disabling");
4294 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4295 	}
4296 	dev->is_busmaster = enable;
4297 }
4298 
4299 /**
4300  * pcibios_setup - process "pci=" kernel boot arguments
4301  * @str: string used to pass in "pci=" kernel boot arguments
4302  *
4303  * Process kernel boot arguments.  This is the default implementation.
4304  * Architecture specific implementations can override this as necessary.
4305  */
pcibios_setup(char * str)4306 char * __weak __init pcibios_setup(char *str)
4307 {
4308 	return str;
4309 }
4310 
4311 /**
4312  * pcibios_set_master - enable PCI bus-mastering for device dev
4313  * @dev: the PCI device to enable
4314  *
4315  * Enables PCI bus-mastering for the device.  This is the default
4316  * implementation.  Architecture specific implementations can override
4317  * this if necessary.
4318  */
pcibios_set_master(struct pci_dev * dev)4319 void __weak pcibios_set_master(struct pci_dev *dev)
4320 {
4321 	u8 lat;
4322 
4323 	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
4324 	if (pci_is_pcie(dev))
4325 		return;
4326 
4327 	pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat);
4328 	if (lat < 16)
4329 		lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
4330 	else if (lat > pcibios_max_latency)
4331 		lat = pcibios_max_latency;
4332 	else
4333 		return;
4334 
4335 	pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat);
4336 }
4337 
4338 /**
4339  * pci_set_master - enables bus-mastering for device dev
4340  * @dev: the PCI device to enable
4341  *
4342  * Enables bus-mastering on the device and calls pcibios_set_master()
4343  * to do the needed arch specific settings.
4344  */
pci_set_master(struct pci_dev * dev)4345 void pci_set_master(struct pci_dev *dev)
4346 {
4347 	__pci_set_master(dev, true);
4348 	pcibios_set_master(dev);
4349 }
4350 EXPORT_SYMBOL(pci_set_master);
4351 
4352 /**
4353  * pci_clear_master - disables bus-mastering for device dev
4354  * @dev: the PCI device to disable
4355  */
pci_clear_master(struct pci_dev * dev)4356 void pci_clear_master(struct pci_dev *dev)
4357 {
4358 	__pci_set_master(dev, false);
4359 }
4360 EXPORT_SYMBOL(pci_clear_master);
4361 
4362 /**
4363  * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
4364  * @dev: the PCI device for which MWI is to be enabled
4365  *
4366  * Helper function for pci_set_mwi.
4367  * Originally copied from drivers/net/acenic.c.
4368  * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
4369  *
4370  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4371  */
pci_set_cacheline_size(struct pci_dev * dev)4372 int pci_set_cacheline_size(struct pci_dev *dev)
4373 {
4374 	u8 cacheline_size;
4375 
4376 	if (!pci_cache_line_size)
4377 		return -EINVAL;
4378 
4379 	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
4380 	   equal to or multiple of the right value. */
4381 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4382 	if (cacheline_size >= pci_cache_line_size &&
4383 	    (cacheline_size % pci_cache_line_size) == 0)
4384 		return 0;
4385 
4386 	/* Write the correct value. */
4387 	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size);
4388 	/* Read it back. */
4389 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4390 	if (cacheline_size == pci_cache_line_size)
4391 		return 0;
4392 
4393 	pci_dbg(dev, "cache line size of %d is not supported\n",
4394 		   pci_cache_line_size << 2);
4395 
4396 	return -EINVAL;
4397 }
4398 EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
4399 
4400 /**
4401  * pci_set_mwi - enables memory-write-invalidate PCI transaction
4402  * @dev: the PCI device for which MWI is enabled
4403  *
4404  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4405  *
4406  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4407  */
pci_set_mwi(struct pci_dev * dev)4408 int pci_set_mwi(struct pci_dev *dev)
4409 {
4410 #ifdef PCI_DISABLE_MWI
4411 	return 0;
4412 #else
4413 	int rc;
4414 	u16 cmd;
4415 
4416 	rc = pci_set_cacheline_size(dev);
4417 	if (rc)
4418 		return rc;
4419 
4420 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4421 	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
4422 		pci_dbg(dev, "enabling Mem-Wr-Inval\n");
4423 		cmd |= PCI_COMMAND_INVALIDATE;
4424 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4425 	}
4426 	return 0;
4427 #endif
4428 }
4429 EXPORT_SYMBOL(pci_set_mwi);
4430 
4431 /**
4432  * pci_try_set_mwi - enables memory-write-invalidate PCI transaction
4433  * @dev: the PCI device for which MWI is enabled
4434  *
4435  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4436  * Callers are not required to check the return value.
4437  *
4438  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4439  */
pci_try_set_mwi(struct pci_dev * dev)4440 int pci_try_set_mwi(struct pci_dev *dev)
4441 {
4442 #ifdef PCI_DISABLE_MWI
4443 	return 0;
4444 #else
4445 	return pci_set_mwi(dev);
4446 #endif
4447 }
4448 EXPORT_SYMBOL(pci_try_set_mwi);
4449 
4450 /**
4451  * pci_clear_mwi - disables Memory-Write-Invalidate for device dev
4452  * @dev: the PCI device to disable
4453  *
4454  * Disables PCI Memory-Write-Invalidate transaction on the device
4455  */
pci_clear_mwi(struct pci_dev * dev)4456 void pci_clear_mwi(struct pci_dev *dev)
4457 {
4458 #ifndef PCI_DISABLE_MWI
4459 	u16 cmd;
4460 
4461 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4462 	if (cmd & PCI_COMMAND_INVALIDATE) {
4463 		cmd &= ~PCI_COMMAND_INVALIDATE;
4464 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4465 	}
4466 #endif
4467 }
4468 EXPORT_SYMBOL(pci_clear_mwi);
4469 
4470 /**
4471  * pci_disable_parity - disable parity checking for device
4472  * @dev: the PCI device to operate on
4473  *
4474  * Disable parity checking for device @dev
4475  */
pci_disable_parity(struct pci_dev * dev)4476 void pci_disable_parity(struct pci_dev *dev)
4477 {
4478 	u16 cmd;
4479 
4480 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4481 	if (cmd & PCI_COMMAND_PARITY) {
4482 		cmd &= ~PCI_COMMAND_PARITY;
4483 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4484 	}
4485 }
4486 
4487 /**
4488  * pci_intx - enables/disables PCI INTx for device dev
4489  * @pdev: the PCI device to operate on
4490  * @enable: boolean: whether to enable or disable PCI INTx
4491  *
4492  * Enables/disables PCI INTx for device @pdev
4493  */
pci_intx(struct pci_dev * pdev,int enable)4494 void pci_intx(struct pci_dev *pdev, int enable)
4495 {
4496 	u16 pci_command, new;
4497 
4498 	pci_read_config_word(pdev, PCI_COMMAND, &pci_command);
4499 
4500 	if (enable)
4501 		new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
4502 	else
4503 		new = pci_command | PCI_COMMAND_INTX_DISABLE;
4504 
4505 	if (new == pci_command)
4506 		return;
4507 
4508 	pci_write_config_word(pdev, PCI_COMMAND, new);
4509 }
4510 EXPORT_SYMBOL_GPL(pci_intx);
4511 
4512 /**
4513  * pci_wait_for_pending_transaction - wait for pending transaction
4514  * @dev: the PCI device to operate on
4515  *
4516  * Return 0 if transaction is pending 1 otherwise.
4517  */
pci_wait_for_pending_transaction(struct pci_dev * dev)4518 int pci_wait_for_pending_transaction(struct pci_dev *dev)
4519 {
4520 	if (!pci_is_pcie(dev))
4521 		return 1;
4522 
4523 	return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
4524 				    PCI_EXP_DEVSTA_TRPND);
4525 }
4526 EXPORT_SYMBOL(pci_wait_for_pending_transaction);
4527 
4528 /**
4529  * pcie_flr - initiate a PCIe function level reset
4530  * @dev: device to reset
4531  *
4532  * Initiate a function level reset unconditionally on @dev without
4533  * checking any flags and DEVCAP
4534  */
pcie_flr(struct pci_dev * dev)4535 int pcie_flr(struct pci_dev *dev)
4536 {
4537 	if (!pci_wait_for_pending_transaction(dev))
4538 		pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
4539 
4540 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
4541 
4542 	if (dev->imm_ready)
4543 		return 0;
4544 
4545 	/*
4546 	 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
4547 	 * 100ms, but may silently discard requests while the FLR is in
4548 	 * progress.  Wait 100ms before trying to access the device.
4549 	 */
4550 	msleep(100);
4551 
4552 	return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
4553 }
4554 EXPORT_SYMBOL_GPL(pcie_flr);
4555 
4556 /**
4557  * pcie_reset_flr - initiate a PCIe function level reset
4558  * @dev: device to reset
4559  * @probe: if true, return 0 if device can be reset this way
4560  *
4561  * Initiate a function level reset on @dev.
4562  */
pcie_reset_flr(struct pci_dev * dev,bool probe)4563 int pcie_reset_flr(struct pci_dev *dev, bool probe)
4564 {
4565 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4566 		return -ENOTTY;
4567 
4568 	if (!(dev->devcap & PCI_EXP_DEVCAP_FLR))
4569 		return -ENOTTY;
4570 
4571 	if (probe)
4572 		return 0;
4573 
4574 	return pcie_flr(dev);
4575 }
4576 EXPORT_SYMBOL_GPL(pcie_reset_flr);
4577 
pci_af_flr(struct pci_dev * dev,bool probe)4578 static int pci_af_flr(struct pci_dev *dev, bool probe)
4579 {
4580 	int pos;
4581 	u8 cap;
4582 
4583 	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
4584 	if (!pos)
4585 		return -ENOTTY;
4586 
4587 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4588 		return -ENOTTY;
4589 
4590 	pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
4591 	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
4592 		return -ENOTTY;
4593 
4594 	if (probe)
4595 		return 0;
4596 
4597 	/*
4598 	 * Wait for Transaction Pending bit to clear.  A word-aligned test
4599 	 * is used, so we use the control offset rather than status and shift
4600 	 * the test bit to match.
4601 	 */
4602 	if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL,
4603 				 PCI_AF_STATUS_TP << 8))
4604 		pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
4605 
4606 	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
4607 
4608 	if (dev->imm_ready)
4609 		return 0;
4610 
4611 	/*
4612 	 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
4613 	 * updated 27 July 2006; a device must complete an FLR within
4614 	 * 100ms, but may silently discard requests while the FLR is in
4615 	 * progress.  Wait 100ms before trying to access the device.
4616 	 */
4617 	msleep(100);
4618 
4619 	return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
4620 }
4621 
4622 /**
4623  * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
4624  * @dev: Device to reset.
4625  * @probe: if true, return 0 if the device can be reset this way.
4626  *
4627  * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
4628  * unset, it will be reinitialized internally when going from PCI_D3hot to
4629  * PCI_D0.  If that's the case and the device is not in a low-power state
4630  * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
4631  *
4632  * NOTE: This causes the caller to sleep for twice the device power transition
4633  * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
4634  * by default (i.e. unless the @dev's d3hot_delay field has a different value).
4635  * Moreover, only devices in D0 can be reset by this function.
4636  */
pci_pm_reset(struct pci_dev * dev,bool probe)4637 static int pci_pm_reset(struct pci_dev *dev, bool probe)
4638 {
4639 	u16 csr;
4640 
4641 	if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
4642 		return -ENOTTY;
4643 
4644 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr);
4645 	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
4646 		return -ENOTTY;
4647 
4648 	if (probe)
4649 		return 0;
4650 
4651 	if (dev->current_state != PCI_D0)
4652 		return -EINVAL;
4653 
4654 	csr &= ~PCI_PM_CTRL_STATE_MASK;
4655 	csr |= PCI_D3hot;
4656 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4657 	pci_dev_d3_sleep(dev);
4658 
4659 	csr &= ~PCI_PM_CTRL_STATE_MASK;
4660 	csr |= PCI_D0;
4661 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4662 	pci_dev_d3_sleep(dev);
4663 
4664 	return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
4665 }
4666 
4667 /**
4668  * pcie_wait_for_link_status - Wait for link status change
4669  * @pdev: Device whose link to wait for.
4670  * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE.
4671  * @active: Waiting for active or inactive?
4672  *
4673  * Return 0 if successful, or -ETIMEDOUT if status has not changed within
4674  * PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4675  */
pcie_wait_for_link_status(struct pci_dev * pdev,bool use_lt,bool active)4676 static int pcie_wait_for_link_status(struct pci_dev *pdev,
4677 				     bool use_lt, bool active)
4678 {
4679 	u16 lnksta_mask, lnksta_match;
4680 	unsigned long end_jiffies;
4681 	u16 lnksta;
4682 
4683 	lnksta_mask = use_lt ? PCI_EXP_LNKSTA_LT : PCI_EXP_LNKSTA_DLLLA;
4684 	lnksta_match = active ? lnksta_mask : 0;
4685 
4686 	end_jiffies = jiffies + msecs_to_jiffies(PCIE_LINK_RETRAIN_TIMEOUT_MS);
4687 	do {
4688 		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
4689 		if ((lnksta & lnksta_mask) == lnksta_match)
4690 			return 0;
4691 		msleep(1);
4692 	} while (time_before(jiffies, end_jiffies));
4693 
4694 	return -ETIMEDOUT;
4695 }
4696 
4697 /**
4698  * pcie_retrain_link - Request a link retrain and wait for it to complete
4699  * @pdev: Device whose link to retrain.
4700  * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE, for status.
4701  *
4702  * Retrain completion status is retrieved from the Link Status Register
4703  * according to @use_lt.  It is not verified whether the use of the DLLLA
4704  * bit is valid.
4705  *
4706  * Return 0 if successful, or -ETIMEDOUT if training has not completed
4707  * within PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4708  */
pcie_retrain_link(struct pci_dev * pdev,bool use_lt)4709 int pcie_retrain_link(struct pci_dev *pdev, bool use_lt)
4710 {
4711 	int rc;
4712 
4713 	/*
4714 	 * Ensure the updated LNKCTL parameters are used during link
4715 	 * training by checking that there is no ongoing link training that
4716 	 * may have started before link parameters were changed, so as to
4717 	 * avoid LTSSM race as recommended in Implementation Note at the end
4718 	 * of PCIe r6.1 sec 7.5.3.7.
4719 	 */
4720 	rc = pcie_wait_for_link_status(pdev, true, false);
4721 	if (rc)
4722 		return rc;
4723 
4724 	pcie_capability_set_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4725 	if (pdev->clear_retrain_link) {
4726 		/*
4727 		 * Due to an erratum in some devices the Retrain Link bit
4728 		 * needs to be cleared again manually to allow the link
4729 		 * training to succeed.
4730 		 */
4731 		pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4732 	}
4733 
4734 	rc = pcie_wait_for_link_status(pdev, use_lt, !use_lt);
4735 
4736 	/*
4737 	 * Clear LBMS after a manual retrain so that the bit can be used
4738 	 * to track link speed or width changes made by hardware itself
4739 	 * in attempt to correct unreliable link operation.
4740 	 */
4741 	pcie_reset_lbms_count(pdev);
4742 	return rc;
4743 }
4744 
4745 /**
4746  * pcie_wait_for_link_delay - Wait until link is active or inactive
4747  * @pdev: Bridge device
4748  * @active: waiting for active or inactive?
4749  * @delay: Delay to wait after link has become active (in ms)
4750  *
4751  * Use this to wait till link becomes active or inactive.
4752  */
pcie_wait_for_link_delay(struct pci_dev * pdev,bool active,int delay)4753 static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
4754 				     int delay)
4755 {
4756 	int rc;
4757 
4758 	/*
4759 	 * Some controllers might not implement link active reporting. In this
4760 	 * case, we wait for 1000 ms + any delay requested by the caller.
4761 	 */
4762 	if (!pdev->link_active_reporting) {
4763 		msleep(PCIE_LINK_RETRAIN_TIMEOUT_MS + delay);
4764 		return true;
4765 	}
4766 
4767 	/*
4768 	 * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms,
4769 	 * after which we should expect an link active if the reset was
4770 	 * successful. If so, software must wait a minimum 100ms before sending
4771 	 * configuration requests to devices downstream this port.
4772 	 *
4773 	 * If the link fails to activate, either the device was physically
4774 	 * removed or the link is permanently failed.
4775 	 */
4776 	if (active)
4777 		msleep(20);
4778 	rc = pcie_wait_for_link_status(pdev, false, active);
4779 	if (active) {
4780 		if (rc)
4781 			rc = pcie_failed_link_retrain(pdev);
4782 		if (rc)
4783 			return false;
4784 
4785 		msleep(delay);
4786 		return true;
4787 	}
4788 
4789 	if (rc)
4790 		return false;
4791 
4792 	return true;
4793 }
4794 
4795 /**
4796  * pcie_wait_for_link - Wait until link is active or inactive
4797  * @pdev: Bridge device
4798  * @active: waiting for active or inactive?
4799  *
4800  * Use this to wait till link becomes active or inactive.
4801  */
pcie_wait_for_link(struct pci_dev * pdev,bool active)4802 bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
4803 {
4804 	return pcie_wait_for_link_delay(pdev, active, 100);
4805 }
4806 
4807 /*
4808  * Find maximum D3cold delay required by all the devices on the bus.  The
4809  * spec says 100 ms, but firmware can lower it and we allow drivers to
4810  * increase it as well.
4811  *
4812  * Called with @pci_bus_sem locked for reading.
4813  */
pci_bus_max_d3cold_delay(const struct pci_bus * bus)4814 static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
4815 {
4816 	const struct pci_dev *pdev;
4817 	int min_delay = 100;
4818 	int max_delay = 0;
4819 
4820 	list_for_each_entry(pdev, &bus->devices, bus_list) {
4821 		if (pdev->d3cold_delay < min_delay)
4822 			min_delay = pdev->d3cold_delay;
4823 		if (pdev->d3cold_delay > max_delay)
4824 			max_delay = pdev->d3cold_delay;
4825 	}
4826 
4827 	return max(min_delay, max_delay);
4828 }
4829 
4830 /**
4831  * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible
4832  * @dev: PCI bridge
4833  * @reset_type: reset type in human-readable form
4834  *
4835  * Handle necessary delays before access to the devices on the secondary
4836  * side of the bridge are permitted after D3cold to D0 transition
4837  * or Conventional Reset.
4838  *
4839  * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For
4840  * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section
4841  * 4.3.2.
4842  *
4843  * Return 0 on success or -ENOTTY if the first device on the secondary bus
4844  * failed to become accessible.
4845  */
pci_bridge_wait_for_secondary_bus(struct pci_dev * dev,char * reset_type)4846 int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type)
4847 {
4848 	struct pci_dev *child __free(pci_dev_put) = NULL;
4849 	int delay;
4850 
4851 	if (pci_dev_is_disconnected(dev))
4852 		return 0;
4853 
4854 	if (!pci_is_bridge(dev))
4855 		return 0;
4856 
4857 	down_read(&pci_bus_sem);
4858 
4859 	/*
4860 	 * We only deal with devices that are present currently on the bus.
4861 	 * For any hot-added devices the access delay is handled in pciehp
4862 	 * board_added(). In case of ACPI hotplug the firmware is expected
4863 	 * to configure the devices before OS is notified.
4864 	 */
4865 	if (!dev->subordinate || list_empty(&dev->subordinate->devices)) {
4866 		up_read(&pci_bus_sem);
4867 		return 0;
4868 	}
4869 
4870 	/* Take d3cold_delay requirements into account */
4871 	delay = pci_bus_max_d3cold_delay(dev->subordinate);
4872 	if (!delay) {
4873 		up_read(&pci_bus_sem);
4874 		return 0;
4875 	}
4876 
4877 	child = pci_dev_get(list_first_entry(&dev->subordinate->devices,
4878 					     struct pci_dev, bus_list));
4879 	up_read(&pci_bus_sem);
4880 
4881 	/*
4882 	 * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before
4883 	 * accessing the device after reset (that is 1000 ms + 100 ms).
4884 	 */
4885 	if (!pci_is_pcie(dev)) {
4886 		pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay);
4887 		msleep(1000 + delay);
4888 		return 0;
4889 	}
4890 
4891 	/*
4892 	 * For PCIe downstream and root ports that do not support speeds
4893 	 * greater than 5 GT/s need to wait minimum 100 ms. For higher
4894 	 * speeds (gen3) we need to wait first for the data link layer to
4895 	 * become active.
4896 	 *
4897 	 * However, 100 ms is the minimum and the PCIe spec says the
4898 	 * software must allow at least 1s before it can determine that the
4899 	 * device that did not respond is a broken device. Also device can
4900 	 * take longer than that to respond if it indicates so through Request
4901 	 * Retry Status completions.
4902 	 *
4903 	 * Therefore we wait for 100 ms and check for the device presence
4904 	 * until the timeout expires.
4905 	 */
4906 	if (!pcie_downstream_port(dev))
4907 		return 0;
4908 
4909 	if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) {
4910 		u16 status;
4911 
4912 		pci_dbg(dev, "waiting %d ms for downstream link\n", delay);
4913 		msleep(delay);
4914 
4915 		if (!pci_dev_wait(child, reset_type, PCI_RESET_WAIT - delay))
4916 			return 0;
4917 
4918 		/*
4919 		 * If the port supports active link reporting we now check
4920 		 * whether the link is active and if not bail out early with
4921 		 * the assumption that the device is not present anymore.
4922 		 */
4923 		if (!dev->link_active_reporting)
4924 			return -ENOTTY;
4925 
4926 		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &status);
4927 		if (!(status & PCI_EXP_LNKSTA_DLLLA))
4928 			return -ENOTTY;
4929 
4930 		return pci_dev_wait(child, reset_type,
4931 				    PCIE_RESET_READY_POLL_MS - PCI_RESET_WAIT);
4932 	}
4933 
4934 	pci_dbg(dev, "waiting %d ms for downstream link, after activation\n",
4935 		delay);
4936 	if (!pcie_wait_for_link_delay(dev, true, delay)) {
4937 		/* Did not train, no need to wait any further */
4938 		pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n");
4939 		return -ENOTTY;
4940 	}
4941 
4942 	return pci_dev_wait(child, reset_type,
4943 			    PCIE_RESET_READY_POLL_MS - delay);
4944 }
4945 
pci_reset_secondary_bus(struct pci_dev * dev)4946 void pci_reset_secondary_bus(struct pci_dev *dev)
4947 {
4948 	u16 ctrl;
4949 
4950 	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
4951 	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
4952 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
4953 
4954 	/*
4955 	 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms.  Double
4956 	 * this to 2ms to ensure that we meet the minimum requirement.
4957 	 */
4958 	msleep(2);
4959 
4960 	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
4961 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
4962 }
4963 
pcibios_reset_secondary_bus(struct pci_dev * dev)4964 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
4965 {
4966 	pci_reset_secondary_bus(dev);
4967 }
4968 
4969 /**
4970  * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge.
4971  * @dev: Bridge device
4972  *
4973  * Use the bridge control register to assert reset on the secondary bus.
4974  * Devices on the secondary bus are left in power-on state.
4975  */
pci_bridge_secondary_bus_reset(struct pci_dev * dev)4976 int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
4977 {
4978 	if (!dev->block_cfg_access)
4979 		pci_warn_once(dev, "unlocked secondary bus reset via: %pS\n",
4980 			      __builtin_return_address(0));
4981 	pcibios_reset_secondary_bus(dev);
4982 
4983 	return pci_bridge_wait_for_secondary_bus(dev, "bus reset");
4984 }
4985 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
4986 
pci_parent_bus_reset(struct pci_dev * dev,bool probe)4987 static int pci_parent_bus_reset(struct pci_dev *dev, bool probe)
4988 {
4989 	struct pci_dev *pdev;
4990 
4991 	if (pci_is_root_bus(dev->bus) || dev->subordinate ||
4992 	    !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4993 		return -ENOTTY;
4994 
4995 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4996 		if (pdev != dev)
4997 			return -ENOTTY;
4998 
4999 	if (probe)
5000 		return 0;
5001 
5002 	return pci_bridge_secondary_bus_reset(dev->bus->self);
5003 }
5004 
pci_reset_hotplug_slot(struct hotplug_slot * hotplug,bool probe)5005 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe)
5006 {
5007 	int rc = -ENOTTY;
5008 
5009 	if (!hotplug || !try_module_get(hotplug->owner))
5010 		return rc;
5011 
5012 	if (hotplug->ops->reset_slot)
5013 		rc = hotplug->ops->reset_slot(hotplug, probe);
5014 
5015 	module_put(hotplug->owner);
5016 
5017 	return rc;
5018 }
5019 
pci_dev_reset_slot_function(struct pci_dev * dev,bool probe)5020 static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe)
5021 {
5022 	if (dev->multifunction || dev->subordinate || !dev->slot ||
5023 	    dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
5024 		return -ENOTTY;
5025 
5026 	return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
5027 }
5028 
cxl_port_dvsec(struct pci_dev * dev)5029 static u16 cxl_port_dvsec(struct pci_dev *dev)
5030 {
5031 	return pci_find_dvsec_capability(dev, PCI_VENDOR_ID_CXL,
5032 					 PCI_DVSEC_CXL_PORT);
5033 }
5034 
cxl_sbr_masked(struct pci_dev * dev)5035 static bool cxl_sbr_masked(struct pci_dev *dev)
5036 {
5037 	u16 dvsec, reg;
5038 	int rc;
5039 
5040 	dvsec = cxl_port_dvsec(dev);
5041 	if (!dvsec)
5042 		return false;
5043 
5044 	rc = pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
5045 	if (rc || PCI_POSSIBLE_ERROR(reg))
5046 		return false;
5047 
5048 	/*
5049 	 * Per CXL spec r3.1, sec 8.1.5.2, when "Unmask SBR" is 0, the SBR
5050 	 * bit in Bridge Control has no effect.  When 1, the Port generates
5051 	 * hot reset when the SBR bit is set to 1.
5052 	 */
5053 	if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR)
5054 		return false;
5055 
5056 	return true;
5057 }
5058 
pci_reset_bus_function(struct pci_dev * dev,bool probe)5059 static int pci_reset_bus_function(struct pci_dev *dev, bool probe)
5060 {
5061 	struct pci_dev *bridge = pci_upstream_bridge(dev);
5062 	int rc;
5063 
5064 	/*
5065 	 * If "dev" is below a CXL port that has SBR control masked, SBR
5066 	 * won't do anything, so return error.
5067 	 */
5068 	if (bridge && cxl_sbr_masked(bridge)) {
5069 		if (probe)
5070 			return 0;
5071 
5072 		return -ENOTTY;
5073 	}
5074 
5075 	rc = pci_dev_reset_slot_function(dev, probe);
5076 	if (rc != -ENOTTY)
5077 		return rc;
5078 	return pci_parent_bus_reset(dev, probe);
5079 }
5080 
cxl_reset_bus_function(struct pci_dev * dev,bool probe)5081 static int cxl_reset_bus_function(struct pci_dev *dev, bool probe)
5082 {
5083 	struct pci_dev *bridge;
5084 	u16 dvsec, reg, val;
5085 	int rc;
5086 
5087 	bridge = pci_upstream_bridge(dev);
5088 	if (!bridge)
5089 		return -ENOTTY;
5090 
5091 	dvsec = cxl_port_dvsec(bridge);
5092 	if (!dvsec)
5093 		return -ENOTTY;
5094 
5095 	if (probe)
5096 		return 0;
5097 
5098 	rc = pci_read_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
5099 	if (rc)
5100 		return -ENOTTY;
5101 
5102 	if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) {
5103 		val = reg;
5104 	} else {
5105 		val = reg | PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR;
5106 		pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL,
5107 				      val);
5108 	}
5109 
5110 	rc = pci_reset_bus_function(dev, probe);
5111 
5112 	if (reg != val)
5113 		pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL,
5114 				      reg);
5115 
5116 	return rc;
5117 }
5118 
pci_dev_lock(struct pci_dev * dev)5119 void pci_dev_lock(struct pci_dev *dev)
5120 {
5121 	/* block PM suspend, driver probe, etc. */
5122 	device_lock(&dev->dev);
5123 	pci_cfg_access_lock(dev);
5124 }
5125 EXPORT_SYMBOL_GPL(pci_dev_lock);
5126 
5127 /* Return 1 on successful lock, 0 on contention */
pci_dev_trylock(struct pci_dev * dev)5128 int pci_dev_trylock(struct pci_dev *dev)
5129 {
5130 	if (device_trylock(&dev->dev)) {
5131 		if (pci_cfg_access_trylock(dev))
5132 			return 1;
5133 		device_unlock(&dev->dev);
5134 	}
5135 
5136 	return 0;
5137 }
5138 EXPORT_SYMBOL_GPL(pci_dev_trylock);
5139 
pci_dev_unlock(struct pci_dev * dev)5140 void pci_dev_unlock(struct pci_dev *dev)
5141 {
5142 	pci_cfg_access_unlock(dev);
5143 	device_unlock(&dev->dev);
5144 }
5145 EXPORT_SYMBOL_GPL(pci_dev_unlock);
5146 
pci_dev_save_and_disable(struct pci_dev * dev)5147 static void pci_dev_save_and_disable(struct pci_dev *dev)
5148 {
5149 	const struct pci_error_handlers *err_handler =
5150 			dev->driver ? dev->driver->err_handler : NULL;
5151 
5152 	/*
5153 	 * dev->driver->err_handler->reset_prepare() is protected against
5154 	 * races with ->remove() by the device lock, which must be held by
5155 	 * the caller.
5156 	 */
5157 	if (err_handler && err_handler->reset_prepare)
5158 		err_handler->reset_prepare(dev);
5159 	else if (dev->driver)
5160 		pci_warn(dev, "resetting");
5161 
5162 	/*
5163 	 * Wake-up device prior to save.  PM registers default to D0 after
5164 	 * reset and a simple register restore doesn't reliably return
5165 	 * to a non-D0 state anyway.
5166 	 */
5167 	pci_set_power_state(dev, PCI_D0);
5168 
5169 	pci_save_state(dev);
5170 	/*
5171 	 * Disable the device by clearing the Command register, except for
5172 	 * INTx-disable which is set.  This not only disables MMIO and I/O port
5173 	 * BARs, but also prevents the device from being Bus Master, preventing
5174 	 * DMA from the device including MSI/MSI-X interrupts.  For PCI 2.3
5175 	 * compliant devices, INTx-disable prevents legacy interrupts.
5176 	 */
5177 	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
5178 }
5179 
pci_dev_restore(struct pci_dev * dev)5180 static void pci_dev_restore(struct pci_dev *dev)
5181 {
5182 	const struct pci_error_handlers *err_handler =
5183 			dev->driver ? dev->driver->err_handler : NULL;
5184 
5185 	pci_restore_state(dev);
5186 
5187 	/*
5188 	 * dev->driver->err_handler->reset_done() is protected against
5189 	 * races with ->remove() by the device lock, which must be held by
5190 	 * the caller.
5191 	 */
5192 	if (err_handler && err_handler->reset_done)
5193 		err_handler->reset_done(dev);
5194 	else if (dev->driver)
5195 		pci_warn(dev, "reset done");
5196 }
5197 
5198 /* dev->reset_methods[] is a 0-terminated list of indices into this array */
5199 const struct pci_reset_fn_method pci_reset_fn_methods[] = {
5200 	{ },
5201 	{ pci_dev_specific_reset, .name = "device_specific" },
5202 	{ pci_dev_acpi_reset, .name = "acpi" },
5203 	{ pcie_reset_flr, .name = "flr" },
5204 	{ pci_af_flr, .name = "af_flr" },
5205 	{ pci_pm_reset, .name = "pm" },
5206 	{ pci_reset_bus_function, .name = "bus" },
5207 	{ cxl_reset_bus_function, .name = "cxl_bus" },
5208 };
5209 
5210 /**
5211  * __pci_reset_function_locked - reset a PCI device function while holding
5212  * the @dev mutex lock.
5213  * @dev: PCI device to reset
5214  *
5215  * Some devices allow an individual function to be reset without affecting
5216  * other functions in the same device.  The PCI device must be responsive
5217  * to PCI config space in order to use this function.
5218  *
5219  * The device function is presumed to be unused and the caller is holding
5220  * the device mutex lock when this function is called.
5221  *
5222  * Resetting the device will make the contents of PCI configuration space
5223  * random, so any caller of this must be prepared to reinitialise the
5224  * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
5225  * etc.
5226  *
5227  * Returns 0 if the device function was successfully reset or negative if the
5228  * device doesn't support resetting a single function.
5229  */
__pci_reset_function_locked(struct pci_dev * dev)5230 int __pci_reset_function_locked(struct pci_dev *dev)
5231 {
5232 	int i, m, rc;
5233 
5234 	might_sleep();
5235 
5236 	/*
5237 	 * A reset method returns -ENOTTY if it doesn't support this device and
5238 	 * we should try the next method.
5239 	 *
5240 	 * If it returns 0 (success), we're finished.  If it returns any other
5241 	 * error, we're also finished: this indicates that further reset
5242 	 * mechanisms might be broken on the device.
5243 	 */
5244 	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5245 		m = dev->reset_methods[i];
5246 		if (!m)
5247 			return -ENOTTY;
5248 
5249 		rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET);
5250 		if (!rc)
5251 			return 0;
5252 		if (rc != -ENOTTY)
5253 			return rc;
5254 	}
5255 
5256 	return -ENOTTY;
5257 }
5258 EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
5259 
5260 /**
5261  * pci_init_reset_methods - check whether device can be safely reset
5262  * and store supported reset mechanisms.
5263  * @dev: PCI device to check for reset mechanisms
5264  *
5265  * Some devices allow an individual function to be reset without affecting
5266  * other functions in the same device.  The PCI device must be in D0-D3hot
5267  * state.
5268  *
5269  * Stores reset mechanisms supported by device in reset_methods byte array
5270  * which is a member of struct pci_dev.
5271  */
pci_init_reset_methods(struct pci_dev * dev)5272 void pci_init_reset_methods(struct pci_dev *dev)
5273 {
5274 	int m, i, rc;
5275 
5276 	BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS);
5277 
5278 	might_sleep();
5279 
5280 	i = 0;
5281 	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5282 		rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE);
5283 		if (!rc)
5284 			dev->reset_methods[i++] = m;
5285 		else if (rc != -ENOTTY)
5286 			break;
5287 	}
5288 
5289 	dev->reset_methods[i] = 0;
5290 }
5291 
5292 /**
5293  * pci_reset_function - quiesce and reset a PCI device function
5294  * @dev: PCI device to reset
5295  *
5296  * Some devices allow an individual function to be reset without affecting
5297  * other functions in the same device.  The PCI device must be responsive
5298  * to PCI config space in order to use this function.
5299  *
5300  * This function does not just reset the PCI portion of a device, but
5301  * clears all the state associated with the device.  This function differs
5302  * from __pci_reset_function_locked() in that it saves and restores device state
5303  * over the reset and takes the PCI device lock.
5304  *
5305  * Returns 0 if the device function was successfully reset or negative if the
5306  * device doesn't support resetting a single function.
5307  */
pci_reset_function(struct pci_dev * dev)5308 int pci_reset_function(struct pci_dev *dev)
5309 {
5310 	struct pci_dev *bridge;
5311 	int rc;
5312 
5313 	if (!pci_reset_supported(dev))
5314 		return -ENOTTY;
5315 
5316 	/*
5317 	 * If there's no upstream bridge, no locking is needed since there is
5318 	 * no upstream bridge configuration to hold consistent.
5319 	 */
5320 	bridge = pci_upstream_bridge(dev);
5321 	if (bridge)
5322 		pci_dev_lock(bridge);
5323 
5324 	pci_dev_lock(dev);
5325 	pci_dev_save_and_disable(dev);
5326 
5327 	rc = __pci_reset_function_locked(dev);
5328 
5329 	pci_dev_restore(dev);
5330 	pci_dev_unlock(dev);
5331 
5332 	if (bridge)
5333 		pci_dev_unlock(bridge);
5334 
5335 	return rc;
5336 }
5337 EXPORT_SYMBOL_GPL(pci_reset_function);
5338 
5339 /**
5340  * pci_reset_function_locked - quiesce and reset a PCI device function
5341  * @dev: PCI device to reset
5342  *
5343  * Some devices allow an individual function to be reset without affecting
5344  * other functions in the same device.  The PCI device must be responsive
5345  * to PCI config space in order to use this function.
5346  *
5347  * This function does not just reset the PCI portion of a device, but
5348  * clears all the state associated with the device.  This function differs
5349  * from __pci_reset_function_locked() in that it saves and restores device state
5350  * over the reset.  It also differs from pci_reset_function() in that it
5351  * requires the PCI device lock to be held.
5352  *
5353  * Returns 0 if the device function was successfully reset or negative if the
5354  * device doesn't support resetting a single function.
5355  */
pci_reset_function_locked(struct pci_dev * dev)5356 int pci_reset_function_locked(struct pci_dev *dev)
5357 {
5358 	int rc;
5359 
5360 	if (!pci_reset_supported(dev))
5361 		return -ENOTTY;
5362 
5363 	pci_dev_save_and_disable(dev);
5364 
5365 	rc = __pci_reset_function_locked(dev);
5366 
5367 	pci_dev_restore(dev);
5368 
5369 	return rc;
5370 }
5371 EXPORT_SYMBOL_GPL(pci_reset_function_locked);
5372 
5373 /**
5374  * pci_try_reset_function - quiesce and reset a PCI device function
5375  * @dev: PCI device to reset
5376  *
5377  * Same as above, except return -EAGAIN if unable to lock device.
5378  */
pci_try_reset_function(struct pci_dev * dev)5379 int pci_try_reset_function(struct pci_dev *dev)
5380 {
5381 	int rc;
5382 
5383 	if (!pci_reset_supported(dev))
5384 		return -ENOTTY;
5385 
5386 	if (!pci_dev_trylock(dev))
5387 		return -EAGAIN;
5388 
5389 	pci_dev_save_and_disable(dev);
5390 	rc = __pci_reset_function_locked(dev);
5391 	pci_dev_restore(dev);
5392 	pci_dev_unlock(dev);
5393 
5394 	return rc;
5395 }
5396 EXPORT_SYMBOL_GPL(pci_try_reset_function);
5397 
5398 /* Do any devices on or below this bus prevent a bus reset? */
pci_bus_resettable(struct pci_bus * bus)5399 static bool pci_bus_resettable(struct pci_bus *bus)
5400 {
5401 	struct pci_dev *dev;
5402 
5403 
5404 	if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5405 		return false;
5406 
5407 	list_for_each_entry(dev, &bus->devices, bus_list) {
5408 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5409 		    (dev->subordinate && !pci_bus_resettable(dev->subordinate)))
5410 			return false;
5411 	}
5412 
5413 	return true;
5414 }
5415 
5416 /* Lock devices from the top of the tree down */
pci_bus_lock(struct pci_bus * bus)5417 static void pci_bus_lock(struct pci_bus *bus)
5418 {
5419 	struct pci_dev *dev;
5420 
5421 	pci_dev_lock(bus->self);
5422 	list_for_each_entry(dev, &bus->devices, bus_list) {
5423 		if (dev->subordinate)
5424 			pci_bus_lock(dev->subordinate);
5425 		else
5426 			pci_dev_lock(dev);
5427 	}
5428 }
5429 
5430 /* Unlock devices from the bottom of the tree up */
pci_bus_unlock(struct pci_bus * bus)5431 static void pci_bus_unlock(struct pci_bus *bus)
5432 {
5433 	struct pci_dev *dev;
5434 
5435 	list_for_each_entry(dev, &bus->devices, bus_list) {
5436 		if (dev->subordinate)
5437 			pci_bus_unlock(dev->subordinate);
5438 		else
5439 			pci_dev_unlock(dev);
5440 	}
5441 	pci_dev_unlock(bus->self);
5442 }
5443 
5444 /* Return 1 on successful lock, 0 on contention */
pci_bus_trylock(struct pci_bus * bus)5445 static int pci_bus_trylock(struct pci_bus *bus)
5446 {
5447 	struct pci_dev *dev;
5448 
5449 	if (!pci_dev_trylock(bus->self))
5450 		return 0;
5451 
5452 	list_for_each_entry(dev, &bus->devices, bus_list) {
5453 		if (dev->subordinate) {
5454 			if (!pci_bus_trylock(dev->subordinate))
5455 				goto unlock;
5456 		} else if (!pci_dev_trylock(dev))
5457 			goto unlock;
5458 	}
5459 	return 1;
5460 
5461 unlock:
5462 	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
5463 		if (dev->subordinate)
5464 			pci_bus_unlock(dev->subordinate);
5465 		else
5466 			pci_dev_unlock(dev);
5467 	}
5468 	pci_dev_unlock(bus->self);
5469 	return 0;
5470 }
5471 
5472 /* Do any devices on or below this slot prevent a bus reset? */
pci_slot_resettable(struct pci_slot * slot)5473 static bool pci_slot_resettable(struct pci_slot *slot)
5474 {
5475 	struct pci_dev *dev;
5476 
5477 	if (slot->bus->self &&
5478 	    (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5479 		return false;
5480 
5481 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5482 		if (!dev->slot || dev->slot != slot)
5483 			continue;
5484 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5485 		    (dev->subordinate && !pci_bus_resettable(dev->subordinate)))
5486 			return false;
5487 	}
5488 
5489 	return true;
5490 }
5491 
5492 /* Lock devices from the top of the tree down */
pci_slot_lock(struct pci_slot * slot)5493 static void pci_slot_lock(struct pci_slot *slot)
5494 {
5495 	struct pci_dev *dev;
5496 
5497 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5498 		if (!dev->slot || dev->slot != slot)
5499 			continue;
5500 		if (dev->subordinate)
5501 			pci_bus_lock(dev->subordinate);
5502 		else
5503 			pci_dev_lock(dev);
5504 	}
5505 }
5506 
5507 /* Unlock devices from the bottom of the tree up */
pci_slot_unlock(struct pci_slot * slot)5508 static void pci_slot_unlock(struct pci_slot *slot)
5509 {
5510 	struct pci_dev *dev;
5511 
5512 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5513 		if (!dev->slot || dev->slot != slot)
5514 			continue;
5515 		if (dev->subordinate)
5516 			pci_bus_unlock(dev->subordinate);
5517 		pci_dev_unlock(dev);
5518 	}
5519 }
5520 
5521 /* Return 1 on successful lock, 0 on contention */
pci_slot_trylock(struct pci_slot * slot)5522 static int pci_slot_trylock(struct pci_slot *slot)
5523 {
5524 	struct pci_dev *dev;
5525 
5526 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5527 		if (!dev->slot || dev->slot != slot)
5528 			continue;
5529 		if (dev->subordinate) {
5530 			if (!pci_bus_trylock(dev->subordinate)) {
5531 				pci_dev_unlock(dev);
5532 				goto unlock;
5533 			}
5534 		} else if (!pci_dev_trylock(dev))
5535 			goto unlock;
5536 	}
5537 	return 1;
5538 
5539 unlock:
5540 	list_for_each_entry_continue_reverse(dev,
5541 					     &slot->bus->devices, bus_list) {
5542 		if (!dev->slot || dev->slot != slot)
5543 			continue;
5544 		if (dev->subordinate)
5545 			pci_bus_unlock(dev->subordinate);
5546 		else
5547 			pci_dev_unlock(dev);
5548 	}
5549 	return 0;
5550 }
5551 
5552 /*
5553  * Save and disable devices from the top of the tree down while holding
5554  * the @dev mutex lock for the entire tree.
5555  */
pci_bus_save_and_disable_locked(struct pci_bus * bus)5556 static void pci_bus_save_and_disable_locked(struct pci_bus *bus)
5557 {
5558 	struct pci_dev *dev;
5559 
5560 	list_for_each_entry(dev, &bus->devices, bus_list) {
5561 		pci_dev_save_and_disable(dev);
5562 		if (dev->subordinate)
5563 			pci_bus_save_and_disable_locked(dev->subordinate);
5564 	}
5565 }
5566 
5567 /*
5568  * Restore devices from top of the tree down while holding @dev mutex lock
5569  * for the entire tree.  Parent bridges need to be restored before we can
5570  * get to subordinate devices.
5571  */
pci_bus_restore_locked(struct pci_bus * bus)5572 static void pci_bus_restore_locked(struct pci_bus *bus)
5573 {
5574 	struct pci_dev *dev;
5575 
5576 	list_for_each_entry(dev, &bus->devices, bus_list) {
5577 		pci_dev_restore(dev);
5578 		if (dev->subordinate) {
5579 			pci_bridge_wait_for_secondary_bus(dev, "bus reset");
5580 			pci_bus_restore_locked(dev->subordinate);
5581 		}
5582 	}
5583 }
5584 
5585 /*
5586  * Save and disable devices from the top of the tree down while holding
5587  * the @dev mutex lock for the entire tree.
5588  */
pci_slot_save_and_disable_locked(struct pci_slot * slot)5589 static void pci_slot_save_and_disable_locked(struct pci_slot *slot)
5590 {
5591 	struct pci_dev *dev;
5592 
5593 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5594 		if (!dev->slot || dev->slot != slot)
5595 			continue;
5596 		pci_dev_save_and_disable(dev);
5597 		if (dev->subordinate)
5598 			pci_bus_save_and_disable_locked(dev->subordinate);
5599 	}
5600 }
5601 
5602 /*
5603  * Restore devices from top of the tree down while holding @dev mutex lock
5604  * for the entire tree.  Parent bridges need to be restored before we can
5605  * get to subordinate devices.
5606  */
pci_slot_restore_locked(struct pci_slot * slot)5607 static void pci_slot_restore_locked(struct pci_slot *slot)
5608 {
5609 	struct pci_dev *dev;
5610 
5611 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5612 		if (!dev->slot || dev->slot != slot)
5613 			continue;
5614 		pci_dev_restore(dev);
5615 		if (dev->subordinate) {
5616 			pci_bridge_wait_for_secondary_bus(dev, "slot reset");
5617 			pci_bus_restore_locked(dev->subordinate);
5618 		}
5619 	}
5620 }
5621 
pci_slot_reset(struct pci_slot * slot,bool probe)5622 static int pci_slot_reset(struct pci_slot *slot, bool probe)
5623 {
5624 	int rc;
5625 
5626 	if (!slot || !pci_slot_resettable(slot))
5627 		return -ENOTTY;
5628 
5629 	if (!probe)
5630 		pci_slot_lock(slot);
5631 
5632 	might_sleep();
5633 
5634 	rc = pci_reset_hotplug_slot(slot->hotplug, probe);
5635 
5636 	if (!probe)
5637 		pci_slot_unlock(slot);
5638 
5639 	return rc;
5640 }
5641 
5642 /**
5643  * pci_probe_reset_slot - probe whether a PCI slot can be reset
5644  * @slot: PCI slot to probe
5645  *
5646  * Return 0 if slot can be reset, negative if a slot reset is not supported.
5647  */
pci_probe_reset_slot(struct pci_slot * slot)5648 int pci_probe_reset_slot(struct pci_slot *slot)
5649 {
5650 	return pci_slot_reset(slot, PCI_RESET_PROBE);
5651 }
5652 EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
5653 
5654 /**
5655  * __pci_reset_slot - Try to reset a PCI slot
5656  * @slot: PCI slot to reset
5657  *
5658  * A PCI bus may host multiple slots, each slot may support a reset mechanism
5659  * independent of other slots.  For instance, some slots may support slot power
5660  * control.  In the case of a 1:1 bus to slot architecture, this function may
5661  * wrap the bus reset to avoid spurious slot related events such as hotplug.
5662  * Generally a slot reset should be attempted before a bus reset.  All of the
5663  * function of the slot and any subordinate buses behind the slot are reset
5664  * through this function.  PCI config space of all devices in the slot and
5665  * behind the slot is saved before and restored after reset.
5666  *
5667  * Same as above except return -EAGAIN if the slot cannot be locked
5668  */
__pci_reset_slot(struct pci_slot * slot)5669 static int __pci_reset_slot(struct pci_slot *slot)
5670 {
5671 	int rc;
5672 
5673 	rc = pci_slot_reset(slot, PCI_RESET_PROBE);
5674 	if (rc)
5675 		return rc;
5676 
5677 	if (pci_slot_trylock(slot)) {
5678 		pci_slot_save_and_disable_locked(slot);
5679 		might_sleep();
5680 		rc = pci_reset_hotplug_slot(slot->hotplug, PCI_RESET_DO_RESET);
5681 		pci_slot_restore_locked(slot);
5682 		pci_slot_unlock(slot);
5683 	} else
5684 		rc = -EAGAIN;
5685 
5686 	return rc;
5687 }
5688 
pci_bus_reset(struct pci_bus * bus,bool probe)5689 static int pci_bus_reset(struct pci_bus *bus, bool probe)
5690 {
5691 	int ret;
5692 
5693 	if (!bus->self || !pci_bus_resettable(bus))
5694 		return -ENOTTY;
5695 
5696 	if (probe)
5697 		return 0;
5698 
5699 	pci_bus_lock(bus);
5700 
5701 	might_sleep();
5702 
5703 	ret = pci_bridge_secondary_bus_reset(bus->self);
5704 
5705 	pci_bus_unlock(bus);
5706 
5707 	return ret;
5708 }
5709 
5710 /**
5711  * pci_bus_error_reset - reset the bridge's subordinate bus
5712  * @bridge: The parent device that connects to the bus to reset
5713  *
5714  * This function will first try to reset the slots on this bus if the method is
5715  * available. If slot reset fails or is not available, this will fall back to a
5716  * secondary bus reset.
5717  */
pci_bus_error_reset(struct pci_dev * bridge)5718 int pci_bus_error_reset(struct pci_dev *bridge)
5719 {
5720 	struct pci_bus *bus = bridge->subordinate;
5721 	struct pci_slot *slot;
5722 
5723 	if (!bus)
5724 		return -ENOTTY;
5725 
5726 	mutex_lock(&pci_slot_mutex);
5727 	if (list_empty(&bus->slots))
5728 		goto bus_reset;
5729 
5730 	list_for_each_entry(slot, &bus->slots, list)
5731 		if (pci_probe_reset_slot(slot))
5732 			goto bus_reset;
5733 
5734 	list_for_each_entry(slot, &bus->slots, list)
5735 		if (pci_slot_reset(slot, PCI_RESET_DO_RESET))
5736 			goto bus_reset;
5737 
5738 	mutex_unlock(&pci_slot_mutex);
5739 	return 0;
5740 bus_reset:
5741 	mutex_unlock(&pci_slot_mutex);
5742 	return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET);
5743 }
5744 
5745 /**
5746  * pci_probe_reset_bus - probe whether a PCI bus can be reset
5747  * @bus: PCI bus to probe
5748  *
5749  * Return 0 if bus can be reset, negative if a bus reset is not supported.
5750  */
pci_probe_reset_bus(struct pci_bus * bus)5751 int pci_probe_reset_bus(struct pci_bus *bus)
5752 {
5753 	return pci_bus_reset(bus, PCI_RESET_PROBE);
5754 }
5755 EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
5756 
5757 /**
5758  * __pci_reset_bus - Try to reset a PCI bus
5759  * @bus: top level PCI bus to reset
5760  *
5761  * Same as above except return -EAGAIN if the bus cannot be locked
5762  */
__pci_reset_bus(struct pci_bus * bus)5763 int __pci_reset_bus(struct pci_bus *bus)
5764 {
5765 	int rc;
5766 
5767 	rc = pci_bus_reset(bus, PCI_RESET_PROBE);
5768 	if (rc)
5769 		return rc;
5770 
5771 	if (pci_bus_trylock(bus)) {
5772 		pci_bus_save_and_disable_locked(bus);
5773 		might_sleep();
5774 		rc = pci_bridge_secondary_bus_reset(bus->self);
5775 		pci_bus_restore_locked(bus);
5776 		pci_bus_unlock(bus);
5777 	} else
5778 		rc = -EAGAIN;
5779 
5780 	return rc;
5781 }
5782 
5783 /**
5784  * pci_reset_bus - Try to reset a PCI bus
5785  * @pdev: top level PCI device to reset via slot/bus
5786  *
5787  * Same as above except return -EAGAIN if the bus cannot be locked
5788  */
pci_reset_bus(struct pci_dev * pdev)5789 int pci_reset_bus(struct pci_dev *pdev)
5790 {
5791 	return (!pci_probe_reset_slot(pdev->slot)) ?
5792 	    __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus);
5793 }
5794 EXPORT_SYMBOL_GPL(pci_reset_bus);
5795 
5796 /**
5797  * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
5798  * @dev: PCI device to query
5799  *
5800  * Returns mmrbc: maximum designed memory read count in bytes or
5801  * appropriate error value.
5802  */
pcix_get_max_mmrbc(struct pci_dev * dev)5803 int pcix_get_max_mmrbc(struct pci_dev *dev)
5804 {
5805 	int cap;
5806 	u32 stat;
5807 
5808 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5809 	if (!cap)
5810 		return -EINVAL;
5811 
5812 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5813 		return -EINVAL;
5814 
5815 	return 512 << FIELD_GET(PCI_X_STATUS_MAX_READ, stat);
5816 }
5817 EXPORT_SYMBOL(pcix_get_max_mmrbc);
5818 
5819 /**
5820  * pcix_get_mmrbc - get PCI-X maximum memory read byte count
5821  * @dev: PCI device to query
5822  *
5823  * Returns mmrbc: maximum memory read count in bytes or appropriate error
5824  * value.
5825  */
pcix_get_mmrbc(struct pci_dev * dev)5826 int pcix_get_mmrbc(struct pci_dev *dev)
5827 {
5828 	int cap;
5829 	u16 cmd;
5830 
5831 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5832 	if (!cap)
5833 		return -EINVAL;
5834 
5835 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5836 		return -EINVAL;
5837 
5838 	return 512 << FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5839 }
5840 EXPORT_SYMBOL(pcix_get_mmrbc);
5841 
5842 /**
5843  * pcix_set_mmrbc - set PCI-X maximum memory read byte count
5844  * @dev: PCI device to query
5845  * @mmrbc: maximum memory read count in bytes
5846  *    valid values are 512, 1024, 2048, 4096
5847  *
5848  * If possible sets maximum memory read byte count, some bridges have errata
5849  * that prevent this.
5850  */
pcix_set_mmrbc(struct pci_dev * dev,int mmrbc)5851 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
5852 {
5853 	int cap;
5854 	u32 stat, v, o;
5855 	u16 cmd;
5856 
5857 	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc))
5858 		return -EINVAL;
5859 
5860 	v = ffs(mmrbc) - 10;
5861 
5862 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5863 	if (!cap)
5864 		return -EINVAL;
5865 
5866 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5867 		return -EINVAL;
5868 
5869 	if (v > FIELD_GET(PCI_X_STATUS_MAX_READ, stat))
5870 		return -E2BIG;
5871 
5872 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5873 		return -EINVAL;
5874 
5875 	o = FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5876 	if (o != v) {
5877 		if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
5878 			return -EIO;
5879 
5880 		cmd &= ~PCI_X_CMD_MAX_READ;
5881 		cmd |= FIELD_PREP(PCI_X_CMD_MAX_READ, v);
5882 		if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
5883 			return -EIO;
5884 	}
5885 	return 0;
5886 }
5887 EXPORT_SYMBOL(pcix_set_mmrbc);
5888 
5889 /**
5890  * pcie_get_readrq - get PCI Express read request size
5891  * @dev: PCI device to query
5892  *
5893  * Returns maximum memory read request in bytes or appropriate error value.
5894  */
pcie_get_readrq(struct pci_dev * dev)5895 int pcie_get_readrq(struct pci_dev *dev)
5896 {
5897 	u16 ctl;
5898 
5899 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
5900 
5901 	return 128 << FIELD_GET(PCI_EXP_DEVCTL_READRQ, ctl);
5902 }
5903 EXPORT_SYMBOL(pcie_get_readrq);
5904 
5905 /**
5906  * pcie_set_readrq - set PCI Express maximum memory read request
5907  * @dev: PCI device to query
5908  * @rq: maximum memory read count in bytes
5909  *    valid values are 128, 256, 512, 1024, 2048, 4096
5910  *
5911  * If possible sets maximum memory read request in bytes
5912  */
pcie_set_readrq(struct pci_dev * dev,int rq)5913 int pcie_set_readrq(struct pci_dev *dev, int rq)
5914 {
5915 	u16 v;
5916 	int ret;
5917 	struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
5918 
5919 	if (rq < 128 || rq > 4096 || !is_power_of_2(rq))
5920 		return -EINVAL;
5921 
5922 	/*
5923 	 * If using the "performance" PCIe config, we clamp the read rq
5924 	 * size to the max packet size to keep the host bridge from
5925 	 * generating requests larger than we can cope with.
5926 	 */
5927 	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
5928 		int mps = pcie_get_mps(dev);
5929 
5930 		if (mps < rq)
5931 			rq = mps;
5932 	}
5933 
5934 	v = FIELD_PREP(PCI_EXP_DEVCTL_READRQ, ffs(rq) - 8);
5935 
5936 	if (bridge->no_inc_mrrs) {
5937 		int max_mrrs = pcie_get_readrq(dev);
5938 
5939 		if (rq > max_mrrs) {
5940 			pci_info(dev, "can't set Max_Read_Request_Size to %d; max is %d\n", rq, max_mrrs);
5941 			return -EINVAL;
5942 		}
5943 	}
5944 
5945 	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5946 						  PCI_EXP_DEVCTL_READRQ, v);
5947 
5948 	return pcibios_err_to_errno(ret);
5949 }
5950 EXPORT_SYMBOL(pcie_set_readrq);
5951 
5952 /**
5953  * pcie_get_mps - get PCI Express maximum payload size
5954  * @dev: PCI device to query
5955  *
5956  * Returns maximum payload size in bytes
5957  */
pcie_get_mps(struct pci_dev * dev)5958 int pcie_get_mps(struct pci_dev *dev)
5959 {
5960 	u16 ctl;
5961 
5962 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
5963 
5964 	return 128 << FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, ctl);
5965 }
5966 EXPORT_SYMBOL(pcie_get_mps);
5967 
5968 /**
5969  * pcie_set_mps - set PCI Express maximum payload size
5970  * @dev: PCI device to query
5971  * @mps: maximum payload size in bytes
5972  *    valid values are 128, 256, 512, 1024, 2048, 4096
5973  *
5974  * If possible sets maximum payload size
5975  */
pcie_set_mps(struct pci_dev * dev,int mps)5976 int pcie_set_mps(struct pci_dev *dev, int mps)
5977 {
5978 	u16 v;
5979 	int ret;
5980 
5981 	if (mps < 128 || mps > 4096 || !is_power_of_2(mps))
5982 		return -EINVAL;
5983 
5984 	v = ffs(mps) - 8;
5985 	if (v > dev->pcie_mpss)
5986 		return -EINVAL;
5987 	v = FIELD_PREP(PCI_EXP_DEVCTL_PAYLOAD, v);
5988 
5989 	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5990 						  PCI_EXP_DEVCTL_PAYLOAD, v);
5991 
5992 	return pcibios_err_to_errno(ret);
5993 }
5994 EXPORT_SYMBOL(pcie_set_mps);
5995 
to_pcie_link_speed(u16 lnksta)5996 static enum pci_bus_speed to_pcie_link_speed(u16 lnksta)
5997 {
5998 	return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)];
5999 }
6000 
pcie_link_speed_mbps(struct pci_dev * pdev)6001 int pcie_link_speed_mbps(struct pci_dev *pdev)
6002 {
6003 	u16 lnksta;
6004 	int err;
6005 
6006 	err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
6007 	if (err)
6008 		return err;
6009 
6010 	return pcie_dev_speed_mbps(to_pcie_link_speed(lnksta));
6011 }
6012 EXPORT_SYMBOL(pcie_link_speed_mbps);
6013 
6014 /**
6015  * pcie_bandwidth_available - determine minimum link settings of a PCIe
6016  *			      device and its bandwidth limitation
6017  * @dev: PCI device to query
6018  * @limiting_dev: storage for device causing the bandwidth limitation
6019  * @speed: storage for speed of limiting device
6020  * @width: storage for width of limiting device
6021  *
6022  * Walk up the PCI device chain and find the point where the minimum
6023  * bandwidth is available.  Return the bandwidth available there and (if
6024  * limiting_dev, speed, and width pointers are supplied) information about
6025  * that point.  The bandwidth returned is in Mb/s, i.e., megabits/second of
6026  * raw bandwidth.
6027  */
pcie_bandwidth_available(struct pci_dev * dev,struct pci_dev ** limiting_dev,enum pci_bus_speed * speed,enum pcie_link_width * width)6028 u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
6029 			     enum pci_bus_speed *speed,
6030 			     enum pcie_link_width *width)
6031 {
6032 	u16 lnksta;
6033 	enum pci_bus_speed next_speed;
6034 	enum pcie_link_width next_width;
6035 	u32 bw, next_bw;
6036 
6037 	if (speed)
6038 		*speed = PCI_SPEED_UNKNOWN;
6039 	if (width)
6040 		*width = PCIE_LNK_WIDTH_UNKNOWN;
6041 
6042 	bw = 0;
6043 
6044 	while (dev) {
6045 		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
6046 
6047 		next_speed = to_pcie_link_speed(lnksta);
6048 		next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
6049 
6050 		next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed);
6051 
6052 		/* Check if current device limits the total bandwidth */
6053 		if (!bw || next_bw <= bw) {
6054 			bw = next_bw;
6055 
6056 			if (limiting_dev)
6057 				*limiting_dev = dev;
6058 			if (speed)
6059 				*speed = next_speed;
6060 			if (width)
6061 				*width = next_width;
6062 		}
6063 
6064 		dev = pci_upstream_bridge(dev);
6065 	}
6066 
6067 	return bw;
6068 }
6069 EXPORT_SYMBOL(pcie_bandwidth_available);
6070 
6071 /**
6072  * pcie_get_supported_speeds - query Supported Link Speed Vector
6073  * @dev: PCI device to query
6074  *
6075  * Query @dev supported link speeds.
6076  *
6077  * Implementation Note in PCIe r6.0 sec 7.5.3.18 recommends determining
6078  * supported link speeds using the Supported Link Speeds Vector in the Link
6079  * Capabilities 2 Register (when available).
6080  *
6081  * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18.
6082  *
6083  * Without Link Capabilities 2, i.e., prior to PCIe r3.0, Supported Link
6084  * Speeds field in Link Capabilities is used and only 2.5 GT/s and 5.0 GT/s
6085  * speeds were defined.
6086  *
6087  * For @dev without Supported Link Speed Vector, the field is synthesized
6088  * from the Max Link Speed field in the Link Capabilities Register.
6089  *
6090  * Return: Supported Link Speeds Vector (+ reserved 0 at LSB).
6091  */
pcie_get_supported_speeds(struct pci_dev * dev)6092 u8 pcie_get_supported_speeds(struct pci_dev *dev)
6093 {
6094 	u32 lnkcap2, lnkcap;
6095 	u8 speeds;
6096 
6097 	/*
6098 	 * Speeds retain the reserved 0 at LSB before PCIe Supported Link
6099 	 * Speeds Vector to allow using SLS Vector bit defines directly.
6100 	 */
6101 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2);
6102 	speeds = lnkcap2 & PCI_EXP_LNKCAP2_SLS;
6103 
6104 	/* Ignore speeds higher than Max Link Speed */
6105 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
6106 	speeds &= GENMASK(lnkcap & PCI_EXP_LNKCAP_SLS, 0);
6107 
6108 	/* PCIe r3.0-compliant */
6109 	if (speeds)
6110 		return speeds;
6111 
6112 	/* Synthesize from the Max Link Speed field */
6113 	if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB)
6114 		speeds = PCI_EXP_LNKCAP2_SLS_5_0GB | PCI_EXP_LNKCAP2_SLS_2_5GB;
6115 	else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB)
6116 		speeds = PCI_EXP_LNKCAP2_SLS_2_5GB;
6117 
6118 	return speeds;
6119 }
6120 
6121 /**
6122  * pcie_get_speed_cap - query for the PCI device's link speed capability
6123  * @dev: PCI device to query
6124  *
6125  * Query the PCI device speed capability.
6126  *
6127  * Return: the maximum link speed supported by the device.
6128  */
pcie_get_speed_cap(struct pci_dev * dev)6129 enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev)
6130 {
6131 	return PCIE_LNKCAP2_SLS2SPEED(dev->supported_speeds);
6132 }
6133 EXPORT_SYMBOL(pcie_get_speed_cap);
6134 
6135 /**
6136  * pcie_get_width_cap - query for the PCI device's link width capability
6137  * @dev: PCI device to query
6138  *
6139  * Query the PCI device width capability.  Return the maximum link width
6140  * supported by the device.
6141  */
pcie_get_width_cap(struct pci_dev * dev)6142 enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev)
6143 {
6144 	u32 lnkcap;
6145 
6146 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
6147 	if (lnkcap)
6148 		return FIELD_GET(PCI_EXP_LNKCAP_MLW, lnkcap);
6149 
6150 	return PCIE_LNK_WIDTH_UNKNOWN;
6151 }
6152 EXPORT_SYMBOL(pcie_get_width_cap);
6153 
6154 /**
6155  * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability
6156  * @dev: PCI device
6157  * @speed: storage for link speed
6158  * @width: storage for link width
6159  *
6160  * Calculate a PCI device's link bandwidth by querying for its link speed
6161  * and width, multiplying them, and applying encoding overhead.  The result
6162  * is in Mb/s, i.e., megabits/second of raw bandwidth.
6163  */
pcie_bandwidth_capable(struct pci_dev * dev,enum pci_bus_speed * speed,enum pcie_link_width * width)6164 static u32 pcie_bandwidth_capable(struct pci_dev *dev,
6165 				  enum pci_bus_speed *speed,
6166 				  enum pcie_link_width *width)
6167 {
6168 	*speed = pcie_get_speed_cap(dev);
6169 	*width = pcie_get_width_cap(dev);
6170 
6171 	if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
6172 		return 0;
6173 
6174 	return *width * PCIE_SPEED2MBS_ENC(*speed);
6175 }
6176 
6177 /**
6178  * __pcie_print_link_status - Report the PCI device's link speed and width
6179  * @dev: PCI device to query
6180  * @verbose: Print info even when enough bandwidth is available
6181  *
6182  * If the available bandwidth at the device is less than the device is
6183  * capable of, report the device's maximum possible bandwidth and the
6184  * upstream link that limits its performance.  If @verbose, always print
6185  * the available bandwidth, even if the device isn't constrained.
6186  */
__pcie_print_link_status(struct pci_dev * dev,bool verbose)6187 void __pcie_print_link_status(struct pci_dev *dev, bool verbose)
6188 {
6189 	enum pcie_link_width width, width_cap;
6190 	enum pci_bus_speed speed, speed_cap;
6191 	struct pci_dev *limiting_dev = NULL;
6192 	u32 bw_avail, bw_cap;
6193 
6194 	bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap);
6195 	bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width);
6196 
6197 	if (bw_avail >= bw_cap && verbose)
6198 		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n",
6199 			 bw_cap / 1000, bw_cap % 1000,
6200 			 pci_speed_string(speed_cap), width_cap);
6201 	else if (bw_avail < bw_cap)
6202 		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n",
6203 			 bw_avail / 1000, bw_avail % 1000,
6204 			 pci_speed_string(speed), width,
6205 			 limiting_dev ? pci_name(limiting_dev) : "<unknown>",
6206 			 bw_cap / 1000, bw_cap % 1000,
6207 			 pci_speed_string(speed_cap), width_cap);
6208 }
6209 
6210 /**
6211  * pcie_print_link_status - Report the PCI device's link speed and width
6212  * @dev: PCI device to query
6213  *
6214  * Report the available bandwidth at the device.
6215  */
pcie_print_link_status(struct pci_dev * dev)6216 void pcie_print_link_status(struct pci_dev *dev)
6217 {
6218 	__pcie_print_link_status(dev, true);
6219 }
6220 EXPORT_SYMBOL(pcie_print_link_status);
6221 
6222 /**
6223  * pci_select_bars - Make BAR mask from the type of resource
6224  * @dev: the PCI device for which BAR mask is made
6225  * @flags: resource type mask to be selected
6226  *
6227  * This helper routine makes bar mask from the type of resource.
6228  */
pci_select_bars(struct pci_dev * dev,unsigned long flags)6229 int pci_select_bars(struct pci_dev *dev, unsigned long flags)
6230 {
6231 	int i, bars = 0;
6232 	for (i = 0; i < PCI_NUM_RESOURCES; i++)
6233 		if (pci_resource_flags(dev, i) & flags)
6234 			bars |= (1 << i);
6235 	return bars;
6236 }
6237 EXPORT_SYMBOL(pci_select_bars);
6238 
6239 /* Some architectures require additional programming to enable VGA */
6240 static arch_set_vga_state_t arch_set_vga_state;
6241 
pci_register_set_vga_state(arch_set_vga_state_t func)6242 void __init pci_register_set_vga_state(arch_set_vga_state_t func)
6243 {
6244 	arch_set_vga_state = func;	/* NULL disables */
6245 }
6246 
pci_set_vga_state_arch(struct pci_dev * dev,bool decode,unsigned int command_bits,u32 flags)6247 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
6248 				  unsigned int command_bits, u32 flags)
6249 {
6250 	if (arch_set_vga_state)
6251 		return arch_set_vga_state(dev, decode, command_bits,
6252 						flags);
6253 	return 0;
6254 }
6255 
6256 /**
6257  * pci_set_vga_state - set VGA decode state on device and parents if requested
6258  * @dev: the PCI device
6259  * @decode: true = enable decoding, false = disable decoding
6260  * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
6261  * @flags: traverse ancestors and change bridges
6262  * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
6263  */
pci_set_vga_state(struct pci_dev * dev,bool decode,unsigned int command_bits,u32 flags)6264 int pci_set_vga_state(struct pci_dev *dev, bool decode,
6265 		      unsigned int command_bits, u32 flags)
6266 {
6267 	struct pci_bus *bus;
6268 	struct pci_dev *bridge;
6269 	u16 cmd;
6270 	int rc;
6271 
6272 	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
6273 
6274 	/* ARCH specific VGA enables */
6275 	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
6276 	if (rc)
6277 		return rc;
6278 
6279 	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
6280 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
6281 		if (decode)
6282 			cmd |= command_bits;
6283 		else
6284 			cmd &= ~command_bits;
6285 		pci_write_config_word(dev, PCI_COMMAND, cmd);
6286 	}
6287 
6288 	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
6289 		return 0;
6290 
6291 	bus = dev->bus;
6292 	while (bus) {
6293 		bridge = bus->self;
6294 		if (bridge) {
6295 			pci_read_config_word(bridge, PCI_BRIDGE_CONTROL,
6296 					     &cmd);
6297 			if (decode)
6298 				cmd |= PCI_BRIDGE_CTL_VGA;
6299 			else
6300 				cmd &= ~PCI_BRIDGE_CTL_VGA;
6301 			pci_write_config_word(bridge, PCI_BRIDGE_CONTROL,
6302 					      cmd);
6303 		}
6304 		bus = bus->parent;
6305 	}
6306 	return 0;
6307 }
6308 
6309 #ifdef CONFIG_ACPI
pci_pr3_present(struct pci_dev * pdev)6310 bool pci_pr3_present(struct pci_dev *pdev)
6311 {
6312 	struct acpi_device *adev;
6313 
6314 	if (acpi_disabled)
6315 		return false;
6316 
6317 	adev = ACPI_COMPANION(&pdev->dev);
6318 	if (!adev)
6319 		return false;
6320 
6321 	return adev->power.flags.power_resources &&
6322 		acpi_has_method(adev->handle, "_PR3");
6323 }
6324 EXPORT_SYMBOL_GPL(pci_pr3_present);
6325 #endif
6326 
6327 /**
6328  * pci_add_dma_alias - Add a DMA devfn alias for a device
6329  * @dev: the PCI device for which alias is added
6330  * @devfn_from: alias slot and function
6331  * @nr_devfns: number of subsequent devfns to alias
6332  *
6333  * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask
6334  * which is used to program permissible bus-devfn source addresses for DMA
6335  * requests in an IOMMU.  These aliases factor into IOMMU group creation
6336  * and are useful for devices generating DMA requests beyond or different
6337  * from their logical bus-devfn.  Examples include device quirks where the
6338  * device simply uses the wrong devfn, as well as non-transparent bridges
6339  * where the alias may be a proxy for devices in another domain.
6340  *
6341  * IOMMU group creation is performed during device discovery or addition,
6342  * prior to any potential DMA mapping and therefore prior to driver probing
6343  * (especially for userspace assigned devices where IOMMU group definition
6344  * cannot be left as a userspace activity).  DMA aliases should therefore
6345  * be configured via quirks, such as the PCI fixup header quirk.
6346  */
pci_add_dma_alias(struct pci_dev * dev,u8 devfn_from,unsigned int nr_devfns)6347 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from,
6348 		       unsigned int nr_devfns)
6349 {
6350 	int devfn_to;
6351 
6352 	nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from);
6353 	devfn_to = devfn_from + nr_devfns - 1;
6354 
6355 	if (!dev->dma_alias_mask)
6356 		dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL);
6357 	if (!dev->dma_alias_mask) {
6358 		pci_warn(dev, "Unable to allocate DMA alias mask\n");
6359 		return;
6360 	}
6361 
6362 	bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns);
6363 
6364 	if (nr_devfns == 1)
6365 		pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n",
6366 				PCI_SLOT(devfn_from), PCI_FUNC(devfn_from));
6367 	else if (nr_devfns > 1)
6368 		pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n",
6369 				PCI_SLOT(devfn_from), PCI_FUNC(devfn_from),
6370 				PCI_SLOT(devfn_to), PCI_FUNC(devfn_to));
6371 }
6372 
pci_devs_are_dma_aliases(struct pci_dev * dev1,struct pci_dev * dev2)6373 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
6374 {
6375 	return (dev1->dma_alias_mask &&
6376 		test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
6377 	       (dev2->dma_alias_mask &&
6378 		test_bit(dev1->devfn, dev2->dma_alias_mask)) ||
6379 	       pci_real_dma_dev(dev1) == dev2 ||
6380 	       pci_real_dma_dev(dev2) == dev1;
6381 }
6382 
pci_device_is_present(struct pci_dev * pdev)6383 bool pci_device_is_present(struct pci_dev *pdev)
6384 {
6385 	u32 v;
6386 
6387 	/* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */
6388 	pdev = pci_physfn(pdev);
6389 	if (pci_dev_is_disconnected(pdev))
6390 		return false;
6391 	return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0);
6392 }
6393 EXPORT_SYMBOL_GPL(pci_device_is_present);
6394 
pci_ignore_hotplug(struct pci_dev * dev)6395 void pci_ignore_hotplug(struct pci_dev *dev)
6396 {
6397 	struct pci_dev *bridge = dev->bus->self;
6398 
6399 	dev->ignore_hotplug = 1;
6400 	/* Propagate the "ignore hotplug" setting to the parent bridge. */
6401 	if (bridge)
6402 		bridge->ignore_hotplug = 1;
6403 }
6404 EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
6405 
6406 /**
6407  * pci_real_dma_dev - Get PCI DMA device for PCI device
6408  * @dev: the PCI device that may have a PCI DMA alias
6409  *
6410  * Permits the platform to provide architecture-specific functionality to
6411  * devices needing to alias DMA to another PCI device on another PCI bus. If
6412  * the PCI device is on the same bus, it is recommended to use
6413  * pci_add_dma_alias(). This is the default implementation. Architecture
6414  * implementations can override this.
6415  */
pci_real_dma_dev(struct pci_dev * dev)6416 struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev)
6417 {
6418 	return dev;
6419 }
6420 
pcibios_default_alignment(void)6421 resource_size_t __weak pcibios_default_alignment(void)
6422 {
6423 	return 0;
6424 }
6425 
6426 /*
6427  * Arches that don't want to expose struct resource to userland as-is in
6428  * sysfs and /proc can implement their own pci_resource_to_user().
6429  */
pci_resource_to_user(const struct pci_dev * dev,int bar,const struct resource * rsrc,resource_size_t * start,resource_size_t * end)6430 void __weak pci_resource_to_user(const struct pci_dev *dev, int bar,
6431 				 const struct resource *rsrc,
6432 				 resource_size_t *start, resource_size_t *end)
6433 {
6434 	*start = rsrc->start;
6435 	*end = rsrc->end;
6436 }
6437 
6438 static char *resource_alignment_param;
6439 static DEFINE_SPINLOCK(resource_alignment_lock);
6440 
6441 /**
6442  * pci_specified_resource_alignment - get resource alignment specified by user.
6443  * @dev: the PCI device to get
6444  * @resize: whether or not to change resources' size when reassigning alignment
6445  *
6446  * RETURNS: Resource alignment if it is specified.
6447  *          Zero if it is not specified.
6448  */
pci_specified_resource_alignment(struct pci_dev * dev,bool * resize)6449 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
6450 							bool *resize)
6451 {
6452 	int align_order, count;
6453 	resource_size_t align = pcibios_default_alignment();
6454 	const char *p;
6455 	int ret;
6456 
6457 	spin_lock(&resource_alignment_lock);
6458 	p = resource_alignment_param;
6459 	if (!p || !*p)
6460 		goto out;
6461 	if (pci_has_flag(PCI_PROBE_ONLY)) {
6462 		align = 0;
6463 		pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
6464 		goto out;
6465 	}
6466 
6467 	while (*p) {
6468 		count = 0;
6469 		if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
6470 		    p[count] == '@') {
6471 			p += count + 1;
6472 			if (align_order > 63) {
6473 				pr_err("PCI: Invalid requested alignment (order %d)\n",
6474 				       align_order);
6475 				align_order = PAGE_SHIFT;
6476 			}
6477 		} else {
6478 			align_order = PAGE_SHIFT;
6479 		}
6480 
6481 		ret = pci_dev_str_match(dev, p, &p);
6482 		if (ret == 1) {
6483 			*resize = true;
6484 			align = 1ULL << align_order;
6485 			break;
6486 		} else if (ret < 0) {
6487 			pr_err("PCI: Can't parse resource_alignment parameter: %s\n",
6488 			       p);
6489 			break;
6490 		}
6491 
6492 		if (*p != ';' && *p != ',') {
6493 			/* End of param or invalid format */
6494 			break;
6495 		}
6496 		p++;
6497 	}
6498 out:
6499 	spin_unlock(&resource_alignment_lock);
6500 	return align;
6501 }
6502 
pci_request_resource_alignment(struct pci_dev * dev,int bar,resource_size_t align,bool resize)6503 static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
6504 					   resource_size_t align, bool resize)
6505 {
6506 	struct resource *r = &dev->resource[bar];
6507 	const char *r_name = pci_resource_name(dev, bar);
6508 	resource_size_t size;
6509 
6510 	if (!(r->flags & IORESOURCE_MEM))
6511 		return;
6512 
6513 	if (r->flags & IORESOURCE_PCI_FIXED) {
6514 		pci_info(dev, "%s %pR: ignoring requested alignment %#llx\n",
6515 			 r_name, r, (unsigned long long)align);
6516 		return;
6517 	}
6518 
6519 	size = resource_size(r);
6520 	if (size >= align)
6521 		return;
6522 
6523 	/*
6524 	 * Increase the alignment of the resource.  There are two ways we
6525 	 * can do this:
6526 	 *
6527 	 * 1) Increase the size of the resource.  BARs are aligned on their
6528 	 *    size, so when we reallocate space for this resource, we'll
6529 	 *    allocate it with the larger alignment.  This also prevents
6530 	 *    assignment of any other BARs inside the alignment region, so
6531 	 *    if we're requesting page alignment, this means no other BARs
6532 	 *    will share the page.
6533 	 *
6534 	 *    The disadvantage is that this makes the resource larger than
6535 	 *    the hardware BAR, which may break drivers that compute things
6536 	 *    based on the resource size, e.g., to find registers at a
6537 	 *    fixed offset before the end of the BAR.
6538 	 *
6539 	 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
6540 	 *    set r->start to the desired alignment.  By itself this
6541 	 *    doesn't prevent other BARs being put inside the alignment
6542 	 *    region, but if we realign *every* resource of every device in
6543 	 *    the system, none of them will share an alignment region.
6544 	 *
6545 	 * When the user has requested alignment for only some devices via
6546 	 * the "pci=resource_alignment" argument, "resize" is true and we
6547 	 * use the first method.  Otherwise we assume we're aligning all
6548 	 * devices and we use the second.
6549 	 */
6550 
6551 	pci_info(dev, "%s %pR: requesting alignment to %#llx\n",
6552 		 r_name, r, (unsigned long long)align);
6553 
6554 	if (resize) {
6555 		r->start = 0;
6556 		r->end = align - 1;
6557 	} else {
6558 		r->flags &= ~IORESOURCE_SIZEALIGN;
6559 		r->flags |= IORESOURCE_STARTALIGN;
6560 		resource_set_range(r, align, size);
6561 	}
6562 	r->flags |= IORESOURCE_UNSET;
6563 }
6564 
6565 /*
6566  * This function disables memory decoding and releases memory resources
6567  * of the device specified by kernel's boot parameter 'pci=resource_alignment='.
6568  * It also rounds up size to specified alignment.
6569  * Later on, the kernel will assign page-aligned memory resource back
6570  * to the device.
6571  */
pci_reassigndev_resource_alignment(struct pci_dev * dev)6572 void pci_reassigndev_resource_alignment(struct pci_dev *dev)
6573 {
6574 	int i;
6575 	struct resource *r;
6576 	resource_size_t align;
6577 	u16 command;
6578 	bool resize = false;
6579 
6580 	/*
6581 	 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec
6582 	 * 3.4.1.11.  Their resources are allocated from the space
6583 	 * described by the VF BARx register in the PF's SR-IOV capability.
6584 	 * We can't influence their alignment here.
6585 	 */
6586 	if (dev->is_virtfn)
6587 		return;
6588 
6589 	/* check if specified PCI is target device to reassign */
6590 	align = pci_specified_resource_alignment(dev, &resize);
6591 	if (!align)
6592 		return;
6593 
6594 	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
6595 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
6596 		pci_warn(dev, "Can't reassign resources to host bridge\n");
6597 		return;
6598 	}
6599 
6600 	pci_read_config_word(dev, PCI_COMMAND, &command);
6601 	command &= ~PCI_COMMAND_MEMORY;
6602 	pci_write_config_word(dev, PCI_COMMAND, command);
6603 
6604 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
6605 		pci_request_resource_alignment(dev, i, align, resize);
6606 
6607 	/*
6608 	 * Need to disable bridge's resource window,
6609 	 * to enable the kernel to reassign new resource
6610 	 * window later on.
6611 	 */
6612 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
6613 		for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
6614 			r = &dev->resource[i];
6615 			if (!(r->flags & IORESOURCE_MEM))
6616 				continue;
6617 			r->flags |= IORESOURCE_UNSET;
6618 			r->end = resource_size(r) - 1;
6619 			r->start = 0;
6620 		}
6621 		pci_disable_bridge_window(dev);
6622 	}
6623 }
6624 
resource_alignment_show(const struct bus_type * bus,char * buf)6625 static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf)
6626 {
6627 	size_t count = 0;
6628 
6629 	spin_lock(&resource_alignment_lock);
6630 	if (resource_alignment_param)
6631 		count = sysfs_emit(buf, "%s\n", resource_alignment_param);
6632 	spin_unlock(&resource_alignment_lock);
6633 
6634 	return count;
6635 }
6636 
resource_alignment_store(const struct bus_type * bus,const char * buf,size_t count)6637 static ssize_t resource_alignment_store(const struct bus_type *bus,
6638 					const char *buf, size_t count)
6639 {
6640 	char *param, *old, *end;
6641 
6642 	if (count >= (PAGE_SIZE - 1))
6643 		return -EINVAL;
6644 
6645 	param = kstrndup(buf, count, GFP_KERNEL);
6646 	if (!param)
6647 		return -ENOMEM;
6648 
6649 	end = strchr(param, '\n');
6650 	if (end)
6651 		*end = '\0';
6652 
6653 	spin_lock(&resource_alignment_lock);
6654 	old = resource_alignment_param;
6655 	if (strlen(param)) {
6656 		resource_alignment_param = param;
6657 	} else {
6658 		kfree(param);
6659 		resource_alignment_param = NULL;
6660 	}
6661 	spin_unlock(&resource_alignment_lock);
6662 
6663 	kfree(old);
6664 
6665 	return count;
6666 }
6667 
6668 static BUS_ATTR_RW(resource_alignment);
6669 
pci_resource_alignment_sysfs_init(void)6670 static int __init pci_resource_alignment_sysfs_init(void)
6671 {
6672 	return bus_create_file(&pci_bus_type,
6673 					&bus_attr_resource_alignment);
6674 }
6675 late_initcall(pci_resource_alignment_sysfs_init);
6676 
pci_no_domains(void)6677 static void pci_no_domains(void)
6678 {
6679 #ifdef CONFIG_PCI_DOMAINS
6680 	pci_domains_supported = 0;
6681 #endif
6682 }
6683 
6684 #ifdef CONFIG_PCI_DOMAINS_GENERIC
6685 static DEFINE_IDA(pci_domain_nr_static_ida);
6686 static DEFINE_IDA(pci_domain_nr_dynamic_ida);
6687 
of_pci_reserve_static_domain_nr(void)6688 static void of_pci_reserve_static_domain_nr(void)
6689 {
6690 	struct device_node *np;
6691 	int domain_nr;
6692 
6693 	for_each_node_by_type(np, "pci") {
6694 		domain_nr = of_get_pci_domain_nr(np);
6695 		if (domain_nr < 0)
6696 			continue;
6697 		/*
6698 		 * Permanently allocate domain_nr in dynamic_ida
6699 		 * to prevent it from dynamic allocation.
6700 		 */
6701 		ida_alloc_range(&pci_domain_nr_dynamic_ida,
6702 				domain_nr, domain_nr, GFP_KERNEL);
6703 	}
6704 }
6705 
of_pci_bus_find_domain_nr(struct device * parent)6706 static int of_pci_bus_find_domain_nr(struct device *parent)
6707 {
6708 	static bool static_domains_reserved = false;
6709 	int domain_nr;
6710 
6711 	/* On the first call scan device tree for static allocations. */
6712 	if (!static_domains_reserved) {
6713 		of_pci_reserve_static_domain_nr();
6714 		static_domains_reserved = true;
6715 	}
6716 
6717 	if (parent) {
6718 		/*
6719 		 * If domain is in DT, allocate it in static IDA.  This
6720 		 * prevents duplicate static allocations in case of errors
6721 		 * in DT.
6722 		 */
6723 		domain_nr = of_get_pci_domain_nr(parent->of_node);
6724 		if (domain_nr >= 0)
6725 			return ida_alloc_range(&pci_domain_nr_static_ida,
6726 					       domain_nr, domain_nr,
6727 					       GFP_KERNEL);
6728 	}
6729 
6730 	/*
6731 	 * If domain was not specified in DT, choose a free ID from dynamic
6732 	 * allocations. All domain numbers from DT are permanently in
6733 	 * dynamic allocations to prevent assigning them to other DT nodes
6734 	 * without static domain.
6735 	 */
6736 	return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL);
6737 }
6738 
of_pci_bus_release_domain_nr(struct device * parent,int domain_nr)6739 static void of_pci_bus_release_domain_nr(struct device *parent, int domain_nr)
6740 {
6741 	if (domain_nr < 0)
6742 		return;
6743 
6744 	/* Release domain from IDA where it was allocated. */
6745 	if (of_get_pci_domain_nr(parent->of_node) == domain_nr)
6746 		ida_free(&pci_domain_nr_static_ida, domain_nr);
6747 	else
6748 		ida_free(&pci_domain_nr_dynamic_ida, domain_nr);
6749 }
6750 
pci_bus_find_domain_nr(struct pci_bus * bus,struct device * parent)6751 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
6752 {
6753 	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
6754 			       acpi_pci_bus_find_domain_nr(bus);
6755 }
6756 
pci_bus_release_domain_nr(struct device * parent,int domain_nr)6757 void pci_bus_release_domain_nr(struct device *parent, int domain_nr)
6758 {
6759 	if (!acpi_disabled)
6760 		return;
6761 	of_pci_bus_release_domain_nr(parent, domain_nr);
6762 }
6763 #endif
6764 
6765 /**
6766  * pci_ext_cfg_avail - can we access extended PCI config space?
6767  *
6768  * Returns 1 if we can access PCI extended config space (offsets
6769  * greater than 0xff). This is the default implementation. Architecture
6770  * implementations can override this.
6771  */
pci_ext_cfg_avail(void)6772 int __weak pci_ext_cfg_avail(void)
6773 {
6774 	return 1;
6775 }
6776 
pci_fixup_cardbus(struct pci_bus * bus)6777 void __weak pci_fixup_cardbus(struct pci_bus *bus)
6778 {
6779 }
6780 EXPORT_SYMBOL(pci_fixup_cardbus);
6781 
pci_setup(char * str)6782 static int __init pci_setup(char *str)
6783 {
6784 	while (str) {
6785 		char *k = strchr(str, ',');
6786 		if (k)
6787 			*k++ = 0;
6788 		if (*str && (str = pcibios_setup(str)) && *str) {
6789 			if (!strcmp(str, "nomsi")) {
6790 				pci_no_msi();
6791 			} else if (!strncmp(str, "noats", 5)) {
6792 				pr_info("PCIe: ATS is disabled\n");
6793 				pcie_ats_disabled = true;
6794 			} else if (!strcmp(str, "noaer")) {
6795 				pci_no_aer();
6796 			} else if (!strcmp(str, "earlydump")) {
6797 				pci_early_dump = true;
6798 			} else if (!strncmp(str, "realloc=", 8)) {
6799 				pci_realloc_get_opt(str + 8);
6800 			} else if (!strncmp(str, "realloc", 7)) {
6801 				pci_realloc_get_opt("on");
6802 			} else if (!strcmp(str, "nodomains")) {
6803 				pci_no_domains();
6804 			} else if (!strncmp(str, "noari", 5)) {
6805 				pcie_ari_disabled = true;
6806 			} else if (!strncmp(str, "notph", 5)) {
6807 				pci_no_tph();
6808 			} else if (!strncmp(str, "cbiosize=", 9)) {
6809 				pci_cardbus_io_size = memparse(str + 9, &str);
6810 			} else if (!strncmp(str, "cbmemsize=", 10)) {
6811 				pci_cardbus_mem_size = memparse(str + 10, &str);
6812 			} else if (!strncmp(str, "resource_alignment=", 19)) {
6813 				resource_alignment_param = str + 19;
6814 			} else if (!strncmp(str, "ecrc=", 5)) {
6815 				pcie_ecrc_get_policy(str + 5);
6816 			} else if (!strncmp(str, "hpiosize=", 9)) {
6817 				pci_hotplug_io_size = memparse(str + 9, &str);
6818 			} else if (!strncmp(str, "hpmmiosize=", 11)) {
6819 				pci_hotplug_mmio_size = memparse(str + 11, &str);
6820 			} else if (!strncmp(str, "hpmmioprefsize=", 15)) {
6821 				pci_hotplug_mmio_pref_size = memparse(str + 15, &str);
6822 			} else if (!strncmp(str, "hpmemsize=", 10)) {
6823 				pci_hotplug_mmio_size = memparse(str + 10, &str);
6824 				pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size;
6825 			} else if (!strncmp(str, "hpbussize=", 10)) {
6826 				pci_hotplug_bus_size =
6827 					simple_strtoul(str + 10, &str, 0);
6828 				if (pci_hotplug_bus_size > 0xff)
6829 					pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
6830 			} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
6831 				pcie_bus_config = PCIE_BUS_TUNE_OFF;
6832 			} else if (!strncmp(str, "pcie_bus_safe", 13)) {
6833 				pcie_bus_config = PCIE_BUS_SAFE;
6834 			} else if (!strncmp(str, "pcie_bus_perf", 13)) {
6835 				pcie_bus_config = PCIE_BUS_PERFORMANCE;
6836 			} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
6837 				pcie_bus_config = PCIE_BUS_PEER2PEER;
6838 			} else if (!strncmp(str, "pcie_scan_all", 13)) {
6839 				pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS);
6840 			} else if (!strncmp(str, "disable_acs_redir=", 18)) {
6841 				disable_acs_redir_param = str + 18;
6842 			} else if (!strncmp(str, "config_acs=", 11)) {
6843 				config_acs_param = str + 11;
6844 			} else {
6845 				pr_err("PCI: Unknown option `%s'\n", str);
6846 			}
6847 		}
6848 		str = k;
6849 	}
6850 	return 0;
6851 }
6852 early_param("pci", pci_setup);
6853 
6854 /*
6855  * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized
6856  * in pci_setup(), above, to point to data in the __initdata section which
6857  * will be freed after the init sequence is complete. We can't allocate memory
6858  * in pci_setup() because some architectures do not have any memory allocation
6859  * service available during an early_param() call. So we allocate memory and
6860  * copy the variable here before the init section is freed.
6861  *
6862  */
pci_realloc_setup_params(void)6863 static int __init pci_realloc_setup_params(void)
6864 {
6865 	resource_alignment_param = kstrdup(resource_alignment_param,
6866 					   GFP_KERNEL);
6867 	disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL);
6868 	config_acs_param = kstrdup(config_acs_param, GFP_KERNEL);
6869 
6870 	return 0;
6871 }
6872 pure_initcall(pci_realloc_setup_params);
6873